mirror of
https://github.com/netzbegruenung/green-spider.git
synced 2024-05-08 03:43:41 +02:00
Detect and report responsive layout details
This commit is contained in:
parent
10cf0de2ab
commit
914fb1f35e
|
@ -17,3 +17,5 @@ Wir prüfen Sites nach den folgenden Kriterien:
|
||||||
- `FEEDS`: Die Site verweist auf RSS oder Atom Feeds via `rel=alternate` Link Tag.
|
- `FEEDS`: Die Site verweist auf RSS oder Atom Feeds via `rel=alternate` Link Tag.
|
||||||
|
|
||||||
- `HTTP_RESPONSE_DURATION`: Zeit, die vom Absenden des HTTP-Request bis zum Empfang der Response-Header vergangen ist.
|
- `HTTP_RESPONSE_DURATION`: Zeit, die vom Absenden des HTTP-Request bis zum Empfang der Response-Header vergangen ist.
|
||||||
|
|
||||||
|
- `RESPONSIVE`: Die Seite besitzt ein `viewport` Meta-Tag und die Breite der Inhalte passt sich an verschiedene Fenster- bzw. Gerätegrößen an.
|
||||||
|
|
|
@ -6,5 +6,6 @@ GitPython==2.1.9
|
||||||
idna==2.6
|
idna==2.6
|
||||||
PyYAML==3.12
|
PyYAML==3.12
|
||||||
requests==2.18.4
|
requests==2.18.4
|
||||||
|
selenium==3.11.0
|
||||||
smmap2==2.0.3
|
smmap2==2.0.3
|
||||||
urllib3==1.22
|
urllib3==1.22
|
||||||
|
|
72
spider.py
72
spider.py
|
@ -3,6 +3,7 @@
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from git import Repo
|
from git import Repo
|
||||||
from multiprocessing import Pool
|
from multiprocessing import Pool
|
||||||
|
from selenium import webdriver
|
||||||
from socket import gethostbyname_ex
|
from socket import gethostbyname_ex
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urljoin
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
@ -132,6 +133,45 @@ def normalize_title(s):
|
||||||
s = s.strip()
|
s = s.strip()
|
||||||
return s
|
return s
|
||||||
|
|
||||||
|
def check_responsiveness(url):
|
||||||
|
"""
|
||||||
|
Checks
|
||||||
|
- whether a page adapts to different viewport sizes
|
||||||
|
- whether a viewport meta tag exists
|
||||||
|
and returns details
|
||||||
|
"""
|
||||||
|
details = {
|
||||||
|
'document_width': {},
|
||||||
|
'viewport_meta_tag': None,
|
||||||
|
}
|
||||||
|
|
||||||
|
# sizes we check for (width, height)
|
||||||
|
sizes = (
|
||||||
|
(320,480), # old smartphone
|
||||||
|
(768,1024), # older tablet or newer smartphone
|
||||||
|
(1024,768), # older desktop or horiz. tablet
|
||||||
|
(1920, 1080), # Full HD horizontal
|
||||||
|
)
|
||||||
|
|
||||||
|
# Our selenium user agent using PhantomJS/Webkit as an engine
|
||||||
|
driver = webdriver.PhantomJS()
|
||||||
|
driver.set_window_size(sizes[0][0], sizes[0][1])
|
||||||
|
driver.get(url)
|
||||||
|
|
||||||
|
for (width, height) in sizes:
|
||||||
|
driver.set_window_size(width, height)
|
||||||
|
key = "%sx%s" % (width, height)
|
||||||
|
width = driver.execute_script("return document.body.scrollWidth")
|
||||||
|
details['document_width'][key] = int(width)
|
||||||
|
|
||||||
|
try:
|
||||||
|
element = driver.find_element_by_xpath("//meta[@name='viewport']")
|
||||||
|
details['viewport_meta_tag'] = element.get_attribute('content')
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return details
|
||||||
|
|
||||||
def check_content(r):
|
def check_content(r):
|
||||||
"""
|
"""
|
||||||
Adds details to check regarding content of the page
|
Adds details to check regarding content of the page
|
||||||
|
@ -261,6 +301,7 @@ def check_site(entry):
|
||||||
'icons': [],
|
'icons': [],
|
||||||
'feeds': [],
|
'feeds': [],
|
||||||
'cms': None,
|
'cms': None,
|
||||||
|
'responsive': None,
|
||||||
},
|
},
|
||||||
# The actual report criteria
|
# The actual report criteria
|
||||||
'result': {
|
'result': {
|
||||||
|
@ -272,6 +313,7 @@ def check_site(entry):
|
||||||
'FAVICON': {'type': 'boolean', 'value': False, 'score': 0},
|
'FAVICON': {'type': 'boolean', 'value': False, 'score': 0},
|
||||||
'FEEDS': {'type': 'boolean', 'value': False, 'score': 0},
|
'FEEDS': {'type': 'boolean', 'value': False, 'score': 0},
|
||||||
'HTTP_RESPONSE_DURATION': {'type': 'number', 'value': None, 'score': 0},
|
'HTTP_RESPONSE_DURATION': {'type': 'number', 'value': None, 'score': 0},
|
||||||
|
'RESPONSIVE': {'type': 'boolean', 'value': False, 'score': 0},
|
||||||
},
|
},
|
||||||
'score': 0.0,
|
'score': 0.0,
|
||||||
}
|
}
|
||||||
|
@ -357,6 +399,7 @@ def check_site(entry):
|
||||||
'duration': None,
|
'duration': None,
|
||||||
'error': None,
|
'error': None,
|
||||||
'content': None,
|
'content': None,
|
||||||
|
'responsive': None,
|
||||||
}
|
}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -368,6 +411,12 @@ def check_site(entry):
|
||||||
if r.status_code < 300:
|
if r.status_code < 300:
|
||||||
check['content'] = check_content(r)
|
check['content'] = check_content(r)
|
||||||
|
|
||||||
|
# Responsiveness check
|
||||||
|
try:
|
||||||
|
check['responsive'] = check_responsiveness(check_url)
|
||||||
|
except Exception as e:
|
||||||
|
logging.error("Error when checking responsiveness for '%s': %s" % (check_url, e))
|
||||||
|
|
||||||
except requests.exceptions.ConnectionError as e:
|
except requests.exceptions.ConnectionError as e:
|
||||||
logging.error(str(e) + " " + check_url)
|
logging.error(str(e) + " " + check_url)
|
||||||
check['error'] = "connection"
|
check['error'] = "connection"
|
||||||
|
@ -409,6 +458,22 @@ def check_site(entry):
|
||||||
feeds.add(feed)
|
feeds.add(feed)
|
||||||
result['details']['feeds'] = sorted(list(feeds))
|
result['details']['feeds'] = sorted(list(feeds))
|
||||||
|
|
||||||
|
# detect responsive
|
||||||
|
viewports = set()
|
||||||
|
min_width = 2000
|
||||||
|
for c in result['details']['urlchecks']:
|
||||||
|
if c['responsive'] is None:
|
||||||
|
continue
|
||||||
|
if c['responsive']['viewport_meta_tag'] is not None:
|
||||||
|
viewports.add(c['responsive']['viewport_meta_tag'])
|
||||||
|
widths = c['responsive']['document_width'].values()
|
||||||
|
if min(widths) < min_width:
|
||||||
|
min_width = min(widths)
|
||||||
|
result['details']['responsive'] = {
|
||||||
|
'viewport_meta_tag': list(viewports),
|
||||||
|
'min_width': min_width,
|
||||||
|
}
|
||||||
|
|
||||||
# detect CMS
|
# detect CMS
|
||||||
for c in result['details']['urlchecks']:
|
for c in result['details']['urlchecks']:
|
||||||
if c['content'] is None:
|
if c['content'] is None:
|
||||||
|
@ -503,6 +568,13 @@ def check_site(entry):
|
||||||
elif val < 1000:
|
elif val < 1000:
|
||||||
result['result']['HTTP_RESPONSE_DURATION']['score'] = 0.5
|
result['result']['HTTP_RESPONSE_DURATION']['score'] = 0.5
|
||||||
|
|
||||||
|
# RESPONSIVE
|
||||||
|
if result['details']['responsive'] is not None:
|
||||||
|
if (result['details']['responsive']['min_width'] < 500 and
|
||||||
|
len(result['details']['responsive']['viewport_meta_tag']) > 0):
|
||||||
|
result['result']['RESPONSIVE']['value'] = True
|
||||||
|
result['result']['RESPONSIVE']['score'] = 1
|
||||||
|
|
||||||
# Overall score
|
# Overall score
|
||||||
for item in result['result'].keys():
|
for item in result['result'].keys():
|
||||||
result['score'] += result['result'][item]['score']
|
result['score'] += result['result'][item]['score']
|
||||||
|
|
3
webapp/dist/index.html
vendored
3
webapp/dist/index.html
vendored
|
@ -61,9 +61,10 @@
|
||||||
<th scope="col">Erreichbar</th>
|
<th scope="col">Erreichbar</th>
|
||||||
<th scope="col">Antwortzeit</th>
|
<th scope="col">Antwortzeit</th>
|
||||||
<th scope="col">Icon</th>
|
<th scope="col">Icon</th>
|
||||||
|
<th scope="col"><abbr title="Site nutzt HTTP-Verschlüsselung">HTTPS</abbr></th>
|
||||||
<th scope="col"><abbr title="Site ist sowohl mit www. als auch ohne www. in URL erreichbar">www. optional</abbr></th>
|
<th scope="col"><abbr title="Site ist sowohl mit www. als auch ohne www. in URL erreichbar">www. optional</abbr></th>
|
||||||
<th scope="col"><abbr title="URL-Varianten leiten auf eine einzige Startseiten-URL weiter">Kanonische URL</abbr></th>
|
<th scope="col"><abbr title="URL-Varianten leiten auf eine einzige Startseiten-URL weiter">Kanonische URL</abbr></th>
|
||||||
<th scope="col"><abbr title="Site nutzt HTTP-Verschlüsselung">HTTPS</abbr></th>
|
<th scope="col">Responsive</th>
|
||||||
<th scope="col">Feed</th>
|
<th scope="col">Feed</th>
|
||||||
<th scope="col">Screenshots</th>
|
<th scope="col">Screenshots</th>
|
||||||
<th scope="col">CMS</th>
|
<th scope="col">CMS</th>
|
||||||
|
|
|
@ -68,6 +68,10 @@ $(function(){
|
||||||
var icon = item.result.FAVICON.value;
|
var icon = item.result.FAVICON.value;
|
||||||
row.append('<td class="' + (icon ? 'good' : 'bad') + ' text-center">' + (icon ? ('<img src="' + item.details.icons[0] + '" class="icon">') : '❌') + '</td>');
|
row.append('<td class="' + (icon ? 'good' : 'bad') + ' text-center">' + (icon ? ('<img src="' + item.details.icons[0] + '" class="icon">') : '❌') + '</td>');
|
||||||
|
|
||||||
|
// HTTPS
|
||||||
|
var hasHTTPS = item.result.HTTPS.value;
|
||||||
|
row.append('<td class="'+ (hasHTTPS ? 'good' : 'bad') +' text-center">' + (hasHTTPS ? '✅' : '❌') + '</td>');
|
||||||
|
|
||||||
// WWW_OPTIONAL
|
// WWW_OPTIONAL
|
||||||
var wwwOptional = item.result.WWW_OPTIONAL.value;
|
var wwwOptional = item.result.WWW_OPTIONAL.value;
|
||||||
row.append('<td class="'+ (wwwOptional ? 'good' : 'bad') +' text-center">' + (wwwOptional ? '✅' : '❌') + '</td>');
|
row.append('<td class="'+ (wwwOptional ? 'good' : 'bad') +' text-center">' + (wwwOptional ? '✅' : '❌') + '</td>');
|
||||||
|
@ -76,9 +80,8 @@ $(function(){
|
||||||
var canonical = item.result.CANONICAL_URL.value;
|
var canonical = item.result.CANONICAL_URL.value;
|
||||||
row.append('<td class="'+ (canonical ? 'good' : 'bad') +' text-center">' + (canonical ? '✅' : '❌') + '</td>');
|
row.append('<td class="'+ (canonical ? 'good' : 'bad') +' text-center">' + (canonical ? '✅' : '❌') + '</td>');
|
||||||
|
|
||||||
// https
|
var responsive = item.result.RESPONSIVE.value;
|
||||||
var hasHTTPS = item.result.HTTPS.value;
|
row.append('<td class="'+ (responsive ? 'good' : 'bad') +' text-center">' + (responsive ? '✅' : '❌') + '</td>');
|
||||||
row.append('<td class="'+ (hasHTTPS ? 'good' : 'bad') +' text-center">' + (hasHTTPS ? '✅' : '❌') + '</td>');
|
|
||||||
|
|
||||||
// feeds
|
// feeds
|
||||||
var feeds = item.result.FEEDS.value;
|
var feeds = item.result.FEEDS.value;
|
||||||
|
|
Loading…
Reference in a new issue