Browse Source

Detect and report responsive layout details

pull/29/head
Marian Steinbach 4 years ago
parent
commit
914fb1f35e
  1. 2
      KRITERIEN.md
  2. 1
      requirements.txt
  3. 72
      spider.py
  4. 3
      webapp/dist/index.html
  5. 9
      webapp/src/index.js

2
KRITERIEN.md

@ -17,3 +17,5 @@ Wir prüfen Sites nach den folgenden Kriterien:
- `FEEDS`: Die Site verweist auf RSS oder Atom Feeds via `rel=alternate` Link Tag.
- `HTTP_RESPONSE_DURATION`: Zeit, die vom Absenden des HTTP-Request bis zum Empfang der Response-Header vergangen ist.
- `RESPONSIVE`: Die Seite besitzt ein `viewport` Meta-Tag und die Breite der Inhalte passt sich an verschiedene Fenster- bzw. Gerätegrößen an.

1
requirements.txt

@ -6,5 +6,6 @@ GitPython==2.1.9
idna==2.6
PyYAML==3.12
requests==2.18.4
selenium==3.11.0
smmap2==2.0.3
urllib3==1.22

72
spider.py

@ -3,6 +3,7 @@
from bs4 import BeautifulSoup
from git import Repo
from multiprocessing import Pool
from selenium import webdriver
from socket import gethostbyname_ex
from urllib.parse import urljoin
from urllib.parse import urlparse
@ -132,6 +133,45 @@ def normalize_title(s):
s = s.strip()
return s
def check_responsiveness(url):
"""
Checks
- whether a page adapts to different viewport sizes
- whether a viewport meta tag exists
and returns details
"""
details = {
'document_width': {},
'viewport_meta_tag': None,
}
# sizes we check for (width, height)
sizes = (
(320,480), # old smartphone
(768,1024), # older tablet or newer smartphone
(1024,768), # older desktop or horiz. tablet
(1920, 1080), # Full HD horizontal
)
# Our selenium user agent using PhantomJS/Webkit as an engine
driver = webdriver.PhantomJS()
driver.set_window_size(sizes[0][0], sizes[0][1])
driver.get(url)
for (width, height) in sizes:
driver.set_window_size(width, height)
key = "%sx%s" % (width, height)
width = driver.execute_script("return document.body.scrollWidth")
details['document_width'][key] = int(width)
try:
element = driver.find_element_by_xpath("//meta[@name='viewport']")
details['viewport_meta_tag'] = element.get_attribute('content')
except:
pass
return details
def check_content(r):
"""
Adds details to check regarding content of the page
@ -261,6 +301,7 @@ def check_site(entry):
'icons': [],
'feeds': [],
'cms': None,
'responsive': None,
},
# The actual report criteria
'result': {
@ -272,6 +313,7 @@ def check_site(entry):
'FAVICON': {'type': 'boolean', 'value': False, 'score': 0},
'FEEDS': {'type': 'boolean', 'value': False, 'score': 0},
'HTTP_RESPONSE_DURATION': {'type': 'number', 'value': None, 'score': 0},
'RESPONSIVE': {'type': 'boolean', 'value': False, 'score': 0},
},
'score': 0.0,
}
@ -357,6 +399,7 @@ def check_site(entry):
'duration': None,
'error': None,
'content': None,
'responsive': None,
}
try:
@ -368,6 +411,12 @@ def check_site(entry):
if r.status_code < 300:
check['content'] = check_content(r)
# Responsiveness check
try:
check['responsive'] = check_responsiveness(check_url)
except Exception as e:
logging.error("Error when checking responsiveness for '%s': %s" % (check_url, e))
except requests.exceptions.ConnectionError as e:
logging.error(str(e) + " " + check_url)
check['error'] = "connection"
@ -409,6 +458,22 @@ def check_site(entry):
feeds.add(feed)
result['details']['feeds'] = sorted(list(feeds))
# detect responsive
viewports = set()
min_width = 2000
for c in result['details']['urlchecks']:
if c['responsive'] is None:
continue
if c['responsive']['viewport_meta_tag'] is not None:
viewports.add(c['responsive']['viewport_meta_tag'])
widths = c['responsive']['document_width'].values()
if min(widths) < min_width:
min_width = min(widths)
result['details']['responsive'] = {
'viewport_meta_tag': list(viewports),
'min_width': min_width,
}
# detect CMS
for c in result['details']['urlchecks']:
if c['content'] is None:
@ -503,6 +568,13 @@ def check_site(entry):
elif val < 1000:
result['result']['HTTP_RESPONSE_DURATION']['score'] = 0.5
# RESPONSIVE
if result['details']['responsive'] is not None:
if (result['details']['responsive']['min_width'] < 500 and
len(result['details']['responsive']['viewport_meta_tag']) > 0):
result['result']['RESPONSIVE']['value'] = True
result['result']['RESPONSIVE']['score'] = 1
# Overall score
for item in result['result'].keys():
result['score'] += result['result'][item]['score']

3
webapp/dist/index.html vendored

@ -61,9 +61,10 @@
<th scope="col">Erreichbar</th>
<th scope="col">Antwortzeit</th>
<th scope="col">Icon</th>
<th scope="col"><abbr title="Site nutzt HTTP-Verschlüsselung">HTTPS</abbr></th>
<th scope="col"><abbr title="Site ist sowohl mit www. als auch ohne www. in URL erreichbar">www. optional</abbr></th>
<th scope="col"><abbr title="URL-Varianten leiten auf eine einzige Startseiten-URL weiter">Kanonische URL</abbr></th>
<th scope="col"><abbr title="Site nutzt HTTP-Verschlüsselung">HTTPS</abbr></th>
<th scope="col">Responsive</th>
<th scope="col">Feed</th>
<th scope="col">Screenshots</th>
<th scope="col">CMS</th>

9
webapp/src/index.js

@ -68,6 +68,10 @@ $(function(){
var icon = item.result.FAVICON.value;
row.append('<td class="' + (icon ? 'good' : 'bad') + ' text-center">' + (icon ? ('<img src="' + item.details.icons[0] + '" class="icon">') : '❌') + '</td>');
// HTTPS
var hasHTTPS = item.result.HTTPS.value;
row.append('<td class="'+ (hasHTTPS ? 'good' : 'bad') +' text-center">' + (hasHTTPS ? '✅' : '❌') + '</td>');
// WWW_OPTIONAL
var wwwOptional = item.result.WWW_OPTIONAL.value;
row.append('<td class="'+ (wwwOptional ? 'good' : 'bad') +' text-center">' + (wwwOptional ? '✅' : '❌') + '</td>');
@ -76,9 +80,8 @@ $(function(){
var canonical = item.result.CANONICAL_URL.value;
row.append('<td class="'+ (canonical ? 'good' : 'bad') +' text-center">' + (canonical ? '✅' : '❌') + '</td>');
// https
var hasHTTPS = item.result.HTTPS.value;
row.append('<td class="'+ (hasHTTPS ? 'good' : 'bad') +' text-center">' + (hasHTTPS ? '✅' : '❌') + '</td>');
var responsive = item.result.RESPONSIVE.value;
row.append('<td class="'+ (responsive ? 'good' : 'bad') +' text-center">' + (responsive ? '✅' : '❌') + '</td>');
// feeds
var feeds = item.result.FEEDS.value;

Loading…
Cancel
Save