Update results, spider logic and webapp

pull/1/head
Marian Steinbach 4 years ago
parent 56995067ec
commit 3513eddfe6
  1. 1
      .gitignore
  2. 4
      README.md
  3. 10
      spider.py
  4. 85
      webapp/dist/bundle.js
  5. 28805
      webapp/dist/data/spider_result.json
  6. 60
      webapp/dist/index.html
  7. 7782
      webapp/package-lock.json
  8. 31
      webapp/package.json
  9. 50
      webapp/src/index.js
  10. 9
      webapp/webpack.config.js

1
.gitignore vendored

@ -1,2 +1,3 @@
venv
cache
webapp/node_modules

@ -12,9 +12,9 @@ Written and tested in Python3
- Check which cookies are set and with what settings (expiry, domain)
- submit the URL against a service like Google Page Speed and retrieve the score
- Check against our own webpagetest.org instance
- Detect which one of the well-known CMS is used?
- Detect which one of the well-known CMS is used
- Certificate expiry warning
- Export and publish the report as a single page web app via GitHub pages
- Favourite icon availability check
### Usage

@ -1,6 +1,5 @@
# coding: utf8
from datetime import datetime
from git import Repo
from multiprocessing import Pool
from socket import gethostbyname_ex
@ -32,7 +31,7 @@ green_directory_repo = 'https://github.com/netzbegruenung/green-directory.git'
green_direcory_data_path = 'data'
green_directory_local_path = './cache/green-directory'
result_path = './webapp/data'
result_path = './webapp/dist/data'
# end configuration
@ -275,16 +274,13 @@ def main():
done = set()
# convert results from ApplyResult to dict
for url in results.keys():
for url in sorted(results.keys()):
if url not in done:
results2.append(results[url].get())
done.add(url)
now = datetime.utcnow()
# Write result as JSON
now_stamp = now.strftime('%Y-%m-%d_%H-%M')
output_filename = os.path.join(result_path, 'check_' + now_stamp + ".json")
output_filename = os.path.join(result_path, "spider_result.json")
with open(output_filename, 'w', encoding="utf8") as jsonfile:
json.dump(results2, jsonfile, indent=2, sort_keys=True)

File diff suppressed because one or more lines are too long

@ -0,0 +1,60 @@
<!DOCTYPE html>
<html lang="de">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<title>green spider report: Auswertung einiger Merkmale von Bündnis 90/Die Grünen Websites</title>
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.16/css/jquery.dataTables.css">
<style type="text/css">
body {
padding: 20px;
}
.bad {
background-color: #ffdbdb;
}
.good {
background-color: #cfeaa8;
}
</style>
</head>
<body>
<div class="container-fluid">
<p>Erläuterungen der Testkriterien am <a href="#docs">Ende der Seite</a>. Gehe zum <a href="https://github.com/netzbegruenung/green-spider/">green-spider GitHub repository</a> um mehr zu erfahren.</p>
<table class="table">
<thead>
<tr>
<th scope="col">URL</th>
<th scope="col"><abbr title="Site ist sowohl mit www. als auch ohne www. in URL erreichbar">www. optional</abbr></th>
<th scope="col"><abbr title="URL-Varianten leiten auf eine einzige Startseiten-URL weiter">Kanonische URL</abbr></th>
<th scope="col"><abbr title="Site nutzt HTTP-Verschlüsselung">HTTPS</abbr></th>
</tr>
</thead>
<tbody>
</tbody>
</table>
<hr>
<h3 id="docs">Testkriterien</h3>
<h4>www. optional</h4>
<p>Die Site sollte sowohl mit &quot;www.&quot; als auch ohne &quot;www.&quot; am Anfang der URL erreichbar sein.</p>
<h4>Kanonische URL</h4>
<p>Verschiedene URLs zum Aufruf der Site (mit/ohne &quot;www.&quot; Präfix, mit/ohne HTTPS) sollten alle per HTTP Redirect auf eine einzige &quot;kanonische&quot; URL weiter leiten.</p>
<h4>HTTPS</h4>
<p>Die Site ist per verschlüsselter HTTP-Verbindung (HTTPS-Protokoll) erreichbar.</p>
</div>
<script src="bundle.js"></script>
</body>
</html>

File diff suppressed because it is too large Load Diff

@ -0,0 +1,31 @@
{
"name": "green-spider-webapp",
"version": "0.0.1",
"description": "Displays spider results",
"private": true,
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {
"type": "git",
"url": "git+https://github.com/netzbegruenung/green-spider.git"
},
"author": "",
"license": "Apache-2.0",
"bugs": {
"url": "https://github.com/netzbegruenung/green-spider/issues"
},
"homepage": "https://github.com/netzbegruenung/green-spider#readme",
"devDependencies": {
"webpack": "^4.5.0",
"webpack-cli": "^2.0.14"
},
"dependencies": {
"bootstrap": "^4.0.0",
"datatables.net": "^1.10.16",
"jquery": "^3.3.1",
"lodash": "^4.17.5",
"popper.js": "^1.14.3",
"punycode": "^2.1.0"
}
}

@ -0,0 +1,50 @@
import _ from 'lodash';
import $ from 'jquery';
import 'bootstrap';
import 'popper.js';
import punycode from 'punycode';
import 'datatables.net';
$(function(){
$.getJSON('data/spider_result.json', function(data){
var tbody = $('tbody');
$.each(data, function(index, item) {
var row = $(document.createElement('tr'));
row.append('<td><a href="' + item.input_url + '">' + punycode.toUnicode(item.input_url) + '</a></td>');
// hostnames
var twoHostnames = false;
if (_.filter(item.hostnames, {'resolvable': true}).length === 2) {
twoHostnames = true;
};
row.append('<td class="'+ (twoHostnames ? 'good' : 'bad') +' text-center">' + (twoHostnames ? '✅' : '❌') + '</td>');
// one canonical URL
var canonical = false;
if (item.canonical_urls.length === 1 ) canonical = true;
row.append('<td class="'+ (canonical ? 'good' : 'bad') +' text-center">' + (canonical ? '✅' : '❌') + '</td>');
// https
var hasHTTPS = false;
hasHTTPS = _.find(item.canonical_urls, function(o){
return o.indexOf('https://') !== -1;
});
row.append('<td class="'+ (hasHTTPS ? 'good' : 'bad') +' text-center">' + (hasHTTPS ? '✅' : '❌') + '</td>');
tbody.append(row);
});
// enable data table funcionts (sorting)
$('table.table').DataTable({
order: [[0, "asc"]],
paging: false,
pageLength: 10000,
language: {
"search": "Suche"
}
});
});
});

@ -0,0 +1,9 @@
const path = require('path');
module.exports = {
entry: './src/index.js',
output: {
filename: 'bundle.js',
path: path.resolve(__dirname, 'dist')
}
};
Loading…
Cancel
Save