Browse Source

Neue Such-API verwenden (#35)

* Refactor to use external search API

* Let indexer restart on failure

* Remove serviceWorker
main
Marian Steinbach 3 years ago committed by GitHub
parent
commit
0398e66b33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      .gitignore
  2. 4
      config/nginx/nginx_dev.conf
  3. 6
      config/nginx/nginx_prod.conf
  4. 35
      config/webpack.config.prod.js
  5. 24
      docker-compose-prod.yaml
  6. 23
      docker-compose.yaml
  7. 2
      package.json
  8. 72
      public/serviceWorkerAddon.js
  9. 32
      src/SearchForm.js
  10. 171
      src/SiteDetailsPage.js
  11. 250
      src/SitesSearch.js
  12. 82
      src/index.js
  13. 117
      src/registerServiceWorker.js
  14. 28
      yarn.lock

3
.gitignore vendored

@ -23,6 +23,9 @@ yarn-error.log*
# cache
/proxy-cache
# elasticsearch dev volume
/elasticsearch
# secret key we never want to share
/secrets
/test-certs

4
config/nginx/nginx_dev.conf

@ -27,8 +27,8 @@ http {
proxy_cache_lock on;
proxy_cache_lock_timeout 15s;
proxy_cache_use_stale updating;
proxy_cache_valid 200 6h;
proxy_cache_valid any 60m;
proxy_cache_valid 200 5m;
proxy_cache_valid any 1m;
proxy_set_header X-Real-IP $remote_addr;
}

6
config/nginx/nginx_prod.conf

@ -63,7 +63,7 @@ http {
proxy_set_header X-Real-IP $remote_addr;
}
# All other API calls are cached for 6 hours
# All other API calls are cached for 5 minutes
location /api/ {
proxy_pass http://api:5000;
proxy_cache api;
@ -71,8 +71,8 @@ http {
proxy_cache_lock on;
proxy_cache_lock_timeout 15s;
proxy_cache_use_stale updating;
proxy_cache_valid 200 6h;
proxy_cache_valid any 60m;
proxy_cache_valid 200 5m;
proxy_cache_valid any 1m;
proxy_set_header X-Real-IP $remote_addr;
}

35
config/webpack.config.prod.js

@ -7,7 +7,6 @@ const HtmlWebpackPlugin = require('html-webpack-plugin');
const ExtractTextPlugin = require('extract-text-webpack-plugin');
const ManifestPlugin = require('webpack-manifest-plugin');
const InterpolateHtmlPlugin = require('react-dev-utils/InterpolateHtmlPlugin');
const SWPrecacheWebpackPlugin = require('sw-precache-webpack-plugin');
const eslintFormatter = require('react-dev-utils/eslintFormatter');
const ModuleScopePlugin = require('react-dev-utils/ModuleScopePlugin');
const paths = require('./paths');
@ -293,40 +292,6 @@ module.exports = {
new ManifestPlugin({
fileName: 'asset-manifest.json',
}),
// Generate a service worker script that will precache, and keep up to date,
// the HTML & assets that are part of the Webpack build.
new SWPrecacheWebpackPlugin({
// By default, a cache-busting query parameter is appended to requests
// used to populate the caches, to ensure the responses are fresh.
// If a URL is already hashed by Webpack, then there is no concern
// about it being stale, and the cache-busting can be skipped.
dontCacheBustUrlsMatching: /\.\w{8}\./,
maximumFileSizeToCacheInBytes: 4194304, // 4 MB
filename: 'service-worker.js',
logger(message) {
if (message.indexOf('Total precache size is') === 0) {
// This message occurs for every build and is a bit too noisy.
return;
}
if (message.indexOf('Skipping static resource') === 0) {
// This message obscures real errors so we ignore it.
// https://github.com/facebookincubator/create-react-app/issues/2612
return;
}
console.log(message);
},
minify: true,
importScripts: [
'/serviceWorkerAddon.js',
],
// For unknown URLs, fallback to the index page
navigateFallback: publicUrl + '/index.html',
// Ignores URLs starting from /__ (useful for Firebase):
// https://github.com/facebookincubator/create-react-app/issues/2237#issuecomment-302693219
navigateFallbackWhitelist: [/^(?!\/__).*/],
// Don't precache sourcemaps (they're large) and build asset manifest:
staticFileGlobsIgnorePatterns: [/\.map$/, /asset-manifest\.json$/],
}),
// Moment.js is an extremely popular library that bundles large locale files
// by default due to how Webpack interprets its code. This is a practical
// solution that requires the user to opt into importing specific locales.

24
docker-compose-prod.yaml

@ -22,3 +22,27 @@ services:
GCLOUD_DATASTORE_CREDENTIALS_PATH: /secrets/datastore-reader.json
volumes:
- $PWD/secrets:/secrets
depends_on:
- elasticsearch
elasticsearch:
image: elasticsearch:5.6-alpine
environment:
- cluster.name=green-spider
- discovery.type=single-node
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms256m -Xmx256m"
ports:
- 9200:9200
restart: always
volumes:
- $PWD/elasticsearch:/usr/share/elasticsearch/data
indexer:
image: quay.io/netzbegruenung/green-spider-indexer:latest
volumes:
- $PWD/secrets:/etc/indexer
environment:
- GCLOUD_DATASTORE_CREDENTIALS_PATH=/etc/indexer/datastore-reader.json
restart: on-failure

23
docker-compose.yaml

@ -30,3 +30,26 @@ services:
GCLOUD_DATASTORE_CREDENTIALS_PATH: /secrets/datastore-reader.json
volumes:
- "./secrets:/secrets"
depends_on:
- elasticsearch
elasticsearch:
image: elasticsearch:5.6-alpine
environment:
- cluster.name=green-spider
- discovery.type=single-node
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms256m -Xmx256m"
ports:
- 9200:9200
restart: always
volumes:
- $PWD/elasticsearch:/usr/share/elasticsearch/data
indexer:
image: quay.io/netzbegruenung/green-spider-indexer:latest
volumes:
- $PWD/secrets:/etc/indexer
environment:
- GCLOUD_DATASTORE_CREDENTIALS_PATH=/etc/indexer/datastore-reader.json
restart: on-failure

2
package.json

@ -33,7 +33,6 @@
"html-webpack-plugin": "2.29.0",
"jest": "23.6.0",
"lodash": "^4.17.11",
"lunr": "^2.3.6",
"merge": "1.2.1",
"object-assign": "4.1.1",
"postcss-flexbugs-fixes": "4.1.0",
@ -44,6 +43,7 @@
"react": "^16.8.5",
"react-dev-utils": "^5.0.3",
"react-dom": "^16.8.5",
"react-infinite-scroller": "^1.2.4",
"react-router-dom": "^5.0.0",
"resolve": "^1.10.0",
"style-loader": "^0.23.1",

72
public/serviceWorkerAddon.js

@ -1,72 +0,0 @@
/**
* Will be imported into our service worker
*/
/**
* This handler intercepts all GET requests and either responds with a
* cached version of the resource or fetches the original one and then
* adds it to the cache.
*
* This is called "on network response" in
* https://jakearchibald.com/2014/offline-cookbook/#on-network-response
*
* We whitelist the URLs to cache in this manner.
*/
self.addEventListener('fetch', function(event) {
var shouldRespond = false;
if (event.request.method === 'GET') {
// API
if (event.request.url.indexOf('/api/v1/') !== -1) {
// exclude our freshness check from cache
if (event.request.url.indexOf('/api/v1/spider-results/last-updated/') === -1) {
shouldRespond = true;
}
}
// webfonts
else if (event.request.url.indexOf('https://netzbegruenung.github.io/webfonts/') !== -1) {
shouldRespond = true;
}
// ionicons
else if (event.request.url.indexOf('https://unpkg.com/ionicons') !== -1) {
shouldRespond = true;
}
// If shouldRespond was set to true at any point, then call
// event.respondWith(), using the appropriate cache key.
if (shouldRespond) {
event.respondWith(
caches.open(cacheName).then(function(cache) {
return cache.match(urlsToCacheKeys.get(event.request.url)).then(function(response) {
return response || fetch(event.request).then(function(response) {
console.log("Fetching and caching resource", event.request.url);
cache.put(event.request, response.clone());
return response;
});
});
})
);
}
}
});
/**
* Pre-fetch some static resources on service worker installation.
*/
self.addEventListener('install', function(event) {
event.waitUntil(
caches.open(cacheName).then(function(cache) {
console.log('Pre-fetching some resources on SW install');
return cache.addAll([
'https://netzbegruenung.github.io/webfonts/fonts/Arvo_Gruen_2015_10.woff',
'https://unpkg.com/ionicons@4.4.3/dist/fonts/ionicons.woff2',
'https://netzbegruenung.github.io/webfonts/fonts/arvo_regular.woff',
'https://netzbegruenung.github.io/webfonts/style.css',
'https://unpkg.com/ionicons@4.4.3/dist/css/ionicons.min.css'
]);
})
);
});

32
src/SearchForm.js

@ -0,0 +1,32 @@
import React, { Component } from 'react';
class SearchForm extends Component {
handleChange = (event) => {
this.props.callback(event.target.value);
};
handleSubmit = (event) => {
event.preventDefault();
};
render() {
var hitsInfo = <span>&nbsp;</span>;
if (this.props.hits !== null) {
hitsInfo = <span>{this.props.hits} Treffer</span>;
}
return (
<div className='col-12'>
<form onSubmit={this.handleSubmit}>
<div className='form-group'>
<label htmlFor='queryInput'>Finde Deine Site</label>
<input className='form-control' type='search' name='query' placeholder="z. B. kleinostheim" value={this.props.value} onChange={this.handleChange} id='queryInput' />
<small className='form-text'>{hitsInfo}</small>
</div>
</form>
</div>
);
}
}
export default SearchForm;

171
src/SiteDetailsPage.js

@ -9,6 +9,8 @@ import _ from 'underscore';
class SiteDetailsPage extends Component {
_isMounted = false;
state = {
isLoading: true,
site: null,
@ -16,6 +18,8 @@ class SiteDetailsPage extends Component {
};
componentDidMount() {
this._isMounted = true;
// ensure that this view is opened at the top
// when coming from the SiteSearch
window.scrollTo(0, 0);
@ -23,25 +27,30 @@ class SiteDetailsPage extends Component {
// load data
let url = this.props.match.match.params.siteId;
axios.get(`/api/v1/spider-results/site?url=${url}&date=${this.props.lastUpdated}`)
axios.get(`/api/v1/spider-results/site?url=${url}`)
.then((response) => {
// handle success
this.setState({
isLoading: false,
site: response.data,
url: decodeURIComponent(url),
});
if (this._isMounted) {
// handle success
this.setState({
isLoading: false,
site: response.data,
url: decodeURIComponent(url),
});
}
})
.catch((error) => {
// handle error
console.error(error);
this.setState({isLoading: false});
})
.then(() => {
// always executed
if (this._isMounted) {
this.setState({isLoading: false});
}
});
}
componentWillUnmount() {
this._isMounted = false;
}
render() {
if (this.state.isLoading) {
return <div></div>;
@ -173,7 +182,7 @@ class SiteDetailsPage extends Component {
<hr />
<ScoreComparisonWidget allSites={this.props.sitesHash} thisSite={this.state.site} maxScore={15} />
<ScoreComparisonWidget sitesCount={this.props.sitesCount} thisSite={this.state.site} maxScore={15} />
{
this.state.site.rating.SITE_REACHABLE.value ?
@ -428,12 +437,16 @@ class SocialMediaLinksField extends Component {
class Screenshots extends Component {
_isMounted = false;
state = {
isLoading: true,
screenshots: null,
};
componentDidMount() {
this._isMounted = true;
var baseURL = 'http://green-spider-screenshots.sendung.de';
// load data
@ -462,26 +475,31 @@ class Screenshots extends Component {
// TODO: rewrite screenshot URLs
}
this.setState({
isLoading: false,
screenshots: screenshots,
});
if (this._isMounted) {
this.setState({
isLoading: false,
screenshots: screenshots,
});
}
})
.catch((error) => {
// handle error
console.error(error);
this.setState({isLoading: false});
})
.then(() => {
// always executed
if (this._isMounted) {
this.setState({isLoading: false});
}
});
} else {
this.setState({
isLoading: false,
});
if (this._isMounted) {
this.setState({isLoading: false});
}
}
}
componentWillUnmount() {
this._isMounted = false;
}
render() {
if (this.state.screenshots === null) {
if (this.state.isLoading) {
@ -582,53 +600,82 @@ class NetworkErrorsField extends Component {
}
class ScoreComparisonWidget extends Component {
calculateIndizes() {
var countAll = 0;
var countType = 0;
var countState = 0;
var indexAll = 0;
var indexSiteType = 0;
var indexState = 0;
for (var url of Object.keys(this.props.allSites)) {
countAll++;
var site = this.props.allSites[url];
if (site.meta.type === this.props.thisSite.meta.type && site.meta.level === this.props.thisSite.meta.level) {
countType++;
}
if (site.meta.state === this.props.thisSite.meta.state) {
countState++;
}
state = {
numLowerSites: null,
numSitesOfType: null,
numLowerSitesOfType: null,
numSitesOfState: null,
numLowerSitesOfState: null,
};
if (site.score < this.props.thisSite.score) {
indexAll++;
if (site.meta.type === this.props.thisSite.meta.type && site.meta.level === this.props.thisSite.meta.level) {
indexSiteType++;
}
if (site.meta.state === this.props.thisSite.meta.state) {
indexState++;
}
componentDidMount() {
if (this.props.sitesCount) {
// compare to all sites
var q1 = '+score:[0 TO '+ this.props.thisSite.score +'] -score:'+ this.props.thisSite.score;
axios.get('/api/v1/spider-results/count/?q=' + encodeURIComponent(q1))
.then((response) => {
this.setState({
numLowerSites: response.data.count
});
});
// compare to sites of same type
if (this.props.thisSite.meta.type && this.props.thisSite.meta.level) {
var q2 = '+meta.type:' + this.props.thisSite.meta.type + ' +meta.level:"' + this.props.thisSite.meta.level + '"';
axios.get('/api/v1/spider-results/count/?q=' + encodeURIComponent(q2))
.then((response) => {
this.setState({
numSitesOfType: response.data.count
});
});
var q3 = '+meta.type:' + this.props.thisSite.meta.type + ' +meta.level:"' + this.props.thisSite.meta.level + '" +score:[0 TO '+ this.props.thisSite.score +'] -score:'+ this.props.thisSite.score;
axios.get('/api/v1/spider-results/count/?q=' + encodeURIComponent(q3))
.then((response) => {
this.setState({
numLowerSitesOfType: response.data.count
});
});
}
// compare to sites of same state
if (this.props.thisSite.meta.state) {
var q4 = '+meta.state:"' + this.props.thisSite.meta.state + '"';
axios.get('/api/v1/spider-results/count/?q=' + encodeURIComponent(q4))
.then((response) => {
this.setState({
numSitesOfState: response.data.count
});
});
var q5 = '+meta.state:"' + this.props.thisSite.meta.state + '" +score:[0 TO '+ this.props.thisSite.score +'] -score:'+ this.props.thisSite.score;
axios.get('/api/v1/spider-results/count/?q=' + encodeURIComponent(q5))
.then((response) => {
this.setState({
numLowerSitesOfState: response.data.count
});
});
}
}
indexAll = indexAll / countAll;
indexSiteType = indexSiteType / countType;
indexState = indexState / countState;
return {
all: indexAll,
siteType: indexSiteType,
state: indexState
}
}
render() {
if (this.props.allSites === null) {
if (this.props.sitesCount === null) {
return <div className='row d-flex'></div>;
}
var index = this.calculateIndizes();
var lowerSites = (this.state.numLowerSites !== null) ? (this.state.numLowerSites / this.props.sitesCount * 100).toFixed(1) : '–';
var lowerSitesOfType = (this.state.numSitesOfType !== null && this.state.numLowerSitesOfType !== null) ? (this.state.numLowerSitesOfType / this.state.numSitesOfType * 100).toFixed(1) : '–';
var lowerSitesOfState = (this.state.numSitesOfState !== null && this.state.numLowerSitesOfState !== null) ? (this.state.numLowerSitesOfState / this.state.numSitesOfState * 100).toFixed(1) : '–';
var rows = [<div key='all'>Besser als { lowerSites }% aller Sites</div>];
if (this.state.numSitesOfType !== null) {
rows.push(<div key='type'>Besser als { lowerSitesOfType }% aller <TypeField level={this.props.thisSite.meta.level} type={this.props.thisSite.meta.type} />-Sites</div>);
}
if (this.state.numSitesOfState !== null) {
rows.push(<div key='state'>Besser als { lowerSitesOfState }% aller Sites in <StateField state={this.props.thisSite.meta.state} /></div>);
}
return (
<div className='row d-flex'>
@ -636,9 +683,7 @@ class ScoreComparisonWidget extends Component {
Punkte: <ScoreField score={this.props.thisSite.score} maxScore={this.props.maxScore} />
</div>
<div className='col-8 align-self-center'>
<div>Besser als { Math.round(index.all * 100) }% aller Sites</div>
<div>Besser als { Math.round(index.siteType * 100) }% aller <TypeField level={this.props.thisSite.meta.level} type={this.props.thisSite.meta.type} />-Sites</div>
<div>Besser als { Math.round(index.state * 100) }% aller Sites in <StateField state={this.props.thisSite.meta.state} /></div>
{rows}
</div>
</div>
);

250
src/SitesSearch.js

@ -1,67 +1,142 @@
/**
* The ResultsTable component is a table of results for all websites we checked.
*/
import axios from 'axios';
import React, { Component } from 'react';
import { Link } from "react-router-dom";
import LocationLabel from './LocationLabel';
import SearchForm from './SearchForm';
import ScoreField from './ScoreField';
import URLField from './URLField';
import './SitesSearch.css';
import history from './history';
import InfiniteScroll from 'react-infinite-scroller';
class SitesSearch extends Component {
constructor(props) {
super(props);
itemsPerPage = 20;
this.state = {
loading: true,
sitesHash: null,
searchIndex: null,
searchResult: null,
};
state = {
loading: false,
searchResultItems: [],
query: null,
userQuery: '',
hits: 0,
pageLoaded: null,
};
this.searchResultCallback = this.searchResultCallback.bind(this);
componentDidMount() {
// init search from URL
let params = (new URL(document.location)).searchParams;
if (typeof params === 'object') {
let q = params.get('q');
if (q !== null && q !== '') {
this.doSearch(q);
}
}
}
searchResultCallback(result) {
// sort result by score
if (result) {
result.sort((a, b) => (this.props.sitesHash[b.ref].score > this.props.sitesHash[a.ref].score) ? 1 : ((this.props.sitesHash[a.ref].score > this.props.sitesHash[b.ref].score) ? -1 : 0));
/**
* Performs the search based user input or URL parameter
* and fetches the first results page
*/
doSearch = (q) => {
var minTermLength = 1;
if (q === '') {
history.push(`/`);
} else {
history.push(`/?q=${q}`);
}
this.setState({searchResult: result});
if (q.length > minTermLength) {
// append '*' if last character is not
var esQuery = q.trim();
if (esQuery.substr(esQuery.length - 1) !== '*') {
esQuery += '*';
}
if (q !== this.state.query) {
this.setState({
query: esQuery,
userQuery: q,
searchResultItems: [],
pageLoaded: null,
});
} else {
this.setState({
query: esQuery,
userQuery: q,
});
}
this.getResultsPage(esQuery, 0);
} else if (q.length <= minTermLength) {
this.setState({
query: null,
userQuery: q,
hits: 0,
searchResultItems: [],
});
}
};
getResultsPage = (term, page) => {
var from = page * this.itemsPerPage;
axios.get('/api/v1/spider-results/query/?from=' + from + '&q=' + encodeURI(term))
.then((response) => {
var allResultItems = [];
// if the term has not changed, append result items
if (term === this.state.query) {
allResultItems = this.state.searchResultItems;
}
response.data.hits.hits.forEach((item) => {
allResultItems.push(item);
});
this.setState({
searchResultItems: allResultItems,
hits: response.data.hits.total,
pageLoaded: page,
});
});
}
loadFunc = (pageNum) => {
this.getResultsPage(this.state.query, pageNum);
};
hasMoreFunc = () => {
var result = (this.itemsPerPage * this.state.pageLoaded) < this.state.hits;
return result;
};
render() {
var rows = [];
if (this.state.searchResult) {
for (var site of this.state.searchResult) {
var element = this.props.sitesHash[site.ref];
if (this.state.searchResultItems.length > 0) {
this.state.searchResultItems.forEach((site) => {
var row = (
<Link key={element.input_url} to={`/sites/${ encodeURIComponent(element.input_url) }`} className='SitesSearch'>
<Link key={site._source.url} to={`/sites/${ encodeURIComponent(site._source.url) }`} className='SitesSearch'>
<div className='SitesSearch row'>
<div className='col-9 col-sm-10 col-md-10'>
<LocationLabel level={element.meta.level} type={element.meta.type} district={element.meta.district} city={element.meta.city} state={element.meta.state} truncate={true} />
<URLField url={element.input_url} link={false} />
<LocationLabel level={site._source.meta.level} type={site._source.meta.type} district={site._source.meta.district} city={site._source.meta.city} state={site._source.meta.state} truncate={true} />
<URLField url={site._source.url} link={false} />
</div>
<div className='col-3 col-sm-2 col-md-2 d-flex'>
<ScoreField score={element.score} maxScore={15} />
<ScoreField score={site._source.score} maxScore={15} />
</div>
</div>
</Link>
);
rows.push(row);
}
});
}
var placeholder = (
<div className='row placeholder' key='placeholder'>
<div className='col-12 text-center'>
Vergleiche Deine GRÜNE Website mit { this.props.sitesHash ? Object.keys(this.props.sitesHash).length : 'vielen'} anderen und erfahre, was Du verbessern kannst.
Vergleiche Deine GRÜNE Website mit { this.props.sitesCount ? this.props.sitesCount : 'vielen'} anderen und erfahre, was Du verbessern kannst.
</div>
</div>
);
@ -78,7 +153,15 @@ class SitesSearch extends Component {
var resultFound = (
<div className='row results'>
<div className='col-12'>
{rows}
<InfiniteScroll
pageStart={0}
loadMore={this.loadFunc}
hasMore={this.hasMoreFunc()}
loader={<div className="loader" key={0}>Lade weitere Treffer...</div>}
threshold={250}
>
{rows}
</InfiniteScroll>
</div>
</div>
);
@ -87,11 +170,10 @@ class SitesSearch extends Component {
<div>
<div className='row searchInputRow'>
<div className='col-12'>
{ this.props.searchIndex ?
<SearchField searchIndex={this.props.searchIndex} callback={this.searchResultCallback} />
:
<SearchFieldPlaceholder />
}
<SearchForm
callback={this.doSearch}
value={this.state.userQuery}
hits={this.state.hits} />
</div>
</div>
{ rows.length ? resultFound : noresult }
@ -100,104 +182,4 @@ class SitesSearch extends Component {
}
}
class SearchField extends Component {
state = {
value: '',
lastQuery: '',
hits: 0,
}
constructor(props) {
super(props);
this.handleChange = this.handleChange.bind(this);
this.handleSubmit = this.handleSubmit.bind(this);
this.doSearch = this.doSearch.bind(this);
}
componentDidMount() {
// init search from URL
let params = (new URL(document.location)).searchParams;
if (typeof params === 'object') {
let q = params.get('q');
if (q !== null && q !== '') {
this.doSearch(q);
}
}
}
doSearch(q) {
var minTermLength = 1;
if (q === '') {
history.push(`/`);
} else {
history.push(`/?q=${q}`);
}
this.setState({value: q});
if (q.length > minTermLength && q !== this.state.lastQuery) {
var searchResult = this.props.searchIndex.search(q.trim() + "*");
this.setState({
lastQuery: q,
hits: searchResult.length,
});
this.props.callback(searchResult);
} else if (q.length <= minTermLength) {
this.setState({
lastQuery: q,
hits: 0,
});
this.props.callback(null);
}
}
handleChange(event) {
var q = event.target.value;
this.doSearch(q);
}
handleSubmit(event) {
event.preventDefault();
}
render() {
var hitsInfo = <span>&nbsp;</span>;
if (this.state.lastQuery !== '') {
hitsInfo = <span>{this.state.hits} Treffer</span>;
}
return (
<div className='col-12'>
<form onSubmit={this.handleSubmit}>
<div className='form-group'>
<label htmlFor='queryInput'>Finde Deine Site</label>
<input className='form-control' type='search' name='query' placeholder="Finde Deine Site" value={this.state.value} onChange={this.handleChange} id='queryInput' />
<small className='form-text'>{hitsInfo}</small>
</div>
</form>
</div>
);
}
}
class SearchFieldPlaceholder extends Component {
render() {
return (
<div className='col-12'>
<form onSubmit={this.handleSubmit}>
<div className='form-group'>
<label htmlFor='queryInput'>Finde Deine Site</label>
<input className='form-control' type='search' name='query' placeholder="Daten werden geladen..." value={this.props.value} disabled={true} />
<small className='form-text'>&nbsp;</small>
</div>
</form>
</div>
);
}
}
export default SitesSearch;

82
src/index.js

@ -1,3 +1,4 @@
import axios from 'axios';
import React from 'react';
import ReactDOM from 'react-dom';
import { Router, Route } from "react-router-dom";
@ -6,91 +7,32 @@ import './index.css';
import NavBar from './NavBar';
import SitesSearch from './SitesSearch';
import SiteDetailsPage from './SiteDetailsPage';
import registerServiceWorker from './registerServiceWorker';
import history from './history';
import axios from 'axios';
import lunr from 'lunr';
class App extends React.Component {
state = {
loading: true,
searchIndex: null,
sitesLastUpdated: null,
sitesHash: null,
sitesCount: null,
};
componentDidMount = () => {
// check for fresh data every 5 minutes
window.setInterval(this.loadData, 5 * 60 * 1000);
// and load fresh data now
this.loadData();
}
loadData = () => {
componentDidMount() {
axios.get('/api/v1/spider-results/last-updated/')
.then((response) => {
// load data only of newer than what we have
if (response.data.last_updated !== this.state.sitesLastUpdated) {
this.setState({loading: true});
axios.get('/api/v1/spider-results/compact/?date=' + encodeURIComponent(response.data.last_updated))
.then((response2) => {
// handle success
let sitesHash = {};
for (var site of response2.data) {
sitesHash[site.input_url] = site;
}
this.setState({
loading: false,
sitesHash: sitesHash,
sitesLastUpdated: response.data.last_updated,
searchIndex: this.createSearchIndex(response2.data),
});
})
.catch((error) => {
console.error(error);
this.setState({loading: false});
})
this.setState({sitesLastUpdated: response.data.last_updated});
}
})
.catch((error) => {
console.error('error checking for updates', error);
})
});
axios.get('/api/v1/spider-results/count/')
.then((response) => {
this.setState({sitesCount: response.data.count});
});
}
tokenizeURL = (url) => {
return url.replace(/[:.-/]+/gi, ' ');
}
createSearchIndex = (sites) => {
var tu = this.tokenizeURL;
let searchIndex = lunr(function() {
this.pipeline.remove(lunr.stemmer)
this.searchPipeline.remove(lunr.stemmer)
this.field('url');
this.field('state');
this.field('district');
this.field('city');
for (var site of sites) {
this.add({
"id": site.input_url,
"url": tu(site.input_url),
"state": site.meta.state,
"district": site.meta.district,
"city": site.meta.city,
});
}
});
return searchIndex;
}
render() {
return (
<Router history={history}>
@ -100,8 +42,8 @@ class App extends React.Component {
<div className='row'>
<div className='col-lg'></div>
<div className='col-lg-8 col-sm-12'>
<Route render={() => <SitesSearch sitesHash={this.state.sitesHash} searchIndex={this.state.searchIndex} lastUpdated={this.state.sitesLastUpdated} />} exact path="/" />
<Route component={(match) => <SiteDetailsPage match={match} sitesHash={this.state.sitesHash} lastUpdated={this.state.sitesLastUpdated} />} path="/sites/:siteId" />
<Route render={() => <SitesSearch sitesCount={this.state.sitesCount}/>} exact path="/" />
<Route component={(match) => <SiteDetailsPage match={match} lastUpdated={this.state.sitesLastUpdated} sitesCount={this.state.sitesCount} />} path="/sites/:siteId" />
</div>
<div className='col-lg'></div>
</div>
@ -113,5 +55,3 @@ class App extends React.Component {
}
ReactDOM.render(<App />, document.getElementById('root'));
registerServiceWorker();

117
src/registerServiceWorker.js

@ -1,117 +0,0 @@
// In production, we register a service worker to serve assets from local cache.
// This lets the app load faster on subsequent visits in production, and gives
// it offline capabilities. However, it also means that developers (and users)
// will only see deployed updates on the "N+1" visit to a page, since previously
// cached resources are updated in the background.
// To learn more about the benefits of this model, read https://goo.gl/KwvDNy.
// This link also includes instructions on opting out of this behavior.
const isLocalhost = Boolean(
window.location.hostname === 'localhost' ||
// [::1] is the IPv6 localhost address.
window.location.hostname === '[::1]' ||
// 127.0.0.1/8 is considered localhost for IPv4.
window.location.hostname.match(
/^127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$/
)
);
export default function register() {
if (process.env.NODE_ENV === 'production' && 'serviceWorker' in navigator) {
// The URL constructor is available in all browsers that support SW.
const publicUrl = new URL(process.env.PUBLIC_URL, window.location);
if (publicUrl.origin !== window.location.origin) {
// Our service worker won't work if PUBLIC_URL is on a different origin
// from what our page is served on. This might happen if a CDN is used to
// serve assets; see https://github.com/facebookincubator/create-react-app/issues/2374
return;
}
window.addEventListener('load', () => {
const swUrl = `${process.env.PUBLIC_URL}/service-worker.js`;
if (isLocalhost) {
// This is running on localhost. Lets check if a service worker still exists or not.
checkValidServiceWorker(swUrl);
// Add some additional logging to localhost, pointing developers to the
// service worker/PWA documentation.
navigator.serviceWorker.ready.then(() => {
console.log(
'This web app is being served cache-first by a service ' +
'worker. To learn more, visit https://goo.gl/SC7cgQ'
);
});
} else {
// Is not local host. Just register service worker
registerValidSW(swUrl);
}
});
}
}
function registerValidSW(swUrl) {
navigator.serviceWorker
.register(swUrl)
.then(registration => {
registration.onupdatefound = () => {
const installingWorker = registration.installing;
installingWorker.onstatechange = () => {
if (installingWorker.state === 'installed') {
if (navigator.serviceWorker.controller) {
// At this point, the old content will have been purged and
// the fresh content will have been added to the cache.
// It's the perfect time to display a "New content is
// available; please refresh." message in your web app.
console.log('New content is available; please refresh.');
} else {
// At this point, everything has been precached.
// It's the perfect time to display a
// "Content is cached for offline use." message.
console.log('Content is cached for offline use.');
}
}
};
};
})
.catch(error => {
console.error('Error during service worker registration:', error);
});
}
function checkValidServiceWorker(swUrl) {
// Check if the service worker can be found. If it can't reload the page.
fetch(swUrl)
.then(response => {
// Ensure service worker exists, and that we really are getting a JS file.
if (
response.status === 404 ||
response.headers.get('content-type').indexOf('javascript') === -1
) {
// No service worker found. Probably a different app. Reload the page.
navigator.serviceWorker.ready.then(registration => {
registration.unregister().then(() => {
window.location.reload();
});
});
} else {
// Service worker found. Proceed as normal.
registerValidSW(swUrl);
}
})
.catch(() => {
console.log(
'No internet connection found. App is running in offline mode.'
);
});
}
export function unregister() {
if ('serviceWorker' in navigator) {
navigator.serviceWorker.ready.then(registration => {
registration.unregister();
});
}
}

28
yarn.lock

@ -5322,7 +5322,7 @@ longest@^1.0.1:
resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097"
integrity sha1-MKCy2jj3N3DoKUoNIuZiXtd9AJc=
loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.2.0, loose-envify@^1.3.1:
loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.2.0, loose-envify@^1.3.1, loose-envify@^1.4.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf"
integrity sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==
@ -5355,11 +5355,6 @@ lru-cache@^4.0.1:
pseudomap "^1.0.2"
yallist "^2.1.2"
lunr@^2.3.6:
version "2.3.6"
resolved "https://registry.yarnpkg.com/lunr/-/lunr-2.3.6.tgz#f278beee7ffd56ad86e6e478ce02ab2b98c78dd5"
integrity sha512-swStvEyDqQ85MGpABCMBclZcLI/pBIlu8FFDtmX197+oEgKloJ67QnB+Tidh0340HmLMs39c4GrkPY3cmkXp6Q==
make-dir@^1.0.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c"
@ -6714,6 +6709,15 @@ prompts@^0.1.9:
kleur "^2.0.1"
sisteransi "^0.1.1"
prop-types@^15.5.8:
version "15.7.2"
resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.7.2.tgz#52c41e75b8c87e72b9d9360e0206b99dcbffa6c5"
integrity sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==
dependencies:
loose-envify "^1.4.0"
object-assign "^4.1.1"
react-is "^16.8.1"
prop-types@^15.6.2:
version "15.6.2"
resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.6.2.tgz#05d5ca77b4453e985d60fc7ff8c859094a497102"
@ -6909,11 +6913,23 @@ react-error-overlay@^4.0.1:
resolved "https://registry.yarnpkg.com/react-error-overlay/-/react-error-overlay-4.0.1.tgz#417addb0814a90f3a7082eacba7cee588d00da89"
integrity sha512-xXUbDAZkU08aAkjtUvldqbvI04ogv+a1XdHxvYuHPYKIVk/42BIOD0zSKTHAWV4+gDy3yGm283z2072rA2gdtw==
react-infinite-scroller@^1.2.4:
version "1.2.4"
resolved "https://registry.yarnpkg.com/react-infinite-scroller/-/react-infinite-scroller-1.2.4.tgz#f67eaec4940a4ce6417bebdd6e3433bfc38826e9"
integrity sha512-/oOa0QhZjXPqaD6sictN2edFMsd3kkMiE19Vcz5JDgHpzEJVqYcmq+V3mkwO88087kvKGe1URNksHEOt839Ubw==
dependencies:
prop-types "^15.5.8"
react-is@^16.6.0, react-is@^16.7.0:
version "16.8.5"
resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.8.5.tgz#c54ac229dd66b5afe0de5acbe47647c3da692ff8"
integrity sha512-sudt2uq5P/2TznPV4Wtdi+Lnq3yaYW8LfvPKLM9BKD8jJNBkxMVyB0C9/GmVhLw7Jbdmndk/73n7XQGeN9A3QQ==
react-is@^16.8.1:
version "16.8.6"
resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.8.6.tgz#5bbc1e2d29141c9fbdfed456343fe2bc430a6a16"
integrity sha512-aUk3bHfZ2bRSVFFbbeVS4i+lNPZr3/WM5jT2J5omUVV1zzcs1nAaf3l51ctA5FFvCRbhrH0bdAsRRQddFJZPtA==
react-router-dom@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-5.0.0.tgz#542a9b86af269a37f0b87218c4c25ea8dcf0c073"

Loading…
Cancel
Save