Proxy API requests, load data from API (#15)

* Remove nginx.conf from Dockerfile

* Move nginx.conf, add proxy settings

* Add docker-compose.yaml

* Update .gitignore

* Load data from API

* Remove spider result JSON

* Fix some page details

* Remove logging

* Several details page fixes and improvements

* Load screenshots data from API

* Load screenshots via proxy

* Remove site favicons

* Update nginx configs

* Delete CNAME

* Update serviceWorkerAddon.js

* Update docker-compose files

* Update production docker-compose file

* Update Dockerfile

* Update Makefile

* Several functional improvements and fixes

.gitignore

@@ -22,3 +22,4 @@ yarn-error.log*
 
 # secret key we never want to share
 /secrets
+/test-certs

Dockerfile

@@ -4,14 +4,19 @@ WORKDIR /
 
 ADD yarn.lock /
 ADD package.json /
-ADD src /src
 ADD scripts /scripts
-ADD public /public
-ADD config /config
 
 RUN yarn install
+
+ADD config /config
+ADD public /public
+ADD src /src
+
 RUN yarn build
 
 FROM nginx:1.14-alpine
+
+# move this file, as /etc/nginx/ will be masked by a volume
+RUN cp /etc/nginx/mime.types /mime.types
+
 COPY --from=builder /build /usr/share/nginx/html
-COPY config/nginx.conf /etc/nginx/nginx.conf

Dockerfile_node_dev

@@ -0,0 +1,7 @@
+# Dockerfile for the webapp dev server
+
+FROM node:10-slim
+
+WORKDIR /workdir
+
+RUN yarn install

Makefile

@@ -1,10 +1,6 @@
 
 IMAGE := quay.io/netzbegruenung/green-spider:latest
 
-# Run a development server and load it in your default browser.
-# Will reload as soon as some source has changed.
-dev:
-	yarn start
 
 # Build the web application to be served by a static web server
 build:
@@ -15,8 +11,17 @@ dist: build
 	rm -rf docs
 	cp -r build docs
 
+# Run a development server on http://localhost:8000
+dev:
+	docker-compose -f docker-compose.yaml up --build
+
+# Run a staging server at http://localhost:8080
+serve-staging:
+	docker-compose -f docker-compose-staging.yaml up --build
+
+# Run the production setup at https://localhost
 serve-prod:
-	cd docs && python3 -m http.server
+	docker-compose -f docker-compose-prod.yaml up --build
 
 clean:
 	rm -rf build

config/nginx.conf

@@ -1,58 +0,0 @@
-user nginx;
-worker_processes  1;
-
-error_log stderr warn;
-pid /var/run/nginx.pid;
-
-events {
-    worker_connections  1024;
-}
-
-http {
-    server {
-      listen *:8000;
-      rewrite ^/$ https://green-spider.netzbegruenung.de/ redirect;
-    }
-
-    server {
-      listen *:443;
-
-      ssl on;
-      ssl_certificate /etc/letsencrypt/live/green-spider.netzbegruenung.de/fullchain.pem;
-      ssl_certificate_key /etc/letsencrypt/live/green-spider.netzbegruenung.de/privkey.pem;
-
-      root /usr/share/nginx/html;
-
-      # siteicons, css, js never change
-      location /siteicons/ {
-        expires 100d;
-      }
-      location /static/ {
-        expires 100d;
-      }
-
-      location / {
-        try_files $uri $uri/ /index.html;
-      }
-
-    }
-
-    include /etc/nginx/mime.types;
-
-    default_type application/octet-stream;
-
-    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
-                      '$status $body_bytes_sent "$http_referer" '
-                      '"$http_user_agent" "$http_x_forwarded_for"';
-
-    access_log /dev/stdout main;
-
-    sendfile on;
-    #tcp_nopush on;
-
-    keepalive_timeout 65;
-
-    #gzip on;
-
-    include /etc/nginx/conf.d/*.conf;
-}

config/nginx/nginx_dev.conf

@@ -0,0 +1,91 @@
+user nginx;
+worker_processes  1;
+
+error_log stderr warn;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    proxy_cache_path /var/cache/nginx/cache-api keys_zone=api:1m;
+    proxy_cache_path /var/cache/nginx/cache-screenshots keys_zone=screenshots:1m;
+
+    upstream gs {
+      server     storage.googleapis.com:443;
+      keepalive  128;
+    }
+
+    server {
+      listen *:8000;
+
+      location /api/ {
+        proxy_pass http://api:5000;
+        proxy_cache api;
+        proxy_cache_background_update on;
+        proxy_cache_lock on;
+        proxy_cache_lock_timeout 15s;
+        proxy_cache_use_stale updating;
+        proxy_cache_valid 200 6h;
+        proxy_cache_valid any 60m;
+        proxy_set_header X-Real-IP $remote_addr;
+      }
+
+      location /screenshots/ {
+        resolver 8.8.8.8 ipv6=off;
+        
+        rewrite /screenshots/(.*) /$1 break;
+        proxy_pass https://gs/green-spider-screenshots.sendung.de/$1;
+        proxy_http_version 1.1;
+        proxy_set_header Host storage.googleapis.com;
+        proxy_set_header Connection "";
+        proxy_intercept_errors on;
+        
+        proxy_cache screenshots;
+        proxy_cache_background_update on;
+        proxy_cache_lock on;
+        proxy_cache_lock_timeout 5s;
+        proxy_cache_use_stale updating;
+        proxy_cache_valid 200 24h;
+        proxy_cache_valid any 3h;
+
+        proxy_hide_header       alt-svc;
+        proxy_hide_header       X-GUploader-UploadID;
+        proxy_hide_header       alternate-protocol;
+        proxy_hide_header       x-goog-hash;
+        proxy_hide_header       x-goog-generation;
+        proxy_hide_header       x-goog-metageneration;
+        proxy_hide_header       x-goog-stored-content-encoding;
+        proxy_hide_header       x-goog-stored-content-length;
+        proxy_hide_header       x-goog-storage-class;
+        proxy_hide_header       x-xss-protection;
+        proxy_hide_header       accept-ranges;
+        proxy_hide_header       Set-Cookie;
+        proxy_ignore_headers    Set-Cookie;
+      }
+
+      location / {
+        proxy_pass http://webapp:3000;
+
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header Host $host;
+        proxy_redirect off;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+      }
+
+    }
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for" $request_time';
+
+    access_log /dev/stdout main;
+
+    sendfile on;
+
+    keepalive_timeout 65;
+}

config/nginx/nginx_prod.conf

@@ -0,0 +1,124 @@
+user nginx;
+worker_processes  1;
+
+error_log stderr warn;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    proxy_cache_path /var/cache/nginx/cache-api keys_zone=api:1m;
+    proxy_cache_path /var/cache/nginx/cache-screenshots keys_zone=screenshots:1m;
+
+    upstream gs {
+      server     storage.googleapis.com:443;
+      keepalive  128;
+    }
+    
+    # This server redirects all requests to the HTTPS server
+    server {
+      listen *:80;
+      rewrite ^/$ https://green-spider.netzbegruenung.de/ redirect;
+    }
+
+    server {
+      listen *:443;
+
+      ssl on;
+      ssl_certificate /etc/letsencrypt/live/green-spider.netzbegruenung.de/fullchain.pem;
+      ssl_certificate_key /etc/letsencrypt/live/green-spider.netzbegruenung.de/privkey.pem;
+
+      root /usr/share/nginx/html;
+
+      # css, js never change
+      location /static/ {
+        expires 100d;
+      }
+
+      # The last-updated resource will be cached for 5 minutes max.
+      location /api/v1/spider-results/last-updated/ {
+        proxy_pass http://api:5000/api/v1/spider-results/last-updated/;
+        proxy_cache api;
+        proxy_cache_background_update on;
+        proxy_cache_lock on;
+        proxy_cache_lock_timeout 3s;
+        proxy_cache_use_stale updating;
+        proxy_cache_valid 200 5m;
+        proxy_cache_valid any 30m;
+        proxy_set_header X-Real-IP $remote_addr;
+      }
+
+      # All other API calls are cached for 6 hours
+      location /api/ {
+        proxy_pass http://api:5000;
+        proxy_cache api;
+        proxy_cache_background_update on;
+        proxy_cache_lock on;
+        proxy_cache_lock_timeout 15s;
+        proxy_cache_use_stale updating;
+        proxy_cache_valid 200 6h;
+        proxy_cache_valid any 60m;
+        proxy_set_header X-Real-IP $remote_addr;
+      }
+
+      location /screenshots/ {
+        resolver 8.8.8.8 ipv6=off;
+
+        rewrite /screenshots/(.*) /$1 break;
+        proxy_pass https://gs/green-spider-screenshots.sendung.de/$1;
+        proxy_http_version 1.1;
+        proxy_set_header Host storage.googleapis.com;
+        proxy_set_header Connection "";
+        proxy_intercept_errors on;
+        
+        proxy_cache screenshots;
+        proxy_cache_background_update on;
+        proxy_cache_lock on;
+        proxy_cache_lock_timeout 5s;
+        proxy_cache_use_stale updating;
+        proxy_cache_valid 200 24h;
+        proxy_cache_valid any 3h;
+
+        proxy_hide_header       alt-svc;
+        proxy_hide_header       X-GUploader-UploadID;
+        proxy_hide_header       alternate-protocol;
+        proxy_hide_header       x-goog-hash;
+        proxy_hide_header       x-goog-generation;
+        proxy_hide_header       x-goog-metageneration;
+        proxy_hide_header       x-goog-stored-content-encoding;
+        proxy_hide_header       x-goog-stored-content-length;
+        proxy_hide_header       x-goog-storage-class;
+        proxy_hide_header       x-xss-protection;
+        proxy_hide_header       accept-ranges;
+        proxy_hide_header       Set-Cookie;
+        proxy_ignore_headers    Set-Cookie;
+      }
+
+      location / {
+        try_files $uri $uri/ /index.html;
+      }
+
+    }
+
+    include /mime.types;
+
+    default_type application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for" $request_time';
+
+    access_log /dev/stdout main;
+
+    sendfile on;
+    #tcp_nopush on;
+
+    keepalive_timeout 65;
+
+    gzip on;
+    gzip_types application/json application/javascript application/x-javascript text/css;
+
+    include /etc/nginx/conf.d/*.conf;
+}

config/nginx/nginx_staging.conf

@@ -0,0 +1,114 @@
+user nginx;
+worker_processes  1;
+
+error_log stderr info;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    proxy_cache_path /var/cache/nginx/cache-api keys_zone=api:1m;
+    proxy_cache_path /var/cache/nginx/cache-screenshots keys_zone=screenshots:1m;
+
+    upstream gs {
+      server     storage.googleapis.com:443;
+      keepalive  128;
+    }
+
+    server {
+      listen *:8080;
+
+      root /usr/share/nginx/html;
+
+      # css, js never change
+      location /static/ {
+        expires 100d;
+      }
+
+      # The last-updated resource will be cached for 5 minutes max.
+      location /api/v1/spider-results/last-updated/ {
+        proxy_pass http://api:5000/api/v1/spider-results/last-updated/;
+        proxy_cache api;
+        proxy_cache_background_update on;
+        proxy_cache_lock on;
+        proxy_cache_lock_timeout 3s;
+        proxy_cache_use_stale updating;
+        proxy_cache_valid 200 5m;
+        proxy_cache_valid any 30m;
+        proxy_set_header X-Real-IP $remote_addr;
+      }
+
+      # All other API calls are cached for 6 hours
+      location /api/ {
+        proxy_pass http://api:5000;
+        proxy_cache api;
+        proxy_cache_background_update on;
+        proxy_cache_lock on;
+        proxy_cache_lock_timeout 15s;
+        proxy_cache_use_stale updating;
+        proxy_cache_valid 200 6h;
+        proxy_cache_valid any 60m;
+        proxy_set_header X-Real-IP $remote_addr;
+      }
+
+      location /screenshots/ {
+        resolver 8.8.8.8 ipv6=off;
+
+        rewrite /screenshots/(.*) /$1 break;
+        proxy_pass https://gs/green-spider-screenshots.sendung.de/$1;
+        proxy_http_version 1.1;
+        proxy_set_header Host storage.googleapis.com;
+        proxy_set_header Connection "";
+        proxy_intercept_errors on;
+        
+        proxy_cache screenshots;
+        proxy_cache_background_update on;
+        proxy_cache_lock on;
+        proxy_cache_lock_timeout 5s;
+        proxy_cache_use_stale updating;
+        proxy_cache_valid 200 24h;
+        proxy_cache_valid any 3h;
+
+        proxy_hide_header       alt-svc;
+        proxy_hide_header       X-GUploader-UploadID;
+        proxy_hide_header       alternate-protocol;
+        proxy_hide_header       x-goog-hash;
+        proxy_hide_header       x-goog-generation;
+        proxy_hide_header       x-goog-metageneration;
+        proxy_hide_header       x-goog-stored-content-encoding;
+        proxy_hide_header       x-goog-stored-content-length;
+        proxy_hide_header       x-goog-storage-class;
+        proxy_hide_header       x-xss-protection;
+        proxy_hide_header       accept-ranges;
+        proxy_hide_header       Set-Cookie;
+        proxy_ignore_headers    Set-Cookie;
+      }
+
+      location / {
+        try_files $uri $uri/ /index.html;
+      }
+
+    }
+
+    include /mime.types;
+
+    default_type application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for" $request_time';
+
+    access_log /dev/stdout main;
+
+    sendfile on;
+    #tcp_nopush on;
+
+    keepalive_timeout 65;
+
+    gzip on;
+    gzip_types application/json application/javascript application/x-javascript text/css;
+
+    include /etc/nginx/conf.d/*.conf;
+}

config/webpack.config.dev.js

@@ -1,5 +1,3 @@
-'use strict';
-
 const autoprefixer = require('autoprefixer');
 const path = require('path');
 const webpack = require('webpack');

docker-compose-prod.yaml

@@ -0,0 +1,24 @@
+version: "3"
+services:
+
+  # The HTTPS server and proxy on port 443
+  webapp:
+    image: quay.io/netzbegruenung/green-spider-webapp:latest
+    restart: always
+    ports:
+    - "443:443"
+    volumes:
+      - $PWD:/workdir
+      - $PWD/config/nginx/nginx_prod.conf:/etc/nginx/nginx.conf
+      - $PWD/test-certs:/etc/letsencrypt/live/green-spider.netzbegruenung.de
+      - $PWD/proxy-cache-prod:/var/cache/nginx
+
+  api:
+    image: quay.io/netzbegruenung/green-spider-api:latest
+    restart: always
+    ports:
+    - 5000:5000
+    environment:
+      GCLOUD_DATASTORE_CREDENTIALS_PATH: /secrets/datastore-reader.json
+    volumes:
+      - $PWD/secrets:/secrets

docker-compose-staging.yaml

@@ -0,0 +1,34 @@
+version: "3"
+services:
+
+  # The static file server and proxy on port 80
+  webapp:
+    image: quay.io/netzbegruenung/green-spider-webapp:latest
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - "8080:8080"
+    volumes:
+      - ".:/workdir"
+      - ${PWD}/config/nginx/nginx_staging.conf:/etc/nginx/nginx.conf
+      - ${PWD}/proxy-cache-staging:/var/cache/nginx
+    depends_on:
+      - api
+    networks:
+      - staging
+
+  api:
+    image: quay.io/netzbegruenung/green-spider-api:latest
+    ports:
+      - 5000:5000
+    environment:
+      GCLOUD_DATASTORE_CREDENTIALS_PATH: /secrets/datastore-reader.json
+    volumes:
+      - "./secrets:/secrets"
+    networks:
+      - staging
+
+networks:
+  staging:
+    driver: bridge

docker-compose.yaml

@@ -0,0 +1,32 @@
+version: "3"
+services:
+  
+  # proxy passing through webapp and api
+  proxy:
+    image: nginx:1.14-alpine
+    ports:
+      - "8000:8000"
+    volumes:
+      - ${PWD}/config/nginx/nginx_dev.conf:/etc/nginx/nginx.conf
+      - ${PWD}/proxy-cache:/var/cache/nginx
+
+  # The webapp dev server on port 3000
+  webapp:
+    image: green-spider-webapp:dev
+    build:
+      context: .
+      dockerfile: Dockerfile_node_dev
+    command: yarn start
+    ports:
+    - "3000:3000"
+    volumes:
+      - ".:/workdir"
+  
+  api:
+    image: quay.io/netzbegruenung/green-spider-api:latest
+    ports:
+    - 5000:5000
+    environment:
+      GCLOUD_DATASTORE_CREDENTIALS_PATH: /secrets/datastore-reader.json
+    volumes:
+      - "./secrets:/secrets"

package.json

@@ -4,6 +4,7 @@
   "private": true,
   "dependencies": {
     "autoprefixer": "7.1.6",
+    "axios": "^0.18.0",
     "babel-core": "6.26.0",
     "babel-eslint": "7.2.3",
     "babel-jest": "20.0.3",

public/CNAME

@@ -1 +0,0 @@
-green-spider.netzbegruenung.de

public/index.html

@@ -18,16 +18,7 @@
       JavaScript muss aktiviert sein, um diese Anwendung nutzen zu können.
     </noscript>
 
-    <div class="d-flex flex-column flex-md-row align-items-center p-3 px-md-4 mb-3 header border-bottom box-shadow">
-      <h5 class="my-0 mr-md-auto"><a href="/">Green Spider</a></h5>
-      <nav class="my-2 my-md-0 mr-md-3">
-        <span id="status" class="p-2">Lade Daten...</span>
-        <a class="p-2 text-light" href="https://github.com/netzbegruenung/green-spider/"><i class="ion-logo-github"></i> GitHub</a>
-        <a class="p-2 text-light" href="https://blog.netzbegruenung.de/projekte/green-spider/"><i class="ion-md-information-circle"></i> Über</a>
-      </nav>
-    </div>
-
-    <div id="root" class="container"></div>
+    <div id="root"></div>
 
   </body>
 </html>

public/serviceWorkerAddon.js

@@ -3,16 +3,29 @@
  */
 
 
+/**
+ * This handler intercepts all GET requests and either responds with a
+ * cached version of the resource or fetches the original one and then
+ * adds it to the cache.
+ * 
+ * This is called "on network response" in
+ * https://jakearchibald.com/2014/offline-cookbook/#on-network-response
+ * 
+ * We whitelist the URLs to cache in this manner.
+ */
 self.addEventListener('fetch', function(event) {
   var shouldRespond = false;
   if (event.request.method === 'GET') {
-    // icon
-    if (event.request.url.indexOf('/siteicons/') !== -1) {
-      shouldRespond = true;
+    // API
+    if (event.request.url.indexOf('/api/v1/') !== -1) {
+      // exclude our freshness check from cache
+      if (event.request.url.indexOf('/api/v1/spider-results/last-updated/') === -1) {
+        shouldRespond = true;
+      }
     }
 
     // webfonts
-    else if (event.request.url.indexOf('https://netzbegruenung.github.io') !== -1) {
+    else if (event.request.url.indexOf('https://netzbegruenung.github.io/webfonts/') !== -1) {
       shouldRespond = true;
     }
 
@@ -37,4 +50,23 @@ self.addEventListener('fetch', function(event) {
       );
     }
   }
-});
+});
+
+
+/**
+ * Pre-fetch some static resources on service worker installation.
+ */
+self.addEventListener('install', function(event) {
+  event.waitUntil(
+    caches.open(cacheName).then(function(cache) {
+      console.log('Pre-fetching some resources on SW install');
+      return cache.addAll([
+        'https://netzbegruenung.github.io/webfonts/fonts/Arvo_Gruen_2015_10.woff',
+        'https://unpkg.com/ionicons@4.4.3/dist/fonts/ionicons.woff2',
+        'https://netzbegruenung.github.io/webfonts/fonts/arvo_regular.woff',
+        'https://netzbegruenung.github.io/webfonts/style.css',
+        'https://unpkg.com/ionicons@4.4.3/dist/css/ionicons.min.css'
+      ]);
+    })
+  );
+});