* Remove nginx.conf from Dockerfile * Move nginx.conf, add proxy settings * Add docker-compose.yaml * Update .gitignore * Load data from API * Remove spider result JSON * Fix some page details * Remove logging * Several details page fixes and improvements * Load screenshots data from API * Load screenshots via proxy * Remove site favicons * Update nginx configs * Delete CNAME * Update serviceWorkerAddon.js * Update docker-compose files * Update production docker-compose file * Update Dockerfile * Update Makefile * Several functional improvements and fixesmain


@ -0,0 +1,7 @@
|
||||
# Dockerfile for the webapp dev server |
||||
|
||||
FROM node:10-slim |
||||
|
||||
WORKDIR /workdir |
||||
|
||||
RUN yarn install |
@ -1,58 +0,0 @@
|
||||
user nginx; |
||||
worker_processes 1; |
||||
|
||||
error_log stderr warn; |
||||
pid /var/run/nginx.pid; |
||||
|
||||
events { |
||||
worker_connections 1024; |
||||
} |
||||
|
||||
http { |
||||
server { |
||||
listen *:8000; |
||||
rewrite ^/$ https://green-spider.netzbegruenung.de/ redirect; |
||||
} |
||||
|
||||
server { |
||||
listen *:443; |
||||
|
||||
ssl on; |
||||
ssl_certificate /etc/letsencrypt/live/green-spider.netzbegruenung.de/fullchain.pem; |
||||
ssl_certificate_key /etc/letsencrypt/live/green-spider.netzbegruenung.de/privkey.pem; |
||||
|
||||
root /usr/share/nginx/html; |
||||
|
||||
# siteicons, css, js never change |
||||
location /siteicons/ { |
||||
expires 100d; |
||||
} |
||||
location /static/ { |
||||
expires 100d; |
||||
} |
||||
|
||||
location / { |
||||
try_files $uri $uri/ /index.html; |
||||
} |
||||
|
||||
} |
||||
|
||||
include /etc/nginx/mime.types; |
||||
|
||||
default_type application/octet-stream; |
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" ' |
||||
'$status $body_bytes_sent "$http_referer" ' |
||||
'"$http_user_agent" "$http_x_forwarded_for"'; |
||||
|
||||
access_log /dev/stdout main; |
||||
|
||||
sendfile on; |
||||
#tcp_nopush on; |
||||
|
||||
keepalive_timeout 65; |
||||
|
||||
#gzip on; |
||||
|
||||
include /etc/nginx/conf.d/*.conf; |
||||
} |
@ -0,0 +1,91 @@
|
||||
user nginx; |
||||
worker_processes 1; |
||||
|
||||
error_log stderr warn; |
||||
pid /var/run/nginx.pid; |
||||
|
||||
events { |
||||
worker_connections 1024; |
||||
} |
||||
|
||||
http { |
||||
proxy_cache_path /var/cache/nginx/cache-api keys_zone=api:1m; |
||||
proxy_cache_path /var/cache/nginx/cache-screenshots keys_zone=screenshots:1m; |
||||
|
||||
upstream gs { |
||||
server storage.googleapis.com:443; |
||||
keepalive 128; |
||||
} |
||||
|
||||
server { |
||||
listen *:8000; |
||||
|
||||
location /api/ { |
||||
proxy_pass http://api:5000; |
||||
proxy_cache api; |
||||
proxy_cache_background_update on; |
||||
proxy_cache_lock on; |
||||
proxy_cache_lock_timeout 15s; |
||||
proxy_cache_use_stale updating; |
||||
proxy_cache_valid 200 6h; |
||||
proxy_cache_valid any 60m; |
||||
proxy_set_header X-Real-IP $remote_addr; |
||||
} |
||||
|
||||
location /screenshots/ { |
||||
resolver 8.8.8.8 ipv6=off; |
||||
|
||||
rewrite /screenshots/(.*) /$1 break; |
||||
proxy_pass https://gs/green-spider-screenshots.sendung.de/$1; |
||||
proxy_http_version 1.1; |
||||
proxy_set_header Host storage.googleapis.com; |
||||
proxy_set_header Connection ""; |
||||
proxy_intercept_errors on; |
||||
|
||||
proxy_cache screenshots; |
||||
proxy_cache_background_update on; |
||||
proxy_cache_lock on; |
||||
proxy_cache_lock_timeout 5s; |
||||
proxy_cache_use_stale updating; |
||||
proxy_cache_valid 200 24h; |
||||
proxy_cache_valid any 3h; |
||||
|
||||
proxy_hide_header alt-svc; |
||||
proxy_hide_header X-GUploader-UploadID; |
||||
proxy_hide_header alternate-protocol; |
||||
proxy_hide_header x-goog-hash; |
||||
proxy_hide_header x-goog-generation; |
||||
proxy_hide_header x-goog-metageneration; |
||||
proxy_hide_header x-goog-stored-content-encoding; |
||||
proxy_hide_header x-goog-stored-content-length; |
||||
proxy_hide_header x-goog-storage-class; |
||||
proxy_hide_header x-xss-protection; |
||||
proxy_hide_header accept-ranges; |
||||
proxy_hide_header Set-Cookie; |
||||
proxy_ignore_headers Set-Cookie; |
||||
} |
||||
|
||||
location / { |
||||
proxy_pass http://webapp:3000; |
||||
|
||||
proxy_set_header X-Real-IP $remote_addr; |
||||
proxy_set_header X-Forwarded-For $remote_addr; |
||||
proxy_set_header Host $host; |
||||
proxy_redirect off; |
||||
proxy_http_version 1.1; |
||||
proxy_set_header Upgrade $http_upgrade; |
||||
proxy_set_header Connection "upgrade"; |
||||
} |
||||
|
||||
} |
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" ' |
||||
'$status $body_bytes_sent "$http_referer" ' |
||||
'"$http_user_agent" "$http_x_forwarded_for" $request_time'; |
||||
|
||||
access_log /dev/stdout main; |
||||
|
||||
sendfile on; |
||||
|
||||
keepalive_timeout 65; |
||||
} |
@ -0,0 +1,124 @@
|
||||
user nginx; |
||||
worker_processes 1; |
||||
|
||||
error_log stderr warn; |
||||
pid /var/run/nginx.pid; |
||||
|
||||
events { |
||||
worker_connections 1024; |
||||
} |
||||
|
||||
http { |
||||
proxy_cache_path /var/cache/nginx/cache-api keys_zone=api:1m; |
||||
proxy_cache_path /var/cache/nginx/cache-screenshots keys_zone=screenshots:1m; |
||||
|
||||
upstream gs { |
||||
server storage.googleapis.com:443; |
||||
keepalive 128; |
||||
} |
||||
|
||||
# This server redirects all requests to the HTTPS server |
||||
server { |
||||
listen *:80; |
||||
rewrite ^/$ https://green-spider.netzbegruenung.de/ redirect; |
||||
} |
||||
|
||||
server { |
||||
listen *:443; |
||||
|
||||
ssl on; |
||||
ssl_certificate /etc/letsencrypt/live/green-spider.netzbegruenung.de/fullchain.pem; |
||||
ssl_certificate_key /etc/letsencrypt/live/green-spider.netzbegruenung.de/privkey.pem; |
||||
|
||||
root /usr/share/nginx/html; |
||||
|
||||
# css, js never change |
||||
location /static/ { |
||||
expires 100d; |
||||
} |
||||
|
||||
# The last-updated resource will be cached for 5 minutes max. |
||||
location /api/v1/spider-results/last-updated/ { |
||||
proxy_pass http://api:5000/api/v1/spider-results/last-updated/; |
||||
proxy_cache api; |
||||
proxy_cache_background_update on; |
||||
proxy_cache_lock on; |
||||
proxy_cache_lock_timeout 3s; |
||||
proxy_cache_use_stale updating; |
||||
proxy_cache_valid 200 5m; |
||||
proxy_cache_valid any 30m; |
||||
proxy_set_header X-Real-IP $remote_addr; |
||||
} |
||||
|
||||
# All other API calls are cached for 6 hours |
||||
location /api/ { |
||||
proxy_pass http://api:5000; |
||||
proxy_cache api; |
||||
proxy_cache_background_update on; |
||||
proxy_cache_lock on; |
||||
proxy_cache_lock_timeout 15s; |
||||
proxy_cache_use_stale updating; |
||||
proxy_cache_valid 200 6h; |
||||
proxy_cache_valid any 60m; |
||||
proxy_set_header X-Real-IP $remote_addr; |
||||
} |
||||
|
||||
location /screenshots/ { |
||||
resolver 8.8.8.8 ipv6=off; |
||||
|
||||
rewrite /screenshots/(.*) /$1 break; |
||||
proxy_pass https://gs/green-spider-screenshots.sendung.de/$1; |
||||
proxy_http_version 1.1; |
||||
proxy_set_header Host storage.googleapis.com; |
||||
proxy_set_header Connection ""; |
||||
proxy_intercept_errors on; |
||||
|
||||
proxy_cache screenshots; |
||||
proxy_cache_background_update on; |
||||
proxy_cache_lock on; |
||||
proxy_cache_lock_timeout 5s; |
||||
proxy_cache_use_stale updating; |
||||
proxy_cache_valid 200 24h; |
||||
proxy_cache_valid any 3h; |
||||
|
||||
proxy_hide_header alt-svc; |
||||
proxy_hide_header X-GUploader-UploadID; |
||||
proxy_hide_header alternate-protocol; |
||||
proxy_hide_header x-goog-hash; |
||||
proxy_hide_header x-goog-generation; |
||||
proxy_hide_header x-goog-metageneration; |
||||
proxy_hide_header x-goog-stored-content-encoding; |
||||
proxy_hide_header x-goog-stored-content-length; |
||||
proxy_hide_header x-goog-storage-class; |
||||
proxy_hide_header x-xss-protection; |
||||
proxy_hide_header accept-ranges; |
||||
proxy_hide_header Set-Cookie; |
||||
proxy_ignore_headers Set-Cookie; |
||||
} |
||||
|
||||
location / { |
||||
try_files $uri $uri/ /index.html; |
||||
} |
||||
|
||||
} |
||||
|
||||
include /mime.types; |
||||
|
||||
default_type application/octet-stream; |
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" ' |
||||
'$status $body_bytes_sent "$http_referer" ' |
||||
'"$http_user_agent" "$http_x_forwarded_for" $request_time'; |
||||
|
||||
access_log /dev/stdout main; |
||||
|
||||
sendfile on; |
||||
#tcp_nopush on; |
||||
|
||||
keepalive_timeout 65; |
||||
|
||||
gzip on; |
||||
gzip_types application/json application/javascript application/x-javascript text/css; |
||||
|
||||
include /etc/nginx/conf.d/*.conf; |
||||
} |
@ -0,0 +1,114 @@
|
||||
user nginx; |
||||
worker_processes 1; |
||||
|
||||
error_log stderr info; |
||||
pid /var/run/nginx.pid; |
||||
|
||||
events { |
||||
worker_connections 1024; |
||||
} |
||||
|
||||
http { |
||||
proxy_cache_path /var/cache/nginx/cache-api keys_zone=api:1m; |
||||
proxy_cache_path /var/cache/nginx/cache-screenshots keys_zone=screenshots:1m; |
||||
|
||||
upstream gs { |
||||
server storage.googleapis.com:443; |
||||
keepalive 128; |
||||
} |
||||
|
||||
server { |
||||
listen *:8080; |
||||
|
||||
root /usr/share/nginx/html; |
||||
|
||||
# css, js never change |
||||
location /static/ { |
||||
expires 100d; |
||||
} |
||||
|
||||
# The last-updated resource will be cached for 5 minutes max. |
||||
location /api/v1/spider-results/last-updated/ { |
||||
proxy_pass http://api:5000/api/v1/spider-results/last-updated/; |
||||
proxy_cache api; |
||||
proxy_cache_background_update on; |
||||
proxy_cache_lock on; |
||||
proxy_cache_lock_timeout 3s; |
||||
proxy_cache_use_stale updating; |
||||
proxy_cache_valid 200 5m; |
||||
proxy_cache_valid any 30m; |
||||
proxy_set_header X-Real-IP $remote_addr; |
||||
} |
||||
|
||||
# All other API calls are cached for 6 hours |
||||
location /api/ { |
||||
proxy_pass http://api:5000; |
||||
proxy_cache api; |
||||
proxy_cache_background_update on; |
||||
proxy_cache_lock on; |
||||
proxy_cache_lock_timeout 15s; |
||||
proxy_cache_use_stale updating; |
||||
proxy_cache_valid 200 6h; |
||||
proxy_cache_valid any 60m; |
||||
proxy_set_header X-Real-IP $remote_addr; |
||||
} |
||||
|
||||
location /screenshots/ { |
||||
resolver 8.8.8.8 ipv6=off; |
||||
|
||||
rewrite /screenshots/(.*) /$1 break; |
||||
proxy_pass https://gs/green-spider-screenshots.sendung.de/$1; |
||||
proxy_http_version 1.1; |
||||
proxy_set_header Host storage.googleapis.com; |
||||
proxy_set_header Connection ""; |
||||
proxy_intercept_errors on; |
||||
|
||||
proxy_cache screenshots; |
||||
proxy_cache_background_update on; |
||||
proxy_cache_lock on; |
||||
proxy_cache_lock_timeout 5s; |
||||
proxy_cache_use_stale updating; |
||||
proxy_cache_valid 200 24h; |
||||
proxy_cache_valid any 3h; |
||||
|
||||
proxy_hide_header alt-svc; |
||||
proxy_hide_header X-GUploader-UploadID; |
||||
proxy_hide_header alternate-protocol; |
||||
proxy_hide_header x-goog-hash; |
||||
proxy_hide_header x-goog-generation; |
||||
proxy_hide_header x-goog-metageneration; |
||||
proxy_hide_header x-goog-stored-content-encoding; |
||||
proxy_hide_header x-goog-stored-content-length; |
||||
proxy_hide_header x-goog-storage-class; |
||||
proxy_hide_header x-xss-protection; |
||||
proxy_hide_header accept-ranges; |
||||
proxy_hide_header Set-Cookie; |
||||
proxy_ignore_headers Set-Cookie; |
||||
} |
||||
|
||||
location / { |
||||
try_files $uri $uri/ /index.html; |
||||
} |
||||
|
||||
} |
||||
|
||||
include /mime.types; |
||||
|
||||
default_type application/octet-stream; |
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" ' |
||||
'$status $body_bytes_sent "$http_referer" ' |
||||
'"$http_user_agent" "$http_x_forwarded_for" $request_time'; |
||||
|
||||
access_log /dev/stdout main; |
||||
|
||||
sendfile on; |
||||
#tcp_nopush on; |
||||
|
||||
keepalive_timeout 65; |
||||
|
||||
gzip on; |
||||
gzip_types application/json application/javascript application/x-javascript text/css; |
||||
|
||||
include /etc/nginx/conf.d/*.conf; |
||||
} |
@ -0,0 +1,24 @@
|
||||
version: "3" |
||||
services: |
||||
|
||||
# The HTTPS server and proxy on port 443 |
||||
webapp: |
||||
image: quay.io/netzbegruenung/green-spider-webapp:latest |
||||
restart: always |
||||
ports: |
||||
- "443:443" |
||||
volumes: |
||||
- $PWD:/workdir |
||||
- $PWD/config/nginx/nginx_prod.conf:/etc/nginx/nginx.conf |
||||
- $PWD/test-certs:/etc/letsencrypt/live/green-spider.netzbegruenung.de |
||||
- $PWD/proxy-cache-prod:/var/cache/nginx |
||||
|
||||
api: |
||||
image: quay.io/netzbegruenung/green-spider-api:latest |
||||
restart: always |
||||
ports: |
||||
- 5000:5000 |
||||
environment: |
||||
GCLOUD_DATASTORE_CREDENTIALS_PATH: /secrets/datastore-reader.json |
||||
volumes: |
||||
- $PWD/secrets:/secrets |
@ -0,0 +1,34 @@
|
||||
version: "3" |
||||
services: |
||||
|
||||
# The static file server and proxy on port 80 |
||||
webapp: |
||||
image: quay.io/netzbegruenung/green-spider-webapp:latest |
||||
build: |
||||
context: . |
||||
dockerfile: Dockerfile |
||||
ports: |
||||
- "8080:8080" |
||||
volumes: |
||||
- ".:/workdir" |
||||
- ${PWD}/config/nginx/nginx_staging.conf:/etc/nginx/nginx.conf |
||||
- ${PWD}/proxy-cache-staging:/var/cache/nginx |
||||
depends_on: |
||||
- api |
||||
networks: |
||||
- staging |
||||
|
||||
api: |
||||
image: quay.io/netzbegruenung/green-spider-api:latest |
||||
ports: |
||||
- 5000:5000 |
||||
environment: |
||||
GCLOUD_DATASTORE_CREDENTIALS_PATH: /secrets/datastore-reader.json |
||||
volumes: |
||||
- "./secrets:/secrets" |
||||
networks: |
||||
- staging |
||||
|
||||
networks: |
||||
staging: |
||||
driver: bridge |
@ -0,0 +1,32 @@
|
||||
version: "3" |
||||
services: |
||||
|
||||
# proxy passing through webapp and api |
||||
proxy: |
||||
image: nginx:1.14-alpine |
||||
ports: |
||||
- "8000:8000" |
||||
volumes: |
||||
- ${PWD}/config/nginx/nginx_dev.conf:/etc/nginx/nginx.conf |
||||
- ${PWD}/proxy-cache:/var/cache/nginx |
||||
|
||||
# The webapp dev server on port 3000 |
||||
webapp: |
||||
image: green-spider-webapp:dev |
||||
build: |
||||
context: . |
||||
dockerfile: Dockerfile_node_dev |
||||
command: yarn start |
||||
ports: |
||||
- "3000:3000" |
||||
volumes: |
||||
- ".:/workdir" |
||||
|
||||
api: |
||||
image: quay.io/netzbegruenung/green-spider-api:latest |
||||
ports: |
||||
- 5000:5000 |
||||
environment: |
||||
GCLOUD_DATASTORE_CREDENTIALS_PATH: /secrets/datastore-reader.json |
||||
volumes: |
||||
- "./secrets:/secrets" |
Before Width: | Height: | Size: 1.7 KiB |
Before Width: | Height: | Size: 570 B |
Before Width: | Height: | Size: 1.1 KiB |
Before Width: | Height: | Size: 1.6 KiB |
Before Width: | Height: | Size: 1.1 KiB |
Before Width: | Height: | Size: 34 KiB |
Before Width: | Height: | Size: 1.1 KiB |
Before Width: | Height: | Size: 1.4 KiB |
Before Width: | Height: | Size: 31 KiB |
Before Width: | Height: | Size: 7.4 KiB |
Before Width: | Height: | Size: 2.0 KiB |
Before Width: | Height: | Size: 1.1 KiB |
Before Width: | Height: | Size: 6.8 KiB |
Before Width: | Height: | Size: 1.5 KiB |
Before Width: | Height: | Size: 4.2 KiB |
Before Width: | Height: | Size: 4.9 KiB |
Before Width: | Height: | Size: 3.2 KiB |
Before Width: | Height: | Size: 2.2 KiB |
Before Width: | Height: | Size: 1.6 KiB |
Before Width: | Height: | Size: 2.7 KiB |
Before Width: | Height: | Size: 17 KiB |
Before Width: | Height: | Size: 4.5 KiB |
Before Width: | Height: | Size: 3.2 KiB |
Before Width: | Height: | Size: 46 KiB |
Before Width: | Height: | Size: 171 B |
Before Width: | Height: | Size: 1.4 KiB |
Before Width: | Height: | Size: 1.3 KiB |
Before Width: | Height: | Size: 3.0 KiB |
Before Width: | Height: | Size: 1.1 KiB |
Before Width: | Height: | Size: 3.4 KiB |
Before Width: | Height: | Size: 20 KiB |
Before Width: | Height: | Size: 8.0 KiB |
Before Width: | Height: | Size: 21 KiB |
Before Width: | Height: | Size: 50 KiB |
Before Width: | Height: | Size: 4.4 KiB |
Before Width: | Height: | Size: 5.3 KiB |
Before Width: | Height: | Size: 1.1 KiB |
Before Width: | Height: | Size: 25 KiB |
Before Width: | Height: | Size: 894 B |
Before Width: | Height: | Size: 1.0 KiB |
Before Width: | Height: | Size: 24 KiB |
Before Width: | Height: | Size: 1.1 KiB |
Before Width: | Height: | Size: 894 B |
Before Width: | Height: | Size: 99 KiB |
Before Width: | Height: | Size: 2.3 KiB |
Before Width: | Height: | Size: 2.2 KiB |
Before Width: | Height: | Size: 2.4 KiB |
Before Width: | Height: | Size: 846 B |
Before Width: | Height: | Size: 686 B |
Before Width: | Height: | Size: 2.6 KiB |
Before Width: | Height: | Size: 3.3 KiB |
Before Width: | Height: | Size: 1.8 KiB |
Before Width: | Height: | Size: 1.4 KiB |
Before Width: | Height: | Size: 48 KiB |
Before Width: | Height: | Size: 15 KiB |
Before Width: | Height: | Size: 2.2 KiB |
Before Width: | Height: | Size: 2.9 KiB |
Before Width: | Height: | Size: 1.1 KiB |
Before Width: | Height: | Size: 894 B |
Before Width: | Height: | Size: 1.4 KiB |
Before Width: | Height: | Size: 1.1 KiB |
Before Width: | Height: | Size: 1.1 KiB |
Before Width: | Height: | Size: 1.1 KiB |
Before Width: | Height: | Size: 98 KiB |
Before Width: | Height: | Size: 2.6 KiB |
Before Width: | Height: | Size: 597 B |
Before Width: | Height: | Size: 1.4 KiB |
Before Width: | Height: | Size: 17 KiB |
Before Width: | Height: | Size: 2.2 KiB |
Before Width: | Height: | Size: 6.4 KiB |
Before Width: | Height: | Size: 1.4 KiB |
Before Width: | Height: | Size: 1.1 KiB |
Before Width: | Height: | Size: 1.6 KiB |
Before Width: | Height: | Size: 918 B |
Before Width: | Height: | Size: 2.7 KiB |
Before Width: | Height: | Size: 13 KiB |
Before Width: | Height: | Size: 4.2 KiB |
Before Width: | Height: | Size: 2.2 KiB |
Before Width: | Height: | Size: 9.3 KiB |
Before Width: | Height: | Size: 3.6 KiB |
Before Width: | Height: | Size: 894 B |
Before Width: | Height: | Size: 2.1 KiB |
Before Width: | Height: | Size: 1.1 KiB |
Before Width: | Height: | Size: 610 B |