Adapt Dockerfile, Makefile, script

Dockerfile

@@ -4,18 +4,17 @@ FROM python:3.6-alpine3.8
 RUN echo "http://dl-4.alpinelinux.org/alpine/v3.7/main" >> /etc/apk/repositories && \
     echo "http://dl-4.alpinelinux.org/alpine/v3.7/community" >> /etc/apk/repositories && \
     apk update && \
-    apk --no-cache add chromium chromium-chromedriver python3-dev build-base git py3-lxml libxml2 libxml2-dev libxslt libxslt-dev && \
+    apk --no-cache add chromium chromium-chromedriver python3-dev build-base git py3-lxml libxml2 libxml2-dev libxslt libxslt-dev libffi-dev openssl-dev && \
     pip3 install --upgrade pip && \
-    pip3 install selenium==3.8.0 GitPython PyYAML beautifulsoup4==4.6.0 html-similarity==0.3.2 requests==2.18.4 responses==0.9.0 smmap2==2.0.3 urllib3==1.22 google-cloud-datastore==1.7.0 tenacity==5.0.2 && \
+    pip3 install selenium==3.8.0 GitPython PyYAML beautifulsoup4==4.6.0 html-similarity==0.3.2 pyopenssl==18.0.0 requests==2.18.4 responses==0.9.0 smmap2==2.0.3 urllib3==1.22 google-cloud-datastore==1.7.0 tenacity==5.0.2 && \
     apk del python3-dev build-base
 
-ADD spider.py /
-ADD spider_test.py /
-ADD data_export.py /
+ADD cli.py /
 ADD config /config
 ADD jobs /jobs
 ADD checks /checks
 ADD rating /rating
+ADD spider /spider
+ADD export /export
 
-ENTRYPOINT ["python3"]
-CMD ["/spider.py"]
+ENTRYPOINT ["python3", "/cli.py"]

Makefile

@@ -4,13 +4,13 @@
 
 # Build docker image
 dockerimage:
-	docker build -t quay.io/netzbegruenung/green-spider:latest .
+	docker build -t quay.io/netzbegruenung/green-spider:dev .
 
 # Create spider job queue
 spiderjobs: dockerimage
 	docker run --rm -ti \
 		-v $(PWD)/secrets:/secrets \
-		quay.io/netzbegruenung/green-spider:latest spider.py \
+		quay.io/netzbegruenung/green-spider:dev \
 		--credentials-path /secrets/datastore-writer.json \
 		--loglevel info \
 		jobs
@@ -21,19 +21,22 @@ spider: dockerimage
 	  -v $(PWD)/dev-shm:/dev/shm \
 		-v $(PWD)/webapp/dist/data:/out \
 		-v $(PWD)/secrets:/secrets \
-		quay.io/netzbegruenung/green-spider:latest spider.py \
+		quay.io/netzbegruenung/green-spider:dev \
 		--credentials-path /secrets/datastore-writer.json \
 		--loglevel debug \
-		spider
+		spider --kind spider-results-dev
 
 export: dockerimage
 	docker run --rm -ti \
 		-v $(PWD)/export-json:/out \
 		-v $(PWD)/secrets:/secrets \
 		-v $(PWD)/export-siteicons:/icons \
-		quay.io/netzbegruenung/green-spider:latest \
-		data_export.py /secrets/datastore-reader.json
+		quay.io/netzbegruenung/green-spider:dev \
+		--credentials-path /secrets/datastore-reader.json \
+		--loglevel debug \
+		export --kind spider-results-dev
 
 # run spider tests
+# FIXME
 test: dockerimage
 	docker run --rm -ti quay.io/netzbegruenung/green-spider:latest /spider_test.py

devops/run-job.sh

@@ -19,6 +19,8 @@
 #   secrets/datastore-writer.json
 
 
+DOCKERIMAGE="quay.io/netzbegruenung/green-spider:dev"
+
 API_TOKEN_SECRET="secrets/hetzner-api-token.sh"
 test -f $API_TOKEN_SECRET || { echo >&2 "File $API_TOKEN_SECRET does not exist."; exit 1; }
 source $API_TOKEN_SECRET
@@ -29,10 +31,14 @@ if [[ "$1" == "" ]]; then
   exit 1
 fi
 
+SERVERNAME="$1-$(date | md5 | cut -c1-3)"
+
+# possible values: cx11 (1 core 2 GB), cx21 (2 cores, 4 GB), cx31 (2 cores, 8 GB)
+SERVERTYPE="cx21"
 
 function create_server()
 {
-  echo "Creating server $1"
+  echo "Creating server $SERVERNAME"
 
   # server_type 'cx11' is the smallest, cheapest category.
   # location 'nbg1' is Nürnberg/Nuremberg, Germany.
@@ -44,8 +50,8 @@ function create_server()
     -H "Content-Type: application/json" \
     -H "Authorization: Bearer $API_TOKEN" \
     -d "{
-      \"name\": \"$1\",
-      \"server_type\": \"cx11\",
+      \"name\": \"$SERVERNAME\",
+      \"server_type\": \"$SERVERTYPE\",
       \"location\": \"nbg1\",
       \"start_after_create\": true,
       \"image\": \"debian-9\",
@@ -61,7 +67,7 @@ function create_server()
   # Get IP:
   SERVER_IP=$(echo $CREATE_RESPONSE | jq -r .server.public_net.ipv4.ip)
 
-  echo "Created server with ID $SERVER_ID and IP $SERVER_IP"
+  echo "Created server $SERVERNAME with ID $SERVER_ID and IP $SERVER_IP"
 }
 
 
@@ -142,22 +148,25 @@ else
 
   # Run docker job
   echo "Starting Docker Job"
-  ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
-    -v /root/secrets:/secrets \
-    quay.io/netzbegruenung/green-spider spider.py \
-    --credentials-path /secrets/datastore-writer.json \
-    jobs
+  #ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
+  #  -v /root/secrets:/secrets \
+  #  quay.io/netzbegruenung/green-spider spider.py \
+  #  --credentials-path /secrets/datastore-writer.json \
+  #  jobs
 
+  ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP mkdir -p /dev-shm
   ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
+    -v /dev-shm:/dev/shm \
     -v /root/secrets:/secrets \
-    quay.io/netzbegruenung/green-spider spider.py \
+    $DOCKERIMAGE \
     --credentials-path /secrets/datastore-writer.json \
-    spider
+    --loglevel info \
+    spider --kind spider-results-dev
 
 fi
 
 # Delete the box
-echo "Deleting server $SERVER_ID"
+echo "Deleting server $SERVERNAME with ID $SERVER_ID"
 curl -s -X DELETE -H "Content-Type: application/json" \
   -H "Authorization: Bearer $API_TOKEN" \
   https://api.hetzner.cloud/v1/servers/$SERVER_ID