From 84e62b218ae0f4fcc4285639d19ee60f69a42257 Mon Sep 17 00:00:00 2001
From: Marian Steinbach <marian@giantswarm.io>
Date: Mon, 27 Aug 2018 20:50:18 +0200
Subject: [PATCH] Consolidate scripts

---
 devops/README.md            |   4 +-
 devops/run-job.sh           | 163 ++++++++++++++++++++++++++++++++++++
 devops/run-screenshotter.sh | 120 --------------------------
 devops/run-spider.sh        | 129 ----------------------------
 4 files changed, 165 insertions(+), 251 deletions(-)
 create mode 100755 devops/run-job.sh
 delete mode 100755 devops/run-screenshotter.sh
 delete mode 100755 devops/run-spider.sh

diff --git a/devops/README.md b/devops/README.md
index 3e9dcaa..eb5b826 100644
--- a/devops/README.md
+++ b/devops/README.md
@@ -23,11 +23,11 @@ Die Scripte werden aus dem Root-Verzeichnis des Repositories ausgeführt.
 
 ```
 # Spidern
-devops/run-spider.sh
+devops/run-job.sh spider
 
 
 # Screenshots erzeugen
-devops/run-screenshotter.sh
+devops/run-job.sh screenshotter
 ```
 
 Der Terminal muss bis zum Ende der Ausführung geöffnet bleiben.
diff --git a/devops/run-job.sh b/devops/run-job.sh
new file mode 100755
index 0000000..207c205
--- /dev/null
+++ b/devops/run-job.sh
@@ -0,0 +1,163 @@
+#!/bin/bash
+
+# Creates a server, installs Docker, runs the screenshots job, tears down the server.
+#
+# This will take several hours. For a complete, clean run it is required to leave the
+# terminal running the script open. Otherwise the server won't be deleted properly
+# which will result in extra cost.
+#
+# When stopping the script at any point (Ctrl+C), please make sure that the server
+# gets deleted afterwards.
+#
+# Requirements:
+#
+# - curl
+# - jq (https://stedolan.github.io/jq/)
+# - ssh
+# - SSH key referenced in the server details ("ssh_keys")
+# - Service account with write permission for Storage and Datastore in 
+#   secrets/datastore-writer.json
+
+
+API_TOKEN_SECRET="secrets/hetzner-api-token.sh"
+test -f $API_TOKEN_SECRET || { echo >&2 "File $API_TOKEN_SECRET does not exist."; exit 1; }
+source $API_TOKEN_SECRET
+
+
+if [[ "$1" == "" ]]; then
+  echo "No argument given. Please use 'screenshotter' or 'spider' as arguments."
+  exit 1
+fi
+
+
+function create_server()
+{
+  echo "Creating server $1"
+
+  # server_type 'cx11' is the smallest, cheapest category.
+  # location 'nbg1' is Nürnberg/Nuremberg, Germany.
+  # image 'debian-9' is a plain Debian stretch.
+  # ssh_keys ['Marian'] adds Marian's public key to the server and can be extended.
+  # user_data: Ensures that we can detect when the cloud-init setup is done.
+  #
+  CREATE_RESPONSE=$(curl -s -X POST https://api.hetzner.cloud/v1/servers \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Bearer $API_TOKEN" \
+    -d "{
+      \"name\": \"$1\",
+      \"server_type\": \"cx11\",
+      \"location\": \"nbg1\",
+      \"start_after_create\": true,
+      \"image\": \"debian-9\",
+      \"ssh_keys\": [
+        \"Marian\"
+      ],
+      \"user_data\": \"#cloud-config\nruncmd:\n  - touch /cloud-init-done\n\"
+    }")
+
+  # Get ID:
+  SERVER_ID=$(echo $CREATE_RESPONSE | jq -r .server.id)
+
+  # Get IP:
+  SERVER_IP=$(echo $CREATE_RESPONSE | jq -r .server.public_net.ipv4.ip)
+
+  echo "Created server with ID $SERVER_ID and IP $SERVER_IP"
+}
+
+
+function wait_for_server()
+{
+  echo -n "Waiting for the server to be reachable via SSH "
+
+  sleep 30
+
+  STATUS="255"
+  while [ "$STATUS" != "0" ]; do
+    echo -n "."
+    sleep 5
+    ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP ls /cloud-init-done &> /dev/null
+    STATUS=$?
+  done
+
+  echo ""
+}
+
+
+create_server $1
+wait_for_server
+
+echo "Executing remote commands..."
+
+ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP << EOF
+  DEBIAN_FRONTEND=noninteractive
+  
+  echo ""
+  echo "Update package sources"
+  apt-get update -q
+
+  echo ""
+  echo "Install dependencies"
+  apt-get install -y curl apt-transport-https gnupg2 software-properties-common
+
+  echo ""
+  echo "Add docker repo key"
+  curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
+
+  echo ""
+  echo "Add repo"
+  add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable"
+
+  echo ""
+  echo "Update package sources again"
+  apt-get update -q
+
+  echo ""
+  echo "Install docker"
+  apt-get install -y docker-ce
+
+  mkdir /root/secrets
+EOF
+
+echo "Done with remote setup."
+
+if [[ $1 == "screenshotter" ]]; then
+  ### screenshotter
+
+  # Copy service account secret to server
+  echo "Copying secret to /root/secrets/service-account.json"
+  scp -o StrictHostKeyChecking=no -q secrets/datastore-writer.json root@$SERVER_IP:/root/secrets/service-account.json
+
+  # Run docker job
+  echo "Starting Docker Job"
+  ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
+    -v /root/secrets:/secrets \
+    quay.io/netzbegruenung/green-spider-screenshotter
+
+else
+  ### spider
+
+  # Copy service account secret to server
+  echo "Copying secret to /root/secrets/datastore-writer.json"
+  scp -o StrictHostKeyChecking=no -q secrets/datastore-writer.json root@$SERVER_IP:/root/secrets/datastore-writer.json
+
+  # Run docker job
+  echo "Starting Docker Job"
+  ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
+    -v /root/secrets:/secrets \
+    quay.io/netzbegruenung/green-spider spider.py \
+    --credentials-path /secrets/datastore-writer.json \
+    jobs
+
+  ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
+    -v /root/secrets:/secrets \
+    quay.io/netzbegruenung/green-spider spider.py \
+    --credentials-path /secrets/datastore-writer.json \
+    spider
+
+fi
+
+# Delete the box
+echo "Deleting server $SERVER_ID"
+curl -s -X DELETE -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $API_TOKEN" \
+  https://api.hetzner.cloud/v1/servers/$SERVER_ID
diff --git a/devops/run-screenshotter.sh b/devops/run-screenshotter.sh
deleted file mode 100755
index c696bcc..0000000
--- a/devops/run-screenshotter.sh
+++ /dev/null
@@ -1,120 +0,0 @@
-#!/bin/bash
-
-# Creates a server, installs Docker, runs the screenshots job, tears down the server.
-#
-# This will take several hours. For a complete, clean run it is required to leave the
-# terminal running the script open. Otherwise the server won't be deleted properly
-# which will result in extra cost.
-#
-# When stopping the script at any point (Ctrl+C), please make sure that the server
-# gets deleted afterwards.
-#
-# Requirements:
-#
-# - curl
-# - jq (https://stedolan.github.io/jq/)
-# - ssh
-# - SSH key referenced in the server details ("ssh_keys")
-# - Service account with write permission for Storage and Datastore in 
-#   secrets/datastore-writer.json
-
-
-API_TOKEN_SECRET="secrets/hetzner-api-token.sh"
-
-test -f $API_TOKEN_SECRET || { echo >&2 "File $API_TOKEN_SECRET does not exist."; exit 1; }
-
-source $API_TOKEN_SECRET
-
-echo "Creating server 'screenshotter'"
-
-# server_type 'cx11' is the smallest, cheapest category.
-# location 'nbg1' is Nürnberg/Nuremberg, Germany.
-# image 'debian-9' is a plain Debian stretch.
-# ssh_keys ['Marian'] adds Marian's public key to the server and can be extended.
-# user_data: Ensures that we can detect when the cloud-init setup is done.
-#
-CREATE_RESPONSE=$(curl -s -X POST https://api.hetzner.cloud/v1/servers \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $API_TOKEN" \
-  -d '{
-    "name": "screenshotter",
-    "server_type": "cx11",
-    "location": "nbg1",
-    "start_after_create": true,
-    "image": "debian-9",
-    "ssh_keys": [
-      "Marian"
-    ],
-    "user_data": "#cloud-config\nruncmd:\n  - touch /cloud-init-done\n"
-  }')
-
-# Get ID:
-SERVER_ID=$(echo $CREATE_RESPONSE | jq -r .server.id)
-
-# Get IP:
-SERVER_IP=$(echo $CREATE_RESPONSE | jq -r .server.public_net.ipv4.ip)
-
-echo "Created server with ID $SERVER_ID and IP $SERVER_IP"
-echo -n "Waiting for the server to be reachable via SSH "
-
-sleep 30
-
-STATUS="255"
-while [ "$STATUS" != "0" ]; do
-  echo -n "."
-  sleep 5
-  ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP ls /cloud-init-done &> /dev/null
-  STATUS=$?
-done
-
-echo ""
-
-echo "Executing remote commands..."
-
-ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP << EOF
-  DEBIAN_FRONTEND=noninteractive
-  
-  echo ""
-  echo "Update package sources"
-  apt-get update -q
-
-  echo ""
-  echo "Install dependencies"
-  apt-get install -y curl apt-transport-https gnupg2 software-properties-common
-
-  echo ""
-  echo "Add docker repo key"
-  curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
-
-  echo ""
-  echo "Add repo"
-  add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable"
-
-  echo ""
-  echo "Update package sources again"
-  apt-get update -q
-
-  echo ""
-  echo "Install docker"
-  apt-get install -y docker-ce
-
-  mkdir /root/secrets
-EOF
-
-echo "Done with remote setup."
-
-# Copy service account secret to server
-echo "Copying secret to /root/secrets/datastore-writer.json"
-scp -o StrictHostKeyChecking=no -q secrets/datastore-writer.json root@$SERVER_IP:/root/secrets/service-account.json
-
-# Run docker job
-echo "Starting Docker Job"
-ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
-	-v /root/secrets:/secrets \
-	quay.io/netzbegruenung/green-spider-screenshotter
-
-# Delete the box
-echo "Deleting server $SERVER_ID"
-curl -s -X DELETE -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $API_TOKEN" \
-  https://api.hetzner.cloud/v1/servers/$SERVER_ID
diff --git a/devops/run-spider.sh b/devops/run-spider.sh
deleted file mode 100755
index 7fa0c7e..0000000
--- a/devops/run-spider.sh
+++ /dev/null
@@ -1,129 +0,0 @@
-#!/bin/bash
-
-# Creates a server, installs Docker, runs the spider job, tears down the server.
-#
-# This will take several hours. For a complete, clean run it is required to leave the
-# terminal running the script open. Otherwise the server won't be deleted properly
-# which will result in extra cost.
-#
-# When stopping the script at any point (Ctrl+C), please make sure that the server
-# gets deleted afterwards.
-#
-# Requirements:
-#
-# - curl
-# - jq (https://stedolan.github.io/jq/)
-# - ssh
-# - SSH key referenced in the server details ("ssh_keys")
-# - Service account with write permission for Storage and Datastore in 
-#   secrets/datastore-writer.json
-
-
-API_TOKEN_SECRET="secrets/hetzner-api-token.sh"
-
-test -f $API_TOKEN_SECRET || { echo >&2 "File $API_TOKEN_SECRET does not exist."; exit 1; }
-
-source $API_TOKEN_SECRET
-
-echo "Creating server 'spider'"
-
-# server_type 'cx11' is the smallest, cheapest category.
-# location 'nbg1' is Nürnberg/Nuremberg, Germany.
-# image 'debian-9' is a plain Debian stretch.
-# ssh_keys ['Marian'] adds Marian's public key to the server and can be extended.
-# user_data: Ensures that we can detect when the cloud-init setup is done.
-#
-CREATE_RESPONSE=$(curl -s -X POST https://api.hetzner.cloud/v1/servers \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $API_TOKEN" \
-  -d '{
-    "name": "spider",
-    "server_type": "cx11",
-    "location": "nbg1",
-    "start_after_create": true,
-    "image": "debian-9",
-    "ssh_keys": [
-      "Marian"
-    ],
-    "user_data": "#cloud-config\nruncmd:\n  - touch /cloud-init-done\n"
-  }')
-
-# Get ID:
-SERVER_ID=$(echo $CREATE_RESPONSE | jq -r .server.id)
-
-# Get IP:
-SERVER_IP=$(echo $CREATE_RESPONSE | jq -r .server.public_net.ipv4.ip)
-
-echo "Created server with ID $SERVER_ID and IP $SERVER_IP"
-echo -n "Waiting for the server to be reachable via SSH "
-
-sleep 30
-
-STATUS="255"
-while [ "$STATUS" != "0" ]; do
-  echo -n "."
-  sleep 5
-  ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP ls /cloud-init-done &> /dev/null
-  STATUS=$?
-done
-
-echo ""
-
-echo "Executing remote commands..."
-
-ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP << EOF
-  DEBIAN_FRONTEND=noninteractive
-  
-  echo ""
-  echo "Update package sources"
-  apt-get update -q
-
-  echo ""
-  echo "Install dependencies"
-  apt-get install -y curl apt-transport-https gnupg2 software-properties-common
-
-  echo ""
-  echo "Add docker repo key"
-  curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
-
-  echo ""
-  echo "Add repo"
-  add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable"
-
-  echo ""
-  echo "Update package sources again"
-  apt-get update -q
-
-  echo ""
-  echo "Install docker"
-  apt-get install -y docker-ce
-
-  mkdir /root/secrets
-EOF
-
-echo "Done with remote setup."
-
-# Copy service account secret to server
-echo "Copying secret to /root/secrets/datastore-writer.json"
-scp -o StrictHostKeyChecking=no -q secrets/datastore-writer.json root@$SERVER_IP:/root/secrets/datastore-writer.json
-
-# Run docker job
-echo "Starting Docker Job"
-ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
-	-v /root/secrets:/secrets \
-	quay.io/netzbegruenung/green-spider \
-  spider spider.py \
-	--credentials-path /secrets/datastore-writer.json \
-	jobs
-ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
-	-v /root/secrets:/secrets \
-	quay.io/netzbegruenung/green-spider \
-  spider spider.py \
-	--credentials-path /secrets/datastore-writer.json \
-	spider
-
-# Delete the box
-echo "Deleting server $SERVER_ID"
-curl -s -X DELETE -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $API_TOKEN" \
-  https://api.hetzner.cloud/v1/servers/$SERVER_ID