Adding devops scripts

pull/59/head
Marian Steinbach 4 years ago
parent d9cd0c9a7a
commit aec2ce47f0
  1. 33
      devops/README.md
  2. 120
      devops/run-screenshotter.sh
  3. 129
      devops/run-spider.sh

@ -0,0 +1,33 @@
# DevOps
Die Scripte in diesem Verzeichnis erlauben das weitgehend automatisierte
Provisionieren eines Servers, Ausführen von Jobs wie Spider und Screenshotter
und Entfernen des Servers.
**Warnung**: Die Scripte sind aktuell so einfach, dass die erzeugten Server nur nach erfolgreicher
Ausführung des Jobs entfernt werden. Im Fall eines Fehlers muss der provisionierte
Server unbedingt manuell entfernt werden, um unnötige Kosten zu vermeiden.
## Voraussetzungen
- SSH Public Key ist bei Hetzner hinterlegt und in den scripten eingetragen (siehe 'ssh_keys')
- API Token für Hetzner in Datei 'secrets/hetzner-api-token.sh' im Format 'export API_TOKEN=<token>'
- Service account JSON Datei mit Schreibrechten in 'secrets/datastore-writer.json'
- curl
- jq (https://stedolan.github.io/jq/)
- ssh
## Ausführung
Die Scripte werden aus dem Root-Verzeichnis des Repositories ausgeführt.
```
# Spidern
devops/run-spider.sh
# Screenshots erzeugen
devops/run-screenshotter.sh
```
Der Terminal muss bis zum Ende der Ausführung geöffnet bleiben.

@ -0,0 +1,120 @@
#!/bin/bash
# Creates a server, installs Docker, runs the screenshots job, tears down the server.
#
# This will take several hours. For a complete, clean run it is required to leave the
# terminal running the script open. Otherwise the server won't be deleted properly
# which will result in extra cost.
#
# When stopping the script at any point (Ctrl+C), please make sure that the server
# gets deleted afterwards.
#
# Requirements:
#
# - curl
# - jq (https://stedolan.github.io/jq/)
# - ssh
# - SSH key referenced in the server details ("ssh_keys")
# - Service account with write permission for Storage and Datastore in
# secrets/datastore-writer.json
API_TOKEN_SECRET="secrets/hetzner-api-token.sh"
test -f $API_TOKEN_SECRET || { echo >&2 "File $API_TOKEN_SECRET does not exist."; exit 1; }
source $API_TOKEN_SECRET
echo "Creating server 'screenshotter'"
# server_type 'cx11' is the smallest, cheapest category.
# location 'nbg1' is Nürnberg/Nuremberg, Germany.
# image 'debian-9' is a plain Debian stretch.
# ssh_keys ['Marian'] adds Marian's public key to the server and can be extended.
# user_data: Ensures that we can detect when the cloud-init setup is done.
#
CREATE_RESPONSE=$(curl -s -X POST https://api.hetzner.cloud/v1/servers \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $API_TOKEN" \
-d '{
"name": "screenshotter",
"server_type": "cx11",
"location": "nbg1",
"start_after_create": true,
"image": "debian-9",
"ssh_keys": [
"Marian"
],
"user_data": "#cloud-config\nruncmd:\n - touch /cloud-init-done\n"
}')
# Get ID:
SERVER_ID=$(echo $CREATE_RESPONSE | jq -r .server.id)
# Get IP:
SERVER_IP=$(echo $CREATE_RESPONSE | jq -r .server.public_net.ipv4.ip)
echo "Created server with ID $SERVER_ID and IP $SERVER_IP"
echo -n "Waiting for the server to be reachable via SSH "
sleep 30
STATUS="255"
while [ "$STATUS" != "0" ]; do
echo -n "."
sleep 5
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP ls /cloud-init-done &> /dev/null
STATUS=$?
done
echo ""
echo "Executing remote commands..."
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP << EOF
DEBIAN_FRONTEND=noninteractive
echo ""
echo "Update package sources"
apt-get update -q
echo ""
echo "Install dependencies"
apt-get install -y curl apt-transport-https gnupg2 software-properties-common
echo ""
echo "Add docker repo key"
curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
echo ""
echo "Add repo"
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable"
echo ""
echo "Update package sources again"
apt-get update -q
echo ""
echo "Install docker"
apt-get install -y docker-ce
mkdir /root/secrets
EOF
echo "Done with remote setup."
# Copy service account secret to server
echo "Copying secret to /root/secrets/datastore-writer.json"
scp -o StrictHostKeyChecking=no -q secrets/datastore-writer.json root@$SERVER_IP:/root/secrets/service-account.json
# Run docker job
echo "Starting Docker Job"
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
-v /root/secrets:/secrets \
quay.io/netzbegruenung/green-spider-screenshotter
# Delete the box
echo "Deleting server $SERVER_ID"
curl -s -X DELETE -H "Content-Type: application/json" \
-H "Authorization: Bearer $API_TOKEN" \
https://api.hetzner.cloud/v1/servers/$SERVER_ID

@ -0,0 +1,129 @@
#!/bin/bash
# Creates a server, installs Docker, runs the spider job, tears down the server.
#
# This will take several hours. For a complete, clean run it is required to leave the
# terminal running the script open. Otherwise the server won't be deleted properly
# which will result in extra cost.
#
# When stopping the script at any point (Ctrl+C), please make sure that the server
# gets deleted afterwards.
#
# Requirements:
#
# - curl
# - jq (https://stedolan.github.io/jq/)
# - ssh
# - SSH key referenced in the server details ("ssh_keys")
# - Service account with write permission for Storage and Datastore in
# secrets/datastore-writer.json
API_TOKEN_SECRET="secrets/hetzner-api-token.sh"
test -f $API_TOKEN_SECRET || { echo >&2 "File $API_TOKEN_SECRET does not exist."; exit 1; }
source $API_TOKEN_SECRET
echo "Creating server 'spider'"
# server_type 'cx11' is the smallest, cheapest category.
# location 'nbg1' is Nürnberg/Nuremberg, Germany.
# image 'debian-9' is a plain Debian stretch.
# ssh_keys ['Marian'] adds Marian's public key to the server and can be extended.
# user_data: Ensures that we can detect when the cloud-init setup is done.
#
CREATE_RESPONSE=$(curl -s -X POST https://api.hetzner.cloud/v1/servers \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $API_TOKEN" \
-d '{
"name": "spider",
"server_type": "cx11",
"location": "nbg1",
"start_after_create": true,
"image": "debian-9",
"ssh_keys": [
"Marian"
],
"user_data": "#cloud-config\nruncmd:\n - touch /cloud-init-done\n"
}')
# Get ID:
SERVER_ID=$(echo $CREATE_RESPONSE | jq -r .server.id)
# Get IP:
SERVER_IP=$(echo $CREATE_RESPONSE | jq -r .server.public_net.ipv4.ip)
echo "Created server with ID $SERVER_ID and IP $SERVER_IP"
echo -n "Waiting for the server to be reachable via SSH "
sleep 30
STATUS="255"
while [ "$STATUS" != "0" ]; do
echo -n "."
sleep 5
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP ls /cloud-init-done &> /dev/null
STATUS=$?
done
echo ""
echo "Executing remote commands..."
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP << EOF
DEBIAN_FRONTEND=noninteractive
echo ""
echo "Update package sources"
apt-get update -q
echo ""
echo "Install dependencies"
apt-get install -y curl apt-transport-https gnupg2 software-properties-common
echo ""
echo "Add docker repo key"
curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
echo ""
echo "Add repo"
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable"
echo ""
echo "Update package sources again"
apt-get update -q
echo ""
echo "Install docker"
apt-get install -y docker-ce
mkdir /root/secrets
EOF
echo "Done with remote setup."
# Copy service account secret to server
echo "Copying secret to /root/secrets/datastore-writer.json"
scp -o StrictHostKeyChecking=no -q secrets/datastore-writer.json root@$SERVER_IP:/root/secrets/datastore-writer.json
# Run docker job
echo "Starting Docker Job"
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
-v /root/secrets:/secrets \
quay.io/netzbegruenung/green-spider \
spider spider.py \
--credentials-path /secrets/datastore-writer.json \
jobs
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
-v /root/secrets:/secrets \
quay.io/netzbegruenung/green-spider \
spider spider.py \
--credentials-path /secrets/datastore-writer.json \
spider
# Delete the box
echo "Deleting server $SERVER_ID"
curl -s -X DELETE -H "Content-Type: application/json" \
-H "Authorization: Bearer $API_TOKEN" \
https://api.hetzner.cloud/v1/servers/$SERVER_ID
Loading…
Cancel
Save