Marian Steinbach
4 years ago
parent
d9cd0c9a7a
commit
aec2ce47f0
3 changed files with 282 additions and 0 deletions
@ -0,0 +1,33 @@ |
||||
# DevOps |
||||
|
||||
Die Scripte in diesem Verzeichnis erlauben das weitgehend automatisierte |
||||
Provisionieren eines Servers, Ausführen von Jobs wie Spider und Screenshotter |
||||
und Entfernen des Servers. |
||||
|
||||
**Warnung**: Die Scripte sind aktuell so einfach, dass die erzeugten Server nur nach erfolgreicher |
||||
Ausführung des Jobs entfernt werden. Im Fall eines Fehlers muss der provisionierte |
||||
Server unbedingt manuell entfernt werden, um unnötige Kosten zu vermeiden. |
||||
|
||||
## Voraussetzungen |
||||
|
||||
- SSH Public Key ist bei Hetzner hinterlegt und in den scripten eingetragen (siehe 'ssh_keys') |
||||
- API Token für Hetzner in Datei 'secrets/hetzner-api-token.sh' im Format 'export API_TOKEN=<token>' |
||||
- Service account JSON Datei mit Schreibrechten in 'secrets/datastore-writer.json' |
||||
- curl |
||||
- jq (https://stedolan.github.io/jq/) |
||||
- ssh |
||||
|
||||
## Ausführung |
||||
|
||||
Die Scripte werden aus dem Root-Verzeichnis des Repositories ausgeführt. |
||||
|
||||
``` |
||||
# Spidern |
||||
devops/run-spider.sh |
||||
|
||||
|
||||
# Screenshots erzeugen |
||||
devops/run-screenshotter.sh |
||||
``` |
||||
|
||||
Der Terminal muss bis zum Ende der Ausführung geöffnet bleiben. |
||||
@ -0,0 +1,120 @@ |
||||
#!/bin/bash |
||||
|
||||
# Creates a server, installs Docker, runs the screenshots job, tears down the server. |
||||
# |
||||
# This will take several hours. For a complete, clean run it is required to leave the |
||||
# terminal running the script open. Otherwise the server won't be deleted properly |
||||
# which will result in extra cost. |
||||
# |
||||
# When stopping the script at any point (Ctrl+C), please make sure that the server |
||||
# gets deleted afterwards. |
||||
# |
||||
# Requirements: |
||||
# |
||||
# - curl |
||||
# - jq (https://stedolan.github.io/jq/) |
||||
# - ssh |
||||
# - SSH key referenced in the server details ("ssh_keys") |
||||
# - Service account with write permission for Storage and Datastore in |
||||
# secrets/datastore-writer.json |
||||
|
||||
|
||||
API_TOKEN_SECRET="secrets/hetzner-api-token.sh" |
||||
|
||||
test -f $API_TOKEN_SECRET || { echo >&2 "File $API_TOKEN_SECRET does not exist."; exit 1; } |
||||
|
||||
source $API_TOKEN_SECRET |
||||
|
||||
echo "Creating server 'screenshotter'" |
||||
|
||||
# server_type 'cx11' is the smallest, cheapest category. |
||||
# location 'nbg1' is Nürnberg/Nuremberg, Germany. |
||||
# image 'debian-9' is a plain Debian stretch. |
||||
# ssh_keys ['Marian'] adds Marian's public key to the server and can be extended. |
||||
# user_data: Ensures that we can detect when the cloud-init setup is done. |
||||
# |
||||
CREATE_RESPONSE=$(curl -s -X POST https://api.hetzner.cloud/v1/servers \ |
||||
-H "Content-Type: application/json" \ |
||||
-H "Authorization: Bearer $API_TOKEN" \ |
||||
-d '{ |
||||
"name": "screenshotter", |
||||
"server_type": "cx11", |
||||
"location": "nbg1", |
||||
"start_after_create": true, |
||||
"image": "debian-9", |
||||
"ssh_keys": [ |
||||
"Marian" |
||||
], |
||||
"user_data": "#cloud-config\nruncmd:\n - touch /cloud-init-done\n" |
||||
}') |
||||
|
||||
# Get ID: |
||||
SERVER_ID=$(echo $CREATE_RESPONSE | jq -r .server.id) |
||||
|
||||
# Get IP: |
||||
SERVER_IP=$(echo $CREATE_RESPONSE | jq -r .server.public_net.ipv4.ip) |
||||
|
||||
echo "Created server with ID $SERVER_ID and IP $SERVER_IP" |
||||
echo -n "Waiting for the server to be reachable via SSH " |
||||
|
||||
sleep 30 |
||||
|
||||
STATUS="255" |
||||
while [ "$STATUS" != "0" ]; do |
||||
echo -n "." |
||||
sleep 5 |
||||
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP ls /cloud-init-done &> /dev/null |
||||
STATUS=$? |
||||
done |
||||
|
||||
echo "" |
||||
|
||||
echo "Executing remote commands..." |
||||
|
||||
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP << EOF |
||||
DEBIAN_FRONTEND=noninteractive |
||||
|
||||
echo "" |
||||
echo "Update package sources" |
||||
apt-get update -q |
||||
|
||||
echo "" |
||||
echo "Install dependencies" |
||||
apt-get install -y curl apt-transport-https gnupg2 software-properties-common |
||||
|
||||
echo "" |
||||
echo "Add docker repo key" |
||||
curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - |
||||
|
||||
echo "" |
||||
echo "Add repo" |
||||
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable" |
||||
|
||||
echo "" |
||||
echo "Update package sources again" |
||||
apt-get update -q |
||||
|
||||
echo "" |
||||
echo "Install docker" |
||||
apt-get install -y docker-ce |
||||
|
||||
mkdir /root/secrets |
||||
EOF |
||||
|
||||
echo "Done with remote setup." |
||||
|
||||
# Copy service account secret to server |
||||
echo "Copying secret to /root/secrets/datastore-writer.json" |
||||
scp -o StrictHostKeyChecking=no -q secrets/datastore-writer.json root@$SERVER_IP:/root/secrets/service-account.json |
||||
|
||||
# Run docker job |
||||
echo "Starting Docker Job" |
||||
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \ |
||||
-v /root/secrets:/secrets \ |
||||
quay.io/netzbegruenung/green-spider-screenshotter |
||||
|
||||
# Delete the box |
||||
echo "Deleting server $SERVER_ID" |
||||
curl -s -X DELETE -H "Content-Type: application/json" \ |
||||
-H "Authorization: Bearer $API_TOKEN" \ |
||||
https://api.hetzner.cloud/v1/servers/$SERVER_ID |
||||
@ -0,0 +1,129 @@ |
||||
#!/bin/bash |
||||
|
||||
# Creates a server, installs Docker, runs the spider job, tears down the server. |
||||
# |
||||
# This will take several hours. For a complete, clean run it is required to leave the |
||||
# terminal running the script open. Otherwise the server won't be deleted properly |
||||
# which will result in extra cost. |
||||
# |
||||
# When stopping the script at any point (Ctrl+C), please make sure that the server |
||||
# gets deleted afterwards. |
||||
# |
||||
# Requirements: |
||||
# |
||||
# - curl |
||||
# - jq (https://stedolan.github.io/jq/) |
||||
# - ssh |
||||
# - SSH key referenced in the server details ("ssh_keys") |
||||
# - Service account with write permission for Storage and Datastore in |
||||
# secrets/datastore-writer.json |
||||
|
||||
|
||||
API_TOKEN_SECRET="secrets/hetzner-api-token.sh" |
||||
|
||||
test -f $API_TOKEN_SECRET || { echo >&2 "File $API_TOKEN_SECRET does not exist."; exit 1; } |
||||
|
||||
source $API_TOKEN_SECRET |
||||
|
||||
echo "Creating server 'spider'" |
||||
|
||||
# server_type 'cx11' is the smallest, cheapest category. |
||||
# location 'nbg1' is Nürnberg/Nuremberg, Germany. |
||||
# image 'debian-9' is a plain Debian stretch. |
||||
# ssh_keys ['Marian'] adds Marian's public key to the server and can be extended. |
||||
# user_data: Ensures that we can detect when the cloud-init setup is done. |
||||
# |
||||
CREATE_RESPONSE=$(curl -s -X POST https://api.hetzner.cloud/v1/servers \ |
||||
-H "Content-Type: application/json" \ |
||||
-H "Authorization: Bearer $API_TOKEN" \ |
||||
-d '{ |
||||
"name": "spider", |
||||
"server_type": "cx11", |
||||
"location": "nbg1", |
||||
"start_after_create": true, |
||||
"image": "debian-9", |
||||
"ssh_keys": [ |
||||
"Marian" |
||||
], |
||||
"user_data": "#cloud-config\nruncmd:\n - touch /cloud-init-done\n" |
||||
}') |
||||
|
||||
# Get ID: |
||||
SERVER_ID=$(echo $CREATE_RESPONSE | jq -r .server.id) |
||||
|
||||
# Get IP: |
||||
SERVER_IP=$(echo $CREATE_RESPONSE | jq -r .server.public_net.ipv4.ip) |
||||
|
||||
echo "Created server with ID $SERVER_ID and IP $SERVER_IP" |
||||
echo -n "Waiting for the server to be reachable via SSH " |
||||
|
||||
sleep 30 |
||||
|
||||
STATUS="255" |
||||
while [ "$STATUS" != "0" ]; do |
||||
echo -n "." |
||||
sleep 5 |
||||
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP ls /cloud-init-done &> /dev/null |
||||
STATUS=$? |
||||
done |
||||
|
||||
echo "" |
||||
|
||||
echo "Executing remote commands..." |
||||
|
||||
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP << EOF |
||||
DEBIAN_FRONTEND=noninteractive |
||||
|
||||
echo "" |
||||
echo "Update package sources" |
||||
apt-get update -q |
||||
|
||||
echo "" |
||||
echo "Install dependencies" |
||||
apt-get install -y curl apt-transport-https gnupg2 software-properties-common |
||||
|
||||
echo "" |
||||
echo "Add docker repo key" |
||||
curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - |
||||
|
||||
echo "" |
||||
echo "Add repo" |
||||
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable" |
||||
|
||||
echo "" |
||||
echo "Update package sources again" |
||||
apt-get update -q |
||||
|
||||
echo "" |
||||
echo "Install docker" |
||||
apt-get install -y docker-ce |
||||
|
||||
mkdir /root/secrets |
||||
EOF |
||||
|
||||
echo "Done with remote setup." |
||||
|
||||
# Copy service account secret to server |
||||
echo "Copying secret to /root/secrets/datastore-writer.json" |
||||
scp -o StrictHostKeyChecking=no -q secrets/datastore-writer.json root@$SERVER_IP:/root/secrets/datastore-writer.json |
||||
|
||||
# Run docker job |
||||
echo "Starting Docker Job" |
||||
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \ |
||||
-v /root/secrets:/secrets \ |
||||
quay.io/netzbegruenung/green-spider \ |
||||
spider spider.py \ |
||||
--credentials-path /secrets/datastore-writer.json \ |
||||
jobs |
||||
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \ |
||||
-v /root/secrets:/secrets \ |
||||
quay.io/netzbegruenung/green-spider \ |
||||
spider spider.py \ |
||||
--credentials-path /secrets/datastore-writer.json \ |
||||
spider |
||||
|
||||
# Delete the box |
||||
echo "Deleting server $SERVER_ID" |
||||
curl -s -X DELETE -H "Content-Type: application/json" \ |
||||
-H "Authorization: Bearer $API_TOKEN" \ |
||||
https://api.hetzner.cloud/v1/servers/$SERVER_ID |
||||
Loading…
Reference in new issue