mirror of
https://github.com/netzbegruenung/green-spider.git
synced 2024-04-26 22:34:52 +02:00
Adding devops scripts
This commit is contained in:
parent
d9cd0c9a7a
commit
aec2ce47f0
33
devops/README.md
Normal file
33
devops/README.md
Normal file
|
@ -0,0 +1,33 @@
|
||||||
|
# DevOps
|
||||||
|
|
||||||
|
Die Scripte in diesem Verzeichnis erlauben das weitgehend automatisierte
|
||||||
|
Provisionieren eines Servers, Ausführen von Jobs wie Spider und Screenshotter
|
||||||
|
und Entfernen des Servers.
|
||||||
|
|
||||||
|
**Warnung**: Die Scripte sind aktuell so einfach, dass die erzeugten Server nur nach erfolgreicher
|
||||||
|
Ausführung des Jobs entfernt werden. Im Fall eines Fehlers muss der provisionierte
|
||||||
|
Server unbedingt manuell entfernt werden, um unnötige Kosten zu vermeiden.
|
||||||
|
|
||||||
|
## Voraussetzungen
|
||||||
|
|
||||||
|
- SSH Public Key ist bei Hetzner hinterlegt und in den scripten eingetragen (siehe 'ssh_keys')
|
||||||
|
- API Token für Hetzner in Datei 'secrets/hetzner-api-token.sh' im Format 'export API_TOKEN=<token>'
|
||||||
|
- Service account JSON Datei mit Schreibrechten in 'secrets/datastore-writer.json'
|
||||||
|
- curl
|
||||||
|
- jq (https://stedolan.github.io/jq/)
|
||||||
|
- ssh
|
||||||
|
|
||||||
|
## Ausführung
|
||||||
|
|
||||||
|
Die Scripte werden aus dem Root-Verzeichnis des Repositories ausgeführt.
|
||||||
|
|
||||||
|
```
|
||||||
|
# Spidern
|
||||||
|
devops/run-spider.sh
|
||||||
|
|
||||||
|
|
||||||
|
# Screenshots erzeugen
|
||||||
|
devops/run-screenshotter.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Der Terminal muss bis zum Ende der Ausführung geöffnet bleiben.
|
120
devops/run-screenshotter.sh
Executable file
120
devops/run-screenshotter.sh
Executable file
|
@ -0,0 +1,120 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Creates a server, installs Docker, runs the screenshots job, tears down the server.
|
||||||
|
#
|
||||||
|
# This will take several hours. For a complete, clean run it is required to leave the
|
||||||
|
# terminal running the script open. Otherwise the server won't be deleted properly
|
||||||
|
# which will result in extra cost.
|
||||||
|
#
|
||||||
|
# When stopping the script at any point (Ctrl+C), please make sure that the server
|
||||||
|
# gets deleted afterwards.
|
||||||
|
#
|
||||||
|
# Requirements:
|
||||||
|
#
|
||||||
|
# - curl
|
||||||
|
# - jq (https://stedolan.github.io/jq/)
|
||||||
|
# - ssh
|
||||||
|
# - SSH key referenced in the server details ("ssh_keys")
|
||||||
|
# - Service account with write permission for Storage and Datastore in
|
||||||
|
# secrets/datastore-writer.json
|
||||||
|
|
||||||
|
|
||||||
|
API_TOKEN_SECRET="secrets/hetzner-api-token.sh"
|
||||||
|
|
||||||
|
test -f $API_TOKEN_SECRET || { echo >&2 "File $API_TOKEN_SECRET does not exist."; exit 1; }
|
||||||
|
|
||||||
|
source $API_TOKEN_SECRET
|
||||||
|
|
||||||
|
echo "Creating server 'screenshotter'"
|
||||||
|
|
||||||
|
# server_type 'cx11' is the smallest, cheapest category.
|
||||||
|
# location 'nbg1' is Nürnberg/Nuremberg, Germany.
|
||||||
|
# image 'debian-9' is a plain Debian stretch.
|
||||||
|
# ssh_keys ['Marian'] adds Marian's public key to the server and can be extended.
|
||||||
|
# user_data: Ensures that we can detect when the cloud-init setup is done.
|
||||||
|
#
|
||||||
|
CREATE_RESPONSE=$(curl -s -X POST https://api.hetzner.cloud/v1/servers \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "Authorization: Bearer $API_TOKEN" \
|
||||||
|
-d '{
|
||||||
|
"name": "screenshotter",
|
||||||
|
"server_type": "cx11",
|
||||||
|
"location": "nbg1",
|
||||||
|
"start_after_create": true,
|
||||||
|
"image": "debian-9",
|
||||||
|
"ssh_keys": [
|
||||||
|
"Marian"
|
||||||
|
],
|
||||||
|
"user_data": "#cloud-config\nruncmd:\n - touch /cloud-init-done\n"
|
||||||
|
}')
|
||||||
|
|
||||||
|
# Get ID:
|
||||||
|
SERVER_ID=$(echo $CREATE_RESPONSE | jq -r .server.id)
|
||||||
|
|
||||||
|
# Get IP:
|
||||||
|
SERVER_IP=$(echo $CREATE_RESPONSE | jq -r .server.public_net.ipv4.ip)
|
||||||
|
|
||||||
|
echo "Created server with ID $SERVER_ID and IP $SERVER_IP"
|
||||||
|
echo -n "Waiting for the server to be reachable via SSH "
|
||||||
|
|
||||||
|
sleep 30
|
||||||
|
|
||||||
|
STATUS="255"
|
||||||
|
while [ "$STATUS" != "0" ]; do
|
||||||
|
echo -n "."
|
||||||
|
sleep 5
|
||||||
|
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP ls /cloud-init-done &> /dev/null
|
||||||
|
STATUS=$?
|
||||||
|
done
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
echo "Executing remote commands..."
|
||||||
|
|
||||||
|
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP << EOF
|
||||||
|
DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Update package sources"
|
||||||
|
apt-get update -q
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Install dependencies"
|
||||||
|
apt-get install -y curl apt-transport-https gnupg2 software-properties-common
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Add docker repo key"
|
||||||
|
curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Add repo"
|
||||||
|
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Update package sources again"
|
||||||
|
apt-get update -q
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Install docker"
|
||||||
|
apt-get install -y docker-ce
|
||||||
|
|
||||||
|
mkdir /root/secrets
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo "Done with remote setup."
|
||||||
|
|
||||||
|
# Copy service account secret to server
|
||||||
|
echo "Copying secret to /root/secrets/datastore-writer.json"
|
||||||
|
scp -o StrictHostKeyChecking=no -q secrets/datastore-writer.json root@$SERVER_IP:/root/secrets/service-account.json
|
||||||
|
|
||||||
|
# Run docker job
|
||||||
|
echo "Starting Docker Job"
|
||||||
|
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
|
||||||
|
-v /root/secrets:/secrets \
|
||||||
|
quay.io/netzbegruenung/green-spider-screenshotter
|
||||||
|
|
||||||
|
# Delete the box
|
||||||
|
echo "Deleting server $SERVER_ID"
|
||||||
|
curl -s -X DELETE -H "Content-Type: application/json" \
|
||||||
|
-H "Authorization: Bearer $API_TOKEN" \
|
||||||
|
https://api.hetzner.cloud/v1/servers/$SERVER_ID
|
129
devops/run-spider.sh
Executable file
129
devops/run-spider.sh
Executable file
|
@ -0,0 +1,129 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Creates a server, installs Docker, runs the spider job, tears down the server.
|
||||||
|
#
|
||||||
|
# This will take several hours. For a complete, clean run it is required to leave the
|
||||||
|
# terminal running the script open. Otherwise the server won't be deleted properly
|
||||||
|
# which will result in extra cost.
|
||||||
|
#
|
||||||
|
# When stopping the script at any point (Ctrl+C), please make sure that the server
|
||||||
|
# gets deleted afterwards.
|
||||||
|
#
|
||||||
|
# Requirements:
|
||||||
|
#
|
||||||
|
# - curl
|
||||||
|
# - jq (https://stedolan.github.io/jq/)
|
||||||
|
# - ssh
|
||||||
|
# - SSH key referenced in the server details ("ssh_keys")
|
||||||
|
# - Service account with write permission for Storage and Datastore in
|
||||||
|
# secrets/datastore-writer.json
|
||||||
|
|
||||||
|
|
||||||
|
API_TOKEN_SECRET="secrets/hetzner-api-token.sh"
|
||||||
|
|
||||||
|
test -f $API_TOKEN_SECRET || { echo >&2 "File $API_TOKEN_SECRET does not exist."; exit 1; }
|
||||||
|
|
||||||
|
source $API_TOKEN_SECRET
|
||||||
|
|
||||||
|
echo "Creating server 'spider'"
|
||||||
|
|
||||||
|
# server_type 'cx11' is the smallest, cheapest category.
|
||||||
|
# location 'nbg1' is Nürnberg/Nuremberg, Germany.
|
||||||
|
# image 'debian-9' is a plain Debian stretch.
|
||||||
|
# ssh_keys ['Marian'] adds Marian's public key to the server and can be extended.
|
||||||
|
# user_data: Ensures that we can detect when the cloud-init setup is done.
|
||||||
|
#
|
||||||
|
CREATE_RESPONSE=$(curl -s -X POST https://api.hetzner.cloud/v1/servers \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "Authorization: Bearer $API_TOKEN" \
|
||||||
|
-d '{
|
||||||
|
"name": "spider",
|
||||||
|
"server_type": "cx11",
|
||||||
|
"location": "nbg1",
|
||||||
|
"start_after_create": true,
|
||||||
|
"image": "debian-9",
|
||||||
|
"ssh_keys": [
|
||||||
|
"Marian"
|
||||||
|
],
|
||||||
|
"user_data": "#cloud-config\nruncmd:\n - touch /cloud-init-done\n"
|
||||||
|
}')
|
||||||
|
|
||||||
|
# Get ID:
|
||||||
|
SERVER_ID=$(echo $CREATE_RESPONSE | jq -r .server.id)
|
||||||
|
|
||||||
|
# Get IP:
|
||||||
|
SERVER_IP=$(echo $CREATE_RESPONSE | jq -r .server.public_net.ipv4.ip)
|
||||||
|
|
||||||
|
echo "Created server with ID $SERVER_ID and IP $SERVER_IP"
|
||||||
|
echo -n "Waiting for the server to be reachable via SSH "
|
||||||
|
|
||||||
|
sleep 30
|
||||||
|
|
||||||
|
STATUS="255"
|
||||||
|
while [ "$STATUS" != "0" ]; do
|
||||||
|
echo -n "."
|
||||||
|
sleep 5
|
||||||
|
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP ls /cloud-init-done &> /dev/null
|
||||||
|
STATUS=$?
|
||||||
|
done
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
echo "Executing remote commands..."
|
||||||
|
|
||||||
|
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP << EOF
|
||||||
|
DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Update package sources"
|
||||||
|
apt-get update -q
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Install dependencies"
|
||||||
|
apt-get install -y curl apt-transport-https gnupg2 software-properties-common
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Add docker repo key"
|
||||||
|
curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Add repo"
|
||||||
|
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Update package sources again"
|
||||||
|
apt-get update -q
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "Install docker"
|
||||||
|
apt-get install -y docker-ce
|
||||||
|
|
||||||
|
mkdir /root/secrets
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo "Done with remote setup."
|
||||||
|
|
||||||
|
# Copy service account secret to server
|
||||||
|
echo "Copying secret to /root/secrets/datastore-writer.json"
|
||||||
|
scp -o StrictHostKeyChecking=no -q secrets/datastore-writer.json root@$SERVER_IP:/root/secrets/datastore-writer.json
|
||||||
|
|
||||||
|
# Run docker job
|
||||||
|
echo "Starting Docker Job"
|
||||||
|
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
|
||||||
|
-v /root/secrets:/secrets \
|
||||||
|
quay.io/netzbegruenung/green-spider \
|
||||||
|
spider spider.py \
|
||||||
|
--credentials-path /secrets/datastore-writer.json \
|
||||||
|
jobs
|
||||||
|
ssh -o StrictHostKeyChecking=no -q root@$SERVER_IP docker run -t \
|
||||||
|
-v /root/secrets:/secrets \
|
||||||
|
quay.io/netzbegruenung/green-spider \
|
||||||
|
spider spider.py \
|
||||||
|
--credentials-path /secrets/datastore-writer.json \
|
||||||
|
spider
|
||||||
|
|
||||||
|
# Delete the box
|
||||||
|
echo "Deleting server $SERVER_ID"
|
||||||
|
curl -s -X DELETE -H "Content-Type: application/json" \
|
||||||
|
-H "Authorization: Bearer $API_TOKEN" \
|
||||||
|
https://api.hetzner.cloud/v1/servers/$SERVER_ID
|
Loading…
Reference in a new issue