mirror of
https://github.com/netzbegruenung/green-spider.git
synced 2024-04-26 14:24:51 +02:00
Marian Steinbach
618e29d763
* CLI: remove 'jobs' command, add 'manager' * Add job definition * Move jobs to manage folder * Rename jobs to manager * Add rq and redis dependencies * Add docker-compose YAML * Downgrade to alpine 3.8 * Adjust paths in Dockerfile, remove entrypoint * Rename 'make spiderjobs' to 'make jobs' * Fix docker exectution * Adapt 'make jobs' * Fix metadata scheme * Add docker dependency * Rendomize queue (a bit) * Use latest image, remove debug output * Make docker-compose file downwards-compatible * Use latest instead of dev image tag * Update docker-compose.yaml * Adapt job start script * Fix redis connection in manager * Add support for increasing timeout via environment variable * Adapt load_in_browser to cookies table schema change * Fix execution * Mitigate yaml warning * Bump some dependency versions * Report resource usage stats for each job * checks/load_in_browser: Return DOM size, prevent multiple page loads * Update .dockerignore * Code update * Script update * Update README.md * WIP * WIP commit * Update Dockerfile to alpine:edge and chromium v90 * Update TestCertificateChecker * Set defaults for __init__ function * Detect sunflower theme * Update unit test for new datetime (zero-basing) * Set logging prefs from Chromium in a new way * Move datastore client instantiation As it is not needed for all commands * Change green-directory repository URL * Add git settings for cloning green-directory * Pin alpine version 3.14, fix py3-cryptography * Use plain docker build progress output * Add volumes to 'make test' docker run command * Fix bug * Update example command in README * Update dependencies * Add creation of Kubernetes jobs
68 lines
1.7 KiB
YAML
68 lines
1.7 KiB
YAML
---
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: green-spider-job-1
|
|
namespace: marian
|
|
labels:
|
|
app: green-spider
|
|
spec:
|
|
activeDeadlineSeconds: 120
|
|
ttlSecondsAfterFinished: 600
|
|
completions: 1
|
|
backoffLimit: 3
|
|
|
|
# Pod template
|
|
template:
|
|
metadata:
|
|
name: green-spider-job
|
|
namespace: marian
|
|
labels:
|
|
app: green-spider
|
|
spec:
|
|
restartPolicy: Never
|
|
nodeSelector:
|
|
giantswarm.io/machine-pool: 5n27k
|
|
affinity:
|
|
podAntiAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector:
|
|
matchExpressions:
|
|
- key: app
|
|
operator: In
|
|
values:
|
|
- green-spider
|
|
topologyKey: topology.kubernetes.io/region
|
|
containers:
|
|
- name: spider
|
|
image: quay.io/netzbegruenung/green-spider:kubernetes
|
|
imagePullPolicy: IfNotPresent
|
|
command:
|
|
- python
|
|
- cli.py
|
|
- --credentials-path=/secrets/datastore-writer.json
|
|
- --loglevel=debug
|
|
- spider
|
|
- '--job={"url":"https://www.gruene.de/","type":"PARTY","level":"DE:BUNDESVERBAND","state":null,"district":null,"city":null}'
|
|
volumeMounts:
|
|
- name: secrets
|
|
mountPath: "/secrets"
|
|
readOnly: true
|
|
- name: shared
|
|
mountPath: /dev/shm
|
|
resources:
|
|
requests:
|
|
cpu: 1000m
|
|
memory: 5000M
|
|
volumes:
|
|
- name: secrets
|
|
secret:
|
|
secretName: green-spider
|
|
items:
|
|
- key: datastore-writer.json
|
|
path: datastore-writer.json
|
|
- key: screenshots-uploader.json
|
|
path: screenshots-uploader.json
|
|
- name: shared
|
|
emptyDir: {}
|