Hilft Dir dabei, Deine BÜNDNIS 90/DIE GRÜNEN Website zu optimieren
https://green-spider.netzbegruenung.de/
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
52 lines
1.3 KiB
52 lines
1.3 KiB
IMAGE := quay.io/netzbegruenung/green-spider:latest |
|
|
|
DB_ENTITY := spider-results |
|
|
|
.PHONY: dockerimage spider export |
|
|
|
# Build docker image |
|
dockerimage: |
|
docker build --progress plain -t $(IMAGE) . |
|
|
|
# Fill the queue with spider jobs, one for each site. |
|
jobs: |
|
docker run --rm -ti \ |
|
-v $(PWD)/secrets:/secrets \ |
|
$(IMAGE) \ |
|
python cli.py \ |
|
--credentials-path /secrets/datastore-writer.json \ |
|
--loglevel debug \ |
|
manager |
|
|
|
# Run spider in docker image |
|
spider: |
|
docker run --rm -ti \ |
|
-v $(PWD)/volumes/dev-shm:/dev/shm \ |
|
-v $(PWD)/secrets:/secrets \ |
|
-v $(PWD)/volumes/chrome-userdir:/opt/chrome-userdir \ |
|
--shm-size=2g \ |
|
$(IMAGE) \ |
|
--credentials-path /secrets/datastore-writer.json \ |
|
--loglevel debug \ |
|
spider --kind $(DB_ENTITY) ${ARGS} |
|
|
|
export: |
|
docker run --rm -ti \ |
|
-v $(PWD)/secrets:/secrets \ |
|
-v $(PWD)/volumes/json-export:/json-export \ |
|
$(IMAGE) \ |
|
--credentials-path /secrets/datastore-reader.json \ |
|
--loglevel debug \ |
|
export --kind $(DB_ENTITY) |
|
|
|
# run spider tests |
|
test: |
|
docker run --rm -ti \ |
|
-v $(PWD)/volumes/dev-shm:/dev/shm \ |
|
-v $(PWD)/secrets:/secrets \ |
|
-v $(PWD)/screenshots:/screenshots \ |
|
-v $(PWD)/volumes/chrome-userdir:/opt/chrome-userdir \ |
|
--entrypoint "python3" \ |
|
$(IMAGE) \ |
|
-m unittest discover -p '*_test.py' -v |
|
|
|
|