2018-10-03 11:05:42 +02:00
|
|
|
IMAGE := quay.io/netzbegruenung/green-spider:latest
|
2018-04-05 19:39:09 +02:00
|
|
|
|
2018-10-03 11:05:42 +02:00
|
|
|
DB_ENTITY := spider-results
|
2018-04-05 19:39:09 +02:00
|
|
|
|
2018-11-20 19:54:23 +01:00
|
|
|
.PHONY: dockerimage spider export
|
2018-04-05 19:39:09 +02:00
|
|
|
|
2018-05-03 12:01:30 +02:00
|
|
|
# Build docker image
|
|
|
|
dockerimage:
|
2018-10-03 11:05:42 +02:00
|
|
|
docker build -t $(IMAGE) .
|
2018-05-03 12:01:30 +02:00
|
|
|
|
2018-08-23 09:38:30 +02:00
|
|
|
# Create spider job queue
|
2018-11-20 19:54:23 +01:00
|
|
|
spiderjobs:
|
2018-08-23 09:38:30 +02:00
|
|
|
docker run --rm -ti \
|
|
|
|
-v $(PWD)/secrets:/secrets \
|
2018-10-03 11:05:42 +02:00
|
|
|
$(IMAGE) \
|
2018-08-23 09:38:30 +02:00
|
|
|
--credentials-path /secrets/datastore-writer.json \
|
2018-11-20 19:54:23 +01:00
|
|
|
--loglevel debug \
|
2018-08-23 09:38:30 +02:00
|
|
|
jobs
|
|
|
|
|
2018-05-03 12:01:30 +02:00
|
|
|
# Run spider in docker image
|
2018-11-20 19:54:23 +01:00
|
|
|
spider:
|
2018-05-25 19:10:54 +02:00
|
|
|
docker run --rm -ti \
|
2019-06-03 08:08:55 +02:00
|
|
|
-v $(PWD)/volumes/dev-shm:/dev/shm \
|
2018-08-23 10:02:34 +02:00
|
|
|
-v $(PWD)/secrets:/secrets \
|
2019-06-03 08:08:55 +02:00
|
|
|
-v $(PWD)/volumes/chrome-userdir:/opt/chrome-userdir \
|
2019-11-22 08:39:56 +01:00
|
|
|
--shm-size=2g \
|
2018-10-03 11:05:42 +02:00
|
|
|
$(IMAGE) \
|
2018-08-23 10:02:34 +02:00
|
|
|
--credentials-path /secrets/datastore-writer.json \
|
2018-10-03 11:05:42 +02:00
|
|
|
--loglevel debug \
|
2019-04-19 00:35:28 +02:00
|
|
|
spider --kind $(DB_ENTITY) ${ARGS}
|
2018-10-03 11:05:42 +02:00
|
|
|
|
2018-11-20 19:54:23 +01:00
|
|
|
export:
|
2018-10-03 11:05:42 +02:00
|
|
|
docker run --rm -ti \
|
|
|
|
-v $(PWD)/secrets:/secrets \
|
2019-06-03 08:08:55 +02:00
|
|
|
-v $(PWD)/volumes/json-export:/json-export \
|
2018-10-03 11:05:42 +02:00
|
|
|
$(IMAGE) \
|
|
|
|
--credentials-path /secrets/datastore-reader.json \
|
|
|
|
--loglevel debug \
|
|
|
|
export --kind $(DB_ENTITY)
|
2018-04-05 19:39:09 +02:00
|
|
|
|
2018-08-27 21:17:04 +02:00
|
|
|
# run spider tests
|
2018-11-20 19:54:23 +01:00
|
|
|
test:
|
2018-10-03 11:05:42 +02:00
|
|
|
docker run --rm -ti \
|
2019-06-03 08:08:55 +02:00
|
|
|
-v $(PWD)/volumes/chrome-userdir:/opt/chrome-userdir \
|
2018-10-03 11:05:42 +02:00
|
|
|
--entrypoint "python3" \
|
|
|
|
$(IMAGE) \
|
2018-10-03 21:01:52 +02:00
|
|
|
-m unittest discover -p '*_test.py' -v
|
2018-10-03 11:05:42 +02:00
|
|
|
|