mirror of
https://github.com/netzbegruenung/green-spider.git
synced 2024-05-03 17:43:40 +02:00
5e8347916c
* Fix detection of redirects to bad domains * Fix bad domain check * Add --url flag to spider for faster debugging * Pass args to make spider * Add spidering of a single URL for debugging purposes * Fix tests * Fix test in CI * Remove pip upgrade
48 lines
999 B
Makefile
48 lines
999 B
Makefile
IMAGE := quay.io/netzbegruenung/green-spider:latest
|
|
|
|
DB_ENTITY := spider-results
|
|
|
|
.PHONY: dockerimage spider export
|
|
|
|
# Build docker image
|
|
dockerimage:
|
|
docker build -t $(IMAGE) .
|
|
|
|
# Create spider job queue
|
|
spiderjobs:
|
|
docker run --rm -ti \
|
|
-v $(PWD)/secrets:/secrets \
|
|
$(IMAGE) \
|
|
--credentials-path /secrets/datastore-writer.json \
|
|
--loglevel debug \
|
|
jobs
|
|
|
|
# Run spider in docker image
|
|
spider:
|
|
docker run --rm -ti \
|
|
-v $(PWD)/dev-shm:/dev/shm \
|
|
-v $(PWD)/secrets:/secrets \
|
|
$(IMAGE) \
|
|
--credentials-path /secrets/datastore-writer.json \
|
|
--loglevel debug \
|
|
spider --kind $(DB_ENTITY) ${ARGS}
|
|
|
|
export:
|
|
docker run --rm -ti \
|
|
-w $(PWD)/export-json \
|
|
-v $(PWD)/secrets:/secrets \
|
|
-v $(PWD)/export-siteicons:/icons \
|
|
$(IMAGE) \
|
|
--credentials-path /secrets/datastore-reader.json \
|
|
--loglevel debug \
|
|
export --kind $(DB_ENTITY)
|
|
|
|
# run spider tests
|
|
# FIXME
|
|
test:
|
|
docker run --rm -ti \
|
|
--entrypoint "python3" \
|
|
$(IMAGE) \
|
|
-m unittest discover -p '*_test.py' -v
|
|
|