NB-Public/green-spider
14
0
Fork 1
mirror of https://github.com/netzbegruenung/green-spider.git synced 2024-05-01 08:34:51 +02:00
Code Issues 2 Projects Releases Wiki Activity

Move configuration into module

Browse source
This commit is contained in:
Marian Steinbach 2018-09-24 21:19:23 +02:00
parent 7514aeb542
commit 73eff37110
2 changed files with 26 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
config/__init__.py Normal file
View file

@ -0,0 +1,25 @@
# connection timeout for website checks (seconds)
CONNECT_TIMEOUT = 5
# response timeout for website checks
READ_TIMEOUT = 10
# Git repo for our data
GREEN_DIRECTORY_REPO = 'https://github.com/netzbegruenung/green-directory.git'
# folder in that repo that holds the data
GREEN_DIRECTORY_DATA_PATH = 'data/countries/de'
# folder we use locally to clone the repo
GREEN_DIRECTORY_LOCAL_PATH = './cache/green-directory'
# IP address of the newthinking GCMS server
GCMS_IP = "91.102.13.20"
# kind name of the spider job key datastore entities
JOB_DATASTORE_KIND = 'spider-jobs'
# kind name of the spider results datastore entities
RESULTS_DATASTORE_KIND = 'spider-results'

21
spider.py
View file

@ -28,29 +28,10 @@ from google.api_core.exceptions import Aborted
from google.api_core.exceptions import InvalidArgument
# configuration
# connection timeout for website checks (seconds)
CONNECT_TIMEOUT = 5
# response timeout for website checks
READ_TIMEOUT = 10
# Git repo for our data
GREEN_DIRECTORY_REPO = 'https://github.com/netzbegruenung/green-directory.git'
# folder in that repo that holds the data
GREEN_DIRECTORY_DATA_PATH = 'data/countries/de'
GREEN_DIRECTORY_LOCAL_PATH = './cache/green-directory'
RESULT_PATH = '/out'
# IP address of the newthinking GCMS server
GCMS_IP = "91.102.13.20"
JOB_DATASTORE_KIND = 'spider-jobs'
RESULTS_DATASTORE_KIND = 'spider-results'
# end configuration
import config
DATASTORE_CLIENT = None