mirror of
https://github.com/netzbegruenung/green-spider.git
synced 2024-04-27 23:04:52 +02:00
Marian Steinbach
618e29d763
* CLI: remove 'jobs' command, add 'manager' * Add job definition * Move jobs to manage folder * Rename jobs to manager * Add rq and redis dependencies * Add docker-compose YAML * Downgrade to alpine 3.8 * Adjust paths in Dockerfile, remove entrypoint * Rename 'make spiderjobs' to 'make jobs' * Fix docker exectution * Adapt 'make jobs' * Fix metadata scheme * Add docker dependency * Rendomize queue (a bit) * Use latest image, remove debug output * Make docker-compose file downwards-compatible * Use latest instead of dev image tag * Update docker-compose.yaml * Adapt job start script * Fix redis connection in manager * Add support for increasing timeout via environment variable * Adapt load_in_browser to cookies table schema change * Fix execution * Mitigate yaml warning * Bump some dependency versions * Report resource usage stats for each job * checks/load_in_browser: Return DOM size, prevent multiple page loads * Update .dockerignore * Code update * Script update * Update README.md * WIP * WIP commit * Update Dockerfile to alpine:edge and chromium v90 * Update TestCertificateChecker * Set defaults for __init__ function * Detect sunflower theme * Update unit test for new datetime (zero-basing) * Set logging prefs from Chromium in a new way * Move datastore client instantiation As it is not needed for all commands * Change green-directory repository URL * Add git settings for cloning green-directory * Pin alpine version 3.14, fix py3-cryptography * Use plain docker build progress output * Add volumes to 'make test' docker run command * Fix bug * Update example command in README * Update dependencies * Add creation of Kubernetes jobs
56 lines
1.6 KiB
Python
56 lines
1.6 KiB
Python
class Config(object):
|
|
"""
|
|
Our configuration to be passed to checks
|
|
"""
|
|
|
|
def __init__(self,
|
|
urls,
|
|
screenshot_bucket_name='',
|
|
screenshot_datastore_kind='',
|
|
storage_credentials_path='',
|
|
datastore_credentials_path='',
|
|
user_agent='green-spider/1.0'):
|
|
self._urls = set(urls)
|
|
self._user_agent = user_agent
|
|
self._screenshot_bucket_name = screenshot_bucket_name
|
|
self._screenshot_datastore_kind = screenshot_datastore_kind
|
|
self._storage_credentials_path = storage_credentials_path
|
|
self._datastore_credentials_path = datastore_credentials_path
|
|
|
|
def __repr__(self):
|
|
return "Config(urls=%r)" % self._urls
|
|
|
|
@property
|
|
def urls(self):
|
|
return sorted(list(self._urls))
|
|
|
|
def add_url(self, url):
|
|
self._urls.add(url)
|
|
|
|
def remove_url(self, url):
|
|
"""Removes url from urls, if it was in there. Ignores errors."""
|
|
try:
|
|
self._urls.remove(url)
|
|
except KeyError:
|
|
pass
|
|
|
|
@property
|
|
def user_agent(self):
|
|
return self._user_agent
|
|
|
|
@property
|
|
def screenshot_bucket_name(self):
|
|
return self._screenshot_bucket_name
|
|
|
|
@property
|
|
def storage_credentials_path(self):
|
|
return self._storage_credentials_path
|
|
|
|
@property
|
|
def datastore_credentials_path(self):
|
|
return self._datastore_credentials_path
|
|
|
|
@property
|
|
def screenshot_datastore_kind(self):
|
|
return self._screenshot_datastore_kind
|