Web service API für Green Spider https://github.com/netzbegruenung/green-spider
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

94 lines
2.7 KiB

from datetime import datetime
from os import getenv
from wsgiref import simple_server
import falcon
from falcon import media
import jsonhandler
from google.cloud import datastore
credentials_path = getenv('GCLOUD_DATASTORE_CREDENTIALS_PATH')
datastore_client = datastore.Client.from_service_account_json(credentials_path)
entity_kind = 'spider-results'
def get_compact_results(client):
query = client.query(kind=entity_kind,
order=['-created'],
#projection=['created', 'meta', 'score'],
)
out = []
for entity in query.fetch(eventual=True):
# handle creation date in different ways, depending on whether the lib returns
# a str, int, or datetime.datetime
created = entity.get('created')
dt = ''
if type(created) == datetime:
dt = created
elif type(created) == int:
dt = datetime.utcfromtimestamp(created / 1000000)
elif type(created) == str:
dt = datetime.utcfromtimestamp(int(created) / 1000000)
out.append({
'input_url': entity.key.name,
'created': dt.isoformat(),
'meta': entity.get('meta'),
'score': entity.get('score'),
})
return out
class LastUpdated(object):
def on_get(self, req, resp):
"""
Informs about the most recent update to the spider results data
"""
query = datastore_client.query(kind=entity_kind,
order=['-created'],
projection=['created'])
items = list(query.fetch(limit=1, eventual=True))
ts = int(items[0].get('created')) / 1000000
dt = datetime.utcfromtimestamp(ts).isoformat()
maxage = 60 * 60 # one hour in seconds
resp.cache_control = ["max_age=%d" % maxage]
resp.media = {
"last_updated": dt
}
class CompactResults(object):
def on_get(self, req, resp):
"""
Returns compact sites overview and score
"""
out = get_compact_results(datastore_client)
maxage = 6 * 60 * 60 # six hours in seconds
resp.cache_control = ["max_age=%d" % maxage]
resp.media = out
handlers = media.Handlers({
'application/json': jsonhandler.JSONHandler(),
})
app = falcon.API()
app.req_options.media_handlers = handlers
app.resp_options.media_handlers = handlers
app.add_route('/api/v1/spider-results/last-updated/', LastUpdated())
app.add_route('/api/v1/spider-results/compact/', CompactResults())
if __name__ == '__main__':
httpd = simple_server.make_server('127.0.0.1', 5000, app)
httpd.serve_forever()