NB-Public
/
green-spider
mirror of https://github.com/netzbegruenung/green-spider.git
14
0
Fork 1
Code Issues 2 Projects Releases Wiki Activity

Add retry for get_job_from_queue

This commit is contained in:
Marian Steinbach 2018-08-27 22:40:31 +02:00
parent cb58b32c39
commit 545ea671d8
2 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -12,7 +12,7 @@ RUN apt-get update \
&& wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb \ && wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb \
&& dpkg -i google-chrome-stable_current_amd64.deb \ && dpkg -i google-chrome-stable_current_amd64.deb \
&& rm google-chrome-stable_current_amd64.deb \ && rm google-chrome-stable_current_amd64.deb \
&& pip3 install GitPython idna PyYAML beautifulsoup4==4.6.0 requests==2.18.4 responses==0.9.0 selenium==3.11.0 smmap2==2.0.3 urllib3==1.22 google-cloud-datastore==1.7.0 \ && pip3 install GitPython idna PyYAML beautifulsoup4==4.6.0 requests==2.18.4 responses==0.9.0 selenium==3.11.0 smmap2==2.0.3 urllib3==1.22 google-cloud-datastore==1.7.0 tenacity==5.0.2 \
&& wget https://chromedriver.storage.googleapis.com/2.38/chromedriver_linux64.zip \ && wget https://chromedriver.storage.googleapis.com/2.38/chromedriver_linux64.zip \
&& unzip chromedriver_linux64.zip \ && unzip chromedriver_linux64.zip \
&& rm chromedriver_linux64.zip \ && rm chromedriver_linux64.zip \

4
spider.py
View File

@ -18,11 +18,13 @@ from urllib.parse import urlparse
import requests import requests
import yaml import yaml
import tenacity
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from git import Repo from git import Repo
from selenium import webdriver from selenium import webdriver
from google.cloud import datastore from google.cloud import datastore
from google.api_core.exceptions import Aborted
from google.api_core.exceptions import InvalidArgument from google.api_core.exceptions import InvalidArgument
@ -713,6 +715,8 @@ def check_site(entry):
return result return result
@tenacity.retry(wait=tenacity.wait_exponential(),
retry=tenacity.retry_if_exception_type(Aborted))
def get_job_from_queue(): def get_job_from_queue():
""" """
Returns a URL from the queue Returns a URL from the queue