From eb9a29ac1ce299e8bbce2e66d6e1380ae7d88126 Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Tue, 11 Sep 2018 23:58:53 +0200 Subject: [PATCH] Tweak chromedriver usage --- spider.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spider.py b/spider.py index dd9f834..4e4f6e6 100644 --- a/spider.py +++ b/spider.py @@ -10,6 +10,7 @@ import random import re import shutil import statistics +import time from datetime import datetime from socket import gethostbyname_ex from urllib.parse import urljoin @@ -266,13 +267,14 @@ def check_responsiveness(url): # Our selenium user agent using Chrome headless as an engine chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--headless') - chrome_options.add_argument('--disable-dev-shm-usage') chrome_options.add_argument('--disable-gpu') chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--disable-extensions') driver = webdriver.Chrome(chrome_options=chrome_options) + driver.set_page_load_timeout(60) driver.set_window_size(sizes[0][0], sizes[0][1]) driver.get(url) + time.sleep(1) for (width, height) in sizes: driver.set_window_size(width, height)