From 8df5c668135c7d176d81e885459a7d920569f9ce Mon Sep 17 00:00:00 2001 From: Christian Tramnitz Date: Thu, 10 May 2018 22:47:49 +0200 Subject: [PATCH] Move append into for-loop. This allows mutliple web sites per entry to be added. --- spider.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/spider.py b/spider.py index 3cba123..ef32490 100644 --- a/spider.py +++ b/spider.py @@ -622,16 +622,16 @@ def main(): try: if entry['urls'][n]['type'] == "WEBSITE": website_url = entry['urls'][n]['url'] + if website_url: + input_entries.append({ + "url": website_url, + "level": entry.get("level"), + "state": entry.get("state"), + "district": entry.get("district"), + "city": entry.get("city"), + }) except NameError as ne: logging.error("Error in %s: 'url' key missing (%s)" % (repr_entry(entry), entry['urls'][n])) - if website_url: - input_entries.append({ - "url": website_url, - "level": entry.get("level"), - "state": entry.get("state"), - "district": entry.get("district"), - "city": entry.get("city"), - }) # randomize order, to distribute requests over servers