mirror of
https://github.com/netzbegruenung/green-spider.git
synced 2024-05-04 18:13:40 +02:00
Move append into for-loop. This allows mutliple web sites per entry to be added.
This commit is contained in:
parent
3aec59d500
commit
8df5c66813
16
spider.py
16
spider.py
|
@ -622,16 +622,16 @@ def main():
|
||||||
try:
|
try:
|
||||||
if entry['urls'][n]['type'] == "WEBSITE":
|
if entry['urls'][n]['type'] == "WEBSITE":
|
||||||
website_url = entry['urls'][n]['url']
|
website_url = entry['urls'][n]['url']
|
||||||
|
if website_url:
|
||||||
|
input_entries.append({
|
||||||
|
"url": website_url,
|
||||||
|
"level": entry.get("level"),
|
||||||
|
"state": entry.get("state"),
|
||||||
|
"district": entry.get("district"),
|
||||||
|
"city": entry.get("city"),
|
||||||
|
})
|
||||||
except NameError as ne:
|
except NameError as ne:
|
||||||
logging.error("Error in %s: 'url' key missing (%s)" % (repr_entry(entry), entry['urls'][n]))
|
logging.error("Error in %s: 'url' key missing (%s)" % (repr_entry(entry), entry['urls'][n]))
|
||||||
if website_url:
|
|
||||||
input_entries.append({
|
|
||||||
"url": website_url,
|
|
||||||
"level": entry.get("level"),
|
|
||||||
"state": entry.get("state"),
|
|
||||||
"district": entry.get("district"),
|
|
||||||
"city": entry.get("city"),
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
# randomize order, to distribute requests over servers
|
# randomize order, to distribute requests over servers
|
||||||
|
|
Loading…
Reference in a new issue