NB-Public/green-spider
14
0
Fork 1
mirror of https://github.com/netzbegruenung/green-spider.git synced 2024-05-02 00:54:52 +02:00
Code Issues 2 Projects Releases Wiki Activity

Merge pull request #41 from ctr49/master

Browse source 
Importiere alle Website-URLs je Eintrag aus green-directory, statt nur einen
This commit is contained in:
Marian Steinbach 2018-05-11 08:28:22 +02:00 committed by GitHub
parent 3aec59d500 8df5c66813
commit b1d0a517bb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
spider.py
View file

@ -622,16 +622,16 @@ def main():
try:
if entry['urls'][n]['type'] == "WEBSITE":
website_url = entry['urls'][n]['url']
if website_url:
input_entries.append({
"url": website_url,
"level": entry.get("level"),
"state": entry.get("state"),
"district": entry.get("district"),
"city": entry.get("city"),
})
except NameError as ne:
logging.error("Error in %s: 'url' key missing (%s)" % (repr_entry(entry), entry['urls'][n]))
if website_url:
input_entries.append({
"url": website_url,
"level": entry.get("level"),
"state": entry.get("state"),
"district": entry.get("district"),
"city": entry.get("city"),
})
# randomize order, to distribute requests over servers