NB-Public/social-spider
14
0
Fork 0
mirror of https://github.com/netzbegruenung/social-spider.git synced 2024-05-04 19:53:40 +02:00
Code Issues Projects Releases Wiki Activity

initial working version of table visualization

Browse source
This commit is contained in:
Lukas Mehl 2018-07-17 22:51:00 +02:00
parent af53fb4d3c
commit 3c12e67b97
4 changed files with 61 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
docs/index.html
View file

@ -7,9 +7,40 @@
</head>
<body>
<div class="d-flex flex-column flex-md-row align-items-center p-3 px-md-4 mb-3 header border-bottom box-shadow">
<div>
<h5>Social Spider</h5>
Hello World!
<table id="datatable">
<thead>
<tr>
<th>Typ</th>
<th>Land</th>
<th>Kreis</th>
<th>Stadt</th>
<th>FacebookID</th>
<th>Facebook Likes</th>
<th>TwitterID</th>
<th>Twitter Follower</th>
</tr>
</thead>
</table>
</div>
<script src="jquery-3.3.1.min.js"></script>
<script type="text/javascript">
$.getJSON("result.json" , function(data) {
var tbl_body = document.createElement("tbody");
var odd_even = false;
$.each(data, function() {
var tbl_row = tbl_body.insertRow();
tbl_row.className = odd_even ? "odd" : "even";
$.each(this, function(k , v) {
var cell = tbl_row.insertCell();
cell.appendChild(document.createTextNode(v.toString()));
})
odd_even = !odd_even;
})
$("#datatable").append(tbl_body);
});
</script>
</body>
</html>

2
docs/jquery-3.3.1.min.js vendored Normal file
View file

File diff suppressed because one or more lines are too long

1
docs/result.json Normal file
View file

File diff suppressed because one or more lines are too long

28
spider.py
View file

@ -7,6 +7,7 @@ from pprint import pprint
import sys
import re
import twitter
import json
# Git repo for our data
green_directory_repo = 'https://github.com/netzbegruenung/green-directory.git'
@ -108,9 +109,17 @@ def main():
facebookGraphAPI = facebook.GraphAPI(access_token=facebook_access_token)
# pprint(graph.get_object("B90DieGruenen", fields="fan_count,username,verification_status,website"))
doc = []
result = {}
idx = 0
fbcount = 0
twtcount = 0
for entry in dir_entries():
fbname = "--"
fbLikes = 0
twtname = "--"
twtFollower = 0
if not entry.get("urls"):
continue
for url in entry["urls"]:
@ -123,6 +132,8 @@ def main():
print("FACEBOOK ERROR for", url["url"], "--", fbname, file=sys.stderr)
print(e, file=sys.stderr)
continue
fbLikes = fbdata["fan_count"]
entry.update({"facebookData": fbdata, "facebookID": fbname})
print(fbname)
fbcount += 1
@ -137,13 +148,24 @@ def main():
print("TWITTER ERROR for", url["url"], "--", twtname, file=sys.stderr)
print(e, file=sys.stderr)
continue
twtFollower = twtData["followers_count"]
entry.update({"twitterData": twtData, "twitterName": twtname})
print(twtname)
doc.append(entry)
with open("result.yaml", "w") as f:
yaml.dump_all(doc, f)
typ = entry.get("level").split(":")[1].replace("KREISVERBAND", "KV").replace("ORTSVERBAND", "OV").replace("LANDESVERBAND", "LV").replace("BUNDESVERBAND", "BV")
land = entry.get("state", "")
kreis = entry.get("district", "")
stadt = entry.get("city", "")
if fbname is None:
fbname = ""
result.update({str(idx): [typ, land, kreis, stadt, fbname, fbLikes, twtname, twtFollower]})
idx += 1
with open("docs/result.json", "w") as f:
json.dump(result, f)
# with open("result.yaml", "w") as f:
# yaml.dump_all(doc, f)
print("facebook:", fbcount, "twitter:", twtcount)