mirror of
https://github.com/netzbegruenung/social-spider.git
synced 2024-05-04 19:53:40 +02:00
Add Maps (#2)
* re-add maps part * add name mapping to create maps * finish first version of map rendering * create a unique key * add missing kreise * create basemaps for lv, kv and fill it with data * remove unused maps * remove unused map * use git lfs * update readme * add first version maps * add first version maps * remove one file for testing * remove maps again * remove lfs * include newlines * add maps * add maps html
This commit is contained in:
mehlukas
GitHub
parent
ade59299cf
commit
6683130a4a
|
|
@ -4,7 +4,6 @@ Social Spider erstellt ein Ranking von Grünen Social Media Accounts.
|
|||
|
||||
## Getting started
|
||||
|
||||
```
|
||||
pip install -r requirements.txt
|
||||
python ./spider.py
|
||||
```
|
||||
* Install [Git LFS](https://git-lfs.github.com/) and install it (`git lfs install`).
|
||||
* `pip install -r requirements.txt`
|
||||
* `python ./spider.py`
|
||||
|
|
|
|||
266
createMaps.py
Normal file
266
createMaps.py
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
import json
|
||||
from copy import copy
|
||||
from pprint import pprint
|
||||
from spider import dir_entries
|
||||
import difflib
|
||||
|
||||
COUNTRY_MAP = {"01": "Schleswig-Holstein",
|
||||
"02": "Hamburg",
|
||||
"03": "Niedersachsen",
|
||||
"04": "Bremen",
|
||||
"05": "Nordrhein-Westfalen",
|
||||
"06": "Hessen",
|
||||
"07": "Rheinland-Pfalz",
|
||||
"08": "Baden-Württemberg",
|
||||
"09": "Bayern",
|
||||
"10": "Saarland",
|
||||
"11": "Berlin",
|
||||
"12": "Brandenburg",
|
||||
"13": "Mecklenburg-Vorpommern",
|
||||
"14": "Sachsen",
|
||||
"15": "Sachsen-Anhalt",
|
||||
"16": "Thüringen"}
|
||||
|
||||
MAPPING = {"Neustadt a.d. Aisch-Bad Windsheim": "Neustadt-Aisch",
|
||||
"Märkischer Kreis": "Mark",
|
||||
"Osterode am Harz": "Göttingen",
|
||||
"Göttingen": "Göttingen",
|
||||
"Straubing": "Straubing-Bogen",
|
||||
"Straubing-Bogen": "Straubing-Bogen",
|
||||
"Kaufbeuren": "Ostallgäu",
|
||||
"Ostallgäu": "Ostallgäu"}
|
||||
|
||||
def createLVBasemap():
|
||||
green_data = []
|
||||
for entry in dir_entries():
|
||||
if entry["type"] == "REGIONAL_CHAPTER":
|
||||
if entry["level"] == "DE:LANDESVERBAND":
|
||||
green_data.append(entry)
|
||||
|
||||
with open("maps/bundeslaender_simplify200.geojson", encoding='utf-8') as map_f:
|
||||
maps = json.load(map_f)
|
||||
result_map = copy(maps)
|
||||
result_map["features"] = []
|
||||
for feature in maps["features"]:
|
||||
name = feature["properties"]["GEN"]
|
||||
for d in green_data:
|
||||
if d["state"] == name:
|
||||
feature["properties"]["green-data"] = d
|
||||
result_map["features"].append(feature)
|
||||
with open("maps/lv_basemap.geojson", "w", encoding='utf-8') as output_f:
|
||||
json.dump(result_map, output_f, indent=4)
|
||||
|
||||
def initialize():
|
||||
map_data = []
|
||||
green_data = []
|
||||
with open("maps/landkreise_simplify200.geojson") as map_f:
|
||||
map_data = json.load(map_f)["features"]
|
||||
|
||||
for entry in dir_entries():
|
||||
if entry["type"] == "REGIONAL_CHAPTER":
|
||||
if entry["level"] == "DE:KREISVERBAND":
|
||||
green_data.append(entry)
|
||||
if entry["level"] == "DE:REGIONALVERBAND":
|
||||
entry["district"] = entry["region"]
|
||||
green_data.append(entry)
|
||||
|
||||
print(len(map_data))
|
||||
print(len(green_data))
|
||||
return map_data, green_data
|
||||
|
||||
def generateMapping(map_data_local, green_data_local):
|
||||
# print("-----------------------------")
|
||||
# for d in green_data_local:
|
||||
# print(d["district"])
|
||||
# print("----")
|
||||
# for d in map_data_local:
|
||||
# name = d["properties"]["GEN"]
|
||||
# if d["properties"]["NBD"] == "ja":
|
||||
# fullname = d["properties"]["BEZ"] +" " + d["properties"]["GEN"]
|
||||
# else:
|
||||
# fullname = d["properties"]["GEN"]
|
||||
# print(fullname)
|
||||
# print("-----------------------------")
|
||||
|
||||
mapping = [-1] * len(map_data_local)
|
||||
mapCount = [0] * len(green_data_local)
|
||||
|
||||
# ------------------------------------------
|
||||
# match predefined
|
||||
# ------------------------------------------
|
||||
|
||||
for i, d in enumerate(map_data_local):
|
||||
name = d["properties"]["GEN"]
|
||||
if MAPPING.get(name):
|
||||
mapName = MAPPING[name]
|
||||
for j, entry in enumerate(green_data_local):
|
||||
if mapName == entry["district"]:
|
||||
mapping[i] = j
|
||||
mapCount[j] += 1
|
||||
|
||||
# ------------------------------------------
|
||||
# match full name
|
||||
# ------------------------------------------
|
||||
|
||||
mapCount_copy = copy(mapCount)
|
||||
for i, d in enumerate(map_data_local):
|
||||
if d["properties"]["NBD"] == "ja":
|
||||
fullname = d["properties"]["BEZ"] +" " + d["properties"]["GEN"]
|
||||
else:
|
||||
fullname = d["properties"]["GEN"]
|
||||
for j, entry in enumerate(green_data_local):
|
||||
if mapCount_copy[j] == 0:
|
||||
if fullname == entry["district"]:
|
||||
mapping[i] = j
|
||||
mapCount[j] += 1
|
||||
|
||||
# ------------------------------------------
|
||||
# match -Land and -Stadt entries
|
||||
# ------------------------------------------
|
||||
|
||||
mapCount_copy = copy(mapCount)
|
||||
for i, d in enumerate(map_data_local):
|
||||
if mapping[i] == -1:
|
||||
name = d["properties"]["GEN"]
|
||||
for j, entry in enumerate(green_data_local):
|
||||
if mapCount_copy[j] == 0:
|
||||
district_name = entry["district"]
|
||||
if district_name.endswith("-Stadt") and "stadt" in d["properties"]["BEZ"].lower():
|
||||
if name == district_name[:-6]:
|
||||
mapping[i] = j
|
||||
mapCount[j] += 1
|
||||
if district_name.endswith("-Land") and "land" in d["properties"]["BEZ"].lower():
|
||||
if name == district_name[:-5]:
|
||||
mapping[i] = j
|
||||
mapCount[j] += 1
|
||||
|
||||
# ------------------------------------------
|
||||
# match short name
|
||||
# ------------------------------------------
|
||||
|
||||
mapCount_copy = copy(mapCount)
|
||||
for i, d in enumerate(map_data_local):
|
||||
if mapping[i] == -1:
|
||||
name = d["properties"]["GEN"]
|
||||
for j, entry in enumerate(green_data_local):
|
||||
if mapCount_copy[j] == 0:
|
||||
if name == entry["district"]:
|
||||
mapping[i] = j
|
||||
mapCount[j] += 1
|
||||
|
||||
# ------------------------------------------
|
||||
# match entries ending with "kreis"
|
||||
# ------------------------------------------
|
||||
|
||||
mapCount_copy = copy(mapCount)
|
||||
for i, d in enumerate(map_data_local):
|
||||
if mapping[i] == -1:
|
||||
name = d["properties"]["GEN"]
|
||||
if name.endswith("-Kreis"):
|
||||
name = name[:-6]
|
||||
if name.lower().endswith("kreis"):
|
||||
name = name[:-5]
|
||||
for j, entry in enumerate(green_data_local):
|
||||
if mapCount_copy[j] == 0:
|
||||
if name == entry["district"]:
|
||||
mapping[i] = j
|
||||
mapCount[j] += 1
|
||||
|
||||
# ------------------------------------------
|
||||
# match containing substrings
|
||||
# ------------------------------------------
|
||||
|
||||
mapCount_copy = copy(mapCount)
|
||||
for i, d in enumerate(map_data_local):
|
||||
if mapping[i] == -1:
|
||||
name = d["properties"]["GEN"]
|
||||
for j, entry in enumerate(green_data_local):
|
||||
if mapCount_copy[j] == 0:
|
||||
name2 = entry["district"]
|
||||
if name in name2 or name2 in name:
|
||||
mapping[i] = j
|
||||
mapCount[j] += 1
|
||||
|
||||
# ------------------------------------------
|
||||
# match with string similarity
|
||||
# ------------------------------------------
|
||||
|
||||
mapCount_copy = copy(mapCount)
|
||||
for i, d in enumerate(map_data_local):
|
||||
if mapping[i] == -1:
|
||||
name = d["properties"]["GEN"]
|
||||
for j, entry in enumerate(green_data_local):
|
||||
if mapCount_copy[j] == 0:
|
||||
similarity = difflib.SequenceMatcher(a=entry["district"].lower(), b=name.lower()).ratio()
|
||||
if similarity > 0.6:
|
||||
mapping[i] = j
|
||||
mapCount[j] += 1
|
||||
|
||||
remaining = len([e for e in mapping if e == -1])
|
||||
print("Remaining:", remaining)
|
||||
if remaining == 0:
|
||||
return mapping
|
||||
|
||||
for j, entry in enumerate(green_data_local):
|
||||
if mapCount[j] == 0:
|
||||
print(entry["district"])
|
||||
print("---")
|
||||
for i, d in enumerate(map_data_local):
|
||||
if mapping[i] == -1:
|
||||
if d["properties"]["NBD"] == "ja":
|
||||
fullname = d["properties"]["BEZ"] +" " + d["properties"]["GEN"]
|
||||
else:
|
||||
fullname = d["properties"]["GEN"]
|
||||
|
||||
print(fullname, "--", d["properties"]["GEN"], "--")
|
||||
|
||||
return mapping
|
||||
|
||||
def preprocess():
|
||||
map_data, green_data = initialize()
|
||||
|
||||
all_mapping_data = {}
|
||||
for countrycode in COUNTRY_MAP.keys():
|
||||
countryname = COUNTRY_MAP[countrycode]
|
||||
if countryname in ["Berlin", "Bremen", "Hamburg"]:
|
||||
continue
|
||||
green_data_local = []
|
||||
for d in green_data:
|
||||
if d["state"] == countryname:
|
||||
green_data_local.append(d)
|
||||
map_data_local = []
|
||||
for d in map_data:
|
||||
if d["properties"]["SN_L"] == countrycode:
|
||||
map_data_local.append(d)
|
||||
print("===================================")
|
||||
print(countryname)
|
||||
print("===================================")
|
||||
mapping = generateMapping(map_data_local, green_data_local)
|
||||
all_mapping_data[countrycode] = {"map_data_local": map_data_local,
|
||||
"green_data_local": green_data_local,
|
||||
"mapping": mapping}
|
||||
|
||||
return all_mapping_data
|
||||
|
||||
|
||||
def createKVBasemap():
|
||||
all_mapping_data = preprocess()
|
||||
result_features = []
|
||||
for countrycode in all_mapping_data.keys():
|
||||
for i, feature in enumerate(all_mapping_data[countrycode]["map_data_local"]):
|
||||
entry_idx = all_mapping_data[countrycode]["mapping"][i]
|
||||
if entry_idx != -1:
|
||||
entry = all_mapping_data[countrycode]["green_data_local"][entry_idx]
|
||||
else:
|
||||
# skip not-matched map parts
|
||||
continue
|
||||
feature["properties"]["green-data"] = entry
|
||||
result_features.append(feature)
|
||||
result_map = {"type": "FeatureCollection",
|
||||
"features": result_features}
|
||||
with open("maps/kv_basemap.geojson", "w") as output_f:
|
||||
json.dump(result_map, output_f, indent=4)
|
||||
|
||||
if __name__ == "__main__":
|
||||
createKVBasemap()
|
||||
createLVBasemap()
|
||||
92
docs/maps.html
Normal file
92
docs/maps.html
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="de">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||||
<title>Social Spider</title>
|
||||
<link rel="stylesheet" href="https://netzbegruenung.github.io/webfonts/style.css">
|
||||
<style type="text/css">
|
||||
h1 {
|
||||
font-family: 'Arvo Gruen', sans-serif;
|
||||
font-weight: bold;
|
||||
color: #ffffff;
|
||||
text-transform: uppercase;
|
||||
margin-top: 0;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
h2 {
|
||||
font-family: 'Arvo Gruen', sans-serif;
|
||||
font-weight: bold;
|
||||
color: #ffffff;
|
||||
margin-bottom: 0px;
|
||||
margin-top: 0;
|
||||
}
|
||||
h3 {
|
||||
font-family: 'Arvo', sans-serif;
|
||||
color: #46962b;
|
||||
margin-top: 0;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
body{
|
||||
margin: 0;
|
||||
}
|
||||
.header {
|
||||
background-color: #46962b;
|
||||
padding: 20px;
|
||||
}
|
||||
.map {
|
||||
margin: 10px;
|
||||
float: left;
|
||||
}
|
||||
.mapgroup {
|
||||
margin: 20px;
|
||||
outline-color: red;
|
||||
outline-width: 1px;
|
||||
clear: left;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<h1>Social Spider</h1>
|
||||
</div>
|
||||
<div style="float: left;">
|
||||
<div class="mapgroup">
|
||||
<div class="header">
|
||||
<h2>Landesverbände</h2>
|
||||
</div>
|
||||
<div class="map">
|
||||
<h3>Facebook</h3>
|
||||
<script src="https://embed.github.com/view/geojson/netzbegruenung/social-spider/maps/maps/lv_fb.geojson?height=700&width=500"></script>
|
||||
</div>
|
||||
<div class="map">
|
||||
<h3>Twitter</h3>
|
||||
<script src="https://embed.github.com/view/geojson/netzbegruenung/social-spider/maps/maps/lv_tw.geojson?height=700&width=500"></script>
|
||||
</div>
|
||||
<div class="map">
|
||||
<h3>Instagram</h3>
|
||||
<script src="https://embed.github.com/view/geojson/netzbegruenung/social-spider/maps/maps/lv_in.geojson?height=700&width=500"></script>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mapgroup" style="padding-top: 20px;">
|
||||
<div class="header">
|
||||
<h2>Kreisverbände</h2>
|
||||
</div>
|
||||
<div class="map">
|
||||
<h3>Facebook</h3>
|
||||
<script src="https://embed.github.com/view/geojson/netzbegruenung/social-spider/maps/maps/kv_fb.geojson?height=700&width=500"></script>
|
||||
</div>
|
||||
<div class="map">
|
||||
<h3>Twitter</h3>
|
||||
<script src="https://embed.github.com/view/geojson/netzbegruenung/social-spider/maps/maps/kv_tw.geojson?height=700&width=500"></script>
|
||||
</div>
|
||||
<div class="map">
|
||||
<h3>Instagram</h3>
|
||||
<script src="https://embed.github.com/view/geojson/netzbegruenung/social-spider/maps/maps/kv_in.geojson?height=700&width=500"></script>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
||||
3
maps/bundeslaender_simplify200.geojson
Normal file
3
maps/bundeslaender_simplify200.geojson
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1bbfc58397515a36d4c6acb2cac5e2c54b5c4752e1f1c6aef3866c69e8988d3a
|
||||
size 836395
|
||||
339876
maps/kv_basemap.geojson
Normal file
339876
maps/kv_basemap.geojson
Normal file
File diff suppressed because it is too large
Load diff
343955
maps/kv_fb.geojson
Normal file
343955
maps/kv_fb.geojson
Normal file
File diff suppressed because it is too large
Load diff
343955
maps/kv_in.geojson
Normal file
343955
maps/kv_in.geojson
Normal file
File diff suppressed because it is too large
Load diff
343955
maps/kv_tw.geojson
Normal file
343955
maps/kv_tw.geojson
Normal file
File diff suppressed because it is too large
Load diff
3
maps/landkreise_simplify200.geojson
Normal file
3
maps/landkreise_simplify200.geojson
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:75dab9e093a3e4f0fbe816aa3029079514a44ef3651f14a2dd5430cebb4b5abb
|
||||
size 3279618
|
||||
88170
maps/lv_basemap.geojson
Normal file
88170
maps/lv_basemap.geojson
Normal file
File diff suppressed because it is too large
Load diff
88522
maps/lv_fb.geojson
Normal file
88522
maps/lv_fb.geojson
Normal file
File diff suppressed because it is too large
Load diff
88522
maps/lv_in.geojson
Normal file
88522
maps/lv_in.geojson
Normal file
File diff suppressed because it is too large
Load diff
88522
maps/lv_tw.geojson
Normal file
88522
maps/lv_tw.geojson
Normal file
File diff suppressed because it is too large
Load diff
48
updateMaps.py
Normal file
48
updateMaps.py
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
import json
|
||||
|
||||
def updateMaps(level="lv"):
|
||||
data = {}
|
||||
map_data = {}
|
||||
with open("docs/result.json") as f:
|
||||
data = json.load(f)
|
||||
with open("maps/" + level + "_basemap.geojson") as f:
|
||||
map_data = json.load(f)
|
||||
fb_max = 0
|
||||
tw_max = 0
|
||||
in_max = 0
|
||||
for feature in map_data["features"]:
|
||||
greendata = feature["properties"]["green-data"]
|
||||
key = "//".join([greendata["type"], greendata["level"], greendata["state"], greendata.get("district", ""), greendata.get("city", "")])
|
||||
result = data.get(key)
|
||||
print(result)
|
||||
social_data = {}
|
||||
if result:
|
||||
_, _, _ , _, _, fb_name, fb_count, tw_name, tw_count, in_name, in_count = data.get(key)
|
||||
if fb_count > fb_max:
|
||||
fb_max = fb_count
|
||||
if tw_count > tw_max:
|
||||
tw_max = tw_count
|
||||
if in_count > in_max:
|
||||
in_max = in_count
|
||||
social_data["fb_name"] = fb_name
|
||||
social_data["fb_count"] = fb_count
|
||||
social_data["tw_name"] = tw_name
|
||||
social_data["tw_count"] = tw_count
|
||||
social_data["in_name"] = in_name
|
||||
social_data["in_count"] = in_count
|
||||
feature["properties"]["social-data"] = social_data
|
||||
|
||||
print(fb_max, tw_max, in_max)
|
||||
plattform = "fb"
|
||||
for plattform, maxval, col in zip(["fb", "tw", "in"], [fb_max, tw_max, in_max], ["#3664a2", "#55acee", "#d02e92"]):
|
||||
for feature in map_data["features"]:
|
||||
count = feature["properties"]["social-data"].get(plattform + "_count", 0)
|
||||
feature["properties"]["fill-opacity"] = count/maxval
|
||||
feature["properties"]["fill"] = col
|
||||
feature["properties"]["stroke-width"] = 0
|
||||
with open("maps/" + level + "_" + plattform + ".geojson", "w") as f:
|
||||
json.dump(map_data, f, indent=4)
|
||||
|
||||
if __name__ == "__main__":
|
||||
updateMaps(level="kv")
|
||||
updateMaps(level="lv")
|
||||
Loading…
Reference in a new issue