'sum': {'Summe', 'Berlin'},
}
-# Here only image files are available for the table data.
-unparsable_graphics_fallback = {
+# some pre-filled values
+data = {
+ # For these, only image files are available for the table data.
datetime.datetime(2020, 7, 2): {
'CW': {'growth': 4, 'total': 851},
'FK': {'growth': 10, 'total': 681},
'TS': {'growth': 41, 'total': 362},
'TK': {'growth': 14, 'total': 183},
'sum': {'growth': 263, 'total': 3486}
- }
+ },
+ # This one has no press release but can be reconstructed from
+ # the neighbour ones.
+ datetime.datetime(2020, 3, 13): {
+ 'CW': {'growth': 16, 'total': 47},
+ 'FK': {'growth': 8, 'total': 22},
+ 'Li': {'growth': 2, 'total': 8},
+ 'MH': {'growth': 1, 'total': 4},
+ 'Mi': {'growth': 9, 'total': 29},
+ 'Ne': {'growth': 6, 'total': 16},
+ 'Pa': {'growth': 11, 'total': 26},
+ 'Re': {'growth': 0, 'total': 11},
+ 'Sp': {'growth': 1, 'total': 9},
+ 'SZ': {'growth': 0, 'total': 20},
+ 'TS': {'growth': 1, 'total': 17},
+ 'TK': {'growth': 3, 'total': 7},
+ 'sum': {'growth': 58, 'total': 216}
+ },
+ # Here the growth numbers needed to be reconstructed.
+ datetime.datetime(2020, 3, 10): {
+ 'CW': {'growth': 2, 'total': 15},
+ 'FK': {'growth': 0, 'total': 12},
+ 'Li': {'growth': 4, 'total': 5},
+ 'MH': {'growth': 1, 'total': 3},
+ 'Mi': {'growth': 0, 'total': 8},
+ 'Ne': {'growth': 2, 'total': 5},
+ 'Pa': {'growth': 2, 'total': 8},
+ 'Re': {'growth': 0, 'total': 3},
+ 'Sp': {'growth': 4, 'total': 6},
+ 'SZ': {'growth': 3, 'total': 6},
+ 'TS': {'growth': 2, 'total': 7},
+ 'TK': {'growth': 3, 'total': 3},
+ 'sum': {'growth': 23, 'total': 81}
+ },
+ # Here the totals needed to be reconstructed.
+ datetime.datetime(2020, 3, 9): {
+ 'CW': {'growth': 4, 'total': 13},
+ 'FK': {'growth': 3, 'total': 12},
+ 'Li': {'growth': 0, 'total': 1},
+ 'MH': {'growth': 1, 'total': 2},
+ 'Mi': {'growth': 0, 'total': 8},
+ 'Ne': {'growth': 1, 'total': 3},
+ 'Pa': {'growth': 1, 'total': 6},
+ 'Re': {'growth': 0, 'total': 3},
+ 'Sp': {'growth': 0, 'total': 2},
+ 'SZ': {'growth': 0, 'total': 3},
+ 'TS': {'growth': 0, 'total': 5},
+ 'TK': {'growth': 0, 'total': 0},
+ 'sum': {'growth': 10, 'total': 58}
+ },
+ # Here the growth numbers needed to be reconstructed.
+ datetime.datetime(2020, 3, 8): {
+ 'CW': {'growth': 0, 'total': 9},
+ 'FK': {'growth': 4, 'total': 9},
+ 'Li': {'growth': 1, 'total': 1},
+ 'MH': {'growth': 0, 'total': 1},
+ 'Mi': {'growth': 0, 'total': 8},
+ 'Ne': {'growth': 0, 'total': 2},
+ 'Pa': {'growth': 0, 'total': 5},
+ 'Re': {'growth': 0, 'total': 3},
+ 'Sp': {'growth': 2, 'total': 2},
+ 'SZ': {'growth': 1, 'total': 3},
+ 'TS': {'growth': 0, 'total': 5},
+ 'TK': {'growth': 0, 'total': 0},
+ 'sum': {'growth': 8, 'total': 48}
+ },
+ # Here the growth numbers needed to be reconstructed.
+ datetime.datetime(2020, 3, 7): {
+ 'CW': {'growth': 6, 'total': 9},
+ 'FK': {'growth': 1, 'total': 5},
+ 'Li': {'growth': 0, 'total': 0},
+ 'MH': {'growth': 0, 'total': 1},
+ 'Mi': {'growth': 1, 'total': 8},
+ 'Ne': {'growth': 0, 'total': 2},
+ 'Pa': {'growth': 1, 'total': 5},
+ 'Re': {'growth': 0, 'total': 3},
+ 'Sp': {'growth': 0, 'total': 0},
+ 'SZ': {'growth': 2, 'total': 2},
+ 'TS': {'growth': 1, 'total': 5},
+ 'TK': {'growth': 0, 'total': 0},
+ 'sum': {'growth': 12, 'total': 40}
+ },
}
fixes = {
# Here the official total is 215, while the summation of district
},
}
-# Scan navigation bar for maximum pagination value.
+# Scan navigation bar for maximum pagination value.
url = url_prefix + pm_dir
with urllib.request.urlopen(url) as response:
html = response.read()
continue
day_urls += [link['href']]
-# Collect infection data.
-data = {}
+# Collect infection data.
first_run = True
districts_sorted = []
# TODO: Push limit further back (might need more data fixes for that).
-date_limit = datetime.datetime(2020, 3, 16)
+date_limit = datetime.datetime(2020, 3, 12)
for path in day_urls:
url = url_prefix + path
with urllib.request.urlopen(url) as response:
date = datetime.datetime.strptime(date_formatted , '%d.%m.%Y')
if date_limit > date:
break
- if date in data:
- raise Exception('Double date %s', date)
- #date -= datetime.timedelta(days=1)
- data[date] = {}
+ # On that day, two press releases were released, for that and the prev day.
+ if date == datetime.datetime(2020, 3, 15) and date in data:
+ date = datetime.datetime(2020, 3, 14)
+ # From here on, press releases describe numbers from prev day.
+ if date <= datetime.datetime(2020, 3, 13):
+ date = date - datetime.timedelta(days=1)
table = soup.find('table')
- if table is None:
- data[date] = unparsable_graphics_fallback[date]
+ if table is None and date in data:
continue
+ data[date] = {}
for tr in [tr for tr in table.children if type(tr) == bs4.element.Tag][1:]:
printable_tds = []
for td in [td for td in tr.children if type(td) == bs4.element.Tag][:2]:
raise Exception('Dates not contiguous: %s missing', prev_date)
else:
continue
- prev_total = data[date - datetime.timedelta(days=1)][district]['total']
+ prev_total = data[prev_date][district]['total']
cur_total = data[date][district]['total']
if cur_total - data[date][district]['growth'] != prev_total:
raise Exception('Questionable district infection total in %s/%s' % (district, date))
projects / berlin-corona-table / blobdiff