Turn scrape.py output into daily infections table order format.

[berlin-corona-table] / scrape.py

diff --git a/scrape.py b/scrape.py
index f11f315f5e73e8f3d6b5b3b98c0f7fbfac63267a..d05fe909707bae674362e84b9dfa6ffeeff87073 100755 (executable)
--- a/scrape.py
+++ b/scrape.py
@@ -88,22 +88,6 @@ data = {
         'TK': {'growth': 14, 'total': 183},
         'sum': {'growth': 263, 'total': 3486}
     },
-   # Here the growth numbers needed to be reconstructed.
-   datetime.datetime(2020, 3, 10): {
-        'CW': {'growth': 2, 'total': 15},
-        'FK': {'growth': 0, 'total': 12},
-        'Li': {'growth': 4, 'total': 5},
-        'MH': {'growth': 1, 'total': 3},
-        'Mi': {'growth': 0, 'total': 8},
-        'Ne': {'growth': 2, 'total': 5},
-        'Pa': {'growth': 2, 'total': 8},
-        'Re': {'growth': 0, 'total': 3},
-        'Sp': {'growth': 4, 'total': 6},
-        'SZ': {'growth': 3, 'total': 6},
-        'TS': {'growth': 2, 'total': 7},
-        'TK': {'growth': 3, 'total': 3},
-        'sum': {'growth': 23, 'total': 81}
-   },
    # Here the totals needed to be reconstructed.
    datetime.datetime(2020, 3, 9): {
         'CW': {'growth': 4, 'total': 13},
@@ -277,8 +261,7 @@ for i in range(max_page):
 # Collect infection data.
 first_run = True
 districts_sorted = []
-# TODO: Push limit further back (might need more data fixes for that).
-date_limit = datetime.datetime(2020, 3, 12)
+date_limit = datetime.datetime(2020, 3, 11)
 for path in day_urls:
     url = url_prefix + path
     with urllib.request.urlopen(url) as response:
@@ -302,27 +285,43 @@ for path in day_urls:
         continue
     data[date] = {}
     for tr in [tr for tr in table.children if type(tr) == bs4.element.Tag][1:]:
-        printable_tds = []
-        for td in [td for td in tr.children if type(td) == bs4.element.Tag][:2]:
-            printable_string = ' '.join([s for s in td.strings])
-            printable_tds += [printable_string.strip()]
-        district_long = printable_tds[0]
-        district_short = [k for k in abbrevs if district_long in abbrevs[k]][0]
-        if first_run:
-            districts_sorted += [district_short]
-        split_char = ' '
-        if not split_char in printable_tds[1]:
-            split_char = '('
-        total_str, growth_str = printable_tds[1].split(split_char)
-        growth = int(growth_str.replace('(', '').replace(')', '').replace('+', ''))
-        total = int(total_str.replace('.', ''))
-        data[date][district_short] = {'growth': growth, 'total': total}
+         printable_tds = []
+         for td in [td for td in tr.children if type(td) == bs4.element.Tag][:2]:
+             printable_string = ' '.join([s for s in td.strings])
+             printable_tds += [printable_string.strip()]
+         district_long = printable_tds[0]
+         district_short = [k for k in abbrevs if district_long in abbrevs[k]][0]
+         if first_run:
+             districts_sorted += [district_short]
+         if date == datetime.datetime(2020, 3, 10):
+            # For this date we only get totals.
+            data[date][district_short] = {'total': int(printable_tds[1])}
+         else:
+             split_char = ' '
+             if not split_char in printable_tds[1]:
+                 split_char = '('
+             total_str, growth_str = printable_tds[1].split(split_char)
+             growth = int(growth_str.replace('(', '').replace(')', '').\
+                          replace('+', ''))
+             total = int(total_str.replace('.', ''))
+             data[date][district_short] = {'growth': growth, 'total': total}
     first_run = False
 
+def neighbor_days(day_target):
+   day_delta = datetime.timedelta(days=1)
+   return day_target + day_delta, day_target - day_delta
+
+# Reconstruct growth for 10th of March.
+day_target = datetime.datetime(2020, 3, 10)
+day_after, day_before = neighbor_days(day_target)
+for district in [d for d in districts_sorted]:
+   total_target = data[day_target][district]['total']
+   total_before = data[day_before][district]['total']
+   data[day_target][district]['growth'] = total_target - total_before
+
 # Reconstruct data for 13th of March.
 day_target = datetime.datetime(2020, 3, 13)
-day_after = day_target + datetime.timedelta(days=1)
-day_before = day_target - datetime.timedelta(days=1)
+day_after, day_before = neighbor_days(day_target)
 data[day_target] = {}
 for district in [d for d in districts_sorted]:
    data[day_target][district] = {}
@@ -368,6 +367,7 @@ for date in dates_sorted:
         raise Exception('Questionable district infection growth sum in %s' % date)
 
 # Final output.
+dates_sorted.reverse()
 print(' '*10, ' '.join(['%3s' % d for d in districts_sorted]))
 for date in dates_sorted:
     growths = []