From: Christian Heller Date: Mon, 3 Aug 2020 18:23:49 +0000 (+0200) Subject: De-hardcode March the 10th. X-Git-Url: https://plomlompom.com/repos/?a=commitdiff_plain;h=a4b4a867c3ed12b1b3c0256a505e3f000d969e8f;p=berlin-corona-table De-hardcode March the 10th. --- diff --git a/scrape.py b/scrape.py index f11f315..4448de6 100755 --- a/scrape.py +++ b/scrape.py @@ -88,22 +88,6 @@ data = { 'TK': {'growth': 14, 'total': 183}, 'sum': {'growth': 263, 'total': 3486} }, - # Here the growth numbers needed to be reconstructed. - datetime.datetime(2020, 3, 10): { - 'CW': {'growth': 2, 'total': 15}, - 'FK': {'growth': 0, 'total': 12}, - 'Li': {'growth': 4, 'total': 5}, - 'MH': {'growth': 1, 'total': 3}, - 'Mi': {'growth': 0, 'total': 8}, - 'Ne': {'growth': 2, 'total': 5}, - 'Pa': {'growth': 2, 'total': 8}, - 'Re': {'growth': 0, 'total': 3}, - 'Sp': {'growth': 4, 'total': 6}, - 'SZ': {'growth': 3, 'total': 6}, - 'TS': {'growth': 2, 'total': 7}, - 'TK': {'growth': 3, 'total': 3}, - 'sum': {'growth': 23, 'total': 81} - }, # Here the totals needed to be reconstructed. datetime.datetime(2020, 3, 9): { 'CW': {'growth': 4, 'total': 13}, @@ -277,8 +261,7 @@ for i in range(max_page): # Collect infection data. first_run = True districts_sorted = [] -# TODO: Push limit further back (might need more data fixes for that). -date_limit = datetime.datetime(2020, 3, 12) +date_limit = datetime.datetime(2020, 3, 11) for path in day_urls: url = url_prefix + path with urllib.request.urlopen(url) as response: @@ -302,27 +285,43 @@ for path in day_urls: continue data[date] = {} for tr in [tr for tr in table.children if type(tr) == bs4.element.Tag][1:]: - printable_tds = [] - for td in [td for td in tr.children if type(td) == bs4.element.Tag][:2]: - printable_string = ' '.join([s for s in td.strings]) - printable_tds += [printable_string.strip()] - district_long = printable_tds[0] - district_short = [k for k in abbrevs if district_long in abbrevs[k]][0] - if first_run: - districts_sorted += [district_short] - split_char = ' ' - if not split_char in printable_tds[1]: - split_char = '(' - total_str, growth_str = printable_tds[1].split(split_char) - growth = int(growth_str.replace('(', '').replace(')', '').replace('+', '')) - total = int(total_str.replace('.', '')) - data[date][district_short] = {'growth': growth, 'total': total} + printable_tds = [] + for td in [td for td in tr.children if type(td) == bs4.element.Tag][:2]: + printable_string = ' '.join([s for s in td.strings]) + printable_tds += [printable_string.strip()] + district_long = printable_tds[0] + district_short = [k for k in abbrevs if district_long in abbrevs[k]][0] + if first_run: + districts_sorted += [district_short] + if date == datetime.datetime(2020, 3, 10): + # For this date we only get totals. + data[date][district_short] = {'total': int(printable_tds[1])} + else: + split_char = ' ' + if not split_char in printable_tds[1]: + split_char = '(' + total_str, growth_str = printable_tds[1].split(split_char) + growth = int(growth_str.replace('(', '').replace(')', '').\ + replace('+', '')) + total = int(total_str.replace('.', '')) + data[date][district_short] = {'growth': growth, 'total': total} first_run = False +def neighbor_days(day_target): + day_delta = datetime.timedelta(days=1) + return day_target + day_delta, day_target - day_delta + +# Reconstruct growth for 10th of March. +day_target = datetime.datetime(2020, 3, 10) +day_after, day_before = neighbor_days(day_target) +for district in [d for d in districts_sorted]: + total_target = data[day_target][district]['total'] + total_before = data[day_before][district]['total'] + data[day_target][district]['growth'] = total_target - total_before + # Reconstruct data for 13th of March. day_target = datetime.datetime(2020, 3, 13) -day_after = day_target + datetime.timedelta(days=1) -day_before = day_target - datetime.timedelta(days=1) +day_after, day_before = neighbor_days(day_target) data[day_target] = {} for district in [d for d in districts_sorted]: data[day_target][district] = {}