X-Git-Url: https://plomlompom.com/repos/?p=berlin-corona-table;a=blobdiff_plain;f=scrape.py;h=bbb4835c00fc5eedebe064ab540264d1470b0c05;hp=41a0db9d3c44a5781a66d7dbf02be6200c2df34c;hb=6142dc2ac4ee62b47bf205970a9ee460a49993a6;hpb=402755409e719a9e2cba7a9dcb1482b5834a3aa1 diff --git a/scrape.py b/scrape.py index 41a0db9..bbb4835 100755 --- a/scrape.py +++ b/scrape.py @@ -25,8 +25,9 @@ abbrevs = { 'sum': {'Summe', 'Berlin'}, } -# Here only image files are available for the table data. -unparsable_graphics_fallback = { +# some pre-filled values +data = { + # For these, only image files are available for the table data. datetime.datetime(2020, 7, 2): { 'CW': {'growth': 4, 'total': 851}, 'FK': {'growth': 10, 'total': 681}, @@ -86,7 +87,88 @@ unparsable_graphics_fallback = { 'TS': {'growth': 41, 'total': 362}, 'TK': {'growth': 14, 'total': 183}, 'sum': {'growth': 263, 'total': 3486} - } + }, + # This one has no press release but can be reconstructed from + # the neighbour ones. + datetime.datetime(2020, 3, 13): { + 'CW': {'growth': 16, 'total': 47}, + 'FK': {'growth': 8, 'total': 22}, + 'Li': {'growth': 2, 'total': 8}, + 'MH': {'growth': 1, 'total': 4}, + 'Mi': {'growth': 9, 'total': 29}, + 'Ne': {'growth': 6, 'total': 16}, + 'Pa': {'growth': 11, 'total': 26}, + 'Re': {'growth': 0, 'total': 11}, + 'Sp': {'growth': 1, 'total': 9}, + 'SZ': {'growth': 0, 'total': 20}, + 'TS': {'growth': 1, 'total': 17}, + 'TK': {'growth': 3, 'total': 7}, + 'sum': {'growth': 58, 'total': 216} + }, + # Here the growth numbers needed to be reconstructed. + datetime.datetime(2020, 3, 10): { + 'CW': {'growth': 2, 'total': 15}, + 'FK': {'growth': 0, 'total': 12}, + 'Li': {'growth': 4, 'total': 5}, + 'MH': {'growth': 1, 'total': 3}, + 'Mi': {'growth': 0, 'total': 8}, + 'Ne': {'growth': 2, 'total': 5}, + 'Pa': {'growth': 2, 'total': 8}, + 'Re': {'growth': 0, 'total': 3}, + 'Sp': {'growth': 4, 'total': 6}, + 'SZ': {'growth': 3, 'total': 6}, + 'TS': {'growth': 2, 'total': 7}, + 'TK': {'growth': 3, 'total': 3}, + 'sum': {'growth': 23, 'total': 81} + }, + # Here the totals needed to be reconstructed. + datetime.datetime(2020, 3, 9): { + 'CW': {'growth': 4, 'total': 13}, + 'FK': {'growth': 3, 'total': 12}, + 'Li': {'growth': 0, 'total': 1}, + 'MH': {'growth': 1, 'total': 2}, + 'Mi': {'growth': 0, 'total': 8}, + 'Ne': {'growth': 1, 'total': 3}, + 'Pa': {'growth': 1, 'total': 6}, + 'Re': {'growth': 0, 'total': 3}, + 'Sp': {'growth': 0, 'total': 2}, + 'SZ': {'growth': 0, 'total': 3}, + 'TS': {'growth': 0, 'total': 5}, + 'TK': {'growth': 0, 'total': 0}, + 'sum': {'growth': 10, 'total': 58} + }, + # Here the growth numbers needed to be reconstructed. + datetime.datetime(2020, 3, 8): { + 'CW': {'growth': 0, 'total': 9}, + 'FK': {'growth': 4, 'total': 9}, + 'Li': {'growth': 1, 'total': 1}, + 'MH': {'growth': 0, 'total': 1}, + 'Mi': {'growth': 0, 'total': 8}, + 'Ne': {'growth': 0, 'total': 2}, + 'Pa': {'growth': 0, 'total': 5}, + 'Re': {'growth': 0, 'total': 3}, + 'Sp': {'growth': 2, 'total': 2}, + 'SZ': {'growth': 1, 'total': 3}, + 'TS': {'growth': 0, 'total': 5}, + 'TK': {'growth': 0, 'total': 0}, + 'sum': {'growth': 8, 'total': 48} + }, + # Here the growth numbers needed to be reconstructed. + datetime.datetime(2020, 3, 7): { + 'CW': {'growth': 6, 'total': 9}, + 'FK': {'growth': 1, 'total': 5}, + 'Li': {'growth': 0, 'total': 0}, + 'MH': {'growth': 0, 'total': 1}, + 'Mi': {'growth': 1, 'total': 8}, + 'Ne': {'growth': 0, 'total': 2}, + 'Pa': {'growth': 1, 'total': 5}, + 'Re': {'growth': 0, 'total': 3}, + 'Sp': {'growth': 0, 'total': 0}, + 'SZ': {'growth': 2, 'total': 2}, + 'TS': {'growth': 1, 'total': 5}, + 'TK': {'growth': 0, 'total': 0}, + 'sum': {'growth': 12, 'total': 40} + }, } fixes = { # Here the official total is 215, while the summation of district @@ -104,12 +186,10 @@ fixes = { # compromise to keep as many surrounding numbers stable as possible. datetime.datetime(2020, 3, 26): { 'SZ': { - 'growth': 12, - 'total': 132 + 'growth': 12 }, 'sum': { - 'growth': 286, - 'total': 1931 + 'growth': 286 } }, # Here the official total is 220, while the summation of district @@ -121,7 +201,7 @@ fixes = { }, } -# Scan navigation bar for maximum pagination value. +# Scan navigation bar for maximum pagination value. url = url_prefix + pm_dir with urllib.request.urlopen(url) as response: html = response.read() @@ -147,12 +227,11 @@ for i in range(max_page): continue day_urls += [link['href']] -# Collect infection data. -data = {} +# Collect infection data. first_run = True districts_sorted = [] # TODO: Push limit further back (might need more data fixes for that). -date_limit = datetime.datetime(2020, 3, 16) +date_limit = datetime.datetime(2020, 3, 12) for path in day_urls: url = url_prefix + path with urllib.request.urlopen(url) as response: @@ -164,14 +243,16 @@ for path in day_urls: date = datetime.datetime.strptime(date_formatted , '%d.%m.%Y') if date_limit > date: break - if date in data: - raise Exception('Double date %s', date) - #date -= datetime.timedelta(days=1) - data[date] = {} + # On that day, two press releases were released, for that and the prev day. + if date == datetime.datetime(2020, 3, 15) and date in data: + date = datetime.datetime(2020, 3, 14) + # From here on, press releases describe numbers from prev day. + if date <= datetime.datetime(2020, 3, 13): + date = date - datetime.timedelta(days=1) table = soup.find('table') - if table is None: - data[date] = unparsable_graphics_fallback[date] + if table is None and date in data: continue + data[date] = {} for tr in [tr for tr in table.children if type(tr) == bs4.element.Tag][1:]: printable_tds = [] for td in [td for td in tr.children if type(td) == bs4.element.Tag][:2]: @@ -208,7 +289,7 @@ for date in dates_sorted: raise Exception('Dates not contiguous: %s missing', prev_date) else: continue - prev_total = data[date - datetime.timedelta(days=1)][district]['total'] + prev_total = data[prev_date][district]['total'] cur_total = data[date][district]['total'] if cur_total - data[date][district]['growth'] != prev_total: raise Exception('Questionable district infection total in %s/%s' % (district, date))