X-Git-Url: https://plomlompom.com/repos/%7B%7Bdb.prefix%7D%7D/calendar?a=blobdiff_plain;f=plomlombot.py;h=f7c7e32899704b705baf8b8ca49ea16c0fd7ccb3;hb=a7db2ddce21f684abf6d835e9450fcf370caa5f2;hp=0b605e9946b04f79a8f3e843da56eee2b59b0169;hpb=0d17c0577b6e816dc1c6e02667fc03f74ef5f4a8;p=plomlombot-irc.git diff --git a/plomlombot.py b/plomlombot.py index 0b605e9..f7c7e32 100755 --- a/plomlombot.py +++ b/plomlombot.py @@ -6,9 +6,9 @@ import datetime import select import time import re -import urllib.request -import http.client +import requests import html +import html.parser # Defaults, may be overwritten by command line arguments. SERVER = "irc.freenode.net" @@ -18,6 +18,23 @@ USERNAME = "plomlombot" NICKNAME = USERNAME +class HTMLParser(html.parser.HTMLParser): + def __init__(self, html, tag): + super().__init__() + self._tag_to_check = tag + self._tag = "" + self.data = "" + self.feed(html) + def handle_starttag(self, tag, attrs): + if self.data == "" and tag == self._tag_to_check: + self._tag = tag + def handle_endtag(self, tag): + self._tag = "" + def handle_data(self, data): + if self._tag != "": + self.data = data + + class ExceptionForRestart(Exception): pass @@ -111,26 +128,15 @@ def lineparser_loop(io, nickname): matches = re.findall("(https?://[^\s>]+)", msg) for i in range(len(matches)): url = matches[i] - request = urllib.request.Request(url, headers={ - "User-Agent": "plomlombot" - }) try: - webpage = urllib.request.urlopen(request, timeout=15) - except (urllib.error.HTTPError, urllib.error.URLError, - UnicodeError, http.client.BadStatusLine) as error: + r = requests.get(url, timeout=15) + except (requests.exceptions.TooManyRedirects, + requests.exceptions.ConnectionError, + requests.exceptions.InvalidSchema) as error: notice("TROUBLE FOLLOWING URL: " + str(error)) continue - charset = webpage.info().get_content_charset() - if not charset: - charset = "utf-8" - content_type = webpage.info().get_content_type() - if content_type not in ('text/html', 'text/xml', - 'application/xhtml+xml'): - notice("TROUBLE READING PAGE TITLE: bad content type " - + content_type) - continue - content = webpage.read().decode(charset) - title = str(content).split('