X-Git-Url: https://plomlompom.com/repos/unset_cookie?a=blobdiff_plain;ds=inline;f=plomlombot.py;h=9e9d0a44ce21bc1d07204dcb9780c54ac1b31d0e;hb=9260bc740d86b0b83d90c31eea2f898dfb2cf594;hp=d6feb9377b4e36a1c5af96adb078f480c1c6f5fa;hpb=bb71779ed0fc15241836b3430d9bf365d3f341f9;p=plomlombot-irc.git diff --git a/plomlombot.py b/plomlombot.py index d6feb93..9e9d0a4 100755 --- a/plomlombot.py +++ b/plomlombot.py @@ -9,6 +9,7 @@ import re import urllib.request import http.client import html +import html.parser # Defaults, may be overwritten by command line arguments. SERVER = "irc.freenode.net" @@ -18,6 +19,23 @@ USERNAME = "plomlombot" NICKNAME = USERNAME +class HTMLParser(html.parser.HTMLParser): + def __init__(self, html, tag): + super().__init__() + self._tag_to_check = tag + self._tag = "" + self.data = "" + self.feed(html) + def handle_starttag(self, tag, attrs): + if self.data == "" and tag == self._tag_to_check: + self._tag = tag + def handle_endtag(self, tag): + self._tag = "" + def handle_data(self, data): + if self._tag != "": + self.data = data + + class ExceptionForRestart(Exception): pass @@ -67,7 +85,11 @@ class IO: self._pingtest(send_ping) return None self.last_pong = time.time() - received_runes = self.socket.recv(1024).decode("UTF-8") + received_bytes = self.socket.recv(1024) + try: + received_runes = received_bytes.decode("UTF-8") + except UnicodeDecodeError: + received_runes = received_bytes.decode("latin1") if len(received_runes) == 0: print("SOCKET CONNECTION BROKEN") raise ExceptionForRestart @@ -100,29 +122,36 @@ def lineparser_loop(io, nickname): def act_on_privmsg(tokens): def url_check(msg): + + def notice(msg): + io.send_line("NOTICE " + target + " :" + msg) + matches = re.findall("(https?://[^\s>]+)", msg) for i in range(len(matches)): url = matches[i] + request = urllib.request.Request(url, headers={ + "User-Agent": "plomlombot" + }) try: - webpage = urllib.request.urlopen(url, timeout=15) + webpage = urllib.request.urlopen(request, timeout=15) except (urllib.error.HTTPError, urllib.error.URLError, UnicodeError, http.client.BadStatusLine) as error: - print("TROUBLE FOLLOWING URL: " + str(error)) + notice("TROUBLE FOLLOWING URL: " + str(error)) continue charset = webpage.info().get_content_charset() if not charset: - charset = "utf-8" + notice("TROUBLE READING PAGE TITLE: no charset in header") + continue content_type = webpage.info().get_content_type() if content_type not in ('text/html', 'text/xml', 'application/xhtml+xml'): - print("TROUBLE INTERPRETING URL: bad content type " - + content_type) + notice("TROUBLE READING PAGE TITLE: bad content type " + + content_type) continue content = webpage.read().decode(charset) - title = str(content).split('