X-Git-Url: https://plomlompom.com/repos/%7B%7B%20web_path%20%7D%7D/decks/%7B%7Bdeck_id%7D%7D/cards/%7B%7Bcard_id%7D%7D/static/gitweb.css?a=blobdiff_plain;ds=sidebyside;f=plomlombot.py;h=767d1f71a156c5a744c8d80ed08802ad97af0127;hb=40d48ef337219396e8985eee194f6d21e4173ada;hp=81be31fdb5f438349469e7d63afac9b4b20b1a7e;hpb=3b3ac1e18357e60525a6be9b5665c504e0ec4e6c;p=plomlombot-irc.git diff --git a/plomlombot.py b/plomlombot.py index 81be31f..767d1f7 100755 --- a/plomlombot.py +++ b/plomlombot.py @@ -6,9 +6,8 @@ import datetime import select import time import re -import urllib.request -import http.client -import html +import requests +import bs4 # Defaults, may be overwritten by command line arguments. SERVER = "irc.freenode.net" @@ -67,7 +66,11 @@ class IO: self._pingtest(send_ping) return None self.last_pong = time.time() - received_runes = self.socket.recv(1024).decode("UTF-8") + received_bytes = self.socket.recv(1024) + try: + received_runes = received_bytes.decode("UTF-8") + except UnicodeDecodeError: + received_runes = received_bytes.decode("latin1") if len(received_runes) == 0: print("SOCKET CONNECTION BROKEN") raise ExceptionForRestart @@ -107,28 +110,19 @@ def lineparser_loop(io, nickname): matches = re.findall("(https?://[^\s>]+)", msg) for i in range(len(matches)): url = matches[i] - request = urllib.request.Request(url, headers={ - "User-Agent": "plomlombot" - }) try: - webpage = urllib.request.urlopen(request, timeout=15) - except (urllib.error.HTTPError, urllib.error.URLError, - UnicodeError, http.client.BadStatusLine) as error: + r = requests.get(url, timeout=15) + except (requests.exceptions.TooManyRedirects, + requests.exceptions.ConnectionError, + requests.exceptions.InvalidURL, + requests.exceptions.InvalidSchema) as error: notice("TROUBLE FOLLOWING URL: " + str(error)) continue - charset = webpage.info().get_content_charset() - if not charset: - charset = "utf-8" - content_type = webpage.info().get_content_type() - if content_type not in ('text/html', 'text/xml', - 'application/xhtml+xml'): - notice("TROUBLE READING PAGE TITLE: bad content type " - + content_type) - continue - content = webpage.read().decode(charset) - title = str(content).split('