X-Git-Url: https://plomlompom.com/repos/%7B%7Bdb.prefix%7D%7D/task?a=blobdiff_plain;f=plomlombot.py;h=fe9c6d8d283a3a7d33fec14903c9534819ea2642;hb=60133e10b2df0c671574e294e835d39b8cbf23a2;hp=7d0f5355a1f70eb1cc946249d1d5e76154b33b49;hpb=4a2678aba39982c7912a697bf421d1fefb57ac41;p=plomlombot-irc.git diff --git a/plomlombot.py b/plomlombot.py index 7d0f535..fe9c6d8 100755 --- a/plomlombot.py +++ b/plomlombot.py @@ -6,9 +6,12 @@ import datetime import select import time import re -import urllib.request -import http.client -import html +import requests +import bs4 +import random +import hashlib +import os +import plomsearch # Defaults, may be overwritten by command line arguments. SERVER = "irc.freenode.net" @@ -67,7 +70,11 @@ class IO: self._pingtest(send_ping) return None self.last_pong = time.time() - received_runes = self.socket.recv(1024).decode("UTF-8") + received_bytes = self.socket.recv(1024) + try: + received_runes = received_bytes.decode("UTF-8") + except UnicodeDecodeError: + received_runes = received_bytes.decode("latin1") if len(received_runes) == 0: print("SOCKET CONNECTION BROKEN") raise ExceptionForRestart @@ -99,34 +106,109 @@ def lineparser_loop(io, nickname): def act_on_privmsg(tokens): + def notice(msg): + io.send_line("NOTICE " + target + " :" + msg) + def url_check(msg): + + def handle_url(url, show_url=False): + + def mobile_twitter_hack(url): + re1 = 'https?://(mobile.twitter.com/)[^/]+(/status/)' + re2 = 'https?://mobile.twitter.com/([^/]+)/status/' \ + + '([^\?/]+)' + m = re.search(re1, url) + if m and m.group(1) == 'mobile.twitter.com/' \ + and m.group(2) == '/status/': + m = re.search(re2, url) + url = 'https://twitter.com/' + m.group(1) + '/status/' \ + + m.group(2) + handle_url(url, True) + return True + + try: + r = requests.get(url, timeout=15) + except (requests.exceptions.TooManyRedirects, + requests.exceptions.ConnectionError, + requests.exceptions.InvalidURL, + requests.exceptions.InvalidSchema) as error: + notice("TROUBLE FOLLOWING URL: " + str(error)) + return + if mobile_twitter_hack(url): + return + title = bs4.BeautifulSoup(r.text).title + if title: + prefix = "PAGE TITLE: " + if show_url: + prefix = "PAGE TITLE FOR <" + url + ">: " + notice(prefix + title.string.strip()) + else: + notice("PAGE HAS NO TITLE TAG") + matches = re.findall("(https?://[^\s>]+)", msg) for i in range(len(matches)): - url = matches[i] - request = urllib.request.Request(url, headers={ - "User-Agent": "plomlombot" - }) - try: - webpage = urllib.request.urlopen(request, timeout=15) - except (urllib.error.HTTPError, urllib.error.URLError, - UnicodeError, http.client.BadStatusLine, - UnicodeDecodeError) as error: - print("TROUBLE FOLLOWING URL: " + str(error)) - continue - charset = webpage.info().get_content_charset() - if not charset: - charset = "utf-8" - content_type = webpage.info().get_content_type() - if content_type not in ('text/html', 'text/xml', - 'application/xhtml+xml'): - print("TROUBLE INTERPRETING URL: bad content type " - + content_type) - continue - content = webpage.read().decode(charset) - title = str(content).split('