From d66da409aaafb766eae1e0af1a9c16ae8520510c Mon Sep 17 00:00:00 2001 From: Christian Heller Date: Tue, 19 Jan 2016 22:46:30 +0100 Subject: [PATCH] Use BeautifulSoup for HTML parsing. --- plomlombot.py | 29 ++++++----------------------- requirements.txt | 1 + 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/plomlombot.py b/plomlombot.py index f7c7e32..3e4734f 100755 --- a/plomlombot.py +++ b/plomlombot.py @@ -7,8 +7,7 @@ import select import time import re import requests -import html -import html.parser +import bs4 # Defaults, may be overwritten by command line arguments. SERVER = "irc.freenode.net" @@ -18,23 +17,6 @@ USERNAME = "plomlombot" NICKNAME = USERNAME -class HTMLParser(html.parser.HTMLParser): - def __init__(self, html, tag): - super().__init__() - self._tag_to_check = tag - self._tag = "" - self.data = "" - self.feed(html) - def handle_starttag(self, tag, attrs): - if self.data == "" and tag == self._tag_to_check: - self._tag = tag - def handle_endtag(self, tag): - self._tag = "" - def handle_data(self, data): - if self._tag != "": - self.data = data - - class ExceptionForRestart(Exception): pass @@ -135,10 +117,11 @@ def lineparser_loop(io, nickname): requests.exceptions.InvalidSchema) as error: notice("TROUBLE FOLLOWING URL: " + str(error)) continue - content = r.text - title = HTMLParser(content, "title").data - title = html.unescape(title) - notice("PAGE TITLE FOR URL: " + title) + title = bs4.BeautifulSoup(r.text).title + if title: + notice("PAGE TITLE FOR URL: " + title.string) + else: + notice("PAGE HAS NO TITLE TAG") sender = "" for rune in tokens[0]: diff --git a/requirements.txt b/requirements.txt index ca0dee4..5263dff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ +beautifulsoup4==4.4.1 requests==2.9.1 -- 2.30.2