Use BeautifulSoup for HTML parsing.

diff --git a/plomlombot.py b/plomlombot.py
index f7c7e32899704b705baf8b8ca49ea16c0fd7ccb3..3e4734fa2ec2403aa093768623e1ca9b189d5cd8 100755 (executable)
--- a/plomlombot.py
+++ b/plomlombot.py
@@ -7,8 +7,7 @@ import select
 import time
 import re
 import requests
 import time
 import re
 import requests

-import html
-import html.parser
+import bs4

 
 # Defaults, may be overwritten by command line arguments.
 SERVER = "irc.freenode.net"
 
 # Defaults, may be overwritten by command line arguments.
 SERVER = "irc.freenode.net"


@@ -18,23 +17,6 @@ USERNAME = "plomlombot"
 NICKNAME = USERNAME
 
 
 NICKNAME = USERNAME
 
 

-class HTMLParser(html.parser.HTMLParser):
-    def __init__(self, html, tag):
-        super().__init__()
-        self._tag_to_check = tag
-        self._tag = ""
-        self.data = ""
-        self.feed(html)
-    def handle_starttag(self, tag, attrs):
-        if self.data == "" and tag == self._tag_to_check:
-            self._tag = tag
-    def handle_endtag(self, tag):
-        self._tag = ""
-    def handle_data(self, data):
-        if self._tag != "":
-            self.data = data
-
-

 class ExceptionForRestart(Exception):
     pass
 
 class ExceptionForRestart(Exception):
     pass
 


@@ -135,10 +117,11 @@ def lineparser_loop(io, nickname):
                         requests.exceptions.InvalidSchema) as error:
                     notice("TROUBLE FOLLOWING URL: " + str(error))
                     continue
                         requests.exceptions.InvalidSchema) as error:
                     notice("TROUBLE FOLLOWING URL: " + str(error))
                     continue

-                content = r.text
-                title = HTMLParser(content, "title").data
-                title = html.unescape(title)
-                notice("PAGE TITLE FOR URL: " + title)
+                title = bs4.BeautifulSoup(r.text).title
+                if title:
+                    notice("PAGE TITLE FOR URL: " + title.string)
+                else:
+                    notice("PAGE HAS NO TITLE TAG")

 
         sender = ""
         for rune in tokens[0]:
 
         sender = ""
         for rune in tokens[0]:

diff --git a/requirements.txt b/requirements.txt
index ca0dee4830d57b15effae94aa6b361614bcae613..5263dff39003e50a9e6985d6d52abac8a644c1af 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,2 @@
+beautifulsoup4==4.4.1

 requests==2.9.1
 requests==2.9.1