X-Git-Url: https://plomlompom.com/repos/?p=plomlombot-irc.git;a=blobdiff_plain;f=plomlombot.py;h=4d2f72dc33249e8a6744c8ca3568d52d6d997ad2;hp=66b902f672e978ad0ff5e01961c7fdd94e5e196a;hb=a1530b030975fcaf034acebbf6f502193c534737;hpb=b8009d0d782822b1a5fffe15b1d12ae7c0c2de37

diff --git a/plomlombot.py b/plomlombot.py
index 66b902f..4d2f72d 100755
--- a/plomlombot.py
+++ b/plomlombot.py
@@ -180,9 +180,11 @@ def handle_command(command, argument, notice, target, session):
             if len(results) == 0:
                 notice("NO QUOTES MATCHING QUERY")
             else:
-                for result in results:
-                    notice("QUOTE #" + str(result[0] + 1) + " : "
-                           + result[1][-1])
+                if len(results) > 3:
+                    notice("SHOWING 3 OF " + str(len(results)) + " QUOTES")
+                for result in results[:3]:
+                    notice("QUOTE #" + str(result[0] + 1) + ": "
+                           + result[1][:-1])
             return
         else:
             i = random.randrange(len(lines))
@@ -337,17 +339,23 @@ def handle_url(url, notice, show_url=False):
             return True
 
     try:
-        r = requests.get(url, timeout=15)
+        r = requests.get(url, timeout=15, stream=True)
+        r.raw.decode_content = True
+        text = r.raw.read(10000000+1)
+        if len(text) > 10000000:
+            raise ValueError('Too large a response')
     except (requests.exceptions.TooManyRedirects,
             requests.exceptions.ConnectionError,
             requests.exceptions.InvalidURL,
+            requests.exceptions.ReadTimeout,
             UnicodeError,
+            ValueError,
             requests.exceptions.InvalidSchema) as error:
         notice("TROUBLE FOLLOWING URL: " + str(error))
-        return
+        return False
     if mobile_twitter_hack(url):
-        return
-    title = bs4.BeautifulSoup(r.text, "html5lib").title
+        return True
+    title = bs4.BeautifulSoup(text, "html5lib").title
     if title and title.string:
         prefix = "PAGE TITLE: "
         if show_url:
@@ -355,6 +363,7 @@ def handle_url(url, notice, show_url=False):
         notice(prefix + title.string.strip())
     else:
         notice("PAGE HAS NO TITLE TAG")
+    return True
 
 
 class Session:
@@ -409,8 +418,14 @@ class Session:
                 target = line.receiver
             msg = str.join(" ", line.tokens[3:])[1:]
             matches = re.findall("(https?://[^\s>]+)", msg)
+            url_count = 0
             for i in range(len(matches)):
-                handle_url(matches[i], notice)
+                if handle_url(matches[i], notice):
+                    url_count += 1
+                    if url_count == 3:
+                        notice("MAXIMUM NUMBER OF URLs TO PARSE PER MESSAGE "
+                               "REACHED")
+                        break
             if "!" == msg[0]:
                 tokens = msg[1:].split()
                 argument = str.join(" ", tokens[1:])