Send user agent.

[plomlombot-irc.git] / plomlombot.py

diff --git a/plomlombot.py b/plomlombot.py
index ede4dca9b1dccc6b55861062daea277df26ba415..28cc9ddbfc82aaf489fb7e76227ff7f91948ddc3 100755 (executable)
--- a/plomlombot.py
+++ b/plomlombot.py
@@ -11,6 +11,7 @@ import bs4
 import random
 import hashlib
 import os
+import signal
 import plomsearch
 import irclog
 
@@ -338,8 +339,16 @@ def handle_url(url, notice, show_url=False):
             handle_url(url, notice, True)
             return True
 
+    class TimeOut(Exception):
+        pass
+
+    def timeout_handler(ignore1, ignore2):
+        raise TimeOut("timeout")
+
+    signal.signal(signal.SIGALRM, timeout_handler)
+    signal.alarm(15)
     try:
-        r = requests.get(url, timeout=15, stream=True)
+        r = requests.get(url, headers = {'User-Agent': 'plomlombot'}, stream=True)
         r.raw.decode_content = True
         text = r.raw.read(10000000+1)
         if len(text) > 10000000:
@@ -347,14 +356,16 @@ def handle_url(url, notice, show_url=False):
     except (requests.exceptions.TooManyRedirects,
             requests.exceptions.ConnectionError,
             requests.exceptions.InvalidURL,
-            requests.exceptions.ReadTimeout,
+            TimeOut,
             UnicodeError,
             ValueError,
             requests.exceptions.InvalidSchema) as error:
+        signal.alarm(0)
         notice("TROUBLE FOLLOWING URL: " + str(error))
-        return
+        return False
+    signal.alarm(0)
     if mobile_twitter_hack(url):
-        return
+        return True
     title = bs4.BeautifulSoup(text, "html5lib").title
     if title and title.string:
         prefix = "PAGE TITLE: "
@@ -363,6 +374,7 @@ def handle_url(url, notice, show_url=False):
         notice(prefix + title.string.strip())
     else:
         notice("PAGE HAS NO TITLE TAG")
+    return True
 
 
 class Session:
@@ -417,8 +429,14 @@ class Session:
                 target = line.receiver
             msg = str.join(" ", line.tokens[3:])[1:]
             matches = re.findall("(https?://[^\s>]+)", msg)
+            url_count = 0
             for i in range(len(matches)):
-                handle_url(matches[i], notice)
+                if handle_url(matches[i], notice):
+                    url_count += 1
+                    if url_count == 3:
+                        notice("MAXIMUM NUMBER OF URLs TO PARSE PER MESSAGE "
+                               "REACHED")
+                        break
             if "!" == msg[0]:
                 tokens = msg[1:].split()
                 argument = str.join(" ", tokens[1:])