import random
import hashlib
import os
+import signal
import plomsearch
import irclog
handle_url(url, notice, True)
return True
+ class TimeOut(Exception):
+ pass
+
+ def timeout_handler(ignore1, ignore2):
+ raise TimeOut("timeout")
+
+ signal.signal(signal.SIGALRM, timeout_handler)
+ signal.alarm(15)
try:
- r = requests.get(url, timeout=15)
+ r = requests.get(url, stream=True)
+ r.raw.decode_content = True
+ text = r.raw.read(10000000+1)
+ if len(text) > 10000000:
+ raise ValueError('Too large a response')
except (requests.exceptions.TooManyRedirects,
requests.exceptions.ConnectionError,
requests.exceptions.InvalidURL,
+ TimeOut,
UnicodeError,
+ ValueError,
requests.exceptions.InvalidSchema) as error:
+ signal.alarm(0)
notice("TROUBLE FOLLOWING URL: " + str(error))
- return
+ return False
+ signal.alarm(0)
if mobile_twitter_hack(url):
- return
- title = bs4.BeautifulSoup(r.text, "html5lib").title
+ return True
+ title = bs4.BeautifulSoup(text, "html5lib").title
if title and title.string:
prefix = "PAGE TITLE: "
if show_url:
notice(prefix + title.string.strip())
else:
notice("PAGE HAS NO TITLE TAG")
+ return True
class Session:
target = line.receiver
msg = str.join(" ", line.tokens[3:])[1:]
matches = re.findall("(https?://[^\s>]+)", msg)
+ url_count = 0
for i in range(len(matches)):
- handle_url(matches[i], notice)
+ if handle_url(matches[i], notice):
+ url_count += 1
+ if url_count == 3:
+ notice("MAXIMUM NUMBER OF URLs TO PARSE PER MESSAGE "
+ "REACHED")
+ break
if "!" == msg[0]:
tokens = msg[1:].split()
argument = str.join(" ", tokens[1:])