X-Git-Url: https://plomlompom.com/repos/foo.html?a=blobdiff_plain;f=plomlombot.py;h=aa7061cac656f3f6b5f768e40787ccb6fbc475c4;hb=42c69ab892b5a9d2dc2c68a059e7b506a00197a0;hp=7224b29e2ccff2a90504a113676edd625278253e;hpb=2fc56d6691472c34fc5487c97b11075f0824cf36;p=plomlombot-irc.git
diff --git a/plomlombot.py b/plomlombot.py
index 7224b29..aa7061c 100755
--- a/plomlombot.py
+++ b/plomlombot.py
@@ -339,17 +339,22 @@ def handle_url(url, notice, show_url=False):
return True
try:
- r = requests.get(url, timeout=15)
+ r = requests.get(url, timeout=5, stream=True)
+ r.raw.decode_content = True
+ text = r.raw.read(10000000+1)
+ if len(text) > 10000000:
+ raise ValueError('Too large a response')
except (requests.exceptions.TooManyRedirects,
requests.exceptions.ConnectionError,
requests.exceptions.InvalidURL,
UnicodeError,
+ ValueError,
requests.exceptions.InvalidSchema) as error:
notice("TROUBLE FOLLOWING URL: " + str(error))
return
if mobile_twitter_hack(url):
return
- title = bs4.BeautifulSoup(r.text, "html5lib").title
+ title = bs4.BeautifulSoup(text, "html5lib").title
if title and title.string:
prefix = "PAGE TITLE: "
if show_url: