home
·
contact
·
privacy
projects
/
plomlombot-irc.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Handle URLs for too large websites / files.
[plomlombot-irc.git]
/
plomlombot.py
diff --git
a/plomlombot.py
b/plomlombot.py
index 66b902f672e978ad0ff5e01961c7fdd94e5e196a..aa7061cac656f3f6b5f768e40787ccb6fbc475c4 100755
(executable)
--- a/
plomlombot.py
+++ b/
plomlombot.py
@@
-180,9
+180,11
@@
def handle_command(command, argument, notice, target, session):
if len(results) == 0:
notice("NO QUOTES MATCHING QUERY")
else:
if len(results) == 0:
notice("NO QUOTES MATCHING QUERY")
else:
- for result in results:
- notice("QUOTE #" + str(result[0] + 1) + " : "
- + result[1][-1])
+ if len(results) > 3:
+ notice("SHOWING 3 OF " + str(len(results)) + " QUOTES")
+ for result in results[:3]:
+ notice("QUOTE #" + str(result[0] + 1) + ": "
+ + result[1][:-1])
return
else:
i = random.randrange(len(lines))
return
else:
i = random.randrange(len(lines))
@@
-337,17
+339,22
@@
def handle_url(url, notice, show_url=False):
return True
try:
return True
try:
- r = requests.get(url, timeout=15)
+ r = requests.get(url, timeout=5, stream=True)
+ r.raw.decode_content = True
+ text = r.raw.read(10000000+1)
+ if len(text) > 10000000:
+ raise ValueError('Too large a response')
except (requests.exceptions.TooManyRedirects,
requests.exceptions.ConnectionError,
requests.exceptions.InvalidURL,
UnicodeError,
except (requests.exceptions.TooManyRedirects,
requests.exceptions.ConnectionError,
requests.exceptions.InvalidURL,
UnicodeError,
+ ValueError,
requests.exceptions.InvalidSchema) as error:
notice("TROUBLE FOLLOWING URL: " + str(error))
return
if mobile_twitter_hack(url):
return
requests.exceptions.InvalidSchema) as error:
notice("TROUBLE FOLLOWING URL: " + str(error))
return
if mobile_twitter_hack(url):
return
- title = bs4.BeautifulSoup(
r.
text, "html5lib").title
+ title = bs4.BeautifulSoup(text, "html5lib").title
if title and title.string:
prefix = "PAGE TITLE: "
if show_url:
if title and title.string:
prefix = "PAGE TITLE: "
if show_url: