#!/usr/bin/env python
import time
import random
# for more details see: https://github.com/sickill/reddittools
# https://github.com/sickill/reddittools/tree/master/reddit_scraper
from reddittools.scraper import RedditScraper
from RedditTools.utils import Config, load_config
if __name__ == "__main__":
app_name = "Reddit Scraper - Demo"
parser = Config(app_name, default_config={
"username": "reddit_bot",
"password": "This_is_my_password",
"proxy": "My_proxy",
})
if not parser.validate() or not parser.check_reddit():
parser.print_help()
exit()
reddit_scraper = RedditScraper(parser)
for page in xrange(10):
time.sleep(1)
print "Done", app_name
print reddit_scraper.save_to_file(parser, "Page_%s.html" % page)
print "Saved to", parser.output_dir
print
print reddit_scraper.save_all()
print "Saved all"
print
print
print "Done"
print
print "This is it folks!"
print
print
print "It's done"
print
parser.print_help()
parser.save_config()
exit()
print "Waiting for the server to start"
It runs without any issues on my own computer but when i want to run this using a cloud service like aws instance i get the following error:
C:\Python27\lib\site-packages\requests\packages\chardet\ioc34lz.py:13: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal
if buf.equal(s):
[...]
# The server didn't return enough bytes.
IncompleteRead: 104 bytes read on a total of 4928 bytes.
[...]
# The server disconnected before we could finish reading the data.
socket.error: [Errno 11] Resource temporarily unavailable
During handling of the above exception, another exception occurred:
[...]
[Errno 10054] An existing connection was forcibly closed by the remote host
[...]
Traceback (most recent call last):
File "D:\Bot\scraper.py", line 35, in
reddit_scraper = RedditScraper(parser)
File "D:\Bot\scraper.py", line 10, in __init__
server = reddit_sync.sync(reddit_scraper.get_api_info())
File "D:\Bot\lib\reddit_sync.py", line 24, in sync
raise IOError(msg)
[...]
File "D:\Bot\lib\reddit_sync.py", line 23, in sync
data = self.get_json(url, proxy)
File "D:\Bot\lib\reddit_sync.py", line 13, in get_json
r = requests.get(url, proxies=proxy, timeout
