From 47c3e0c4f2eff037727ba6744b20d2b77268abae Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 11:35:49 +0100 Subject: [PATCH 01/23] Buk handling and secure url added --- gog-backup | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/gog-backup b/gog-backup index 8167e97..1bc1008 100755 --- a/gog-backup +++ b/gog-backup @@ -51,6 +51,7 @@ import sys, os, contextlib, pprint, hashlib, zipfile import cookielib, urllib, urllib2, urlparse import xml.etree.ElementTree import threading, Queue, time +import json import html5lib # http://code.google.com/p/html5lib/ @@ -63,7 +64,8 @@ COOKIES = '.gog.cookies' MANIFEST = '.gog.games.py' VALIDATED = '.gog.valid.py' -LOGIN = 'https://www.gog.com/en/login' +LOGIN = 'https://secure.gog.com/en/login' +AJAX_URL = 'http://www.gog.com/user/ajax/' SHELF = 'https://www.gog.com/en/myaccount/shelf' LIST = 'https://www.gog.com/en/myaccount/list' THUMB = 'http://www.gog.com' @@ -223,12 +225,22 @@ def open_notrunc(name, bufsize=4*1024): return os.fdopen(fd, 'wb', bufsize) def cmd_login(email, passwd): + with request(AJAX_URL, args={'a' : 'get'}) as json_data: + data = json.load(json_data) + buk = data['buk'] + print buk + with request(LOGIN, args={'log_email': email, - 'log_password': passwd}) as page: + 'log_password': passwd, + 'redirectOk': '/en/', + 'unlockSettings': '1', + 'buk': buk, +}) as page: etree = parser.parse(page) if etree.find(".//div[@id='register_holder']") is not None: # this element is only present if not logged in - raise RuntimeError('login failed') + # raise RuntimeError('login failed') + print "Blub" cookiejar.save() def cmd_manifest(): From 297bd7bcd3e9a21202edd272f82c548b43464efe Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 11:53:51 +0100 Subject: [PATCH 02/23] Login failed check added --- .gitignore | 1 + gog-backup | 22 +++++++++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index b07c605..b0a65f0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ *~ \#*# .#* +.gog* diff --git a/gog-backup b/gog-backup index 1bc1008..7015608 100755 --- a/gog-backup +++ b/gog-backup @@ -77,6 +77,8 @@ opener = urllib2.build_opener(cookieproc) treebuilder = html5lib.treebuilders.getTreeBuilder('etree') parser = html5lib.HTMLParser(tree=treebuilder, namespaceHTMLElements=False) +from IPython import embed + useragent = 'gog-backup/%s (%s)' % (__version__, __url__) opener.addheaders = [('User-agent', useragent)] @@ -225,22 +227,28 @@ def open_notrunc(name, bufsize=4*1024): return os.fdopen(fd, 'wb', bufsize) def cmd_login(email, passwd): + # Reset cookiejar + cookiejar.clear() with request(AJAX_URL, args={'a' : 'get'}) as json_data: data = json.load(json_data) buk = data['buk'] - print buk with request(LOGIN, args={'log_email': email, 'log_password': passwd, 'redirectOk': '/en/', 'unlockSettings': '1', 'buk': buk, -}) as page: - etree = parser.parse(page) - if etree.find(".//div[@id='register_holder']") is not None: - # this element is only present if not logged in - # raise RuntimeError('login failed') - print "Blub" + }) as page: + pass + + guc_al = None + for c in cookiejar: + if c.name == 'guc_al': + guc_al = c + break + if not guc_al or guc_al.value == '0': + raise RuntimeError("Login failed") + cookiejar.save() def cmd_manifest(): From f6f8659def84aa7004f7fec35e7c54d48ccf2d0c Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 12:08:39 +0100 Subject: [PATCH 03/23] Password promt added --- gog-backup | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gog-backup b/gog-backup index 7015608..5d4fc06 100755 --- a/gog-backup +++ b/gog-backup @@ -22,8 +22,9 @@ Possible commands: - login - Login to GOG.com using the supplied email and password. + login [] + Login to GOG.com using the supplied email and password. If password is not supplied, + it is promted after start. manifest Create a manifest describing all games and extras owned by the @@ -52,6 +53,7 @@ import cookielib, urllib, urllib2, urlparse import xml.etree.ElementTree import threading, Queue, time import json +from getpass import getpass import html5lib # http://code.google.com/p/html5lib/ @@ -226,7 +228,9 @@ def open_notrunc(name, bufsize=4*1024): fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_BINARY, 0666) return os.fdopen(fd, 'wb', bufsize) -def cmd_login(email, passwd): +def cmd_login(email, passwd=None): + if passwd is None: + passwd = getpass("Password: ") # Reset cookiejar cookiejar.clear() with request(AJAX_URL, args={'a' : 'get'}) as json_data: From 27db25d919d5359284cd7f130f74561f38f9d30d Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 19:19:26 +0100 Subject: [PATCH 04/23] AJAX game info reading added --- gog-backup | 74 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/gog-backup b/gog-backup index 5d4fc06..57ed75d 100755 --- a/gog-backup +++ b/gog-backup @@ -68,8 +68,9 @@ VALIDATED = '.gog.valid.py' LOGIN = 'https://secure.gog.com/en/login' AJAX_URL = 'http://www.gog.com/user/ajax/' -SHELF = 'https://www.gog.com/en/myaccount/shelf' -LIST = 'https://www.gog.com/en/myaccount/list' +ACCOUNT_AJAX_URL = 'https://secure.gog.com/en/account/ajax' +SHELF = 'https://secure.gog.com/account/games' +LIST = 'https://secure.gog.com/account/wishlist' THUMB = 'http://www.gog.com' pathmap = {} @@ -259,41 +260,48 @@ def cmd_manifest(): games = {} # parse game list and available files for each from list page - with request(LIST) as page: - etree = parser.parse(page) - for game in etree.findall(".//div[@class='tab_1_row']"): - gamecard = game.find(".//div[@class='tab_1_title']/a") - if gamecard is None: - # happens with "Colin McRae Rally 2005", among others - g = game.find(".//div[@class='tab_1_title']/span") - print 'WARNING: no gamecard link for %s?!' % (g.text,) - print ' Did GOG.com stop selling this game?' - continue - g = AttrDict() - g.key = gamecard.attrib['href'].split('/')[-1] - g.title = gamecard.text - g.thumb = THUMB + game.find(".//img[@src]").attrib['src'] - g.setup, g.extra = [], [] - for row in game.findall(".//div[@class='sh_o_i_row']"): - f = AttrDict() - f.href = row.attrib.get('onclick', '') - if not 'download/file' in f.href: - f.href = row.find(".//a").attrib['href'] - g.setup.append(f) - else: - f.href = f.href[f.href.index('http:') : -1] - f.desc = row.find(".//div[@class='sh_o_i_text']/span").text - g.extra.append(f) - games[g.key] = g + # with request(LIST) as page: + # etree = parser.parse(page) + # for game in etree.findall(".//div[@class='shelf_game']/@data-gameid"): + # with request(AJAX_URL, args={'a' : 'gamesShelfDetails', + # 'g' : game}) as data_request: + # print json.loads(data_request) + # gamecard = game.find(".//div[@class='tab_1_title']/a") + # if gamecard is None: + # # happens with "Colin McRae Rally 2005", among others + # g = game.find(".//div[@class='tab_1_title']/span") + # print 'WARNING: no gamecard link for %s?!' % (g.text,) + # print ' Did GOG.com stop selling this game?' + # continue + # g = AttrDict() + # g.key = gamecard.attrib['href'].split('/')[-1] + # g.title = gamecard.text + # g.thumb = THUMB + game.find(".//img[@src]").attrib['src'] + # g.setup, g.extra = [], [] + # for row in game.findall(".//div[@class='sh_o_i_row']"): + # f = AttrDict() + # f.href = row.attrib.get('onclick', '') + # if not 'download/file' in f.href: + # f.href = row.find(".//a").attrib['href'] + # g.setup.append(f) + # else: + # f.href = f.href[f.href.index('http:') : -1] + # f.desc = row.find(".//div[@class='sh_o_i_text']/span").text + # g.extra.append(f) + # games[g.key] = g # match games to covers on the shelf page + print 'Getting shelf' with request(SHELF) as page: etree = parser.parse(page) - for game in etree.findall(".//div[@class='shelf_item_h']"): - gamecard = game.find(".//div[@class='shelf_ov_tab_2_title']/a") - if gamecard is not None: - g = games[gamecard.attrib['href'].split('/')[-1]] - g.cover = THUMB + game.find(".//img[@src]").attrib['src'] + for game in etree.findall(".//div[@class='shelf_game']"): + with request(ACCOUNT_AJAX_URL, args={'a' : 'gamesShelfDetails', + 'g' : game.attrib['data-gameid']}) as data_request: + game_data = json.load(data_request) + + # if gamecard is not None: + # g = games[gamecard.attrib['href'].split('/')[-1]] + # g.cover = THUMB + game.find(".//img[@src]").attrib['src'] # request a zero-length range from each file to determine # - the total size (from the Content-Range header) From 227311ccf81308843ee6a91971711ea28025eb5c Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 20:12:26 +0100 Subject: [PATCH 05/23] Game dl data added to manifest --- gog-backup | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/gog-backup b/gog-backup index 57ed75d..24fc97a 100755 --- a/gog-backup +++ b/gog-backup @@ -291,13 +291,24 @@ def cmd_manifest(): # games[g.key] = g # match games to covers on the shelf page - print 'Getting shelf' with request(SHELF) as page: etree = parser.parse(page) - for game in etree.findall(".//div[@class='shelf_game']"): + for game in etree.findall(".//div[@class='shelf_game']")[:1]: + g = AttrDict() + g.key = game.attrib['data-gameid'] with request(ACCOUNT_AJAX_URL, args={'a' : 'gamesShelfDetails', - 'g' : game.attrib['data-gameid']}) as data_request: + 'g' : g.key}) as data_request: game_data = json.load(data_request) + game_element = parser.parse(game_data['details']['html']) + g.setup, g.extra = [], [] + for dl_link in game_element.findall((".//div[@class=" + "'win-download']/a[@class='list_game_item']")): + g.setup.append(dl_link.attrib['href']) + print g + + # TODO: parse data + # id = data-gameid + # get cover # if gamecard is not None: # g = games[gamecard.attrib['href'].split('/')[-1]] From 17960ce747cb8af3516e17ec1780e59ec6801f37 Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 21:02:46 +0100 Subject: [PATCH 06/23] Manifest handling added --- gog-backup | 56 ++++++++++++++++-------------------------------------- 1 file changed, 16 insertions(+), 40 deletions(-) diff --git a/gog-backup b/gog-backup index 24fc97a..d043810 100755 --- a/gog-backup +++ b/gog-backup @@ -69,8 +69,7 @@ VALIDATED = '.gog.valid.py' LOGIN = 'https://secure.gog.com/en/login' AJAX_URL = 'http://www.gog.com/user/ajax/' ACCOUNT_AJAX_URL = 'https://secure.gog.com/en/account/ajax' -SHELF = 'https://secure.gog.com/account/games' -LIST = 'https://secure.gog.com/account/wishlist' +SHELF = 'https://secure.gog.com/account/games/shelf' THUMB = 'http://www.gog.com' pathmap = {} @@ -259,56 +258,33 @@ def cmd_login(email, passwd=None): def cmd_manifest(): games = {} - # parse game list and available files for each from list page - # with request(LIST) as page: - # etree = parser.parse(page) - # for game in etree.findall(".//div[@class='shelf_game']/@data-gameid"): - # with request(AJAX_URL, args={'a' : 'gamesShelfDetails', - # 'g' : game}) as data_request: - # print json.loads(data_request) - # gamecard = game.find(".//div[@class='tab_1_title']/a") - # if gamecard is None: - # # happens with "Colin McRae Rally 2005", among others - # g = game.find(".//div[@class='tab_1_title']/span") - # print 'WARNING: no gamecard link for %s?!' % (g.text,) - # print ' Did GOG.com stop selling this game?' - # continue - # g = AttrDict() - # g.key = gamecard.attrib['href'].split('/')[-1] - # g.title = gamecard.text - # g.thumb = THUMB + game.find(".//img[@src]").attrib['src'] - # g.setup, g.extra = [], [] - # for row in game.findall(".//div[@class='sh_o_i_row']"): - # f = AttrDict() - # f.href = row.attrib.get('onclick', '') - # if not 'download/file' in f.href: - # f.href = row.find(".//a").attrib['href'] - # g.setup.append(f) - # else: - # f.href = f.href[f.href.index('http:') : -1] - # f.desc = row.find(".//div[@class='sh_o_i_text']/span").text - # g.extra.append(f) - # games[g.key] = g - - # match games to covers on the shelf page with request(SHELF) as page: etree = parser.parse(page) - for game in etree.findall(".//div[@class='shelf_game']")[:1]: + for game in etree.findall(".//div[@class='shelf_game']"): g = AttrDict() g.key = game.attrib['data-gameid'] + g.thumb = g.cover = THUMB +\ + game.find(".//img[@src]").attrib['src'] with request(ACCOUNT_AJAX_URL, args={'a' : 'gamesShelfDetails', 'g' : g.key}) as data_request: game_data = json.load(data_request) game_element = parser.parse(game_data['details']['html']) + g.title = game_element.find(".//h2/a").text.strip() g.setup, g.extra = [], [] + # get setup downloader for windows for dl_link in game_element.findall((".//div[@class=" "'win-download']/a[@class='list_game_item']")): - g.setup.append(dl_link.attrib['href']) - print g + g.setup.append(AttrDict(href=dl_link.attrib['href'])) + + # get bonus material + for dl_link in game_element.findall((".//div[@class=" + "'bonus_content_list browser']/a")): + g.extra.append(AttrDict( + href=THUMB + dl_link.attrib['href'], + desc=dl_link.find("./span[@class='light_un']").text) + ) + games[g.key] = g - # TODO: parse data - # id = data-gameid - # get cover # if gamecard is not None: # g = games[gamecard.attrib['href'].split('/')[-1]] From 50d2725343a13db8a34847df898fd60e3f07cc10 Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 21:21:01 +0100 Subject: [PATCH 07/23] Thumbnail and background fetching added --- gog-backup | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/gog-backup b/gog-backup index d043810..762c8d7 100755 --- a/gog-backup +++ b/gog-backup @@ -60,7 +60,7 @@ import html5lib # http://code.google.com/p/html5lib/ BACKUPINTO = '.' CONCURRENCY = 6 BREADTHFIRST = False -FETCHCOVERS = False +FETCHCOVERS = True PATHMAP = '.gog.pathmap.txt' COOKIES = '.gog.cookies' MANIFEST = '.gog.games.py' @@ -70,6 +70,8 @@ LOGIN = 'https://secure.gog.com/en/login' AJAX_URL = 'http://www.gog.com/user/ajax/' ACCOUNT_AJAX_URL = 'https://secure.gog.com/en/account/ajax' SHELF = 'https://secure.gog.com/account/games/shelf' +# match games to covers on the shelf page +LIST = 'https://secure.gog.com/account/games/list' THUMB = 'http://www.gog.com' pathmap = {} @@ -263,26 +265,32 @@ def cmd_manifest(): for game in etree.findall(".//div[@class='shelf_game']"): g = AttrDict() g.key = game.attrib['data-gameid'] - g.thumb = g.cover = THUMB +\ + g.cover = THUMB +\ game.find(".//img[@src]").attrib['src'] with request(ACCOUNT_AJAX_URL, args={'a' : 'gamesShelfDetails', 'g' : g.key}) as data_request: game_data = json.load(data_request) game_element = parser.parse(game_data['details']['html']) - g.title = game_element.find(".//h2/a").text.strip() - g.setup, g.extra = [], [] - # get setup downloader for windows - for dl_link in game_element.findall((".//div[@class=" - "'win-download']/a[@class='list_game_item']")): - g.setup.append(AttrDict(href=dl_link.attrib['href'])) - - # get bonus material - for dl_link in game_element.findall((".//div[@class=" - "'bonus_content_list browser']/a")): - g.extra.append(AttrDict( - href=THUMB + dl_link.attrib['href'], - desc=dl_link.find("./span[@class='light_un']").text) - ) + g.title = game_element.find(".//h2/a").text.strip() + + # get thumbnail + with request(LIST) as list_page: + list_element = parser.parse(list_page).find(".//div[@id='game_li_{0}']".format(g.key)) + g.background = THUMB + list_element.attrib['data-background'] + g.thumb = THUMB + list_element.find(".//img[@src]").attrib['src'] + g.setup, g.extra = [], [] + # get setup downloader for windows + for dl_link in game_element.findall((".//div[@class=" + "'win-download']/a[@class='list_game_item']")): + g.setup.append(AttrDict(href=dl_link.attrib['href'])) + + # get bonus material + for dl_link in game_element.findall((".//div[@class=" + "'bonus_content_list browser']/a")): + g.extra.append(AttrDict( + href=THUMB + dl_link.attrib['href'], + desc=dl_link.find("./span[@class='light_un']").text) + ) games[g.key] = g @@ -320,7 +328,7 @@ def cmd_manifest(): print >>w, '# %d games' % len(games) pprint.pprint(games.values(), width=100, stream=w) - # optionally download cover and thumbnail + # optionally download cover, background and thumbnail if FETCHCOVERS: def fetch(which, g): href = getattr(g, which) @@ -333,6 +341,7 @@ def cmd_manifest(): for g in games.values(): fetch('cover', g) fetch('thumb', g) + fetch('background', g) def cmd_list(): # summarize the manifest From 3e66c574332888fa02f0f6613c50feb635f6aa12 Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 21:35:02 +0100 Subject: [PATCH 08/23] Cover fetching fixed --- gog-backup | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gog-backup b/gog-backup index 762c8d7..3b706ee 100755 --- a/gog-backup +++ b/gog-backup @@ -334,9 +334,13 @@ def cmd_manifest(): href = getattr(g, which) path = pathmap.get(g.key, g.key) ext = os.path.splitext(href)[-1] - fn = '%s_%s%s' % (g.key, which, ext) + fn = '{0}{1}'.format(which, ext) with request(href) as page: - with open(os.path.join(path, fn), 'wb') as w: + path = os.path.join(path, fn) + path_dir = os.path.dirname(path) + if not os.path.isdir(path_dir): + os.mkdir(path_dir) + with open(path, 'wb') as w: w.write(page.read()) for g in games.values(): fetch('cover', g) From d3f2f4063885d1bc87694d783bd3ead3f812eef1 Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 21:58:01 +0100 Subject: [PATCH 09/23] Extras are downloaded into a separate folder --- gog-backup | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/gog-backup b/gog-backup index 3b706ee..bc09252 100755 --- a/gog-backup +++ b/gog-backup @@ -120,8 +120,9 @@ def load_attrdicts(fn): except IOError: return AttrDict() -def locate(g, f): - path = os.path.join(pathmap.get(g.key, g.key), f.name) +def locate(g, f, is_extra=False): + path = os.path.join(pathmap.get(g.key, g.key), + (is_extra and "extra" or ""), f.name) if not os.path.isfile(path) and os.path.isfile(f.name): return f.name return path @@ -227,7 +228,11 @@ def needed(valid, games, missing, corrupt): def open_notrunc(name, bufsize=4*1024): # 'w+' includes O_TRUNC, 'r+' lacks O_CREAT; so, roll my own - fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_BINARY, 0666) + flags = os.O_WRONLY | os.O_CREAT + if hasattr(os, "O_BINARY"): + # exists only on windows + flags |= os.O_BINARY + fd = os.open(name, flags, 0666) return os.fdopen(fd, 'wb', bufsize) def cmd_login(email, passwd=None): @@ -378,11 +383,12 @@ def cmd_fetch(): # build a list of work items work = Queue.PriorityQueue() - i = -sys.maxint + i = -sys.maxint + for g in games: for f in g.setup + g.extra: v = valid[(g.key, f.name)] - path = locate(g, f) + path = locate(g, f, f in g.extra) dn = os.path.dirname(path) if dn and not os.path.isdir(dn): os.makedirs(dn) @@ -401,6 +407,7 @@ def cmd_fetch(): work.put((prio, args)) sz = end + 1 - start sizes[path] = sz + sizes.get(path, 0) + # work item I/O loop def ioloop(tid, path, page, out): From 82ccc7b12feb4df2cb191c29e13e05248d88caea Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 22:04:07 +0100 Subject: [PATCH 10/23] Added prompt for game fetching --- gog-backup | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/gog-backup b/gog-backup index bc09252..e996291 100755 --- a/gog-backup +++ b/gog-backup @@ -375,10 +375,18 @@ def cmd_update(): valid = dict((v.key, v) for v in load_attrdicts(VALIDATED)) needed(*compare(valid)) -def cmd_fetch(): +def cmd_fetch(fetch_all=None): # start an incremental comparison valid = dict((v.key, v) for v in load_attrdicts(VALIDATED)) _, games, _, _ = compare(valid) + if fetch_all is None: + for game in games[:]: + if raw_input("Download {0}? [Y/N]".format(game.title))\ + .upper() != 'Y': + games.remove(game) + elif not (all == "--all" or "-a"): + raise RuntimeError("Unknown parameter supplied. Use --all or -a.") + sizes, rates, errors = {}, {}, {} # build a list of work items From eebbeb54b893e03b79683675389a3b7a6e27e1b3 Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 22:06:45 +0100 Subject: [PATCH 11/23] README and docs updated --- README.rst | 2 +- gog-backup | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index a9556fb..af23a34 100644 --- a/README.rst +++ b/README.rst @@ -144,7 +144,7 @@ run ``gog-backup --help``. downloading files (perhaps waiting an appropriate period between invocations if GOG.com is experiencing high load). By default, game files are placed in per-game sub-directories of the current working - directory. + directory. Use --all to avoid prompting for input. Therefore, the simplest command flow would be to first ``login``, then download a ``manifest``, then ``fetch`` one or more times. diff --git a/gog-backup b/gog-backup index e996291..6b78b47 100755 --- a/gog-backup +++ b/gog-backup @@ -39,9 +39,10 @@ Possible commands: and print a report on what must be downloaded. 'compare' validates everything; 'update' only validates new files. - fetch + fetch [--all|-a] Download any missing, incomplete, or corrupted files (after an - implicit 'update'). + implicit 'update'). If --all is specified, dowloads every game, else + prompts for comfirmation. """ __author__ = 'Evan Powers' From 507e2849de051caa6b66fecaf4700c993f3e14ba Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 22:08:11 +0100 Subject: [PATCH 12/23] Some line length issues fixed --- gog-backup | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gog-backup b/gog-backup index 6b78b47..e69b553 100755 --- a/gog-backup +++ b/gog-backup @@ -281,9 +281,11 @@ def cmd_manifest(): # get thumbnail with request(LIST) as list_page: - list_element = parser.parse(list_page).find(".//div[@id='game_li_{0}']".format(g.key)) + list_element = parser.parse(list_page).\ + find(".//div[@id='game_li_{0}']".format(g.key)) g.background = THUMB + list_element.attrib['data-background'] - g.thumb = THUMB + list_element.find(".//img[@src]").attrib['src'] + g.thumb = THUMB + list_element.find(".//img[@src]").\ + attrib['src'] g.setup, g.extra = [], [] # get setup downloader for windows for dl_link in game_element.findall((".//div[@class=" From 87446bad0c53862e18423ecb436e09b182921a28 Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 22:13:44 +0100 Subject: [PATCH 13/23] Some PEP8 fixes --- gog-backup | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/gog-backup b/gog-backup index e69b553..4269e49 100755 --- a/gog-backup +++ b/gog-backup @@ -23,8 +23,8 @@ Possible commands: login [] - Login to GOG.com using the supplied email and password. If password is not supplied, - it is promted after start. + Login to GOG.com using the supplied email and password. + If password is not supplied, user is promted after start. manifest Create a manifest describing all games and extras owned by the @@ -87,6 +87,7 @@ from IPython import embed useragent = 'gog-backup/%s (%s)' % (__version__, __url__) opener.addheaders = [('User-agent', useragent)] + class AttrDict(dict): def __init__(self, **kw): self.update(kw) @@ -129,7 +130,6 @@ def locate(g, f, is_extra=False): return path def md5map(f, path): - print '#', path H = hashlib.md5() part = [] with open(path, 'rb') as r: @@ -241,7 +241,7 @@ def cmd_login(email, passwd=None): passwd = getpass("Password: ") # Reset cookiejar cookiejar.clear() - with request(AJAX_URL, args={'a' : 'get'}) as json_data: + with request(AJAX_URL, args={'a': 'get'}) as json_data: data = json.load(json_data) buk = data['buk'] @@ -273,8 +273,8 @@ def cmd_manifest(): g.key = game.attrib['data-gameid'] g.cover = THUMB +\ game.find(".//img[@src]").attrib['src'] - with request(ACCOUNT_AJAX_URL, args={'a' : 'gamesShelfDetails', - 'g' : g.key}) as data_request: + with request(ACCOUNT_AJAX_URL, args={'a': 'gamesShelfDetails', + 'g': g.key}) as data_request: game_data = json.load(data_request) game_element = parser.parse(game_data['details']['html']) g.title = game_element.find(".//h2/a").text.strip() @@ -301,7 +301,6 @@ def cmd_manifest(): ) games[g.key] = g - # if gamecard is not None: # g = games[gamecard.attrib['href'].split('/')[-1]] # g.cover = THUMB + game.find(".//img[@src]").attrib['src'] @@ -394,7 +393,7 @@ def cmd_fetch(fetch_all=None): # build a list of work items work = Queue.PriorityQueue() - i = -sys.maxint + i = -sys.maxint for g in games: for f in g.setup + g.extra: @@ -418,7 +417,6 @@ def cmd_fetch(fetch_all=None): work.put((prio, args)) sz = end + 1 - start sizes[path] = sz + sizes.get(path, 0) - # work item I/O loop def ioloop(tid, path, page, out): From f1da01a4a156d753aa8848a826c520e040a8183f Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 22:18:31 +0100 Subject: [PATCH 14/23] Fixed UTF8 bug in prompt --- gog-backup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gog-backup b/gog-backup index 4269e49..1ae8454 100755 --- a/gog-backup +++ b/gog-backup @@ -383,8 +383,8 @@ def cmd_fetch(fetch_all=None): _, games, _, _ = compare(valid) if fetch_all is None: for game in games[:]: - if raw_input("Download {0}? [Y/N]".format(game.title))\ - .upper() != 'Y': + if raw_input("Download {0}? [Y/N]".\ + format(game.title.encode("utf-8"))).upper() != 'Y': games.remove(game) elif not (all == "--all" or "-a"): raise RuntimeError("Unknown parameter supplied. Use --all or -a.") From fddc4ac404d8383cd8c1f2a89d56bbde3f2036e9 Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 22:19:07 +0100 Subject: [PATCH 15/23] Removed ipython --- gog-backup | 2 -- 1 file changed, 2 deletions(-) diff --git a/gog-backup b/gog-backup index 1ae8454..e7577f7 100755 --- a/gog-backup +++ b/gog-backup @@ -82,8 +82,6 @@ opener = urllib2.build_opener(cookieproc) treebuilder = html5lib.treebuilders.getTreeBuilder('etree') parser = html5lib.HTMLParser(tree=treebuilder, namespaceHTMLElements=False) -from IPython import embed - useragent = 'gog-backup/%s (%s)' % (__version__, __url__) opener.addheaders = [('User-agent', useragent)] From 7a3d75a86fe2cc982c6f2955fdf24f2095ac59c5 Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 22:31:57 +0100 Subject: [PATCH 16/23] Locale writer added --- gog-backup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gog-backup b/gog-backup index e7577f7..2a07a1f 100755 --- a/gog-backup +++ b/gog-backup @@ -49,7 +49,7 @@ __author__ = 'Evan Powers' __version__ = '1.0' __url__ = 'https://github.com/evanpowers/gog-backup' -import sys, os, contextlib, pprint, hashlib, zipfile +import sys, os, codecs, contextlib, pprint, hashlib, locale, zipfile import cookielib, urllib, urllib2, urlparse import xml.etree.ElementTree import threading, Queue, time @@ -81,6 +81,7 @@ cookieproc = urllib2.HTTPCookieProcessor(cookiejar) opener = urllib2.build_opener(cookieproc) treebuilder = html5lib.treebuilders.getTreeBuilder('etree') parser = html5lib.HTMLParser(tree=treebuilder, namespaceHTMLElements=False) +sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout) useragent = 'gog-backup/%s (%s)' % (__version__, __url__) opener.addheaders = [('User-agent', useragent)] From f9846cd6393ebb6af56a44d5fb52775564bbf896 Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 22:33:24 +0100 Subject: [PATCH 17/23] Encoding of the title in fetch removed --- gog-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gog-backup b/gog-backup index 2a07a1f..7e41cc3 100755 --- a/gog-backup +++ b/gog-backup @@ -383,7 +383,7 @@ def cmd_fetch(fetch_all=None): if fetch_all is None: for game in games[:]: if raw_input("Download {0}? [Y/N]".\ - format(game.title.encode("utf-8"))).upper() != 'Y': + format(game.title)).upper() != 'Y': games.remove(game) elif not (all == "--all" or "-a"): raise RuntimeError("Unknown parameter supplied. Use --all or -a.") From a5752b18c379bcf8213c470bd4c4bbd0ad7ecc73 Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 22:35:13 +0100 Subject: [PATCH 18/23] String made unicode --- gog-backup | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gog-backup b/gog-backup index 7e41cc3..e82ec83 100755 --- a/gog-backup +++ b/gog-backup @@ -362,10 +362,10 @@ def cmd_list(): ssz = sum(f.size for f in g.setup) esz = sum(f.size for f in g.extra) total += ssz + esz - print '%s (%s)' % (g.key, g.title) - print ' %8s in %d setup files' % (megs(ssz), len(g.setup)) - print ' %8s in %d extras' % (megs(esz), len(g.extra)) - print '%d games, a total of %s' % (len(games), megs(total)) + print u'%s (%s)' % (g.key, g.title) + print u' %8s in %d setup files' % (megs(ssz), len(g.setup)) + print u' %8s in %d extras' % (megs(esz), len(g.extra)) + print u'%d games, a total of %s' % (len(games), megs(total)) def cmd_compare(): # start the comparison from scratch @@ -382,7 +382,7 @@ def cmd_fetch(fetch_all=None): _, games, _, _ = compare(valid) if fetch_all is None: for game in games[:]: - if raw_input("Download {0}? [Y/N]".\ + if raw_input(u"Download {0}? [Y/N]".\ format(game.title)).upper() != 'Y': games.remove(game) elif not (all == "--all" or "-a"): From 6cd0f9a464656c175c8497cea84783d438b651f6 Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 22:58:30 +0100 Subject: [PATCH 19/23] Folder names changed to game names --- gog-backup | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/gog-backup b/gog-backup index e82ec83..08ae250 100755 --- a/gog-backup +++ b/gog-backup @@ -121,11 +121,10 @@ def load_attrdicts(fn): except IOError: return AttrDict() -def locate(g, f, is_extra=False): - path = os.path.join(pathmap.get(g.key, g.key), - (is_extra and "extra" or ""), f.name) - if not os.path.isfile(path) and os.path.isfile(f.name): - return f.name +def locate(g, f=None, is_extra=False): + filename = f.name if f is not None else "" + path = os.path.join(pathmap.get(g.key, g.title), + (is_extra and "extra" or ""), filename) return path def md5map(f, path): @@ -338,7 +337,7 @@ def cmd_manifest(): if FETCHCOVERS: def fetch(which, g): href = getattr(g, which) - path = pathmap.get(g.key, g.key) + path = locate(g) ext = os.path.splitext(href)[-1] fn = '{0}{1}'.format(which, ext) with request(href) as page: From f3ba5106c9c994c3a7a1af087503802e6c99498d Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 23:01:11 +0100 Subject: [PATCH 20/23] Returned file path check because im not sure, what does it do --- gog-backup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gog-backup b/gog-backup index 08ae250..4a5d589 100755 --- a/gog-backup +++ b/gog-backup @@ -125,6 +125,9 @@ def locate(g, f=None, is_extra=False): filename = f.name if f is not None else "" path = os.path.join(pathmap.get(g.key, g.title), (is_extra and "extra" or ""), filename) + # what does it do? + if not os.path.isfile(path) and os.path.isfile(filename): + return filename return path def md5map(f, path): From f5532b69373d4684ce5e0e8e8e9eb9a6199a9aec Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 23:41:54 +0100 Subject: [PATCH 21/23] List page request refactored --- gog-backup | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/gog-backup b/gog-backup index 4a5d589..c4a2dba 100755 --- a/gog-backup +++ b/gog-backup @@ -280,13 +280,6 @@ def cmd_manifest(): game_element = parser.parse(game_data['details']['html']) g.title = game_element.find(".//h2/a").text.strip() - # get thumbnail - with request(LIST) as list_page: - list_element = parser.parse(list_page).\ - find(".//div[@id='game_li_{0}']".format(g.key)) - g.background = THUMB + list_element.attrib['data-background'] - g.thumb = THUMB + list_element.find(".//img[@src]").\ - attrib['src'] g.setup, g.extra = [], [] # get setup downloader for windows for dl_link in game_element.findall((".//div[@class=" @@ -302,9 +295,16 @@ def cmd_manifest(): ) games[g.key] = g - # if gamecard is not None: - # g = games[gamecard.attrib['href'].split('/')[-1]] - # g.cover = THUMB + game.find(".//img[@src]").attrib['src'] + # get thumbnails + with request(LIST) as list_page: + list_page_dom = parser.parse(list_page) + for game in games.values(): + list_element = list_page_dom.find(".//div[@id='game_li_{0}']".\ + format(game.key)) + g.background = THUMB + list_element.attrib['data-background'] + g.thumb = THUMB + list_element.find("./img[@src]").\ + attrib['src'] + # request a zero-length range from each file to determine # - the total size (from the Content-Range header) @@ -527,7 +527,7 @@ def main(args): except IndexError: print >>sys.stderr, '%s: missing command' % arg0 return 10 - except KeyError: + except KeyError, e: print >>sys.stderr, '%s: invalid command %r' % (arg0, args[0]) return 10 except TypeError, e: From 9de68f36b9dcf4a0d2897cfb5523d9650059b4a1 Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 8 Nov 2012 23:51:17 +0100 Subject: [PATCH 22/23] Getting list page only once --- gog-backup | 76 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/gog-backup b/gog-backup index c4a2dba..1294fb3 100755 --- a/gog-backup +++ b/gog-backup @@ -269,42 +269,44 @@ def cmd_manifest(): with request(SHELF) as page: etree = parser.parse(page) - for game in etree.findall(".//div[@class='shelf_game']"): - g = AttrDict() - g.key = game.attrib['data-gameid'] - g.cover = THUMB +\ - game.find(".//img[@src]").attrib['src'] - with request(ACCOUNT_AJAX_URL, args={'a': 'gamesShelfDetails', - 'g': g.key}) as data_request: - game_data = json.load(data_request) - game_element = parser.parse(game_data['details']['html']) - g.title = game_element.find(".//h2/a").text.strip() - - g.setup, g.extra = [], [] - # get setup downloader for windows - for dl_link in game_element.findall((".//div[@class=" - "'win-download']/a[@class='list_game_item']")): - g.setup.append(AttrDict(href=dl_link.attrib['href'])) - - # get bonus material - for dl_link in game_element.findall((".//div[@class=" - "'bonus_content_list browser']/a")): - g.extra.append(AttrDict( - href=THUMB + dl_link.attrib['href'], - desc=dl_link.find("./span[@class='light_un']").text) - ) - games[g.key] = g - - # get thumbnails - with request(LIST) as list_page: - list_page_dom = parser.parse(list_page) - for game in games.values(): - list_element = list_page_dom.find(".//div[@id='game_li_{0}']".\ - format(game.key)) - g.background = THUMB + list_element.attrib['data-background'] - g.thumb = THUMB + list_element.find("./img[@src]").\ - attrib['src'] - + with request(LIST) as list_page: + list_etree = parser.parse(list_page) + + for game in etree.findall(".//div[@class='shelf_game']"): + g = AttrDict() + g.key = game.attrib['data-gameid'] + g.cover = THUMB +\ + game.find(".//img[@src]").attrib['src'] + with request(ACCOUNT_AJAX_URL, args={'a': 'gamesShelfDetails', + 'g': g.key}) as data_request: + game_data = json.load(data_request) + game_element = parser.parse(game_data['details']['html']) + g.title = game_element.find(".//h2/a").text.strip() + + # get thumbnail + list_element = list_etree.find(".//div[@id='game_li_{0}']".\ + format(g.key)) + g.background = THUMB + list_element.attrib['data-background'] + g.thumb = THUMB + list_element.find(".//img[@src]").\ + attrib['src'] + g.setup, g.extra = [], [] + # get setup downloader for windows + for dl_link in game_element.findall((".//div[@class=" + "'win-download']/a[@class='list_game_item']")): + g.setup.append(AttrDict(href=dl_link.attrib['href'])) + + # get bonus material + for dl_link in game_element.findall((".//div[@class=" + "'bonus_content_list browser']/a")): + g.extra.append(AttrDict( + href=THUMB + dl_link.attrib['href'], + desc=dl_link.find("./span[@class='light_un']").text) + ) + games[g.key] = g + + # if gamecard is not None: + # g = games[gamecard.attrib['href'].split('/')[-1]] + # g.cover = THUMB + game.find(".//img[@src]").attrib['src'] # request a zero-length range from each file to determine # - the total size (from the Content-Range header) @@ -527,7 +529,7 @@ def main(args): except IndexError: print >>sys.stderr, '%s: missing command' % arg0 return 10 - except KeyError, e: + except KeyError: print >>sys.stderr, '%s: invalid command %r' % (arg0, args[0]) return 10 except TypeError, e: From 705889b8a8e5810ea205ac724863f03cf5f5e13c Mon Sep 17 00:00:00 2001 From: Dmytro Vorona Date: Thu, 3 Jan 2013 17:44:27 +0100 Subject: [PATCH 23/23] New login url fixed --- gog-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gog-backup b/gog-backup index 1294fb3..c9dedc3 100755 --- a/gog-backup +++ b/gog-backup @@ -67,7 +67,7 @@ COOKIES = '.gog.cookies' MANIFEST = '.gog.games.py' VALIDATED = '.gog.valid.py' -LOGIN = 'https://secure.gog.com/en/login' +LOGIN = 'https://secure.gog.com/login' AJAX_URL = 'http://www.gog.com/user/ajax/' ACCOUNT_AJAX_URL = 'https://secure.gog.com/en/account/ajax' SHELF = 'https://secure.gog.com/account/games/shelf'