Extract forum topics, links

diff --git a/EZSets/EZSets.py b/EZSets/EZSets.py
index c679e3d288bfaaef47c5584eda2947cb66954e9f..881b02ea0c6b0066ac64ced2c8998c736e078343 100644 (file)
--- a/EZSets/EZSets.py
+++ b/EZSets/EZSets.py
@@ -29,7 +29,7 @@ if __name__ == '__main__':
         for i,gid in enumerate(games):
             print(f'Game {i+1} (ID = {gid}): ', end='', flush=True)
             if gid in progressions:
-                print('Already downloaded')
+                print('Already downloaded progression')
             else:
                 response = requests.get(
                     f'https://retroachievements.org/API/API_GetGameProgression.php?y={key}&i={gid}'
@@ -41,10 +41,33 @@ if __name__ == '__main__':
                 fp.flush()
                 print('Downloaded new progression! (Wait for rate limit...)')
                 time.sleep(1)
+    with open(sys.argv[4], 'a+b') as fp:
+        fp.seek(0)
+        summaries = {}
+        for line in fp:
+            g = json.loads(line)
+            summaries[g['ID']] = g
+        for i,gid in enumerate(games):
+            print(f'Game {i+1} (ID = {gid}): ', end='', flush=True)
+            if gid in summaries:
+                print('Already downloaded summary')
+            else:
+                response = requests.get(
+                    f'https://retroachievements.org/API/API_GetGame.php?y={key}&i={gid}'
+                )
+                s = json.loads(response.text)
+                s['ID'] = gid
+                summaries[gid] = s
+                fp.write(json.dumps(s).encode())
+                fp.write(b'\n')
+                fp.flush()
+                print('Downloaded new summary! (Wait for rate limit...)')
+                time.sleep(1)
+
 
-    with open(sys.argv[4], 'wb') as fp:
+    with open(sys.argv[5], 'wb') as fp:
         fp.write('\t'.join([
-            'Game', 'Console', 'Download', 'Points', 'RetroPoints', 'Achievements',
+            'Game', 'Console', 'Forum Topic', 'Download', 'Points', 'RetroPoints', 'Achievements',
             'Number of Masterers', 'Median Time to Master', 
             'Number of Hardcore Beaters', 'Median Time to Hardcore Beat',
             'Number of Completers', 'Median Time to Complete',
@@ -58,6 +81,7 @@ if __name__ == '__main__':
             fp.write('\t'.join([
                 f'=HYPERLINK("https://retroachievements.org/game/{g['ID']}", "{g['Title']}")',
                 f'=HYPERLINK("https://retroachievements.org/system/{g['ConsoleID']}/games", "{g['ConsoleName']}")',
+                f'=HYPERLINK("https://retroachievements.org/forums/topic/{summaries[g['ID']]['ForumTopicID']}", "Link")',
                 'TODO',
                 str(points or 0), str(retropoints or 0),
                 str(g['NumAchievements'] or 0),

diff --git a/EZSets/RAForumScraper.js b/EZSets/RAForumScraper.js
new file mode 100644 (file)
index 0000000..868b560
--- /dev/null
+++ b/EZSets/RAForumScraper.js
@@ -0,0 +1,28 @@
+// Copy this snippet over and over again in the Browser DevTools console
+__RALinks = sessionStorage.getItem('__Links');
+__RAForumTopics = sessionStorage.getItem('__ForumTopics');
+if (!__RAForumTopics && !__RALinks) {
+       // Inline the forum topics into the JS code itself, which isn't great, but works and avoids CORS
+       __RALinks = {};
+       __RAForumTopics = ["17127", "23902", "8560", "17137", "14426", "22498", "24526", "17114", "22417", "25987", "25455", "23451", "20381", "22558", "16269", "37141", "17115", "22282", "37120", "16508", "8242", "19474", "21793", "9652", "16303", "6881", "22930", "8222", "22896", "26846", "22520", "22984", "18014", "20172", "16311", "17139", "21337", "21968", "8185", "20780", "16335", "22348", "31736", "22581", "16450", "9705", "25407", "21781", "24648", "24163", "33643", "25140", "25835", "13759", "7723", "13025", "18535", "18650", "8103", "16883", "36710", "26721", "25950", "13244", "21112", "26105", "34434", "22404", "20388", "16327", "17973", "18557", "16380", "10149", "16491", "21787", "21370", "13211", "35836", "27085", "31564", "27465", "29608", "25774", "19918", "16284", "21963", "16272", "31017", "16390", "12008", "22574", "20163", "15515", "28718", "34214", "28247", "18843", "13190", "23310", "36868", "33736", "9428", "22594", "28875", "20498", "22599", "27104", "22412", "25777", "22422", "21558", "22442", "28876", "13197", "28594", "22453", "30676", "9236", "22409", "5857", "35476", "28749", "24089", "27363", "14454", "21956", "11752", "25372", "17902", "17501", "11187", "26637", "26524", "24947", "22427", "9335", "18029", "12945", "13245", "28621", "18620", "16268", "14722", "16655", "9598", "30210", "28785", "36884", "28421", "28861", "19316", "22510", "22686", "16308", "27105", "19716", "23811", "21259", "16815", "21724", "22928", "24800", "6922", "25508", "23104", "16423", "22437", "25157", "13305", "15779", "31396", "16444", "18540", "27597", "14783", "29803", "24278", "26169", "4977", "16513", "16964", "16845", "20543", "34918", "31940", "22446", "21346", "28771", "28642", "29033", "22338", "23193", "25117", "25974", "26953", "21603", "29448", "17343", "32249", "30774", "22406", "33255", "20072", "24516", "25573", "23506", "34206", "16287", "21428", "25976", "35466", "6208", "23643", "34031", "6375", "7828", "37067", "25954", "33337", "7671", "15887", "21687", "22567", "35477", "16274", "30117", "24534", "14074", "23797", "6974", "21338", "16443", "33063", "17890", "31677", "22481", "13242", "19887", "21728", "13261", "6224", "15917", "20592", "22450", "20439", "20166", "19702", "16289", "21792", "14189", "10795", "31329", "31555", "23246", "22420", "26126", "14459", "34095", "16410", "16297", "28840", "22582", "30647", "22304", "7513", "16277", "20896", "32586", "7433", "17126", "24599", "35736", "21393", "30217", "10163", "26204", "26445", "25958", "28584", "36871", "17468", "12984", "26905", "21822", "33509", "17174", "34555", "16457", "16387", "22859", "16428", "16298", "21732", "16467", "27993", "16425", "18951", "16606", "27075", "22411", "16398", "7981", "16318", "23496", "31368", "26002", "21327", "22432", "14174", "28060", "10904", "30741", "35890", "26046", "23415", "25964", "23253", "22888", "20206", "25026", "32218", "8063", "26373", "16384", "7588", "35837", "12307", "22607", "9562", "35133", "16283", "30743", "2304", "18731", "35432", "22517", "16280", "17901", "21518", "33532", "24267", "16304", "13160", "27199", "11476", "30486", "16299", "26424", "34617", "16891", "14450", "17345", "15763", "22784", "16226", "5216", "22595", "9633", "3226", "26521", "14226", "25993", "16574", "26408", "26882", "12858", "16575", "30009", "24164", "31671", "16385", "25757", "16267", "17452", "16281", "18664", "21572", "17921", "14317", "26020", "25669", "29284", "29731", "20965", "22909", "26508", "9539", "25764", "26009", "23731", "32163", "22423", "11669", "22617", "28072", "14498", "14013", "29121", "32555", "26119", "22630", "15516", "16300", "20933", "23119", "26646", "12504", "23479", "20849", "21740", "9556", "16286", "30228", "28223", "13325", "9414", "16442", "20773", "21962", "16466", "22188", "14518", "22087", "16264", "16314", "19768", "11673", "7678", "10878", "16326", "22822", "29910", "16309", "16337", "18125", "16776", "16622", "7665", "16334", "17450", "24679", "16288", "15335", "30433", "13014", "16312", "23821", "25207", "26549", "8551", "30022", "22346", "31384", "22449", "24515", "10925", "25703", "20906", "16340", "30714", "23321", "30008", "28148", "19200", "33657", "33448", "24751", "11942", "24948", "30880", "18180", "22416", "8234", "21549", "20151", "28602", "23017", "22970", "26496", "16455", "28186", "29061", "9898", "31285", "29783", "16393", "26503", "1466", "26497", "28453", "16285", "15604", "26104", "26216", "27818", "22139", "29970", "6285", "8228", "35880", "31249", "25259", "12416", "33983", "22441", "12946", "18190", "16810", "30222", "32220", "22951", "22099", "22413", "10758", "16296", "28270", "16320", "16321", "25071", "34707", "21907", "13520", "17892", "31257", "15140", "22414", "19366", "23438", "15817", "13982", "19654", "6380", "28546", "12251", "25119", "10861", "28010", "28992", "19728", "33054", "16367", "20396", "22590", "28665", "12954", "13199", "24548", "26283", "35807", "28654", "30736", "20977", "17123", "22440", "26517", "31358", "14686", "16294", "18137", "16472", "16451", "15312", "9729", "16422", "21970", "22426", "15964", "23242", "15496", "15708", "34592", "16276", "36143", "25170", "28071", "24131", "7060", "35535", "23514", "33743", "26371", "27245", "24819", "26820", "16290", "19119", "7823", "22573", "23703", "23505", "13727", "27822", "34523", "32082", "28044", "17784", "35125", "28474", "22424", "16302", "7502", "21151", "27891", "21776", "11729", "19475", "22715", "16386", "23020", "29993", "23699", "20548", "12957", "24406", "26054", "21922", "17469", "26035", "13999", "26838", "17238", "25000", "12631", "26124", "35801", "26411", "27693", "30726", "24771", "24733", "28345", "33553", "21878", "31334", "18946", "21731", "8771", "11049", "22425", "22428", "20127", "982", "20798", "27228", "21501", "16271", "16416", "22559", "23241", "30885", "26280", "9107", "29111", "20045", "24288", "27007", "26837", "16354", "21604", "10435", "21703", "30375", "16306", "19154", "27644", "24886", "21827", "26292", "30878", "21113", "26537", "31378", "13121", "22642", "34497", "15818", "31536", "16293", "26051", "19141", "17199", "17202", "22641", "16345", "30739", "11799", "29018", "24723", "35378", "21576", "28292", "15212", "32543", "30015", "17900", "32877", "16275", "11313", "16439", "37261", "7575", "23457", "22968", "26594", "20515", "21305", "27591", "13466", "22358", "34344", "15840", "21961", "12256", "20073", "20966", "28658", "9821", "28147", "9589", "26073", "19172", "26896", "22658", "25621", "22443", "28230", "22229", "28860", "22678", "8097", "16282", "16292", "26291", "27610", "10787", "10189", "13392", "25953", "15234", "16747", "16336", "34864", "20503", "17929", "28061", "20283", "31500", "14831", "29524", "16301", "22877", "12764", "30977", "20421", "10222", "19413", "34844", "26038", "22410", "34576", "31700", "36036", "20212", "10944", "22980", "31369", "31520", "21919", "16018", "24935", "16310", "18888", "7062", "10935", "31120", "21830", "23959", "35332", "18343", "28577", "17109", "16270", "33089", "27272", "33876", "24268", "24909", "30555", "20952", "21885", "8084", "17853", "12889", "28033", "31374", "19684", "26511", "21003", "30021", "21440", "21295", "26159", "16510", "20853", "23101", "20989", "28576", "10793", "23517", "12294", "16362", "36418", "31301", "27152", "14463", "21474", "10976", "16273", "16333", "31785", "23341", "16571", "20024", "16295", "15836", "15523", "22544", "23892", "25883", "13702", "22396", "31294", "15737", "23697", "22415", "30424", "28343", "21403", "19100", "34957", "22419", "17999", "19362", "23508", "17478", "25075", "28900", "22418", "16324", "31295", "15095", "12819", "36677", "36082", "16370", "16278", "33799", "9306", "34475", "12341", "20444", "16319", "9319", "27538", "16291", "16368", "14108", "7705", "9563", "24682", "27260", "9325", "33423", "26118", "9635", "10841", "9900", "23374", "16454", "9305", "24205", "34904", "34342", "25959", "26037", "13240", "15531", "21763", "25803", "24798"];
+} else {
+       __RAForumTopics = JSON.parse(__RAForumTopics);
+       __RALinks = JSON.parse(__RALinks);
+}
+const __RAForumPrefix = 'https://retroachievements.org/forums/topic/';
+if (window.location.href.startsWith(__RAForumPrefix)) {
+       const __RARedirectPrefix = "https://retroachievements.org/redirect?url=" ;
+       const __RAForumID = window.location.href.substring(__RAForumPrefix.length);
+       if (!__RALinks[__RAForumID]) {
+               __RAPageLinks = Array.from(document.querySelectorAll(`a[href^="${__RARedirectPrefix}"]`))
+                       .map(e => decodeURIComponent(e.href.substring(__RARedirectPrefix.length)))
+                       .filter(href => !href.startsWith('https://www.google.com/search'));
+               __RALinks[__RAForumID] = __RAPageLinks;
+       }
+}
+sessionStorage.setItem('__Links', JSON.stringify(__RALinks));
+if (__RAForumTopics.length) {
+       __RANext = __RAForumTopics[0];
+       sessionStorage.setItem('__ForumTopics', JSON.stringify(__RAForumTopics.slice(1)));
+       window.location.href = 'https://retroachievements.org/forums/topic/' + __RAForumTopics[0];
+}

diff --git a/EZSets/RAHubScraper.js b/EZSets/RAHubScraper.js
index 49609d458d3bdb5783f716e26a1e1250be1a9970..0171a4aefcf37e59a3d91803956161f11592e0e4 100644 (file)
--- a/EZSets/RAHubScraper.js
+++ b/EZSets/RAHubScraper.js
@@ -1,11 +1,11 @@
-// Initialize games as an empty set initially, i.e. games = new Set();
-// Then run this script over and over again in https://retroachievements.org/hub/22750 (Freely Available in ForumTopic)
+// Run this script over and over again in https://retroachievements.org/hub/22750 (Freely Available in ForumTopic)
 // Make sure to also set the pagination to max so you don't have to run this a lot (max was 200 at the time of writing)
-pageGames = new Set(
+__RAGames = (window.__RAGames || new Set());
+__RAPageGames = new Set(
        Array
        .from(document.querySelectorAll('tbody tr td a[href^="https://retroachievements.org/game/"]'))
        .map(e => e.href),
 );
-next = document.querySelector('button[aria-label^="Go to next page"]');
-next.click();
-games = new Set([...games, ...pageGames]);
\ No newline at end of file
+__RANext = document.querySelector('button[aria-label^="Go to next page"]');
+__RANext.click();
+__RAGames = new Set([...__RAGames, ...__RAPageGames]);