Work well version

trentenwen · Mar 30, 2023 · e10aa2a · e10aa2a
1 parent 2a20625
commit e10aa2a
Show file tree

Hide file tree

Showing 16 changed files with 79,819 additions and 46 deletions.
diff --git a/csvoutput/Atomic-Heart.csv b/csvoutput/Atomic-Heart.csv
diff --git a/csvoutput/Cities:-Skylines.csv b/csvoutput/Cities:-Skylines.csv
diff --git a/csvoutput/Cyberpunk-2077.csv b/csvoutput/Cyberpunk-2077.csv
diff --git a/csvoutput/Forza-Horizon-5.csv b/csvoutput/Forza-Horizon-5.csv
diff --git a/csvoutput/Grand-Theft-Auto-V.csv b/csvoutput/Grand-Theft-Auto-V.csv
diff --git a/csvoutput/Hogwarts-Legacy.csv b/csvoutput/Hogwarts-Legacy.csv
diff --git a/csvoutput/It-Takes-Two.csv b/csvoutput/It-Takes-Two.csv
diff --git a/csvoutput/LEGO-Star-Wars:-The-Skywalker-Saga.csv b/csvoutput/LEGO-Star-Wars:-The-Skywalker-Saga.csv
diff --git a/csvoutput/Monster-Hunter:-World.csv b/csvoutput/Monster-Hunter:-World.csv
diff --git a/csvoutput/Overcooked!-2.csv b/csvoutput/Overcooked!-2.csv
diff --git a/csvoutput/Red-Dead-Redemption-2.csv b/csvoutput/Red-Dead-Redemption-2.csv
diff --git a/csvoutput/Sid-Meier’s-Civilization-VI.csv b/csvoutput/Sid-Meier’s-Civilization-VI.csv
diff --git a/csvoutput/Tomb-Raider.csv b/csvoutput/Tomb-Raider.csv
diff --git a/webparse/__pycache__/steam_review_parse.cpython-38.pyc b/webparse/__pycache__/steam_review_parse.cpython-38.pyc
diff --git a/webtools/rolling.py b/webtools/rolling.py
@@ -10,52 +10,70 @@
 from webparse.steam_review_parse import steam_helper
 
 # User Configuration
-infinity_search = 0
+infinity_search = 1
 target_page = 10  # 100 pages contain about 1000 reviews
 waiting_time = 1  # range of waiting in each time
 tolerated_times = 60*10  # number of tolerant times for each page
 
-# Initialize
-driver = webdriver.Chrome()
-# Crawl Target
-driver.implicitly_wait(waiting_time)
-driver.get('https://steamcommunity.com/app/990080/reviews/?browsefilter=toprated&snr=1_5_100010_')
-
-# Start Crawling
-page_counter = 1
-current_tolerated = tolerated_times
-while True:
-
-    try:
-        curr_id = 'page' + str(page_counter)
-        time.sleep(waiting_time)
-        driver.find_element(By.ID, curr_id)
-        print(curr_id)
-    except NoSuchElementException:
-        # Not find
-        # - rolling -> to avoid too much rolling - since one window maybe contain several page block
-        driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
-        # - decrease tolerant
-        if current_tolerated > 0:
-            current_tolerated -= 1
-            continue
+games_dict = {
+    'Hogwarts Legacy': 'https://steamcommunity.com/app/990080/reviews/?browsefilter=toprated&snr=1_5_100010_',
+    'Forza Horizon 5': 'https://steamcommunity.com/app/1551360/reviews/?browsefilter=toprated&snr=1_5_100010_',
+    'Cities: Skylines': 'https://steamcommunity.com/app/255710/reviews/?browsefilter=toprated&snr=1_5_100010_',
+    'Atomic Heart': 'https://steamcommunity.com/app/668580/reviews/?browsefilter=toprated&snr=1_5_100010_',
+    'It Takes Two': 'https://steamcommunity.com/app/1426210/reviews/?browsefilter=toprated&snr=1_5_100010_',
+    'Sid Meier’s Civilization VI': 'https://steamcommunity.com/app/289070/reviews/?browsefilter=toprated&snr=1_5_100010_',
+    'Red Dead Redemption 2': 'https://steamcommunity.com/app/1174180/reviews/?browsefilter=toprated&snr=1_5_100010_',
+    'Grand Theft Auto V': 'https://steamcommunity.com/app/271590/reviews/?browsefilter=toprated&snr=1_5_100010_',
+    'Overcooked! 2': 'https://steamcommunity.com/app/728880/reviews/?browsefilter=toprated&snr=1_5_100010_',
+    'Monster Hunter: World': 'https://steamcommunity.com/app/582010/reviews/?browsefilter=toprated&snr=1_5_100010_',
+    'Tomb Raider': 'https://steamcommunity.com/app/203160/reviews/?browsefilter=toprated&snr=1_5_100010_',
+    'Cyberpunk 2077': 'https://steamcommunity.com/app/1091500/reviews/?browsefilter=toprated&snr=1_5_100010_',
+    'LEGO Star Wars: The Skywalker Saga': 'https://steamcommunity.com/app/920210/reviews/?browsefilter=toprated&snr=1_5_100010_'
+}
+
+for games_title in games_dict:
+
+    # Initialize
+    driver = webdriver.Chrome()
+    # Crawl Target
+    driver.implicitly_wait(waiting_time)
+    driver.get(games_dict[games_title])
+
+    # Start Crawling
+    page_counter = 1
+    current_tolerated = tolerated_times
+    while True:
+
+        try:
+            curr_id = 'page' + str(page_counter)
+            time.sleep(waiting_time)
+            driver.find_element(By.ID, curr_id)
+            print(curr_id)
+        except NoSuchElementException:
+            # Not find
+            # - rolling -> to avoid too much rolling - since one window maybe contain several page block
+            driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
+            # - decrease tolerant
+            if current_tolerated > 0:
+                current_tolerated -= 1
+                continue
+            else:
+                # Reach the end or Network break
+                break
         else:
-            # Reach the end or Network break
-            break
-    else:
-        # Find new page, reset tolerated counter
-        if current_tolerated != tolerated_times:
-            current_tolerated = tolerated_times
-        # Page counter + 1
-        page_counter += 1
-        if (page_counter == target_page) & (infinity_search == 0):
-            print('Finish searching.')
-            break
-
-# Start Parsing
-selector = Selector(text=driver.page_source)
-
-steam_helper(selector, 'Hogwarts-Legacy.csv')
-
-driver.quit()
+            # Find new page, reset tolerated counter
+            if current_tolerated != tolerated_times:
+                current_tolerated = tolerated_times
+            # Page counter + 1
+            page_counter += 1
+            if (page_counter == target_page) & (infinity_search == 0):
+                print('Finish searching.')
+                break
+
+    # Start Parsing
+    selector = Selector(text=driver.page_source)
+
+    steam_helper(selector, '-'.join(games_title.split(' ')) + '.csv')
+
+    driver.quit()
 
diff --git a/webtools/viewer.py b/webtools/viewer.py
@@ -0,0 +1,14 @@
+
+import os
+
+import pandas as pd
+
+csv_path = '../csvoutput'
+
+total_length = 0
+for file in os.listdir(csv_path):
+    df = pd.read_csv(os.path.join(csv_path, file), index_col=0, header=0)
+    total_length += len(df)
+    print(df)
+
+print('Total length:', total_length)