Skip to content

Added a counter for the accepted problems list, and updated the selen… #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 22 additions & 14 deletions crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service

from database import Problem, ProblemTag, Tag, Submission, create_tables, Solution
from utils import destructure, random_wait, do, get
Expand All @@ -21,7 +22,8 @@ class LeetCodeCrawler:
def __init__(self):
# create an http session
self.session = requests.Session()
self.browser = webdriver.Chrome(executable_path="./vendor/chromedriver")
service = Service(r"C:\Users\blau2\workspace\LeetCode-Anki\vendor\chromedriver.exe")
self.browser = webdriver.Chrome(service=service)
self.session.headers.update(
{
'Host': 'leetcode.com',
Expand Down Expand Up @@ -73,21 +75,27 @@ def login(self):
def fetch_accepted_problems(self):
response = self.session.get("https://leetcode.com/api/problems/all/")
all_problems = json.loads(response.content.decode('utf-8'))
# filter AC problems

# Filter the problems with status 'ac'
accepted_problems = [item for item in all_problems['stat_status_pairs'] if item['status'] == 'ac']

# Print the total number of accepted problems
print(f"🤖 Total accepted problems to process: {len(accepted_problems)}")

counter = 0
for item in all_problems['stat_status_pairs']:
if item['status'] == 'ac':
id, slug = destructure(item['stat'], "question_id", "question__title_slug")
# only update problem if not exists
if Problem.get_or_none(Problem.id == id) is None:
counter += 1
# fetch problem
do(self.fetch_problem, args=[slug, True])
# fetch solution
do(self.fetch_solution, args=[slug])
for item in accepted_problems:
id, slug = destructure(item['stat'], "question_id", "question__title_slug")
# Only update problem if it doesn't exist
if Problem.get_or_none(Problem.id == id) is None:
counter += 1
# Fetch problem
do(self.fetch_problem, args=[slug, True])
# Fetch solution
do(self.fetch_solution, args=[slug])

# always try to update submission
do(self.fetch_submission, args=[slug])
# Always try to update submission
do(self.fetch_submission, args=[slug])

print(f"🤖 Updated {counter} problems")

def fetch_problem(self, slug, accepted=False):
Expand Down
Binary file added data/LeetCode.apkg
Binary file not shown.
Binary file modified data/LeetCode.sqlite
Binary file not shown.
7 changes: 4 additions & 3 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
requests~=2.24.0
selenium~=3.141.0
selenium>=4.10.0
peewee~=3.13.3
genanki~=0.8.1
markdown~=3.1.1
python-markdown-math
python-markdown-math
requests>=2.31.0
urllib3>=1.26.5
8 changes: 8 additions & 0 deletions test_script.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
from selenium.webdriver.chrome.service import Service
from selenium import webdriver

service = Service(r"C:\Users\blau2\workspace\LeetCode-Anki\vendor\chromedriver.exe")
driver = webdriver.Chrome(service=service)
driver.get("https://www.google.com")
print("Browser launched successfully!")
driver.quit()
Binary file modified vendor/chromedriver.exe
Binary file not shown.