implement new circular checking method that uses api endpoint

this uses the /new-circular api endpoint instead of caching every single circular and then manually re-checking

Author: Rajdave69

pybpsapi.py

@@ -1,3 +1,4 @@
+import pickle
 import warnings
 import mysql.connector
 import requests
@@ -106,11 +107,11 @@ def getpng(self, url: str) -> list | None:
 
 
 class CircularChecker:
-    def __init__(self, category: str | int, url: str = "https://bpsapi.rajtech.me/", cache_method: str = 'sqlite', **kwargs):
+    def __init__(self, category: str | int = None, url: str = "https://bpsapi.rajtech.me/", cache_method: str = 'sqlite', **kwargs):
         self.url = url
         self.category = category
         self.cache_method = cache_method
-        self._cache = []
+        self.latest_circular_id = self.get_cache()
 
         # Get category names from API
         json = requests.get(self.url + "categories").json()
@@ -120,17 +121,14 @@ def __init__(self, category: str | int, url: str = "https://bpsapi.rajtech.me/",
         else:
             raise ConnectionError("Invalid API Response. API says there are no categories.")
 
-        if kwargs.get("debug"):
-            self.set_cache = self._set_cache
-            self.refresh_cache = self._refresh_cache
-
         # If category id is passed
-        if type(self.category) is int:
-            if not _min_category_id <= self.category:
-                raise ValueError("Invalid category Number")
-        else:   # If category name is passed
-            if self.category not in categories:
-                raise ValueError("Invalid category Name")
+        if category is not None:
+            if type(self.category) is int:
+                if not _min_category_id <= self.category:
+                    raise ValueError("Invalid category Number")
+            else:   # If category name is passed
+                if self.category not in categories:
+                    raise ValueError("Invalid category Name")
 
         # For the sqlite cache method
         if self.cache_method == "sqlite":
@@ -146,8 +144,6 @@ def __init__(self, category: str | int, url: str = "https://bpsapi.rajtech.me/",
             if not os.path.exists(self.db_path + f"/{self.db_name}.db"):
                 os.mkdir(self.db_path)
 
-            self._con = sqlite3.connect(self.db_path + f"/{self.db_name}.db")
-            self._cur = self._con.cursor()
 
         # For the mysql/mariadb cache method
         elif cache_method == "mysql":
@@ -163,163 +159,102 @@ def __init__(self, category: str | int, url: str = "https://bpsapi.rajtech.me/",
                 raise ValueError(
                     "Invalid Database Parameters. One of db_name, db_user, db_host, db_port, db_password, db_table not passed into kwargs")
 
-            self._con = mysql.connector.connect(
-                host=self.db_host, port=self.db_port, password=self.db_password,
-                user=self.db_user, database=self.db_name,
-            )
-            self._cur = self._con.cursor(prepared=True)
+        elif cache_method == 'pickle':
+            try:
+                self.cache_file = kwargs['cache_file']
+            except KeyError:
+                raise ValueError("Invalid cache file path")
 
         else:
             raise ValueError("Invalid cache method. Only mysql and sqlite allowed")
 
         # Create a table to cache circulars if it's not there
-        self._cur.execute(
-            f"""
-        CREATE TABLE IF NOT EXISTS  {self.db_table}  (
-            category	TEXT,
-            id	INTEGER UNIQUE,
-            title	TEXT,
-            link	TEXT
-        )
-        """
-        )
-        self._con.commit()
+        if self.cache_method in ('sqlite', 'mysql'):
+            con, cur = self._get_db()
+
+            cur.execute(
+                f"""
+            CREATE TABLE IF NOT EXISTS  {self.db_table}  (
+                category	TEXT,
+                id	INTEGER UNIQUE,
+                title	TEXT,
+                link	TEXT
+            )
+            """
+            )
+            con.commit()
+            con.close()
 
+        # If cache method is pickle, create a file if it doesn't exist
+        elif self.cache_method == 'pickle':
+            if not os.path.exists(self.cache_file):
+                with open(self.cache_file, 'wb') as f:
+                    f.write(b'')
 
-    # Method to retrieve cache from the database
-    def get_cache(self) -> list[list] | list:
-        self._cur.execute(f"SELECT id, title, link FROM {self.db_table} WHERE category = ?", (self.category,))
-        res = self._cur.fetchall()
+    def _get_db(self):
+        if self.cache_method == 'mysql':
+            con = mysql.connector.connect(
+                host=self.db_host, port=self.db_port, password=self.db_password,
+                user=self.db_user, database=self.db_name,
+            )
+            cur = con.cursor(prepared=True)
 
-        return res
+        elif self.cache_method == 'sqlite':
+            con = sqlite3.connect(self.db_path + f"/{self.db_name}.db")
+            cur = con.cursor()
+        else:
+            raise ValueError("Method not supported for this cache method")
 
-    # Method to add multiple items to cache
-    def _set_cache(self, data):
-        # data [ (id, title, link) ]
-        query = f"INSERT OR IGNORE INTO {self.db_table} (category, id, title, link) VALUES (?, ?, ?, ?)"
+        return con, cur
 
-        if self.cache_method == 'mysql':
-            query = query.replace("OR ", "")
 
-        self._cur.executemany(query, tuple((self.category, *d) for d in data))
-        self._con.commit()
+    # Method to retrieve cache from the database
+    def get_cache(self) -> int | None:
+        if self.cache_method in ('sqlite', 'mysql'):
+            con, cur = self._get_db()
 
-    # Method to add a single item to cache
-    def _add_to_cache(self, id_: int, title: str, link: str):
-        query = f"INSERT OR IGNORE INTO {self.db_table} (id, title, link) VALUES (?, ?, ?, ?)"
+            cur.execute(f"SELECT latest_circular_id FROM {self.db_table} WHERE category = ?", (self.category,))
+            res = cur.fetchone()
 
-        if self.cache_method == 'mysql':
-            query = query.replace("OR ", "")
+        elif self.cache_method == 'pickle':
+            with open(self.cache_file, 'rb') as f:
+                res = pickle.load(f)
 
-        self._cur.execute(query, (self.category, id_, title, link))
+        return res
 
-    # Method to retrieve circulars from the API and insert into cache
-    def _refresh_cache(self):
-        request = requests.get(f"{self.url}list/{self.category}")
-        json: dict = request.json()
+    # Method to add multiple items to cache
+    def _set_cache(self, circular_id: int):
 
-        if json.get("data") is None or json.get("http_status") is None:
-            raise ConnectionError("Invalid API Response, it doesn't contain either 'data' or 'http_code'")
+        if self.cache_method in ('sqlite', 'mysql'):
+            con, cur = self._get_db()
+            query = f"INSERT OR IGNORE INTO {self.db_table} (category, latest_circular_id) VALUES (?, ?)"
 
-        self._cur.execute(f"SELECT id FROM {self.db_table} WHERE category = ?", (self.category,))
+            if self.cache_method == 'mysql':
+                query = query.replace("OR ", "")
 
-        # ((1234,), (4567,), ...) -> ('1234', '4567')
-        cached_ids: list = self._cur.fetchall()
-        cached_ids: tuple[str, ...] = tuple([str(i[0]) for i in cached_ids])
+            cur.execute(query, (self.category, circular_id,))
+            con.commit()
+
+        elif self.cache_method == 'pickle':
+            with open(self.cache_file, 'wb') as f:
+                pickle.dumps(circular_id, f)
 
-        if json['http_status'] == 200:
-            self._set_cache(
-                [
-                    (i['id'], i['title'], i['link'])
-                    for i in json['data']
-                    if i['id'] not in cached_ids    # Add only new circulars to the database
-                ]
-            )
 
     # Method to check for new circular(s)
     def check(self) -> list[dict] | list:
-        # First get cached circulars and store them in a variable 'cached_circular_ids'
-        # Then refresh cache and get the new list of circulars, and then compare and find new ones.
-        self._cur.execute(f"SELECT id FROM {self.db_table} WHERE category = ?", (self.category,))
+        res = requests.get(self.url + f"new-circulars/{self.latest_circular_id}").json()['data']
+        # it's sorted in descending order
 
-        cached_circular_ids = self._cur.fetchall()
-        cached_circular_ids = [i[0] for i in cached_circular_ids]   # [(id, title, link)]
+        if len(res) > 0:
+            self._set_cache(res[0]['id'])
 
-        self._refresh_cache()
-        new_circular_list = self.get_cache() #
+            if self.category:
+                res = [circular for circular in res if circular['category'] == self.category]
 
-        # If there are new circulars
-        if len(new_circular_list) > len(cached_circular_ids):
-            new_circular_objects = [i for i in new_circular_list if i[0] not in cached_circular_ids]
-
-            # (id, title, link) -> {'id': id, 'title': title, 'link': link}
-            new_circular_objects = [
-                {
-                    'id': i[0],
-                    'title': i[1],
-                    'link': i[2]
-                }
-                for i in new_circular_objects
-            ]
-
-            # sort the new_circular_objects by circular id in ascending order
-            new_circular_objects.sort(key=lambda x: x['id'])
-            return new_circular_objects
+        return res
 
-        return []
 
     # Close connections when object is deleted
     def __del__(self):
-        if hasattr(self, '_con'):
-            self._con.close()
-
-
-class CircularCheckerGroup:
-    def __init__(self, *circular_checkers: CircularChecker, **kwargs):
-        self._checkers = []
-
-        # Add each checker to self._checkers
-        for checker in circular_checkers:
-            if type(checker) is not CircularChecker:
-                raise ValueError("Invalid CircularChecker Object")
-            self._checkers.append(checker)
-
-        if bool(kwargs.get("debug")):
-            self.checkers = self._checkers
-
-    # Method to add a circular checker to this group
-    def add(self, checker: CircularChecker, *circular_checkers: CircularChecker):
-        self._checkers.append(checker)
-
-        for checker in circular_checkers:
-            if type(checker) is not CircularChecker:
-                raise ValueError("Invalid CircularChecker Object")
-            self._checkers.append(checker)
-
-    # Method to create a circular checker and add it to the group
-    def create(self, category, url: str = "https://bpsapi.rajtech.me/", cache_method=None, **kwargs):
-        checker = CircularChecker(category, url, cache_method, **kwargs)
-        self._checkers.append(checker)
-
-    # Method to check for new circulars in each one of the checkers
-    def check(self) -> dict[list[dict], ...] | dict:
-        return_dict = {}
-        for checker in self._checkers:
-            return_dict[checker.category] = checker.check()
-        return return_dict
-
-    # Method to refresh (sync) cache from API
-    def refresh_cache(self):
-        for checker in self._checkers:
-            checker.refresh_cache()
-
-    # Method to get the cache of all checkers
-    def get_cache(self) -> dict[list[list]] | dict:
-        return_dict = {}
-        for checker in self._checkers:
-            return_dict[checker.category] = checker.get_cache()
-        return return_dict
+        pass
 
-    def __del__(self):
-        for checker in self._checkers:
-            del checker