oold and opensemantic package integration

setup.cfg

@@ -48,6 +48,9 @@ package_dir =
 # new major versions. This works if the required packages follow Semantic Versioning.
 # For more information, check out https://semver.org/.
 install_requires =
+    oold
+    opensemantic
+    opensemantic.core
     pydantic>=1.10.17
     datamodel-code-generator>=0.25
     mwclient>=0.11.0
@@ -68,7 +71,6 @@ install_requires =
     asyncio
     tqdm
     pybars3-wheel
-    backports.strenum; python_version<"3.11"
 
 [options.packages.find]
 where = src

src/osw/auth.py

@@ -7,10 +7,10 @@
 from warnings import warn
 
 import yaml
+from opensemantic import OswBaseModel
 from pydantic.v1 import PrivateAttr
 
 from osw.defaults import paths as default_paths
-from osw.model.static import OswBaseModel
 
 if TYPE_CHECKING:
     PossibleFilePath = Path

src/osw/controller/database.py

@@ -1,13 +1,13 @@
 from typing import Optional, Union
 
+from opensemantic import OswBaseModel
 from sqlalchemy import URL, create_engine
 from sqlalchemy import text as sql_text
 from sqlalchemy.engine import Engine
 
 import osw.model.entity as model
 from osw.auth import CredentialManager
 from osw.core import OSW
-from osw.model.static import OswBaseModel
 
 
 class DatabaseController(model.Database):

src/osw/controller/page_package.py

@@ -6,14 +6,14 @@
 from typing import Optional, Union
 from warnings import warn
 
+from opensemantic import OswBaseModel
 from pydantic.v1 import FilePath
 from typing_extensions import Dict, List
 
 import osw.model.page_package as model
 from osw.auth import CredentialManager
 from osw.model import page_package as package
 from osw.model.page_package import NAMESPACE_CONST_TO_NAMESPACE_MAPPING
-from osw.model.static import OswBaseModel
 from osw.utils.regex import RegExPatternExtended
 from osw.wtsite import WtPage, WtSite
 
@@ -372,6 +372,17 @@ class CreationConfig(OswBaseModel):
         offline_pages: Optional[Dict[str, WtPage]] = None
         """A dictionary of pages that are already loaded. Pages in this dictionary
         will not be fetched again."""
+        prefer_local_pages: bool = False
+        """Load the pages from the local working directory
+        instead of the server if set to True."""
+        generate_python_code: bool = False
+        """Whether to generate python code for the data models."""
+        python_code_working_dir: Optional[Union[str, Path]] = None
+        """Working directory for python code generation. If set, pydantic v2 data models
+        will be generated in this directory, v1 models in a /v1 subdirectory.
+        """
+        python_code_filename: Optional[str] = "_model_generated.py"
+        """Filename for the generated python code."""
 
         class Config:
             arbitrary_types_allowed = True
@@ -418,6 +429,57 @@ def create(
                 )
             },
         )
+
+        offline_pages = creation_config.offline_pages
+        local_pages = {}
+        if creation_config.prefer_local_pages:
+            # Read the local pages from the package
+            result = wtsite.read_page_package(
+                WtSite.ReadPagePackageParam(
+                    package_name=self.name,
+                    storage_path=Path(creation_config.working_dir),
+                )
+            )
+            local_pages = {p.title: p for p in result.pages}
+            if offline_pages is None:
+                offline_pages = local_pages
+            else:
+                # Merge the local pages with the offline pages
+                offline_pages.update(local_pages)
+
+        if (
+            creation_config.generate_python_code
+            and creation_config.python_code_working_dir is not None
+        ):
+            python_code_path = Path(creation_config.python_code_working_dir)
+            python_code_path /= creation_config.python_code_filename
+            schema_titles = self.page_titles
+            # remove duplicates and entries in ignore_titles
+            schema_titles = list(
+                set(schema_titles)
+                - (
+                    set(creation_config.ignore_titles)
+                    if creation_config.ignore_titles
+                    else set()
+                )
+            )
+            # remove all schemas that do not start with "Category:"
+            schema_titles = [
+                title for title in schema_titles if title.startswith("Category:")
+            ]
+            from osw.core import OSW
+
+            osw_obj = OSW(site=wtsite)
+
+            osw_obj.fetch_schema(
+                fetchSchemaParam=OSW.FetchSchemaParam(
+                    schema_title=schema_titles,
+                    offline_pages=offline_pages,
+                    result_model_path=python_code_path,
+                    mode="replace",
+                )
+            )
+
         # Create a PagePackageConfig instance
         config = package.PagePackageConfig(
             name=self.name,
@@ -432,7 +494,7 @@ def create(
         wtsite.create_page_package(
             WtSite.CreatePagePackageParam(
                 config=config,
-                offline_pages=creation_config.offline_pages,
+                offline_pages=offline_pages,
             )
         )