feat(parity): migrate json libraries to sqlite (#604)

* feat(ui): add PagedPanel widget

* feat(ui): add MigrationModal widget

* feat: add basic json to sql conversion

* fix: chose `poolclass` based on file or memory db

* feat: migrate tag colors from json to sql

* feat: migrate entry fields from json to sql

- fix: tag name column no longer has unique constraint
- fix: tags are referenced by id in db queries
- fix: tag_search_panel no longer queries db on initialization; does not regress to empty search window when shown
- fix: tag name search no longer uses library grid FilterState object
- fix: tag name search now respects tag limit

* set default `is_new` case

* fix: limit correct tag query

* feat: migrate tag aliases and subtags from json to sql

* add migration timer

* fix(tests): fix broken tests

* rename methods, add docstrings

* revert tag id search, split tag name search

* fix: use correct type in sidecar macro

* tests: add json migration tests

* fix: drop leading dot from json extensions

* add special characters to json db test

* tests: add file path and entry field parity checks

* fix(ui): tag manager no longer starts empty

* fix: read old windows paths as posix

Addresses #298

* tests: add posix + windows paths to json library

* tests: add subtag, alias, and shorthand parity tests

* tests: ensure no none values in parity checks

* tests: add tag color test, use tag id in tag tests

* tests: fix and optimize tests

* tests: add discrepancy tracker

* refactor: reduce duplicate UI code

* fix: load non-sequential entry ids

* fix(ui): sort tags in the preview panel

* tests(fix): prioritize `None` check over equality

* fix(tests): fix multi "same tag field type" tests

* ui: increase height of migration modal

* feat: add progress bar to migration ui

* fix(ui): sql values update earlier

* refactor: use `get_color_from_str` in test

* refactor: migrate tags before aliases and subtags

* remove unused assertion

* refactor: use `json_migration_req` flag

Author: CyanVoxel

.gitignore

@@ -55,7 +55,6 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 cover/
-tagstudio/tests/fixtures/library/*
 
 # Translations
 *.mo
@@ -255,11 +254,14 @@ compile_commands.json
 # Ignore all local history of files
 .history
 .ionide
+# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,python,qt
 
 # TagStudio
 .TagStudio
+!*/tests/**/.TagStudio
+tagstudio/tests/fixtures/library/*
+tagstudio/tests/fixtures/json_library/.TagStudio/*.sqlite
 TagStudio.ini
-# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,python,qt
 
 .envrc
 .direnv

tagstudio/src/core/library/alchemy/enums.py

@@ -42,6 +42,13 @@ class TagColor(enum.IntEnum):
     COOL_GRAY = 36
     OLIVE = 37
 
+    @staticmethod
+    def get_color_from_str(color_name: str) -> "TagColor":
+        for color in TagColor:
+            if color.name == color_name.upper().replace(" ", "_"):
+                return color
+        return TagColor.DEFAULT
+
 
 class SearchMode(enum.IntEnum):
     """Operational modes for item searching."""

tagstudio/src/core/library/alchemy/library.py

@@ -1,5 +1,6 @@
 import re
 import shutil
+import time
 import unicodedata
 from dataclasses import dataclass
 from datetime import UTC, datetime
@@ -9,9 +10,11 @@
 from uuid import uuid4
 
 import structlog
+from humanfriendly import format_timespan
 from sqlalchemy import (
     URL,
     Engine,
+    NullPool,
     and_,
     create_engine,
     delete,
@@ -29,6 +32,7 @@
     make_transient,
     selectinload,
 )
+from src.core.library.json.library import Library as JsonLibrary  # type: ignore
 
 from ...constants import (
     BACKUP_FOLDER_NAME,
@@ -122,6 +126,7 @@ class LibraryStatus:
     success: bool
     library_path: Path | None = None
     message: str | None = None
+    json_migration_req: bool = False
 
 
 class Library:
@@ -133,7 +138,8 @@ class Library:
     folder: Folder | None
     included_files: set[Path] = set()
 
-    FILENAME: str = "ts_library.sqlite"
+    SQL_FILENAME: str = "ts_library.sqlite"
+    JSON_FILENAME: str = "ts_library.json"
 
     def close(self):
         if self.engine:
@@ -143,32 +149,119 @@ def close(self):
         self.folder = None
         self.included_files = set()
 
+    def migrate_json_to_sqlite(self, json_lib: JsonLibrary):
+        """Migrate JSON library data to the SQLite database."""
+        logger.info("Starting Library Conversion...")
+        start_time = time.time()
+        folder: Folder = Folder(path=self.library_dir, uuid=str(uuid4()))
+
+        # Tags
+        for tag in json_lib.tags:
+            self.add_tag(
+                Tag(
+                    id=tag.id,
+                    name=tag.name,
+                    shorthand=tag.shorthand,
+                    color=TagColor.get_color_from_str(tag.color),
+                )
+            )
+
+        # Tag Aliases
+        for tag in json_lib.tags:
+            for alias in tag.aliases:
+                self.add_alias(name=alias, tag_id=tag.id)
+
+        # Tag Subtags
+        for tag in json_lib.tags:
+            for subtag_id in tag.subtag_ids:
+                self.add_subtag(parent_id=tag.id, child_id=subtag_id)
+
+        # Entries
+        self.add_entries(
+            [
+                Entry(
+                    path=entry.path / entry.filename,
+                    folder=folder,
+                    fields=[],
+                    id=entry.id + 1,  # JSON IDs start at 0 instead of 1
+                )
+                for entry in json_lib.entries
+            ]
+        )
+        for entry in json_lib.entries:
+            for field in entry.fields:
+                for k, v in field.items():
+                    self.add_entry_field_type(
+                        entry_ids=(entry.id + 1),  # JSON IDs start at 0 instead of 1
+                        field_id=self.get_field_name_from_id(k),
+                        value=v,
+                    )
+
+        # Preferences
+        self.set_prefs(LibraryPrefs.EXTENSION_LIST, [x.strip(".") for x in json_lib.ext_list])
+        self.set_prefs(LibraryPrefs.IS_EXCLUDE_LIST, json_lib.is_exclude_list)
+
+        end_time = time.time()
+        logger.info(f"Library Converted! ({format_timespan(end_time-start_time)})")
+
+    def get_field_name_from_id(self, field_id: int) -> _FieldID:
+        for f in _FieldID:
+            if field_id == f.value.id:
+                return f
+        return None
+
     def open_library(self, library_dir: Path, storage_path: str | None = None) -> LibraryStatus:
+        is_new: bool = True
         if storage_path == ":memory:":
             self.storage_path = storage_path
             is_new = True
+            return self.open_sqlite_library(library_dir, is_new)
         else:
-            self.verify_ts_folders(library_dir)
-            self.storage_path = library_dir / TS_FOLDER_NAME / self.FILENAME
-            is_new = not self.storage_path.exists()
+            self.storage_path = library_dir / TS_FOLDER_NAME / self.SQL_FILENAME
+
+            if self.verify_ts_folder(library_dir) and (is_new := not self.storage_path.exists()):
+                json_path = library_dir / TS_FOLDER_NAME / self.JSON_FILENAME
+                if json_path.exists():
+                    return LibraryStatus(
+                        success=False,
+                        library_path=library_dir,
+                        message="[JSON] Legacy v9.4 library requires conversion to v9.5+",
+                        json_migration_req=True,
+                    )
+
+        return self.open_sqlite_library(library_dir, is_new)
 
+    def open_sqlite_library(
+        self, library_dir: Path, is_new: bool, add_default_data: bool = True
+    ) -> LibraryStatus:
         connection_string = URL.create(
             drivername="sqlite",
             database=str(self.storage_path),
         )
+        # NOTE: File-based databases should use NullPool to create new DB connection in order to
+        # keep connections on separate threads, which prevents the DB files from being locked
+        # even after a connection has been closed.
+        # SingletonThreadPool (the default for :memory:) should still be used for in-memory DBs.
+        # More info can be found on the SQLAlchemy docs:
+        # https://docs.sqlalchemy.org/en/20/changelog/migration_07.html
+        # Under -> sqlite-the-sqlite-dialect-now-uses-nullpool-for-file-based-databases
+        poolclass = None if self.storage_path == ":memory:" else NullPool
 
-        logger.info("opening library", library_dir=library_dir, connection_string=connection_string)
-        self.engine = create_engine(connection_string)
+        logger.info(
+            "Opening SQLite Library", library_dir=library_dir, connection_string=connection_string
+        )
+        self.engine = create_engine(connection_string, poolclass=poolclass)
         with Session(self.engine) as session:
             make_tables(self.engine)
 
-            tags = get_default_tags()
-            try:
-                session.add_all(tags)
-                session.commit()
-            except IntegrityError:
-                # default tags may exist already
-                session.rollback()
+            if add_default_data:
+                tags = get_default_tags()
+                try:
+                    session.add_all(tags)
+                    session.commit()
+                except IntegrityError:
+                    # default tags may exist already
+                    session.rollback()
 
             # dont check db version when creating new library
             if not is_new:
@@ -219,7 +312,6 @@ def open_library(self, library_dir: Path, storage_path: str | None = None) -> Li
                     db_version=db_version.value,
                     expected=LibraryPrefs.DB_VERSION.default,
                 )
-                # TODO - handle migration
                 return LibraryStatus(
                     success=False,
                     message=(
@@ -354,8 +446,12 @@ def tags(self) -> list[Tag]:
 
         return list(tags_list)
 
-    def verify_ts_folders(self, library_dir: Path) -> None:
-        """Verify/create folders required by TagStudio."""
+    def verify_ts_folder(self, library_dir: Path) -> bool:
+        """Verify/create folders required by TagStudio.
+
+        Returns:
+            bool: True if path exists, False if it needed to be created.
+        """
         if library_dir is None:
             raise ValueError("No path set.")
 
@@ -366,6 +462,8 @@ def verify_ts_folders(self, library_dir: Path) -> None:
         if not full_ts_path.exists():
             logger.info("creating library directory", dir=full_ts_path)
             full_ts_path.mkdir(parents=True, exist_ok=True)
+            return False
+        return True
 
     def add_entries(self, items: list[Entry]) -> list[int]:
         """Add multiple Entry records to the Library."""
@@ -507,21 +605,23 @@ def search_library(
 
     def search_tags(
         self,
-        search: FilterState,
+        name: str,
     ) -> list[Tag]:
         """Return a list of Tag records matching the query."""
+        tag_limit = 100
+
         with Session(self.engine) as session:
             query = select(Tag)
             query = query.options(
                 selectinload(Tag.subtags),
                 selectinload(Tag.aliases),
-            )
+            ).limit(tag_limit)
 
-            if search.tag:
+            if name:
                 query = query.where(
                     or_(
-                        Tag.name.icontains(search.tag),
-                        Tag.shorthand.icontains(search.tag),
+                        Tag.name.icontains(name),
+                        Tag.shorthand.icontains(name),
                     )
                 )
 
@@ -531,7 +631,7 @@ def search_tags(
 
             logger.info(
                 "searching tags",
-                search=search,
+                search=name,
                 statement=str(query),
                 results=len(res),
             )
@@ -694,7 +794,7 @@ def add_entry_field_type(
         *,
         field: ValueType | None = None,
         field_id: _FieldID | str | None = None,
-        value: str | datetime | list[str] | None = None,
+        value: str | datetime | list[int] | None = None,
     ) -> bool:
         logger.info(
             "add_field_to_entry",
@@ -727,8 +827,11 @@ def add_entry_field_type(
 
             if value:
                 assert isinstance(value, list)
-                for tag in value:
-                    field_model.tags.add(Tag(name=tag))
+                with Session(self.engine) as session:
+                    for tag_id in list(set(value)):
+                        tag = session.scalar(select(Tag).where(Tag.id == tag_id))
+                        field_model.tags.add(tag)
+                        session.flush()
 
         elif field.type == FieldTypeEnum.DATETIME:
             field_model = DatetimeField(
@@ -760,6 +863,28 @@ def add_entry_field_type(
         )
         return True
 
+    def tag_from_strings(self, strings: list[str] | str) -> list[int]:
+        """Create a Tag from a given string."""
+        # TODO: Port over tag searching with aliases fallbacks
+        # and context clue ranking for string searches.
+        tags: list[int] = []
+
+        if isinstance(strings, str):
+            strings = [strings]
+
+        with Session(self.engine) as session:
+            for string in strings:
+                tag = session.scalar(select(Tag).where(Tag.name == string))
+                if tag:
+                    tags.append(tag.id)
+                else:
+                    new = session.add(Tag(name=string))
+                    if new:
+                        tags.append(new.id)
+                        session.flush()
+            session.commit()
+        return tags
+
     def add_tag(
         self,
         tag: Tag,
@@ -852,7 +977,7 @@ def save_library_backup_to_disk(self) -> Path:
         target_path = self.library_dir / TS_FOLDER_NAME / BACKUP_FOLDER_NAME / filename
 
         shutil.copy2(
-            self.library_dir / TS_FOLDER_NAME / self.FILENAME,
+            self.library_dir / TS_FOLDER_NAME / self.SQL_FILENAME,
             target_path,
         )
 
@@ -879,15 +1004,15 @@ def get_alias(self, tag_id: int, alias_id: int) -> TagAlias:
 
         return alias
 
-    def add_subtag(self, base_id: int, new_tag_id: int) -> bool:
-        if base_id == new_tag_id:
+    def add_subtag(self, parent_id: int, child_id: int) -> bool:
+        if parent_id == child_id:
             return False
 
         # open session and save as parent tag
         with Session(self.engine) as session:
             subtag = TagSubtag(
-                parent_id=base_id,
-                child_id=new_tag_id,
+                parent_id=parent_id,
+                child_id=child_id,
             )
 
             try:
@@ -899,6 +1024,22 @@ def add_subtag(self, base_id: int, new_tag_id: int) -> bool:
                 logger.exception("IntegrityError")
                 return False
 
+    def add_alias(self, name: str, tag_id: int) -> bool:
+        with Session(self.engine) as session:
+            alias = TagAlias(
+                name=name,
+                tag_id=tag_id,
+            )
+
+            try:
+                session.add(alias)
+                session.commit()
+                return True
+            except IntegrityError:
+                session.rollback()
+                logger.exception("IntegrityError")
+                return False
+
     def remove_subtag(self, base_id: int, remove_tag_id: int) -> bool:
         with Session(self.engine) as session:
             p_id = base_id