From f252a86fd546f38760ae556ffd3354d669e088bb Mon Sep 17 00:00:00 2001 From: Jann Stute <46534683+Computerdores@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:36:55 +0200 Subject: [PATCH] refactor: split out sql migrations from library (#1432) * refactor: minor simplification * refactor: split open_library into new and not new case Functionality is entirely unchanged, but entire method is duplicated. Removing dead code from either copy is next. * refactor: remove dead code * doc: add todos and remove trivially true assert * refactor: add assurance 1 version check Assurance 1 was introduced with library version 101, specifically commmit 12e074b71d8860282b44e49e0e1a41b7a2e4bae8. Thus all libraries created on version 101 and above already fullfill it and don't need it. Furthermore, it only fails if the library didn't need the assurance, meaning the try-except and warning catching can be removed * refactor: remove various unnecessary try-except statements in new_lib All of these were introduced to account for the differences between new libraries and existing ones, which makes them unnecessary after this split. * refactor: remove unnecessary check in new_lib * refactor: move folder assurance after migrations The folder assurance has been present since the very first SQL commit e5e7b8afc62c1a958c172b35819caa26995f2060, and thus only has an effect when the library is moved, meaning that is semantically not part of the migrations. * refactor: massively simplify open_library * refactor: move engine creation to static method * refactor: add version check for assurance 3 Assurance 3 was introduced in commit 47c3d5338f4c3ef2cde25d8c1bd4dcaa6d14a28f shortly (24h 20min) after the DB version had been bumped to 200. Since it was also included in the first release with DB version 200, all libraries created on DB version 200 and above can be presumed to have already had this applied on creation. * refactor: add assurance 3 to DB 200 migration * refactor: move assurance 1 to a proper migration method Moving the assurance after the migrations 7, 8, 9, and 100 is fine because it doesn't affect those. * refactor: update version after every successfull migration Since every migration migrates the library to a certain DB_VERSION, we can simply update the library version after such a successfull migration. This way if a migration fails, the previous migrations won't be rerun the next time the library is opened. * refactor: rewrite migration procedure as loop * refactor: apply migration and update version in same transaction None of the content of the migrations is changed, only the `with session:` statements are removed. * refactor: replace all commits in the migrations with flushes Also removes various try-except statements in the migrations. These were introduced to catch the case where the migration is run twice due to a later migration failing, this is not necessary anymore as every migration will only be executed at most once per library. * refactor: make sure the migration log statements are consistent * fix: pass library dir to migrations * fix(db migration 8): only add colors that are actually new DB Migration 8 was previously adding all colors except for the neon ones, however, all but three of those have been around since DB version 4, meaning that they don't need to be added at all (which caused the tests to fail). * fix: json migration used outdated interface * fix(open_library): create TS directory only if not opened in memory * fix: enable sane transaction behaviour By default in SQLAlchemy schema changes are automatically committed, even if auto-commit is turned off. This commit turns off auto-commit completely, which caused some schema to not be applied correctly, but those were fixed here as well. * refactor: hide 'argument is not accessed' notices * fix(db migration 9): filename property wasn't written correctly * fix(db migration 104): include/exclude list was loaded incorrectly * feat: log start and end of DB migrations * fix: don't use double transaction in open_sqlite_library * fix: only commit once when creating new library * doc: add comment on removing Folder logic * refactor: remove dunder naming --- src/tagstudio/core/library/alchemy/db.py | 16 +- src/tagstudio/core/library/alchemy/library.py | 888 +++++++++--------- src/tagstudio/qt/mixed/migration_modal.py | 11 +- 3 files changed, 448 insertions(+), 467 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/db.py b/src/tagstudio/core/library/alchemy/db.py index de119d0a..d8d520d7 100644 --- a/src/tagstudio/core/library/alchemy/db.py +++ b/src/tagstudio/core/library/alchemy/db.py @@ -42,13 +42,17 @@ def make_engine(connection_string: str) -> Engine: def make_tables(engine: Engine) -> None: logger.info("[Library] Creating DB tables...") - Base.metadata.create_all(engine) - - # tag IDs < 1000 are reserved - # create tag and delete it to bump the autoincrement sequence - # TODO - find a better way - # is this the better way? with engine.connect() as conn: + # TODO: this should instead be migrations that create the exact tables that were added in + # the respective DB versions + Base.metadata.create_all(conn) + conn.commit() + + # TODO: this needs to be a migration + # tag IDs < 1000 are reserved + # create tag and delete it to bump the autoincrement sequence + # TODO - find a better way + # is this the better way? result = conn.execute(text("SELECT SEQ FROM sqlite_sequence WHERE name='tags'")) autoincrement_val = result.scalar() if not autoincrement_val or autoincrement_val <= RESERVED_TAG_END: diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index d7cf6905..ea9f2059 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -19,10 +19,10 @@ from os import makedirs from pathlib import Path from typing import TYPE_CHECKING from uuid import uuid4 -from warnings import catch_warnings import sqlalchemy import structlog +import ujson from humanfriendly import format_timespan # pyright: ignore[reportUnknownVariableType] from sqlalchemy import ( URL, @@ -381,20 +381,19 @@ class Library: return tag.name def open_library(self, library_dir: Path, in_memory: bool = False) -> LibraryStatus: - """Wrapper for open_sqlite_library. + """Wrapper for open_sqlite_library and create_sqlite_library. Handles in-memory storage and checks whether a JSON-migration is necessary. """ assert isinstance(library_dir, Path) - if in_memory: - return self.open_sqlite_library(library_dir, is_new=True, storage_path=":memory:") - - is_new = True sql_path = library_dir / TS_FOLDER_NAME / SQL_FILENAME - if self.verify_ts_folder(library_dir) and (is_new := not sql_path.exists()): - json_path = library_dir / TS_FOLDER_NAME / JSON_FILENAME - if json_path.exists(): + json_path = library_dir / TS_FOLDER_NAME / JSON_FILENAME + + is_new = not sql_path.exists() + if not in_memory: + self.verify_ts_folder(library_dir) # ensure .TagStudio directory exists + if is_new and json_path.exists(): return LibraryStatus( success=False, library_path=library_dir, @@ -402,14 +401,18 @@ class Library: json_migration_req=True, ) - return self.open_sqlite_library(library_dir, is_new, str(sql_path)) + if is_new: + return self.create_sqlite_library(library_dir, in_memory) - def open_sqlite_library( - self, library_dir: Path, is_new: bool, storage_path: str - ) -> LibraryStatus: + return self.open_sqlite_library(library_dir, in_memory) + + @staticmethod + def __get_engine(library_dir: Path, in_memory: bool, sql_filename: str): connection_string = URL.create( drivername="sqlite", - database=storage_path, + database=( + ":memory:" if in_memory else str(library_dir / TS_FOLDER_NAME / sql_filename) + ), ) # NOTE: File-based databases should use NullPool to create new DB connection in order to # keep connections on separate threads, which prevents the DB files from being locked @@ -418,169 +421,82 @@ class Library: # More info can be found on the SQLAlchemy docs: # https://docs.sqlalchemy.org/en/20/changelog/migration_07.html # Under -> sqlite-the-sqlite-dialect-now-uses-nullpool-for-file-based-databases - poolclass = None if storage_path == ":memory:" else NullPool + poolclass = None if in_memory else NullPool + + logger.info( + "[Library] Creating SQLAlchemy Engine", + connection_string=connection_string, + poolclass=poolclass, + ) + return create_engine( + connection_string, poolclass=poolclass, connect_args={"autocommit": False} + ) + + def create_sqlite_library( + self, library_dir: Path, in_memory: bool, sql_filename: str = SQL_FILENAME + ) -> LibraryStatus: + self.engine = self.__get_engine(library_dir, in_memory, sql_filename) loaded_db_version: int = 0 - initial_db_version: int = DB_VERSION logger.info( "[Library] Opening SQLite Library", library_dir=library_dir, - connection_string=connection_string, ) - self.engine = create_engine(connection_string, poolclass=poolclass) + + logger.info(f"[Library] Library DB version: {loaded_db_version}") + make_tables(self.engine) + with Session(self.engine) as session: - # Don't check DB version when creating new library - if not is_new: - loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY) - initial_db_version = self.get_version(DB_VERSION_INITIAL_KEY) + # Add default tag color namespaces. + namespaces = default_color_groups.namespaces() - # ======================== Library Database Version Checking ======================= - # DB_VERSION 6 is the first supported SQLite DB version. - # If the DB_VERSION is >= 100, that means it's a compound major + minor version. - # - Dividing by 100 and flooring gives the major (breaking changes) version. - # - If a DB has major version higher than the current program, don't load it. - # - If only the minor version is higher, it's still allowed to load. - if loaded_db_version < 6 or ( - loaded_db_version >= 100 and loaded_db_version // 100 > DB_VERSION // 100 - ): - mismatch_text = Translations["status.library_version_mismatch"] - found_text = Translations["status.library_version_found"] - expected_text = Translations["status.library_version_expected"] - return LibraryStatus( - success=False, - message=( - f"{mismatch_text}\n" - f"{found_text} v{loaded_db_version}, " - f"{expected_text} v{DB_VERSION}" - ), - ) + # TODO: are all of these commits necessary? + session.add_all(namespaces) + session.flush() - logger.info(f"[Library] Library DB version: {loaded_db_version}") - make_tables(self.engine) + # Add default tag colors. + tag_colors: list[TagColorGroup] = default_color_groups.standard() + tag_colors += default_color_groups.pastels() + tag_colors += default_color_groups.shades() + tag_colors += default_color_groups.grayscale() + tag_colors += default_color_groups.earth_tones() + tag_colors += default_color_groups.neon() - if is_new: - # Add default tag color namespaces. - namespaces = default_color_groups.namespaces() - try: - session.add_all(namespaces) - session.commit() - except IntegrityError as e: - logger.error("[Library] Couldn't add default tag color namespaces", error=e) - session.rollback() + session.add_all(tag_colors) + session.flush() - # Add default tag colors. - tag_colors: list[TagColorGroup] = default_color_groups.standard() - tag_colors += default_color_groups.pastels() - tag_colors += default_color_groups.shades() - tag_colors += default_color_groups.grayscale() - tag_colors += default_color_groups.earth_tones() - tag_colors += default_color_groups.neon() - if is_new: - try: - session.add_all(tag_colors) - session.commit() - except IntegrityError as e: - logger.error("[Library] Couldn't add default tag colors", error=e) - session.rollback() - - # Add default tags. - tags = get_default_tags() - try: - session.add_all(tags) - session.commit() - except IntegrityError: - session.rollback() + # Add default tags. + session.add_all(get_default_tags()) + session.flush() # Add default field templates - if is_new: - for template in get_default_field_templates(): - try: - session.add(template) - session.commit() - except IntegrityError: - logger.info( - "[Library] FieldTemplate already exists", field_template=template - ) - session.rollback() + for template in get_default_field_templates(): + session.add(template) + session.flush() # Ensure version rows are present - with catch_warnings(record=True): - try: - initial = DB_VERSION if is_new else 100 - session.add(Version(key=DB_VERSION_INITIAL_KEY, value=initial)) - session.commit() - except IntegrityError: - session.rollback() + session.add(Version(key=DB_VERSION_INITIAL_KEY, value=DB_VERSION)) + session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) + session.flush() - try: - session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) - session.commit() - except IntegrityError: - session.rollback() - - # check if folder matching current path exists already - self.folder = session.scalar(select(Folder).where(Folder.path == library_dir)) - if not self.folder: - folder = Folder( - path=library_dir, - uuid=str(uuid4()), - ) - session.add(folder) - session.expunge(folder) - session.commit() - self.folder = folder + # add folder for current path + folder = Folder( + path=library_dir, + uuid=str(uuid4()), + ) + session.add(folder) + session.expunge(folder) + session.flush() + self.folder = folder # Generate default .ts_ignore file - if is_new: - try: - ts_ignore_template = ( - Path(__file__).parents[3] / "resources/templates/ts_ignore_template.txt" - ) - shutil.copy2(ts_ignore_template, library_dir / TS_FOLDER_NAME / IGNORE_NAME) - except Exception as e: - logger.error("[ERROR][Library] Could not generate '.ts_ignore' file!", error=e) - - # Apply any post-SQL migration patches. - if not is_new: - assert loaded_db_version >= 6 - - # save backup if patches will be applied - if loaded_db_version < DB_VERSION: - self.library_dir = library_dir - self.save_library_backup_to_disk() - self.library_dir = None - - # migrate DB step by step from one version to the next - if loaded_db_version < 7: - # changes: value_type, tags - self.__apply_db7_migration(session) - if loaded_db_version < 8: - # changes: tag_colors - self.__apply_db8_migration(session) - if loaded_db_version < 9: - # changes: entries - self.__apply_db9_migration(session) - if loaded_db_version < 100: - # changes: tag_parents - self.__apply_db100_migration(session) - if loaded_db_version < 102: - # changes: tag_parents - self.__apply_db102_migration(session) - if loaded_db_version < 103: - # changes: tags - self.__apply_db103_migration(session) - if loaded_db_version < 104: - # changes: deletes preferences - self.__apply_db104_migration(session, library_dir) - if loaded_db_version < 200: - # changes: field tables - self.__apply_db200_migration(session) - if initial_db_version < 200 and loaded_db_version < 201: - # changes: field tables - self.__apply_db201_migration(session) - if loaded_db_version < 202: - # changes: tag_parents - self.__apply_db202_migration(session) + try: + ts_ignore_template = ( + Path(__file__).parents[3] / "resources/templates/ts_ignore_template.txt" + ) + shutil.copy2(ts_ignore_template, library_dir / TS_FOLDER_NAME / IGNORE_NAME) + except Exception as e: + logger.error("[ERROR][Library] Could not generate '.ts_ignore' file!", error=e) session.execute( text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") @@ -596,337 +512,399 @@ class Library: ) ) - # Update DB_VERSION - if loaded_db_version < DB_VERSION: - logger.info(f"[Library] Library migrated to DB version {DB_VERSION}") - self.set_version(DB_VERSION_CURRENT_KEY, DB_VERSION) + session.commit() # everything is fine, set the library path self.library_dir = library_dir return LibraryStatus(success=True, library_path=library_dir) - def __apply_db7_migration(self, session: Session): - """Migrate DB from DB_VERSION 6 to 7.""" - logger.info("[Library][Migration] Applying patches to DB_VERSION: 6 library...") - with session: - # Repair tags that may have a disambiguation_id pointing towards a deleted tag. - all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all() - disam_stmt = ( - update(Tag) - .where(Tag.disambiguation_id.not_in(all_tag_ids)) - .values(disambiguation_id=None) - ) - session.execute(disam_stmt) - session.commit() + def open_sqlite_library( + self, library_dir: Path, in_memory: bool, sql_filename: str = SQL_FILENAME + ) -> LibraryStatus: + self.engine = self.__get_engine(library_dir, in_memory, sql_filename) + loaded_db_version: int = 0 + initial_db_version: int = DB_VERSION - def __apply_db8_migration(self, session: Session): + logger.info( + "[Library] Opening SQLite Library", + library_dir=library_dir, + ) + + # Don't check DB version when creating new library + loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY) + initial_db_version = self.get_version(DB_VERSION_INITIAL_KEY) + + # ======================== Library Database Version Checking ======================= + # DB_VERSION 6 is the first supported SQLite DB version. + # If the DB_VERSION is >= 100, that means it's a compound major + minor version. + # - Dividing by 100 and flooring gives the major (breaking changes) version. + # - If a DB has major version higher than the current program, don't load it. + # - If only the minor version is higher, it's still allowed to load. + if loaded_db_version < 6 or ( + loaded_db_version >= 100 and loaded_db_version // 100 > DB_VERSION // 100 + ): + mismatch_text = Translations["status.library_version_mismatch"] + found_text = Translations["status.library_version_found"] + expected_text = Translations["status.library_version_expected"] + return LibraryStatus( + success=False, + message=( + f"{mismatch_text}\n" + f"{found_text} v{loaded_db_version}, " + f"{expected_text} v{DB_VERSION}" + ), + ) + + logger.info(f"[Library] Library DB version: {loaded_db_version}") + # TODO: this is very sketchy; blindly creating all tables the newest DB version should have + # without considering what version the DB is currently on and then doing all of the + # migrations after that seems like it could cause problems in some scenarios. + # instead only have this on creation and create new tables as part of migrations + # Note: this actually produces an error and fails to initialise built-in tags when opening + # a library that doesn't yet have the is_hidden property on the tags table + make_tables(self.engine) + + # save backup if patches will be applied + if loaded_db_version < DB_VERSION: + self.library_dir = library_dir + self.save_library_backup_to_disk() + self.library_dir = None + + # migrate DB step by step from one version to the next + # (migration_method, db_version, initial_db_version) + migrations = [ + (self.__apply_db7_migration, 7, None), # changes: value_type, tags + (self.__apply_db8_migration, 8, None), # changes: tag_colors + (self.__apply_db9_migration, 9, None), # changes: entries + (self.__apply_db100_migration, 100, None), # changes: tag_parents + (self.__apply_db101_migration, 101, None), # changes: versions + (self.__apply_db102_migration, 102, None), # changes: tag_parents + (self.__apply_db103_migration, 103, None), # changes: tags + (self.__apply_db104_migration, 104, None), # changes: deletes preferences + (self.__apply_db200_migration, 200, None), # changes: field tables + (self.__apply_db201_migration, 201, 200), # changes: field tables + (self.__apply_db202_migration, 202, None), # changes: tag_parents + ] + for migration, v, iv in migrations: + if loaded_db_version < v and (iv is None or initial_db_version < iv): + logger.info(f"[Library][Migration][{v}] Starting DB Migration") + with Session(self.engine) as session: + # any error causes transaction to rollback + migration(session, library_dir) + loaded_db_version = v + self.set_version(session, DB_VERSION_CURRENT_KEY, v) + session.commit() + logger.info(f"[Library][Migration][{v}] Completed DB Migration") + + assert loaded_db_version == DB_VERSION, ( + "Ran all migrations, but the DB is still not on the newest version" + ) + logger.info(f"[Library] Library migrated to DB version {DB_VERSION}") + + with Session(self.engine) as session: + # TODO: the folder logic has no use and was never finished, remove it + # check if folder matching current path exists already + # NOTE: this has been causing new Folders to be created when the library is moved, since + # its introduction + self.folder = session.scalar(select(Folder).where(Folder.path == library_dir)) + if not self.folder: + folder = Folder( + path=library_dir, + uuid=str(uuid4()), + ) + session.add(folder) + session.expunge(folder) + session.commit() + self.folder = folder + + # everything is fine, set the library path + self.library_dir = library_dir + return LibraryStatus(success=True, library_path=library_dir) + + def __apply_db7_migration(self, session: Session, _library_dir: Path): + """Migrate DB from DB_VERSION 6 to 7.""" + logger.info("[Library][Migration][7] Applying patches to DB_VERSION: 6 library...") + # Repair tags that may have a disambiguation_id pointing towards a deleted tag. + # TODO: combine into single sql statement + all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all() + disam_stmt = ( + update(Tag) + .where(Tag.disambiguation_id.not_in(all_tag_ids)) + .values(disambiguation_id=None) + ) + session.execute(disam_stmt) + session.flush() + + def __apply_db8_migration(self, session: Session, library_dir: Path): """Migrate DB from DB_VERSION 7 to 8.""" # Add the missing color_border column to the TagColorGroups table. - color_border_stmt = text( - "ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL" + session.execute( + text("ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL") ) - try: - session.execute(color_border_stmt) - session.commit() - logger.info("[Library][Migration] Added color_border column to tag_colors table") - except Exception as e: - logger.error( - "[Library][Migration] Could not create color_border column in tag_colors table!", - error=e, - ) - session.rollback() + session.flush() + logger.info("[Library][Migration][8] Added color_border column to tag_colors table") # collect new default tag colors - tag_colors: list[TagColorGroup] = default_color_groups.standard() - tag_colors += default_color_groups.pastels() - tag_colors += default_color_groups.shades() - tag_colors += default_color_groups.grayscale() - tag_colors += default_color_groups.earth_tones() - # tag_colors += default_color_groups.neon() # NOTE: Neon is handled separately + tag_colors: list[TagColorGroup] = [ + color + for color in default_color_groups.shades() + if color.slug in ["burgundy", "dark-teal", "dark_lavender"] + ] # Add any new default colors introduced in DB_VERSION 8 for color in tag_colors: - try: - session.add(color) - logger.info( - "[Library][Migration] Migrated tag color to DB_VERSION 8+", - color_name=color.name, - ) - session.commit() - except IntegrityError: - session.rollback() + session.add(color) + session.flush() + logger.info( + "[Library][Migration][8] Migrated tag colors to DB_VERSION 8+", + color_name=tag_colors, + ) # Update Neon colors to use the the color_border property for color in default_color_groups.neon(): - try: - neon_stmt = ( - update(TagColorGroup) - .where( - and_( - TagColorGroup.namespace == color.namespace, - TagColorGroup.slug == color.slug, - ) - ) - .values( - slug=color.slug, - namespace=color.namespace, - name=color.name, - primary=color.primary, - secondary=color.secondary, - color_border=color.color_border, + neon_stmt = ( + update(TagColorGroup) + .where( + and_( + TagColorGroup.namespace == color.namespace, + TagColorGroup.slug == color.slug, ) ) - session.execute(neon_stmt) - session.commit() - except IntegrityError as e: - logger.error( - "[Library] Could not migrate Neon colors to DB_VERSION 8+!", - error=e, + .values( + slug=color.slug, + namespace=color.namespace, + name=color.name, + primary=color.primary, + secondary=color.secondary, + color_border=color.color_border, ) - session.rollback() + ) + session.execute(neon_stmt) + session.flush() - def __apply_db9_migration(self, session: Session): + def __apply_db9_migration(self, session: Session, library_dir: Path): """Migrate DB from DB_VERSION 8 to 9.""" # Apply database schema changes add_filename_column = text( "ALTER TABLE entries ADD COLUMN filename TEXT NOT NULL DEFAULT ''" ) - try: - session.execute(add_filename_column) - session.commit() - logger.info("[Library][Migration] Added filename column to entries table") - except Exception as e: - logger.error( - "[Library][Migration] Could not create filename column in entries table!", - error=e, - ) - session.rollback() + session.execute(add_filename_column) + session.flush() + logger.info("[Library][Migration][9] Added filename column to entries table") # Populate the new filename column. - for entry in self.all_entries(): - session.merge(entry).filename = entry.path.name - session.commit() - logger.info("[Library][Migration] Populated filename column in entries table") + for entry in self.__all_entries(session): + entry.filename = entry.path.name + session.merge(entry) + session.flush() + logger.info("[Library][Migration][9] Populated filename column in entries table") - def __apply_db100_migration(self, session: Session): + def __apply_db100_migration(self, session: Session, library_dir: Path): """Migrate DB to DB_VERSION 100.""" - with session: - # Repair parent-child tag relationships that are the wrong way around. - stmt = update(TagParent).values( - parent_id=TagParent.child_id, - child_id=TagParent.parent_id, - ) - session.execute(stmt) - session.commit() - logger.info("[Library][Migration] Refactored TagParent table") + # Repair parent-child tag relationships that are the wrong way around. + stmt = update(TagParent).values( + parent_id=TagParent.child_id, + child_id=TagParent.parent_id, + ) + session.execute(stmt) + session.flush() + logger.info("[Library][Migration][100] Refactored TagParent table") - def __apply_db102_migration(self, session: Session): + def __apply_db101_migration(self, session: Session, library_dir: Path): + """Migrate DB to DB_VERSION 101.""" + # Ensure version rows are present + session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) + session.flush() + + def __apply_db102_migration(self, session: Session, library_dir: Path): """Migrate DB to DB_VERSION 102.""" - with session: - stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct())) - session.execute(stmt) - session.commit() - logger.info("[Library][Migration] Verified TagParent table data") + # delete TagParents with a dangling parent reference + stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct())) + session.execute(stmt) + session.flush() + logger.info("[Library][Migration][102] Verified TagParent table data") - def __apply_db103_migration(self, session: Session): + def __apply_db103_migration(self, session: Session, library_dir: Path): """Migrate DB from DB_VERSION 102 to 103.""" # add the new hidden column for tags - add_is_hidden_column = text( - "ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0" - ) - try: - session.execute(add_is_hidden_column) - session.commit() - logger.info("[Library][Migration] Added is_hidden column to tags table") - except Exception as e: - logger.error( - "[Library][Migration] Could not create is_hidden column in tags table!", - error=e, - ) - session.rollback() + session.execute(text("ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0")) + session.flush() + logger.info("[Library][Migration][103] Added is_hidden column to tags table") # mark the "Archived" tag as hidden - try: - session.query(Tag).filter(Tag.id == TAG_ARCHIVED).update({"is_hidden": True}) - session.commit() - logger.info("[Library][Migration] Updated archived tag to be hidden") - except Exception as e: - logger.error( - "[Library][Migration] Could not update archived tag to be hidden!", - error=e, - ) - session.rollback() + session.query(Tag).filter(Tag.id == TAG_ARCHIVED).update({"is_hidden": True}) + session.flush() + logger.info("[Library][Migration][103] Updated archived tag to be hidden") def __apply_db104_migration(self, session: Session, library_dir: Path): """Migrate DB from DB_VERSION 103 to 104.""" # Convert file extension list to ts_ignore file, if a .ts_ignore file does not exist - self.__migrate_sql_to_ts_ignore(library_dir) + self.__migrate_sql_to_ts_ignore(session, library_dir) session.execute(text("DROP TABLE preferences")) - session.commit() + session.flush() - def __migrate_sql_to_ts_ignore(self, library_dir: Path): + def __migrate_sql_to_ts_ignore(self, session: Session, library_dir: Path): # Do not continue if existing '.ts_ignore' file is found ts_ignore = library_dir / TS_FOLDER_NAME / IGNORE_NAME if Path(ts_ignore).exists(): return # Load legacy extension data - with Session(self.engine) as session: - extensions: list[str] = unwrap( + extensions: list[str] = ujson.loads( + unwrap( session.scalar(text("SELECT value FROM preferences WHERE key = 'EXTENSION_LIST'")) ) - is_exclude_list: bool = unwrap( - session.scalar(text("SELECT value FROM preferences WHERE key = 'IS_EXCLUDE_LIST'")) - ) + ) + is_exclude_list: bool = unwrap( + session.scalar(text("SELECT value FROM preferences WHERE key = 'IS_EXCLUDE_LIST'")) + ) with open(ts_ignore, "w") as f: f.write(migrate_ext_list(extensions, is_exclude_list)) - def __apply_db200_migration(self, session: Session): + def __apply_db200_migration(self, session: Session, library_dir: Path): """Migrate DB to DB_VERSION 200.""" - with session: - # Drop unused 'boolean_fields' and 'value_type' tables - logger.info( - "[Library][Migration][200] Dropping boolean_fields and value_type tables..." - ) - session.execute(text("DROP TABLE boolean_fields")) - session.execute(text("DROP TABLE value_type")) + # Drop unused 'boolean_fields' and 'value_type' tables + logger.info("[Library][Migration][200] Dropping boolean_fields and value_type tables...") + session.execute(text("DROP TABLE boolean_fields")) + session.execute(text("DROP TABLE value_type")) - # Add 'name' column to text_fields and datetime_fields tables - logger.info("[Library][Migration][200] Adding name columns to field tables...") - stmt = text('ALTER TABLE text_fields ADD COLUMN name VARCHAR DEFAULT ""') - session.execute(stmt) - stmt = text('ALTER TABLE datetime_fields ADD COLUMN name VARCHAR DEFAULT ""') - session.execute(stmt) + # Add 'name' column to text_fields and datetime_fields tables + logger.info("[Library][Migration][200] Adding name columns to field tables...") + stmt = text('ALTER TABLE text_fields ADD COLUMN name VARCHAR DEFAULT ""') + session.execute(stmt) + stmt = text('ALTER TABLE datetime_fields ADD COLUMN name VARCHAR DEFAULT ""') + session.execute(stmt) - # Drop unnecessary 'position' columns - logger.info("[Library][Migration][200] Dropping position columns to field tables...") - session.execute(text("ALTER TABLE datetime_fields DROP COLUMN position")) - session.execute(text("ALTER TABLE text_fields DROP COLUMN position")) + # Drop unnecessary 'position' columns + logger.info("[Library][Migration][200] Dropping position columns to field tables...") + session.execute(text("ALTER TABLE datetime_fields DROP COLUMN position")) + session.execute(text("ALTER TABLE text_fields DROP COLUMN position")) - # Add 'is_multiline' column to text_fields table - logger.info("[Library][Migration][200] Adding is_multiline column to text_fields...") - stmt = text( - "ALTER TABLE text_fields ADD COLUMN is_multiline BOOLEAN NOT NULL DEFAULT 0" - ) - session.execute(stmt) - session.flush() + # Add 'is_multiline' column to text_fields table + logger.info("[Library][Migration][200] Adding is_multiline column to text_fields...") + stmt = text("ALTER TABLE text_fields ADD COLUMN is_multiline BOOLEAN NOT NULL DEFAULT 0") + session.execute(stmt) + session.flush() - # Move values from old `type_key` columns into new `name` columns - logger.info("[Library][Migration][200] Moving values from type_key columns to name...") - session.execute(text("UPDATE text_fields SET name = type_key")) - session.execute(text("UPDATE datetime_fields SET name = type_key")) - session.flush() + # Move values from old `type_key` columns into new `name` columns + logger.info("[Library][Migration][200] Moving values from type_key columns to name...") + session.execute(text("UPDATE text_fields SET name = type_key")) + session.execute(text("UPDATE datetime_fields SET name = type_key")) + session.flush() - # Change `name` values to title case - logger.info("[Library][Migration][200] Normalizing TextField names...") - for text_field in session.execute(select(TextField)).scalars(): - # NOTE: The only exception to the "Title Case" conversion is the "URL" field. - text_field.name = text_field.name.title().replace("Url", "URL").replace("_", " ") - logger.info("[Library][Migration][200] Normalizing DatetimeField names...") - for datetime_field in session.execute(select(DatetimeField)).scalars(): - datetime_field.name = datetime_field.name.title().replace("_", " ") - session.flush() + # Change `name` values to title case + logger.info("[Library][Migration][200] Normalizing TextField names...") + for text_field in session.execute(select(TextField)).scalars(): + # NOTE: The only exception to the "Title Case" conversion is the "URL" field. + text_field.name = text_field.name.title().replace("Url", "URL").replace("_", " ") + logger.info("[Library][Migration][200] Normalizing DatetimeField names...") + for datetime_field in session.execute(select(DatetimeField)).scalars(): + datetime_field.name = datetime_field.name.title().replace("_", " ") + session.flush() - # Add correct `is_multiline` values to text_fields table - logger.info("[Library][Migration][200] Updating is_multiline for legacy TEXT_BOXes...") - text_boxes = [ - x.get("name") for x in LEGACY_FIELD_MAP.values() if x.get("is_multiline") is True - ] - update_stmt = ( - update(TextField).where(TextField.name.in_(text_boxes)).values(is_multiline=True) - ) - session.execute(update_stmt) - session.flush() + # Add correct `is_multiline` values to text_fields table + logger.info("[Library][Migration][200] Updating is_multiline for legacy TEXT_BOXes...") + text_boxes = [ + x.get("name") for x in LEGACY_FIELD_MAP.values() if x.get("is_multiline") is True + ] + update_stmt = ( + update(TextField).where(TextField.name.in_(text_boxes)).values(is_multiline=True) + ) + session.execute(update_stmt) + session.flush() - # Repair legacy "Description" fields to use is_multiline = True - logger.info("[Library][Migration][200] Repairing legacy Description fields...") - desc_stmt = ( - update(TextField) - .where(TextField.name == "Description" and TextField.is_multiline == False) # noqa: E712 - .values(is_multiline=True) - ) - session.execute(desc_stmt) + # Repair legacy "Description" fields to use is_multiline = True + logger.info("[Library][Migration][200] Repairing legacy Description fields...") + desc_stmt = ( + update(TextField) + .where(TextField.name == "Description" and TextField.is_multiline == False) # noqa: E712 + .values(is_multiline=True) + ) + session.execute(desc_stmt) - # Repair legacy "Comments" fields to use is_multiline = True - logger.info("[Library][Migration][200] Repairing legacy Comment fields...") - comm_stmt = ( - update(TextField) - .where(TextField.name == "Comments" and TextField.is_multiline == False) # noqa: E712 - .values(is_multiline=True) - ) - session.execute(comm_stmt) + # Repair legacy "Comments" fields to use is_multiline = True + logger.info("[Library][Migration][200] Repairing legacy Comment fields...") + comm_stmt = ( + update(TextField) + .where(TextField.name == "Comments" and TextField.is_multiline == False) # noqa: E712 + .values(is_multiline=True) + ) + session.execute(comm_stmt) - # Add default field templates - logger.info("[Library][Migration][200] Adding default field templates...") - for template in get_default_field_templates(): - try: - session.add(template) - session.flush() - except IntegrityError: - logger.error("[Library] FieldTemplate already exists", field_template=template) - session.rollback() + # Add default field templates + logger.info("[Library][Migration][200] Adding default field templates...") + for template in get_default_field_templates(): + session.add(template) + session.flush() - session.commit() + # DB indices for improved performance + session.execute( + text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") + ) + session.execute( + text("CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id ON tag_parents (child_id)") + ) + session.execute( + text("CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)") + ) - def __apply_db201_migration(self, session: Session): + def __apply_db201_migration(self, session: Session, library_dir: Path): """Migrate DB to DB_VERSION 201.""" - with session: - create_text_fields_table = text(""" - CREATE TABLE text_fields_new ( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - name VARCHAR NOT NULL, - entry_id INTEGER NOT NULL, - value VARCHAR, - is_multiline BOOLEAN NOT NULL, - FOREIGN KEY(entry_id) REFERENCES entries (id) - ) + create_text_fields_table = text(""" + CREATE TABLE text_fields_new ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR NOT NULL, + entry_id INTEGER NOT NULL, + value VARCHAR, + is_multiline BOOLEAN NOT NULL, + FOREIGN KEY(entry_id) REFERENCES entries (id) + ) + """) + create_datetime_fields_table = text(""" + CREATE TABLE datetime_fields_new ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR NOT NULL, + entry_id INTEGER NOT NULL, + value VARCHAR, + FOREIGN KEY(entry_id) REFERENCES entries (id) + ) + """) + + logger.info("[Library][Migration][201] Dropping type_key from text_fields table...") + session.execute(create_text_fields_table) + session.flush() + session.execute( + text(""" + INSERT INTO text_fields_new (id, name, entry_id, value, is_multiline) + SELECT id, name, entry_id, value, is_multiline + FROM text_fields """) - create_datetime_fields_table = text(""" - CREATE TABLE datetime_fields_new ( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - name VARCHAR NOT NULL, - entry_id INTEGER NOT NULL, - value VARCHAR, - FOREIGN KEY(entry_id) REFERENCES entries (id) - ) + ) + session.execute(text("DROP TABLE text_fields")) + session.execute(text("ALTER TABLE text_fields_new RENAME TO text_fields")) + + logger.info("[Library][Migration][201] Dropping type_key from datetime_fields table...") + session.execute(create_datetime_fields_table) + session.flush() + session.execute( + text(""" + INSERT INTO datetime_fields_new (id, name, entry_id, value) + SELECT id, name, entry_id, value + FROM datetime_fields """) + ) + session.execute(text("DROP TABLE datetime_fields")) + session.execute(text("ALTER TABLE datetime_fields_new RENAME TO datetime_fields")) - logger.info("[Library][Migration][201] Dropping type_key from text_fields table...") - session.execute(create_text_fields_table) - session.flush() - session.execute( - text(""" - INSERT INTO text_fields_new (id, name, entry_id, value, is_multiline) - SELECT id, name, entry_id, value, is_multiline - FROM text_fields - """) - ) - session.execute(text("DROP TABLE text_fields")) - session.execute(text("ALTER TABLE text_fields_new RENAME TO text_fields")) + session.flush() - logger.info("[Library][Migration][201] Dropping type_key from datetime_fields table...") - session.execute(create_datetime_fields_table) - session.flush() - session.execute( - text(""" - INSERT INTO datetime_fields_new (id, name, entry_id, value) - SELECT id, name, entry_id, value - FROM datetime_fields - """) - ) - session.execute(text("DROP TABLE datetime_fields")) - session.execute(text("ALTER TABLE datetime_fields_new RENAME TO datetime_fields")) - - session.commit() - - def __apply_db202_migration(self, session: Session): + def __apply_db202_migration(self, session: Session, library_dir: Path): """Migrate DB to DB_VERSION 202.""" - with session: - stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct())) - session.execute(stmt) - session.commit() - logger.info("[Library][Migration] Verified TagParent table data") + stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct())) + session.execute(stmt) + session.flush() + logger.info("[Library][Migration][202] Verified TagParent table data") @property def field_templates(self) -> Sequence[BaseFieldTemplate]: @@ -1075,32 +1053,36 @@ class Library: with Session(self.engine) as session: return unwrap(session.scalar(select(func.count(Entry.id)))) + def __all_entries(self, session: Session, with_joins: bool = False) -> Iterator[Entry]: + """Load entries without joins.""" + stmt = select(Entry) + if with_joins: + # load Entry with all joins and all tags + stmt = ( + stmt.outerjoin(Entry.text_fields) + .outerjoin(Entry.datetime_fields) + .outerjoin(Entry.tags) + ) + stmt = stmt.options( + contains_eager(Entry.text_fields), + contains_eager(Entry.datetime_fields), + contains_eager(Entry.tags), + ) + + stmt = stmt.distinct() + + entries = session.execute(stmt).scalars() + if with_joins: + entries = entries.unique() + + for entry in entries: + yield entry + session.expunge(entry) + def all_entries(self, with_joins: bool = False) -> Iterator[Entry]: """Load entries without joins.""" with Session(self.engine) as session: - stmt = select(Entry) - if with_joins: - # load Entry with all joins and all tags - stmt = ( - stmt.outerjoin(Entry.text_fields) - .outerjoin(Entry.datetime_fields) - .outerjoin(Entry.tags) - ) - stmt = stmt.options( - contains_eager(Entry.text_fields), - contains_eager(Entry.datetime_fields), - contains_eager(Entry.tags), - ) - - stmt = stmt.distinct() - - entries = session.execute(stmt).scalars() - if with_joins: - entries = entries.unique() - - for entry in entries: - yield entry - session.expunge(entry) + return self.__all_entries(session, with_joins) @property def tags(self) -> list[Tag]: @@ -1128,11 +1110,12 @@ class Library: raise ValueError("Invalid library directory.") full_ts_path = library_dir / TS_FOLDER_NAME - if not full_ts_path.exists(): - logger.info("creating library directory", dir=full_ts_path) - full_ts_path.mkdir(parents=True, exist_ok=True) - return False - return True + if full_ts_path.exists(): + return True + + logger.info("creating library directory", dir=full_ts_path) + full_ts_path.mkdir(parents=True, exist_ok=True) + return False def add_entries(self, items: list[Entry]) -> list[int]: """Add multiple Entry records to the Library.""" @@ -2167,23 +2150,16 @@ class Library: except Exception: return 0 - def set_version(self, key: str, value: int) -> None: + def set_version(self, session: Session, key: str, value: int) -> None: """Set a version value to the DB. Args: + session(Session): The SQLAlchemy DB Session to use. key(str): The key for the name of the version type to set. value(int): The version value to set. """ - with Session(self.engine) as session: - try: - version = session.scalar(select(Version).where(Version.key == key)) - assert version - version.value = value - session.add(version) - session.commit() - except (IntegrityError, AssertionError) as e: - logger.error("[Library][ERROR] Couldn't add default tag color namespaces", error=e) - session.rollback() + # Insert if key has no value yet, otherwise update the value + session.merge(Version(key=key, value=value)) def mirror_entry_fields(self, entries: list[Entry]) -> None: """Mirror fields among multiple Entry items.""" diff --git a/src/tagstudio/qt/mixed/migration_modal.py b/src/tagstudio/qt/mixed/migration_modal.py index 93e983a1..77850a72 100644 --- a/src/tagstudio/qt/mixed/migration_modal.py +++ b/src/tagstudio/qt/mixed/migration_modal.py @@ -395,14 +395,15 @@ class JsonMigrationModal(QObject): # Convert JSON Library to SQLite yield Translations["json_migration.creating_database_tables"] self.sql_lib = SqliteLibrary() - self.temp_path: Path = ( - self.json_lib.library_dir / TS_FOLDER_NAME / "migration_ts_library.sqlite" - ) + temp_filename = "migration_ts_library.sqlite" + self.temp_path: Path = self.json_lib.library_dir / TS_FOLDER_NAME / temp_filename if self.temp_path.exists(): logger.info('Temporary migration file "temp_path" already exists. Removing...') self.temp_path.unlink() - self.sql_lib.open_sqlite_library( - self.json_lib.library_dir, is_new=True, storage_path=str(self.temp_path) + self.sql_lib.create_sqlite_library( + self.json_lib.library_dir, + in_memory=False, + sql_filename=temp_filename, ) yield Translations.format( "json_migration.migrating_files_entries", entries=len(self.json_lib.entries)