From b0decac6104046959d5b70fa1e7d09f5db5346f6 Mon Sep 17 00:00:00 2001 From: Jann Stute <46534683+Computerdores@users.noreply.github.com> Date: Sat, 9 May 2026 04:37:09 +0200 Subject: [PATCH] refactor: sql migrations (#1295) * refactor: cleanup parameters of open_library and open_sqlite_library * doc: notes on what tables are affected by which migration steps * refactor(migration order): move DBv6 repairs * refactor(migration order): move DBv8 repairs * refactor(migration order): move DBv9 repairs * refactor(migration order): move DBv100 repairs * refactor(migration order): move DBv102 repairs * refactor: merge migration methods * doc: final comment changes * fix: query tag ids independent of future DB changes --- src/tagstudio/core/library/alchemy/library.py | 81 +++++++++---------- 1 file changed, 39 insertions(+), 42 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 1e763d0c..c5c60f5f 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -423,8 +423,8 @@ class Library: logger.info(f"[Library] DB_VERSION: {loaded_db_version}") make_tables(self.engine) - # Add default tag color namespaces. if is_new: + # Add default tag color namespaces. namespaces = default_color_groups.namespaces() try: session.add_all(namespaces) @@ -433,8 +433,7 @@ class Library: logger.error("[Library] Couldn't add default tag color namespaces", error=e) session.rollback() - # Add default tag colors. - if is_new: + # Add default tag colors. tag_colors: list[TagColorGroup] = default_color_groups.standard() tag_colors += default_color_groups.pastels() tag_colors += default_color_groups.shades() @@ -449,8 +448,7 @@ class Library: logger.error("[Library] Couldn't add default tag colors", error=e) session.rollback() - # Add default tags. - if is_new: + # Add default tags. tags = get_default_tags() try: session.add_all(tags) @@ -531,35 +529,36 @@ class Library: # Apply any post-SQL migration patches. if not is_new: + assert loaded_db_version >= 6 + # save backup if patches will be applied if loaded_db_version < DB_VERSION: self.library_dir = library_dir self.save_library_backup_to_disk() self.library_dir = None - # NOTE: Depending on the data, some data and schema changes need to be applied in - # different orders. This chain of methods can likely be cleaned up and/or moved. + # migrate DB step by step from one version to the next + if loaded_db_version < 7: + # changes: value_type, tags + self.__apply_db7_migration(session) if loaded_db_version < 8: - self.__apply_db8_schema_changes(session) + # changes: tag_colors + self.__apply_db8_migration(session) if loaded_db_version < 9: - self.__apply_db9_schema_changes(session) - if loaded_db_version < 103: - self.__apply_db103_schema_changes(session) - if loaded_db_version == 6: - self.__apply_repairs_for_db6(session) - - if loaded_db_version >= 6 and loaded_db_version < 8: - self.__apply_db8_default_data(session) - if loaded_db_version < 9: - self.__apply_db9_filename_population(session) + # changes: entries + self.__apply_db9_migration(session) if loaded_db_version < 100: - self.__apply_db100_parent_repairs(session) + # changes: tag_parents + self.__apply_db100_migration(session) if loaded_db_version < 102: - self.__apply_db102_repairs(session) + # changes: tag_parents + self.__apply_db102_migration(session) if loaded_db_version < 103: - self.__apply_db103_default_data(session) + # changes: tags + self.__apply_db103_migration(session) # Convert file extension list to ts_ignore file, if a .ts_ignore file does not exist + # TODO: do this in the migration step that will remove the preferences table self.migrate_sql_to_ts_ignore(library_dir) # Update DB_VERSION @@ -570,8 +569,8 @@ class Library: self.library_dir = library_dir return LibraryStatus(success=True, library_path=library_dir) - def __apply_repairs_for_db6(self, session: Session): - """Apply database repairs introduced in DB_VERSION 7.""" + def __apply_db7_migration(self, session: Session): + """Migrate DB from DB_VERSION 6 to 7.""" logger.info("[Library][Migration] Applying patches to DB_VERSION: 6 library...") with session: # Repair "Description" fields with a TEXT_LINE key instead of a TEXT_BOX key. @@ -584,7 +583,7 @@ class Library: session.flush() # Repair tags that may have a disambiguation_id pointing towards a deleted tag. - all_tag_ids: set[int] = {tag.id for tag in self.tags} + all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all() disam_stmt = ( update(Tag) .where(Tag.disambiguation_id.not_in(all_tag_ids)) @@ -593,9 +592,8 @@ class Library: session.execute(disam_stmt) session.commit() - def __apply_db8_schema_changes(self, session: Session): - """Apply database schema changes introduced in DB_VERSION 8.""" - # TODO: Use Alembic for this part instead + def __apply_db8_migration(self, session: Session): + """Migrate DB from DB_VERSION 7 to 8.""" # Add the missing color_border column to the TagColorGroups table. color_border_stmt = text( "ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL" @@ -611,8 +609,7 @@ class Library: ) session.rollback() - def __apply_db8_default_data(self, session: Session): - """Apply default data changes introduced in DB_VERSION 8.""" + # collect new default tag colors tag_colors: list[TagColorGroup] = default_color_groups.standard() tag_colors += default_color_groups.pastels() tag_colors += default_color_groups.shades() @@ -661,8 +658,9 @@ class Library: ) session.rollback() - def __apply_db9_schema_changes(self, session: Session): - """Apply database schema changes introduced in DB_VERSION 9.""" + def __apply_db9_migration(self, session: Session): + """Migrate DB from DB_VERSION 8 to 9.""" + # Apply database schema changes add_filename_column = text( "ALTER TABLE entries ADD COLUMN filename TEXT NOT NULL DEFAULT ''" ) @@ -677,15 +675,14 @@ class Library: ) session.rollback() - def __apply_db9_filename_population(self, session: Session): - """Populate the filename column introduced in DB_VERSION 9.""" + # Populate the new filename column. for entry in self.all_entries(): session.merge(entry).filename = entry.path.name session.commit() logger.info("[Library][Migration] Populated filename column in entries table") - def __apply_db100_parent_repairs(self, session: Session): - """Swap the child_id and parent_id values in the TagParent table.""" + def __apply_db100_migration(self, session: Session): + """Migrate DB to DB_VERSION 100.""" with session: # Repair parent-child tag relationships that are the wrong way around. stmt = update(TagParent).values( @@ -696,17 +693,18 @@ class Library: session.commit() logger.info("[Library][Migration] Refactored TagParent table") - def __apply_db102_repairs(self, session: Session): - """Repair tag_parents rows with references to deleted tags.""" + def __apply_db102_migration(self, session: Session): + """Migrate DB to DB_VERSION 102.""" with session: - all_tag_ids: list[int] = [t.id for t in self.tags] + all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all() stmt = delete(TagParent).where(TagParent.parent_id.not_in(all_tag_ids)) session.execute(stmt) session.commit() logger.info("[Library][Migration] Verified TagParent table data") - def __apply_db103_schema_changes(self, session: Session): - """Apply database schema changes introduced in DB_VERSION 103.""" + def __apply_db103_migration(self, session: Session): + """Migrate DB from DB_VERSION 102 to 103.""" + # add the new hidden column for tags add_is_hidden_column = text( "ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0" ) @@ -721,8 +719,7 @@ class Library: ) session.rollback() - def __apply_db103_default_data(self, session: Session): - """Apply default data changes introduced in DB_VERSION 103.""" + # mark the "Archived" tag as hidden try: session.query(Tag).filter(Tag.id == TAG_ARCHIVED).update({"is_hidden": True}) session.commit()