From 194b2b82cc701ad5f3412163eb7593ad5de1467f Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 01:01:14 +0200 Subject: [PATCH 01/27] refactor: minor simplification --- src/tagstudio/core/library/alchemy/library.py | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index d7cf69050..8fedc1cb8 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -428,36 +428,37 @@ def open_sqlite_library( connection_string=connection_string, ) self.engine = create_engine(connection_string, poolclass=poolclass) - with Session(self.engine) as session: - # Don't check DB version when creating new library - if not is_new: - loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY) - initial_db_version = self.get_version(DB_VERSION_INITIAL_KEY) - - # ======================== Library Database Version Checking ======================= - # DB_VERSION 6 is the first supported SQLite DB version. - # If the DB_VERSION is >= 100, that means it's a compound major + minor version. - # - Dividing by 100 and flooring gives the major (breaking changes) version. - # - If a DB has major version higher than the current program, don't load it. - # - If only the minor version is higher, it's still allowed to load. - if loaded_db_version < 6 or ( - loaded_db_version >= 100 and loaded_db_version // 100 > DB_VERSION // 100 - ): - mismatch_text = Translations["status.library_version_mismatch"] - found_text = Translations["status.library_version_found"] - expected_text = Translations["status.library_version_expected"] - return LibraryStatus( - success=False, - message=( - f"{mismatch_text}\n" - f"{found_text} v{loaded_db_version}, " - f"{expected_text} v{DB_VERSION}" - ), - ) - logger.info(f"[Library] Library DB version: {loaded_db_version}") - make_tables(self.engine) + # Don't check DB version when creating new library + if not is_new: + loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY) + initial_db_version = self.get_version(DB_VERSION_INITIAL_KEY) + + # ======================== Library Database Version Checking ======================= + # DB_VERSION 6 is the first supported SQLite DB version. + # If the DB_VERSION is >= 100, that means it's a compound major + minor version. + # - Dividing by 100 and flooring gives the major (breaking changes) version. + # - If a DB has major version higher than the current program, don't load it. + # - If only the minor version is higher, it's still allowed to load. + if loaded_db_version < 6 or ( + loaded_db_version >= 100 and loaded_db_version // 100 > DB_VERSION // 100 + ): + mismatch_text = Translations["status.library_version_mismatch"] + found_text = Translations["status.library_version_found"] + expected_text = Translations["status.library_version_expected"] + return LibraryStatus( + success=False, + message=( + f"{mismatch_text}\n" + f"{found_text} v{loaded_db_version}, " + f"{expected_text} v{DB_VERSION}" + ), + ) + + logger.info(f"[Library] Library DB version: {loaded_db_version}") + make_tables(self.engine) + with Session(self.engine) as session: if is_new: # Add default tag color namespaces. namespaces = default_color_groups.namespaces() @@ -491,8 +492,7 @@ def open_sqlite_library( except IntegrityError: session.rollback() - # Add default field templates - if is_new: + # Add default field templates for template in get_default_field_templates(): try: session.add(template) From 02a9295743b11b3121b6b6d5e80298b88dc7a46f Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 01:07:50 +0200 Subject: [PATCH 02/27] refactor: split open_library into new and not new case Functionality is entirely unchanged, but entire method is duplicated. Removing dead code from either copy is next. --- src/tagstudio/core/library/alchemy/library.py | 207 ++++++++++++++++++ 1 file changed, 207 insertions(+) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 8fedc1cb8..06c3b54b3 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -407,6 +407,213 @@ def open_library(self, library_dir: Path, in_memory: bool = False) -> LibrarySta def open_sqlite_library( self, library_dir: Path, is_new: bool, storage_path: str ) -> LibraryStatus: + if is_new: + return self.new_lib(library_dir, storage_path) + else: + return self.migrate_lib(library_dir, storage_path) + + def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: + is_new = True + connection_string = URL.create( + drivername="sqlite", + database=storage_path, + ) + # NOTE: File-based databases should use NullPool to create new DB connection in order to + # keep connections on separate threads, which prevents the DB files from being locked + # even after a connection has been closed. + # SingletonThreadPool (the default for :memory:) should still be used for in-memory DBs. + # More info can be found on the SQLAlchemy docs: + # https://docs.sqlalchemy.org/en/20/changelog/migration_07.html + # Under -> sqlite-the-sqlite-dialect-now-uses-nullpool-for-file-based-databases + poolclass = None if storage_path == ":memory:" else NullPool + loaded_db_version: int = 0 + initial_db_version: int = DB_VERSION + + logger.info( + "[Library] Opening SQLite Library", + library_dir=library_dir, + connection_string=connection_string, + ) + self.engine = create_engine(connection_string, poolclass=poolclass) + + # Don't check DB version when creating new library + if not is_new: + loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY) + initial_db_version = self.get_version(DB_VERSION_INITIAL_KEY) + + # ======================== Library Database Version Checking ======================= + # DB_VERSION 6 is the first supported SQLite DB version. + # If the DB_VERSION is >= 100, that means it's a compound major + minor version. + # - Dividing by 100 and flooring gives the major (breaking changes) version. + # - If a DB has major version higher than the current program, don't load it. + # - If only the minor version is higher, it's still allowed to load. + if loaded_db_version < 6 or ( + loaded_db_version >= 100 and loaded_db_version // 100 > DB_VERSION // 100 + ): + mismatch_text = Translations["status.library_version_mismatch"] + found_text = Translations["status.library_version_found"] + expected_text = Translations["status.library_version_expected"] + return LibraryStatus( + success=False, + message=( + f"{mismatch_text}\n" + f"{found_text} v{loaded_db_version}, " + f"{expected_text} v{DB_VERSION}" + ), + ) + + logger.info(f"[Library] Library DB version: {loaded_db_version}") + make_tables(self.engine) + + with Session(self.engine) as session: + if is_new: + # Add default tag color namespaces. + namespaces = default_color_groups.namespaces() + try: + session.add_all(namespaces) + session.commit() + except IntegrityError as e: + logger.error("[Library] Couldn't add default tag color namespaces", error=e) + session.rollback() + + # Add default tag colors. + tag_colors: list[TagColorGroup] = default_color_groups.standard() + tag_colors += default_color_groups.pastels() + tag_colors += default_color_groups.shades() + tag_colors += default_color_groups.grayscale() + tag_colors += default_color_groups.earth_tones() + tag_colors += default_color_groups.neon() + if is_new: + try: + session.add_all(tag_colors) + session.commit() + except IntegrityError as e: + logger.error("[Library] Couldn't add default tag colors", error=e) + session.rollback() + + # Add default tags. + tags = get_default_tags() + try: + session.add_all(tags) + session.commit() + except IntegrityError: + session.rollback() + + # Add default field templates + for template in get_default_field_templates(): + try: + session.add(template) + session.commit() + except IntegrityError: + logger.info( + "[Library] FieldTemplate already exists", field_template=template + ) + session.rollback() + + # Ensure version rows are present + with catch_warnings(record=True): + try: + initial = DB_VERSION if is_new else 100 + session.add(Version(key=DB_VERSION_INITIAL_KEY, value=initial)) + session.commit() + except IntegrityError: + session.rollback() + + try: + session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) + session.commit() + except IntegrityError: + session.rollback() + + # check if folder matching current path exists already + self.folder = session.scalar(select(Folder).where(Folder.path == library_dir)) + if not self.folder: + folder = Folder( + path=library_dir, + uuid=str(uuid4()), + ) + session.add(folder) + session.expunge(folder) + session.commit() + self.folder = folder + + # Generate default .ts_ignore file + if is_new: + try: + ts_ignore_template = ( + Path(__file__).parents[3] / "resources/templates/ts_ignore_template.txt" + ) + shutil.copy2(ts_ignore_template, library_dir / TS_FOLDER_NAME / IGNORE_NAME) + except Exception as e: + logger.error("[ERROR][Library] Could not generate '.ts_ignore' file!", error=e) + + # Apply any post-SQL migration patches. + if not is_new: + assert loaded_db_version >= 6 + + # save backup if patches will be applied + if loaded_db_version < DB_VERSION: + self.library_dir = library_dir + self.save_library_backup_to_disk() + self.library_dir = None + + # migrate DB step by step from one version to the next + if loaded_db_version < 7: + # changes: value_type, tags + self.__apply_db7_migration(session) + if loaded_db_version < 8: + # changes: tag_colors + self.__apply_db8_migration(session) + if loaded_db_version < 9: + # changes: entries + self.__apply_db9_migration(session) + if loaded_db_version < 100: + # changes: tag_parents + self.__apply_db100_migration(session) + if loaded_db_version < 102: + # changes: tag_parents + self.__apply_db102_migration(session) + if loaded_db_version < 103: + # changes: tags + self.__apply_db103_migration(session) + if loaded_db_version < 104: + # changes: deletes preferences + self.__apply_db104_migration(session, library_dir) + if loaded_db_version < 200: + # changes: field tables + self.__apply_db200_migration(session) + if initial_db_version < 200 and loaded_db_version < 201: + # changes: field tables + self.__apply_db201_migration(session) + if loaded_db_version < 202: + # changes: tag_parents + self.__apply_db202_migration(session) + + session.execute( + text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") + ) + session.execute( + text( + "CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id ON tag_parents (child_id)" + ) + ) + session.execute( + text( + "CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)" + ) + ) + + # Update DB_VERSION + if loaded_db_version < DB_VERSION: + logger.info(f"[Library] Library migrated to DB version {DB_VERSION}") + self.set_version(DB_VERSION_CURRENT_KEY, DB_VERSION) + + # everything is fine, set the library path + self.library_dir = library_dir + return LibraryStatus(success=True, library_path=library_dir) + + def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: + is_new = False connection_string = URL.create( drivername="sqlite", database=storage_path, From bdc8e4b59d87b6240192009cf0824399bbb11b9f Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 01:16:41 +0200 Subject: [PATCH 03/27] refactor: remove dead code --- src/tagstudio/core/library/alchemy/library.py | 349 ++++++------------ 1 file changed, 109 insertions(+), 240 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 06c3b54b3..2e3734163 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -413,7 +413,6 @@ def open_sqlite_library( return self.migrate_lib(library_dir, storage_path) def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: - is_new = True connection_string = URL.create( drivername="sqlite", database=storage_path, @@ -427,7 +426,6 @@ def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: # Under -> sqlite-the-sqlite-dialect-now-uses-nullpool-for-file-based-databases poolclass = None if storage_path == ":memory:" else NullPool loaded_db_version: int = 0 - initial_db_version: int = DB_VERSION logger.info( "[Library] Opening SQLite Library", @@ -436,84 +434,58 @@ def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: ) self.engine = create_engine(connection_string, poolclass=poolclass) - # Don't check DB version when creating new library - if not is_new: - loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY) - initial_db_version = self.get_version(DB_VERSION_INITIAL_KEY) - - # ======================== Library Database Version Checking ======================= - # DB_VERSION 6 is the first supported SQLite DB version. - # If the DB_VERSION is >= 100, that means it's a compound major + minor version. - # - Dividing by 100 and flooring gives the major (breaking changes) version. - # - If a DB has major version higher than the current program, don't load it. - # - If only the minor version is higher, it's still allowed to load. - if loaded_db_version < 6 or ( - loaded_db_version >= 100 and loaded_db_version // 100 > DB_VERSION // 100 - ): - mismatch_text = Translations["status.library_version_mismatch"] - found_text = Translations["status.library_version_found"] - expected_text = Translations["status.library_version_expected"] - return LibraryStatus( - success=False, - message=( - f"{mismatch_text}\n" - f"{found_text} v{loaded_db_version}, " - f"{expected_text} v{DB_VERSION}" - ), - ) - logger.info(f"[Library] Library DB version: {loaded_db_version}") make_tables(self.engine) with Session(self.engine) as session: - if is_new: - # Add default tag color namespaces. - namespaces = default_color_groups.namespaces() - try: - session.add_all(namespaces) - session.commit() - except IntegrityError as e: - logger.error("[Library] Couldn't add default tag color namespaces", error=e) - session.rollback() + # Add default tag color namespaces. + namespaces = default_color_groups.namespaces() + + # TODO: do these try excepts even make sense? Shouldn't library creation just fail + # in these cases? + try: + session.add_all(namespaces) + session.commit() + except IntegrityError as e: + logger.error("[Library] Couldn't add default tag color namespaces", error=e) + session.rollback() + + # Add default tag colors. + tag_colors: list[TagColorGroup] = default_color_groups.standard() + tag_colors += default_color_groups.pastels() + tag_colors += default_color_groups.shades() + tag_colors += default_color_groups.grayscale() + tag_colors += default_color_groups.earth_tones() + tag_colors += default_color_groups.neon() - # Add default tag colors. - tag_colors: list[TagColorGroup] = default_color_groups.standard() - tag_colors += default_color_groups.pastels() - tag_colors += default_color_groups.shades() - tag_colors += default_color_groups.grayscale() - tag_colors += default_color_groups.earth_tones() - tag_colors += default_color_groups.neon() - if is_new: - try: - session.add_all(tag_colors) - session.commit() - except IntegrityError as e: - logger.error("[Library] Couldn't add default tag colors", error=e) - session.rollback() - - # Add default tags. - tags = get_default_tags() + try: + session.add_all(tag_colors) + session.commit() + except IntegrityError as e: + logger.error("[Library] Couldn't add default tag colors", error=e) + session.rollback() + + # Add default tags. + tags = get_default_tags() + try: + session.add_all(tags) + session.commit() + except IntegrityError: + session.rollback() + + # Add default field templates + for template in get_default_field_templates(): try: - session.add_all(tags) + session.add(template) session.commit() except IntegrityError: + logger.info("[Library] FieldTemplate already exists", field_template=template) session.rollback() - # Add default field templates - for template in get_default_field_templates(): - try: - session.add(template) - session.commit() - except IntegrityError: - logger.info( - "[Library] FieldTemplate already exists", field_template=template - ) - session.rollback() - # Ensure version rows are present with catch_warnings(record=True): try: - initial = DB_VERSION if is_new else 100 + initial = DB_VERSION session.add(Version(key=DB_VERSION_INITIAL_KEY, value=initial)) session.commit() except IntegrityError: @@ -524,6 +496,7 @@ def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: session.commit() except IntegrityError: session.rollback() + self.set_version(DB_VERSION_CURRENT_KEY, DB_VERSION) # check if folder matching current path exists already self.folder = session.scalar(select(Folder).where(Folder.path == library_dir)) @@ -538,56 +511,13 @@ def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: self.folder = folder # Generate default .ts_ignore file - if is_new: - try: - ts_ignore_template = ( - Path(__file__).parents[3] / "resources/templates/ts_ignore_template.txt" - ) - shutil.copy2(ts_ignore_template, library_dir / TS_FOLDER_NAME / IGNORE_NAME) - except Exception as e: - logger.error("[ERROR][Library] Could not generate '.ts_ignore' file!", error=e) - - # Apply any post-SQL migration patches. - if not is_new: - assert loaded_db_version >= 6 - - # save backup if patches will be applied - if loaded_db_version < DB_VERSION: - self.library_dir = library_dir - self.save_library_backup_to_disk() - self.library_dir = None - - # migrate DB step by step from one version to the next - if loaded_db_version < 7: - # changes: value_type, tags - self.__apply_db7_migration(session) - if loaded_db_version < 8: - # changes: tag_colors - self.__apply_db8_migration(session) - if loaded_db_version < 9: - # changes: entries - self.__apply_db9_migration(session) - if loaded_db_version < 100: - # changes: tag_parents - self.__apply_db100_migration(session) - if loaded_db_version < 102: - # changes: tag_parents - self.__apply_db102_migration(session) - if loaded_db_version < 103: - # changes: tags - self.__apply_db103_migration(session) - if loaded_db_version < 104: - # changes: deletes preferences - self.__apply_db104_migration(session, library_dir) - if loaded_db_version < 200: - # changes: field tables - self.__apply_db200_migration(session) - if initial_db_version < 200 and loaded_db_version < 201: - # changes: field tables - self.__apply_db201_migration(session) - if loaded_db_version < 202: - # changes: tag_parents - self.__apply_db202_migration(session) + try: + ts_ignore_template = ( + Path(__file__).parents[3] / "resources/templates/ts_ignore_template.txt" + ) + shutil.copy2(ts_ignore_template, library_dir / TS_FOLDER_NAME / IGNORE_NAME) + except Exception as e: + logger.error("[ERROR][Library] Could not generate '.ts_ignore' file!", error=e) session.execute( text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") @@ -603,11 +533,6 @@ def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: ) ) - # Update DB_VERSION - if loaded_db_version < DB_VERSION: - logger.info(f"[Library] Library migrated to DB version {DB_VERSION}") - self.set_version(DB_VERSION_CURRENT_KEY, DB_VERSION) - # everything is fine, set the library path self.library_dir = library_dir return LibraryStatus(success=True, library_path=library_dir) @@ -637,79 +562,34 @@ def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: self.engine = create_engine(connection_string, poolclass=poolclass) # Don't check DB version when creating new library - if not is_new: - loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY) - initial_db_version = self.get_version(DB_VERSION_INITIAL_KEY) - - # ======================== Library Database Version Checking ======================= - # DB_VERSION 6 is the first supported SQLite DB version. - # If the DB_VERSION is >= 100, that means it's a compound major + minor version. - # - Dividing by 100 and flooring gives the major (breaking changes) version. - # - If a DB has major version higher than the current program, don't load it. - # - If only the minor version is higher, it's still allowed to load. - if loaded_db_version < 6 or ( - loaded_db_version >= 100 and loaded_db_version // 100 > DB_VERSION // 100 - ): - mismatch_text = Translations["status.library_version_mismatch"] - found_text = Translations["status.library_version_found"] - expected_text = Translations["status.library_version_expected"] - return LibraryStatus( - success=False, - message=( - f"{mismatch_text}\n" - f"{found_text} v{loaded_db_version}, " - f"{expected_text} v{DB_VERSION}" - ), - ) + loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY) + initial_db_version = self.get_version(DB_VERSION_INITIAL_KEY) + + # ======================== Library Database Version Checking ======================= + # DB_VERSION 6 is the first supported SQLite DB version. + # If the DB_VERSION is >= 100, that means it's a compound major + minor version. + # - Dividing by 100 and flooring gives the major (breaking changes) version. + # - If a DB has major version higher than the current program, don't load it. + # - If only the minor version is higher, it's still allowed to load. + if loaded_db_version < 6 or ( + loaded_db_version >= 100 and loaded_db_version // 100 > DB_VERSION // 100 + ): + mismatch_text = Translations["status.library_version_mismatch"] + found_text = Translations["status.library_version_found"] + expected_text = Translations["status.library_version_expected"] + return LibraryStatus( + success=False, + message=( + f"{mismatch_text}\n" + f"{found_text} v{loaded_db_version}, " + f"{expected_text} v{DB_VERSION}" + ), + ) logger.info(f"[Library] Library DB version: {loaded_db_version}") make_tables(self.engine) with Session(self.engine) as session: - if is_new: - # Add default tag color namespaces. - namespaces = default_color_groups.namespaces() - try: - session.add_all(namespaces) - session.commit() - except IntegrityError as e: - logger.error("[Library] Couldn't add default tag color namespaces", error=e) - session.rollback() - - # Add default tag colors. - tag_colors: list[TagColorGroup] = default_color_groups.standard() - tag_colors += default_color_groups.pastels() - tag_colors += default_color_groups.shades() - tag_colors += default_color_groups.grayscale() - tag_colors += default_color_groups.earth_tones() - tag_colors += default_color_groups.neon() - if is_new: - try: - session.add_all(tag_colors) - session.commit() - except IntegrityError as e: - logger.error("[Library] Couldn't add default tag colors", error=e) - session.rollback() - - # Add default tags. - tags = get_default_tags() - try: - session.add_all(tags) - session.commit() - except IntegrityError: - session.rollback() - - # Add default field templates - for template in get_default_field_templates(): - try: - session.add(template) - session.commit() - except IntegrityError: - logger.info( - "[Library] FieldTemplate already exists", field_template=template - ) - session.rollback() - # Ensure version rows are present with catch_warnings(record=True): try: @@ -737,57 +617,46 @@ def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: session.commit() self.folder = folder - # Generate default .ts_ignore file - if is_new: - try: - ts_ignore_template = ( - Path(__file__).parents[3] / "resources/templates/ts_ignore_template.txt" - ) - shutil.copy2(ts_ignore_template, library_dir / TS_FOLDER_NAME / IGNORE_NAME) - except Exception as e: - logger.error("[ERROR][Library] Could not generate '.ts_ignore' file!", error=e) - # Apply any post-SQL migration patches. - if not is_new: - assert loaded_db_version >= 6 - - # save backup if patches will be applied - if loaded_db_version < DB_VERSION: - self.library_dir = library_dir - self.save_library_backup_to_disk() - self.library_dir = None - - # migrate DB step by step from one version to the next - if loaded_db_version < 7: - # changes: value_type, tags - self.__apply_db7_migration(session) - if loaded_db_version < 8: - # changes: tag_colors - self.__apply_db8_migration(session) - if loaded_db_version < 9: - # changes: entries - self.__apply_db9_migration(session) - if loaded_db_version < 100: - # changes: tag_parents - self.__apply_db100_migration(session) - if loaded_db_version < 102: - # changes: tag_parents - self.__apply_db102_migration(session) - if loaded_db_version < 103: - # changes: tags - self.__apply_db103_migration(session) - if loaded_db_version < 104: - # changes: deletes preferences - self.__apply_db104_migration(session, library_dir) - if loaded_db_version < 200: - # changes: field tables - self.__apply_db200_migration(session) - if initial_db_version < 200 and loaded_db_version < 201: - # changes: field tables - self.__apply_db201_migration(session) - if loaded_db_version < 202: - # changes: tag_parents - self.__apply_db202_migration(session) + assert loaded_db_version >= 6 + + # save backup if patches will be applied + if loaded_db_version < DB_VERSION: + self.library_dir = library_dir + self.save_library_backup_to_disk() + self.library_dir = None + + # migrate DB step by step from one version to the next + if loaded_db_version < 7: + # changes: value_type, tags + self.__apply_db7_migration(session) + if loaded_db_version < 8: + # changes: tag_colors + self.__apply_db8_migration(session) + if loaded_db_version < 9: + # changes: entries + self.__apply_db9_migration(session) + if loaded_db_version < 100: + # changes: tag_parents + self.__apply_db100_migration(session) + if loaded_db_version < 102: + # changes: tag_parents + self.__apply_db102_migration(session) + if loaded_db_version < 103: + # changes: tags + self.__apply_db103_migration(session) + if loaded_db_version < 104: + # changes: deletes preferences + self.__apply_db104_migration(session, library_dir) + if loaded_db_version < 200: + # changes: field tables + self.__apply_db200_migration(session) + if initial_db_version < 200 and loaded_db_version < 201: + # changes: field tables + self.__apply_db201_migration(session) + if loaded_db_version < 202: + # changes: tag_parents + self.__apply_db202_migration(session) session.execute( text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") From 6266ba7a065eb46bac77c55b7864f72051a357bd Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 13:57:05 +0200 Subject: [PATCH 04/27] doc: add todos and remove trivially true assert --- src/tagstudio/core/library/alchemy/library.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 2e3734163..b8018871b 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -587,9 +587,14 @@ def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: ) logger.info(f"[Library] Library DB version: {loaded_db_version}") + # TODO: this is very sketchy; blindly creating all tables the newest DB version should have + # without considering what version the DB is currently on and then doing all of the + # migrations after that seems like it could cause problems in some scenarios. + # instead only have this on creation and create new tables as part of migrations make_tables(self.engine) with Session(self.engine) as session: + # TODO ASSURANCE 1: version check + reordering # Ensure version rows are present with catch_warnings(record=True): try: @@ -605,6 +610,7 @@ def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: except IntegrityError: session.rollback() + # TODO ASSURANCE 2: version check + reordering # check if folder matching current path exists already self.folder = session.scalar(select(Folder).where(Folder.path == library_dir)) if not self.folder: @@ -617,9 +623,6 @@ def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: session.commit() self.folder = folder - # Apply any post-SQL migration patches. - assert loaded_db_version >= 6 - # save backup if patches will be applied if loaded_db_version < DB_VERSION: self.library_dir = library_dir @@ -627,6 +630,7 @@ def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: self.library_dir = None # migrate DB step by step from one version to the next + # TODO: every migration step should either complete sucessfully or be aborted fully if loaded_db_version < 7: # changes: value_type, tags self.__apply_db7_migration(session) @@ -658,6 +662,7 @@ def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: # changes: tag_parents self.__apply_db202_migration(session) + # TODO ASSURANCE 3: version check + reordering session.execute( text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") ) @@ -672,6 +677,7 @@ def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: ) ) + # TODO: instead update DB version after every migration and abort on first fail # Update DB_VERSION if loaded_db_version < DB_VERSION: logger.info(f"[Library] Library migrated to DB version {DB_VERSION}") From db520890a2f91c60c09f6993a47180652593d64e Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 14:17:31 +0200 Subject: [PATCH 05/27] refactor: add assurance 1 version check Assurance 1 was introduced with library version 101, specifically commmit 12e074b71d8860282b44e49e0e1a41b7a2e4bae8. Thus all libraries created on version 101 and above already fullfill it and don't need it. Furthermore, it only fails if the library didn't need the assurance, meaning the try-except and warning catching can be removed --- src/tagstudio/core/library/alchemy/library.py | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index b8018871b..731d4f2f9 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -538,7 +538,6 @@ def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: return LibraryStatus(success=True, library_path=library_dir) def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: - is_new = False connection_string = URL.create( drivername="sqlite", database=storage_path, @@ -591,24 +590,18 @@ def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: # without considering what version the DB is currently on and then doing all of the # migrations after that seems like it could cause problems in some scenarios. # instead only have this on creation and create new tables as part of migrations + # Note: this actually produces an error and fails to initialise built-in tags when opening + # a library that doesn't yet have the is_hidden property on the tags table make_tables(self.engine) with Session(self.engine) as session: - # TODO ASSURANCE 1: version check + reordering + # TODO ASSURANCE 1: reordering # Ensure version rows are present - with catch_warnings(record=True): - try: - initial = DB_VERSION if is_new else 100 - session.add(Version(key=DB_VERSION_INITIAL_KEY, value=initial)) - session.commit() - except IntegrityError: - session.rollback() + if loaded_db_version < 101: + session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) - try: - session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) - session.commit() - except IntegrityError: - session.rollback() + session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) + session.commit() # TODO ASSURANCE 2: version check + reordering # check if folder matching current path exists already From fda10ec67c012813e0b9061beaaad37b8f6e6eb8 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 14:28:52 +0200 Subject: [PATCH 06/27] refactor: remove various unnecessary try-except statements in new_lib All of these were introduced to account for the differences between new libraries and existing ones, which makes them unnecessary after this split. --- src/tagstudio/core/library/alchemy/library.py | 55 +++++-------------- 1 file changed, 13 insertions(+), 42 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 731d4f2f9..0977b6877 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -19,7 +19,6 @@ from pathlib import Path from typing import TYPE_CHECKING from uuid import uuid4 -from warnings import catch_warnings import sqlalchemy import structlog @@ -441,14 +440,9 @@ def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: # Add default tag color namespaces. namespaces = default_color_groups.namespaces() - # TODO: do these try excepts even make sense? Shouldn't library creation just fail - # in these cases? - try: - session.add_all(namespaces) - session.commit() - except IntegrityError as e: - logger.error("[Library] Couldn't add default tag color namespaces", error=e) - session.rollback() + # TODO: are all of these commits necessary? + session.add_all(namespaces) + session.commit() # Add default tag colors. tag_colors: list[TagColorGroup] = default_color_groups.standard() @@ -458,45 +452,22 @@ def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: tag_colors += default_color_groups.earth_tones() tag_colors += default_color_groups.neon() - try: - session.add_all(tag_colors) - session.commit() - except IntegrityError as e: - logger.error("[Library] Couldn't add default tag colors", error=e) - session.rollback() + session.add_all(tag_colors) + session.commit() # Add default tags. - tags = get_default_tags() - try: - session.add_all(tags) - session.commit() - except IntegrityError: - session.rollback() + session.add_all(get_default_tags()) + session.commit() # Add default field templates for template in get_default_field_templates(): - try: - session.add(template) - session.commit() - except IntegrityError: - logger.info("[Library] FieldTemplate already exists", field_template=template) - session.rollback() + session.add(template) + session.commit() # Ensure version rows are present - with catch_warnings(record=True): - try: - initial = DB_VERSION - session.add(Version(key=DB_VERSION_INITIAL_KEY, value=initial)) - session.commit() - except IntegrityError: - session.rollback() - - try: - session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) - session.commit() - except IntegrityError: - session.rollback() - self.set_version(DB_VERSION_CURRENT_KEY, DB_VERSION) + session.add(Version(key=DB_VERSION_INITIAL_KEY, value=DB_VERSION)) + session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) + session.commit() # check if folder matching current path exists already self.folder = session.scalar(select(Folder).where(Folder.path == library_dir)) @@ -2257,7 +2228,7 @@ def set_version(self, key: str, value: int) -> None: session.add(version) session.commit() except (IntegrityError, AssertionError) as e: - logger.error("[Library][ERROR] Couldn't add default tag color namespaces", error=e) + logger.error("[Library][ERROR] Couldn't set DB version value", error=e) session.rollback() def mirror_entry_fields(self, entries: list[Entry]) -> None: From a7985b971a0663d835b74d431a99097237ee5ccd Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 14:32:41 +0200 Subject: [PATCH 07/27] refactor: remove unnecessary check in new_lib --- src/tagstudio/core/library/alchemy/library.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 0977b6877..5a8c938d1 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -470,16 +470,14 @@ def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: session.commit() # check if folder matching current path exists already - self.folder = session.scalar(select(Folder).where(Folder.path == library_dir)) - if not self.folder: - folder = Folder( - path=library_dir, - uuid=str(uuid4()), - ) - session.add(folder) - session.expunge(folder) - session.commit() - self.folder = folder + folder = Folder( + path=library_dir, + uuid=str(uuid4()), + ) + session.add(folder) + session.expunge(folder) + session.commit() + self.folder = folder # Generate default .ts_ignore file try: @@ -570,7 +568,6 @@ def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: # Ensure version rows are present if loaded_db_version < 101: session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) - session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) session.commit() From 804bb89b91163fe039f09d9505301da0a0902bbd Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 14:49:00 +0200 Subject: [PATCH 08/27] refactor: move folder assurance after migrations The folder assurance has been present since the very first SQL commit e5e7b8afc62c1a958c172b35819caa26995f2060, and thus only has an effect when the library is moved, meaning that is semantically not part of the migrations. --- src/tagstudio/core/library/alchemy/library.py | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 5a8c938d1..ef3840779 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -469,7 +469,7 @@ def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) session.commit() - # check if folder matching current path exists already + # add folder for current path folder = Folder( path=library_dir, uuid=str(uuid4()), @@ -571,19 +571,6 @@ def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) session.commit() - # TODO ASSURANCE 2: version check + reordering - # check if folder matching current path exists already - self.folder = session.scalar(select(Folder).where(Folder.path == library_dir)) - if not self.folder: - folder = Folder( - path=library_dir, - uuid=str(uuid4()), - ) - session.add(folder) - session.expunge(folder) - session.commit() - self.folder = folder - # save backup if patches will be applied if loaded_db_version < DB_VERSION: self.library_dir = library_dir @@ -644,6 +631,18 @@ def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: logger.info(f"[Library] Library migrated to DB version {DB_VERSION}") self.set_version(DB_VERSION_CURRENT_KEY, DB_VERSION) + # check if folder matching current path exists already + self.folder = session.scalar(select(Folder).where(Folder.path == library_dir)) + if not self.folder: + folder = Folder( + path=library_dir, + uuid=str(uuid4()), + ) + session.add(folder) + session.expunge(folder) + session.commit() + self.folder = folder + # everything is fine, set the library path self.library_dir = library_dir return LibraryStatus(success=True, library_path=library_dir) From 4a8d404905a23858c94d1c05ce05623587a978d5 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 15:13:48 +0200 Subject: [PATCH 09/27] refactor: massively simplify open_library --- src/tagstudio/core/library/alchemy/library.py | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index ef3840779..617e481c4 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -385,36 +385,35 @@ def open_library(self, library_dir: Path, in_memory: bool = False) -> LibrarySta Handles in-memory storage and checks whether a JSON-migration is necessary. """ assert isinstance(library_dir, Path) + self.verify_ts_folder(library_dir) # ensure .TagStudio directory exists - if in_memory: - return self.open_sqlite_library(library_dir, is_new=True, storage_path=":memory:") - - is_new = True sql_path = library_dir / TS_FOLDER_NAME / SQL_FILENAME - if self.verify_ts_folder(library_dir) and (is_new := not sql_path.exists()): - json_path = library_dir / TS_FOLDER_NAME / JSON_FILENAME - if json_path.exists(): - return LibraryStatus( - success=False, - library_path=library_dir, - message="[JSON] Legacy v9.4 library requires conversion to v9.5+", - json_migration_req=True, - ) + json_path = library_dir / TS_FOLDER_NAME / JSON_FILENAME + + is_new = not sql_path.exists() + if not in_memory and is_new and json_path.exists(): + return LibraryStatus( + success=False, + library_path=library_dir, + message="[JSON] Legacy v9.4 library requires conversion to v9.5+", + json_migration_req=True, + ) - return self.open_sqlite_library(library_dir, is_new, str(sql_path)) + return self.open_sqlite_library(library_dir, is_new, in_memory) def open_sqlite_library( - self, library_dir: Path, is_new: bool, storage_path: str + self, library_dir: Path, is_new: bool, in_memory: bool ) -> LibraryStatus: if is_new: - return self.new_lib(library_dir, storage_path) - else: - return self.migrate_lib(library_dir, storage_path) + return self.new_lib(library_dir, in_memory) + return self.migrate_lib(library_dir, in_memory) - def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: + def new_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: connection_string = URL.create( drivername="sqlite", - database=storage_path, + database=( + ":memory:" if in_memory else str(library_dir / TS_FOLDER_NAME / SQL_FILENAME) + ), ) # NOTE: File-based databases should use NullPool to create new DB connection in order to # keep connections on separate threads, which prevents the DB files from being locked @@ -423,7 +422,7 @@ def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: # More info can be found on the SQLAlchemy docs: # https://docs.sqlalchemy.org/en/20/changelog/migration_07.html # Under -> sqlite-the-sqlite-dialect-now-uses-nullpool-for-file-based-databases - poolclass = None if storage_path == ":memory:" else NullPool + poolclass = None if in_memory else NullPool loaded_db_version: int = 0 logger.info( @@ -506,10 +505,12 @@ def new_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: self.library_dir = library_dir return LibraryStatus(success=True, library_path=library_dir) - def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: + def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: connection_string = URL.create( drivername="sqlite", - database=storage_path, + database=( + ":memory:" if in_memory else str(library_dir / TS_FOLDER_NAME / SQL_FILENAME) + ), ) # NOTE: File-based databases should use NullPool to create new DB connection in order to # keep connections on separate threads, which prevents the DB files from being locked @@ -518,7 +519,7 @@ def migrate_lib(self, library_dir: Path, storage_path: str) -> LibraryStatus: # More info can be found on the SQLAlchemy docs: # https://docs.sqlalchemy.org/en/20/changelog/migration_07.html # Under -> sqlite-the-sqlite-dialect-now-uses-nullpool-for-file-based-databases - poolclass = None if storage_path == ":memory:" else NullPool + poolclass = None if in_memory else NullPool loaded_db_version: int = 0 initial_db_version: int = DB_VERSION @@ -1170,11 +1171,12 @@ def verify_ts_folder(self, library_dir: Path | None) -> bool: raise ValueError("Invalid library directory.") full_ts_path = library_dir / TS_FOLDER_NAME - if not full_ts_path.exists(): - logger.info("creating library directory", dir=full_ts_path) - full_ts_path.mkdir(parents=True, exist_ok=True) - return False - return True + if full_ts_path.exists(): + return True + + logger.info("creating library directory", dir=full_ts_path) + full_ts_path.mkdir(parents=True, exist_ok=True) + return False def add_entries(self, items: list[Entry]) -> list[int]: """Add multiple Entry records to the Library.""" From 67fe77a67cc46e7b4339354fa3596571235f986f Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 15:21:20 +0200 Subject: [PATCH 10/27] refactor: move engine creation to static method --- src/tagstudio/core/library/alchemy/library.py | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 617e481c4..38095332b 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -408,7 +408,8 @@ def open_sqlite_library( return self.new_lib(library_dir, in_memory) return self.migrate_lib(library_dir, in_memory) - def new_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: + @staticmethod + def __get_engine(library_dir: Path, in_memory: bool): connection_string = URL.create( drivername="sqlite", database=( @@ -423,14 +424,22 @@ def new_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: # https://docs.sqlalchemy.org/en/20/changelog/migration_07.html # Under -> sqlite-the-sqlite-dialect-now-uses-nullpool-for-file-based-databases poolclass = None if in_memory else NullPool + + logger.info( + "[Library] Creating SQLAlchemy Engine", + connection_string=connection_string, + poolclass=poolclass, + ) + return create_engine(connection_string, poolclass=poolclass) + + def new_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: + self.engine = self.__get_engine(library_dir, in_memory) loaded_db_version: int = 0 logger.info( "[Library] Opening SQLite Library", library_dir=library_dir, - connection_string=connection_string, ) - self.engine = create_engine(connection_string, poolclass=poolclass) logger.info(f"[Library] Library DB version: {loaded_db_version}") make_tables(self.engine) @@ -506,29 +515,14 @@ def new_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: return LibraryStatus(success=True, library_path=library_dir) def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: - connection_string = URL.create( - drivername="sqlite", - database=( - ":memory:" if in_memory else str(library_dir / TS_FOLDER_NAME / SQL_FILENAME) - ), - ) - # NOTE: File-based databases should use NullPool to create new DB connection in order to - # keep connections on separate threads, which prevents the DB files from being locked - # even after a connection has been closed. - # SingletonThreadPool (the default for :memory:) should still be used for in-memory DBs. - # More info can be found on the SQLAlchemy docs: - # https://docs.sqlalchemy.org/en/20/changelog/migration_07.html - # Under -> sqlite-the-sqlite-dialect-now-uses-nullpool-for-file-based-databases - poolclass = None if in_memory else NullPool + self.engine = self.__get_engine(library_dir, in_memory) loaded_db_version: int = 0 initial_db_version: int = DB_VERSION logger.info( "[Library] Opening SQLite Library", library_dir=library_dir, - connection_string=connection_string, ) - self.engine = create_engine(connection_string, poolclass=poolclass) # Don't check DB version when creating new library loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY) From 6679bb92fb91b83126c6025fdfdab32487f4eb22 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 15:30:56 +0200 Subject: [PATCH 11/27] refactor: add version check for assurance 3 Assurance 3 was introduced in commit 47c3d5338f4c3ef2cde25d8c1bd4dcaa6d14a28f shortly (24h 20min) after the DB version had been bumped to 200. Since it was also included in the first release with DB version 200, all libraries created on DB version 200 and above can be presumed to have already had this applied on creation. --- src/tagstudio/core/library/alchemy/library.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 38095332b..1be7411c5 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -598,27 +598,33 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: if loaded_db_version < 200: # changes: field tables self.__apply_db200_migration(session) - if initial_db_version < 200 and loaded_db_version < 201: + if loaded_db_version < 201 and initial_db_version < 200: # changes: field tables self.__apply_db201_migration(session) if loaded_db_version < 202: # changes: tag_parents self.__apply_db202_migration(session) - # TODO ASSURANCE 3: version check + reordering - session.execute( - text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") - ) - session.execute( - text( - "CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id ON tag_parents (child_id)" + # TODO ASSURANCE 3: reordering + if loaded_db_version < 200: + session.execute( + text( + "CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand " + "ON tags (name, shorthand)" + ) ) - ) - session.execute( - text( - "CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)" + session.execute( + text( + "CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id " + "ON tag_parents (child_id)" + ) + ) + session.execute( + text( + "CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id " + "ON tag_entries (entry_id)" + ) ) - ) # TODO: instead update DB version after every migration and abort on first fail # Update DB_VERSION From f4f33b0e015e80093051c5c35b90673211b89611 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 15:35:34 +0200 Subject: [PATCH 12/27] refactor: add assurance 3 to DB 200 migration --- src/tagstudio/core/library/alchemy/library.py | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 1be7411c5..7620a14b2 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -605,27 +605,6 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: # changes: tag_parents self.__apply_db202_migration(session) - # TODO ASSURANCE 3: reordering - if loaded_db_version < 200: - session.execute( - text( - "CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand " - "ON tags (name, shorthand)" - ) - ) - session.execute( - text( - "CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id " - "ON tag_parents (child_id)" - ) - ) - session.execute( - text( - "CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id " - "ON tag_entries (entry_id)" - ) - ) - # TODO: instead update DB version after every migration and abort on first fail # Update DB_VERSION if loaded_db_version < DB_VERSION: @@ -912,6 +891,21 @@ def __apply_db200_migration(self, session: Session): session.commit() + # DB indices for improved performance + session.execute( + text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") + ) + session.execute( + text( + "CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id ON tag_parents (child_id)" + ) + ) + session.execute( + text( + "CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)" + ) + ) + def __apply_db201_migration(self, session: Session): """Migrate DB to DB_VERSION 201.""" with session: From eb41ed0eb909858c0af8c8b8b3b08d8e7ed986f6 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 15:46:13 +0200 Subject: [PATCH 13/27] refactor: move assurance 1 to a proper migration method Moving the assurance after the migrations 7, 8, 9, and 100 is fine because it doesn't affect those. --- src/tagstudio/core/library/alchemy/library.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 7620a14b2..42610ad19 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -559,13 +559,6 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: make_tables(self.engine) with Session(self.engine) as session: - # TODO ASSURANCE 1: reordering - # Ensure version rows are present - if loaded_db_version < 101: - session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) - session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) - session.commit() - # save backup if patches will be applied if loaded_db_version < DB_VERSION: self.library_dir = library_dir @@ -574,6 +567,8 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: # migrate DB step by step from one version to the next # TODO: every migration step should either complete sucessfully or be aborted fully + # - the migration steps seem to have try-except cases to catch a previously failed + # migration if loaded_db_version < 7: # changes: value_type, tags self.__apply_db7_migration(session) @@ -586,6 +581,9 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: if loaded_db_version < 100: # changes: tag_parents self.__apply_db100_migration(session) + if loaded_db_version < 101: + # changes: versions + self.__apply_db101_migration(session) if loaded_db_version < 102: # changes: tag_parents self.__apply_db102_migration(session) @@ -742,6 +740,14 @@ def __apply_db100_migration(self, session: Session): session.commit() logger.info("[Library][Migration] Refactored TagParent table") + def __apply_db101_migration(self, session: Session): + """Migrate DB to DB_VERSION 101.""" + with session: + # Ensure version rows are present + session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) + session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) + session.commit() + def __apply_db102_migration(self, session: Session): """Migrate DB to DB_VERSION 102.""" with session: From e383cab4023e5390e1178f14000bbb5bbc8bfb57 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 16:14:39 +0200 Subject: [PATCH 14/27] refactor: update version after every successfull migration Since every migration migrates the library to a certain DB_VERSION, we can simply update the library version after such a successfull migration. This way if a migration fails, the previous migrations won't be rerun the next time the library is opened. --- src/tagstudio/core/library/alchemy/library.py | 60 +++++++++++++------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 42610ad19..135a81e7d 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -569,45 +569,78 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: # TODO: every migration step should either complete sucessfully or be aborted fully # - the migration steps seem to have try-except cases to catch a previously failed # migration + # TODO: the migration and the version update should have fate-sharing if loaded_db_version < 7: # changes: value_type, tags self.__apply_db7_migration(session) + loaded_db_version = 7 + self.set_version(session, DB_VERSION_CURRENT_KEY, 7) + session.commit() if loaded_db_version < 8: # changes: tag_colors self.__apply_db8_migration(session) + loaded_db_version = 8 + self.set_version(session, DB_VERSION_CURRENT_KEY, 8) + session.commit() if loaded_db_version < 9: # changes: entries self.__apply_db9_migration(session) + loaded_db_version = 9 + self.set_version(session, DB_VERSION_CURRENT_KEY, 9) + session.commit() if loaded_db_version < 100: # changes: tag_parents self.__apply_db100_migration(session) + loaded_db_version = 100 + self.set_version(session, DB_VERSION_CURRENT_KEY, 100) + session.commit() if loaded_db_version < 101: # changes: versions self.__apply_db101_migration(session) + loaded_db_version = 101 + self.set_version(session, DB_VERSION_CURRENT_KEY, 101) + session.commit() if loaded_db_version < 102: # changes: tag_parents self.__apply_db102_migration(session) + loaded_db_version = 102 + self.set_version(session, DB_VERSION_CURRENT_KEY, 102) + session.commit() if loaded_db_version < 103: # changes: tags self.__apply_db103_migration(session) + loaded_db_version = 103 + self.set_version(session, DB_VERSION_CURRENT_KEY, 103) + session.commit() if loaded_db_version < 104: # changes: deletes preferences self.__apply_db104_migration(session, library_dir) + loaded_db_version = 104 + self.set_version(session, DB_VERSION_CURRENT_KEY, 104) + session.commit() if loaded_db_version < 200: # changes: field tables self.__apply_db200_migration(session) + loaded_db_version = 200 + self.set_version(session, DB_VERSION_CURRENT_KEY, 200) + session.commit() if loaded_db_version < 201 and initial_db_version < 200: # changes: field tables self.__apply_db201_migration(session) + loaded_db_version = 201 + self.set_version(session, DB_VERSION_CURRENT_KEY, 201) + session.commit() if loaded_db_version < 202: # changes: tag_parents self.__apply_db202_migration(session) + loaded_db_version = 202 + self.set_version(session, DB_VERSION_CURRENT_KEY, 202) + session.commit() - # TODO: instead update DB version after every migration and abort on first fail - # Update DB_VERSION - if loaded_db_version < DB_VERSION: - logger.info(f"[Library] Library migrated to DB version {DB_VERSION}") - self.set_version(DB_VERSION_CURRENT_KEY, DB_VERSION) + assert loaded_db_version == DB_VERSION, ( + "Ran all migrations, but the DB is still not on the newest version" + ) + logger.info(f"[Library] Library migrated to DB version {DB_VERSION}") # check if folder matching current path exists already self.folder = session.scalar(select(Folder).where(Folder.path == library_dir)) @@ -745,7 +778,6 @@ def __apply_db101_migration(self, session: Session): with session: # Ensure version rows are present session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) - session.add(Version(key=DB_VERSION_CURRENT_KEY, value=DB_VERSION)) session.commit() def __apply_db102_migration(self, session: Session): @@ -2211,23 +2243,17 @@ def get_version(self, key: str) -> int: except Exception: return 0 - def set_version(self, key: str, value: int) -> None: + def set_version(self, session: Session, key: str, value: int) -> None: """Set a version value to the DB. Args: + session(Session): The SQLAlchemy DB Session to use. key(str): The key for the name of the version type to set. value(int): The version value to set. """ - with Session(self.engine) as session: - try: - version = session.scalar(select(Version).where(Version.key == key)) - assert version - version.value = value - session.add(version) - session.commit() - except (IntegrityError, AssertionError) as e: - logger.error("[Library][ERROR] Couldn't set DB version value", error=e) - session.rollback() + with session: + # Insert if key has no value yet, otherwise update the value + session.merge(Version(key=key, value=value)) def mirror_entry_fields(self, entries: list[Entry]) -> None: """Mirror fields among multiple Entry items.""" From cc78c1abb925b332fa04639e82ec6057d590a1de Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 16:29:46 +0200 Subject: [PATCH 15/27] refactor: rewrite migration procedure as loop --- src/tagstudio/core/library/alchemy/library.py | 87 +++++-------------- 1 file changed, 20 insertions(+), 67 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 135a81e7d..c621a9e25 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -569,73 +569,26 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: # TODO: every migration step should either complete sucessfully or be aborted fully # - the migration steps seem to have try-except cases to catch a previously failed # migration - # TODO: the migration and the version update should have fate-sharing - if loaded_db_version < 7: - # changes: value_type, tags - self.__apply_db7_migration(session) - loaded_db_version = 7 - self.set_version(session, DB_VERSION_CURRENT_KEY, 7) - session.commit() - if loaded_db_version < 8: - # changes: tag_colors - self.__apply_db8_migration(session) - loaded_db_version = 8 - self.set_version(session, DB_VERSION_CURRENT_KEY, 8) - session.commit() - if loaded_db_version < 9: - # changes: entries - self.__apply_db9_migration(session) - loaded_db_version = 9 - self.set_version(session, DB_VERSION_CURRENT_KEY, 9) - session.commit() - if loaded_db_version < 100: - # changes: tag_parents - self.__apply_db100_migration(session) - loaded_db_version = 100 - self.set_version(session, DB_VERSION_CURRENT_KEY, 100) - session.commit() - if loaded_db_version < 101: - # changes: versions - self.__apply_db101_migration(session) - loaded_db_version = 101 - self.set_version(session, DB_VERSION_CURRENT_KEY, 101) - session.commit() - if loaded_db_version < 102: - # changes: tag_parents - self.__apply_db102_migration(session) - loaded_db_version = 102 - self.set_version(session, DB_VERSION_CURRENT_KEY, 102) - session.commit() - if loaded_db_version < 103: - # changes: tags - self.__apply_db103_migration(session) - loaded_db_version = 103 - self.set_version(session, DB_VERSION_CURRENT_KEY, 103) - session.commit() - if loaded_db_version < 104: - # changes: deletes preferences - self.__apply_db104_migration(session, library_dir) - loaded_db_version = 104 - self.set_version(session, DB_VERSION_CURRENT_KEY, 104) - session.commit() - if loaded_db_version < 200: - # changes: field tables - self.__apply_db200_migration(session) - loaded_db_version = 200 - self.set_version(session, DB_VERSION_CURRENT_KEY, 200) - session.commit() - if loaded_db_version < 201 and initial_db_version < 200: - # changes: field tables - self.__apply_db201_migration(session) - loaded_db_version = 201 - self.set_version(session, DB_VERSION_CURRENT_KEY, 201) - session.commit() - if loaded_db_version < 202: - # changes: tag_parents - self.__apply_db202_migration(session) - loaded_db_version = 202 - self.set_version(session, DB_VERSION_CURRENT_KEY, 202) - session.commit() + # (migration_method, db_version, initial_db_version) + migrations = [ + (self.__apply_db7_migration, 7, None), # changes: value_type, tags + (self.__apply_db8_migration, 8, None), # changes: tag_colors + (self.__apply_db9_migration, 9, None), # changes: entries + (self.__apply_db100_migration, 100, None), # changes: tag_parents + (self.__apply_db101_migration, 101, None), # changes: versions + (self.__apply_db102_migration, 102, None), # changes: tag_parents + (self.__apply_db103_migration, 103, None), # changes: tags + (self.__apply_db104_migration, 104, None), # changes: deletes preferences + (self.__apply_db200_migration, 200, None), # changes: field tables + (self.__apply_db201_migration, 201, 200), # changes: field tables + (self.__apply_db202_migration, 202, None), # changes: tag_parents + ] + for migration, v, iv in migrations: + if loaded_db_version < v and (iv is None or initial_db_version < iv): + migration(session) + loaded_db_version = v + self.set_version(session, DB_VERSION_CURRENT_KEY, v) + session.commit() assert loaded_db_version == DB_VERSION, ( "Ran all migrations, but the DB is still not on the newest version" From d8b339a17c25637c5c81130b60136c8036d03034 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 16:42:53 +0200 Subject: [PATCH 16/27] refactor: apply migration and update version in same transaction None of the content of the migrations is changed, only the `with session:` statements are removed. --- src/tagstudio/core/library/alchemy/library.py | 365 +++++++++--------- 1 file changed, 175 insertions(+), 190 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index c621a9e25..6ee0f940c 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -570,6 +570,7 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: # - the migration steps seem to have try-except cases to catch a previously failed # migration # (migration_method, db_version, initial_db_version) + # TODO: log statements in migrations are inconsistent migrations = [ (self.__apply_db7_migration, 7, None), # changes: value_type, tags (self.__apply_db8_migration, 8, None), # changes: tag_colors @@ -585,10 +586,11 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: ] for migration, v, iv in migrations: if loaded_db_version < v and (iv is None or initial_db_version < iv): - migration(session) - loaded_db_version = v - self.set_version(session, DB_VERSION_CURRENT_KEY, v) - session.commit() + with session: + migration(session) + loaded_db_version = v + self.set_version(session, DB_VERSION_CURRENT_KEY, v) + session.commit() assert loaded_db_version == DB_VERSION, ( "Ran all migrations, but the DB is still not on the newest version" @@ -614,16 +616,15 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: def __apply_db7_migration(self, session: Session): """Migrate DB from DB_VERSION 6 to 7.""" logger.info("[Library][Migration] Applying patches to DB_VERSION: 6 library...") - with session: - # Repair tags that may have a disambiguation_id pointing towards a deleted tag. - all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all() - disam_stmt = ( - update(Tag) - .where(Tag.disambiguation_id.not_in(all_tag_ids)) - .values(disambiguation_id=None) - ) - session.execute(disam_stmt) - session.commit() + # Repair tags that may have a disambiguation_id pointing towards a deleted tag. + all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all() + disam_stmt = ( + update(Tag) + .where(Tag.disambiguation_id.not_in(all_tag_ids)) + .values(disambiguation_id=None) + ) + session.execute(disam_stmt) + session.commit() def __apply_db8_migration(self, session: Session): """Migrate DB from DB_VERSION 7 to 8.""" @@ -716,30 +717,27 @@ def __apply_db9_migration(self, session: Session): def __apply_db100_migration(self, session: Session): """Migrate DB to DB_VERSION 100.""" - with session: - # Repair parent-child tag relationships that are the wrong way around. - stmt = update(TagParent).values( - parent_id=TagParent.child_id, - child_id=TagParent.parent_id, - ) - session.execute(stmt) - session.commit() - logger.info("[Library][Migration] Refactored TagParent table") + # Repair parent-child tag relationships that are the wrong way around. + stmt = update(TagParent).values( + parent_id=TagParent.child_id, + child_id=TagParent.parent_id, + ) + session.execute(stmt) + session.commit() + logger.info("[Library][Migration] Refactored TagParent table") def __apply_db101_migration(self, session: Session): """Migrate DB to DB_VERSION 101.""" - with session: - # Ensure version rows are present - session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) - session.commit() + # Ensure version rows are present + session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) + session.commit() def __apply_db102_migration(self, session: Session): """Migrate DB to DB_VERSION 102.""" - with session: - stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct())) - session.execute(stmt) - session.commit() - logger.info("[Library][Migration] Verified TagParent table data") + stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct())) + session.execute(stmt) + session.commit() + logger.info("[Library][Migration] Verified TagParent table data") def __apply_db103_migration(self, session: Session): """Migrate DB from DB_VERSION 102 to 103.""" @@ -773,188 +771,176 @@ def __apply_db103_migration(self, session: Session): def __apply_db104_migration(self, session: Session, library_dir: Path): """Migrate DB from DB_VERSION 103 to 104.""" # Convert file extension list to ts_ignore file, if a .ts_ignore file does not exist - self.__migrate_sql_to_ts_ignore(library_dir) + self.__migrate_sql_to_ts_ignore(session, library_dir) session.execute(text("DROP TABLE preferences")) session.commit() - def __migrate_sql_to_ts_ignore(self, library_dir: Path): + def __migrate_sql_to_ts_ignore(self, session: Session, library_dir: Path): # Do not continue if existing '.ts_ignore' file is found ts_ignore = library_dir / TS_FOLDER_NAME / IGNORE_NAME if Path(ts_ignore).exists(): return # Load legacy extension data - with Session(self.engine) as session: - extensions: list[str] = unwrap( - session.scalar(text("SELECT value FROM preferences WHERE key = 'EXTENSION_LIST'")) - ) - is_exclude_list: bool = unwrap( - session.scalar(text("SELECT value FROM preferences WHERE key = 'IS_EXCLUDE_LIST'")) - ) + extensions: list[str] = unwrap( + session.scalar(text("SELECT value FROM preferences WHERE key = 'EXTENSION_LIST'")) + ) + is_exclude_list: bool = unwrap( + session.scalar(text("SELECT value FROM preferences WHERE key = 'IS_EXCLUDE_LIST'")) + ) with open(ts_ignore, "w") as f: f.write(migrate_ext_list(extensions, is_exclude_list)) def __apply_db200_migration(self, session: Session): """Migrate DB to DB_VERSION 200.""" - with session: - # Drop unused 'boolean_fields' and 'value_type' tables - logger.info( - "[Library][Migration][200] Dropping boolean_fields and value_type tables..." - ) - session.execute(text("DROP TABLE boolean_fields")) - session.execute(text("DROP TABLE value_type")) - - # Add 'name' column to text_fields and datetime_fields tables - logger.info("[Library][Migration][200] Adding name columns to field tables...") - stmt = text('ALTER TABLE text_fields ADD COLUMN name VARCHAR DEFAULT ""') - session.execute(stmt) - stmt = text('ALTER TABLE datetime_fields ADD COLUMN name VARCHAR DEFAULT ""') - session.execute(stmt) - - # Drop unnecessary 'position' columns - logger.info("[Library][Migration][200] Dropping position columns to field tables...") - session.execute(text("ALTER TABLE datetime_fields DROP COLUMN position")) - session.execute(text("ALTER TABLE text_fields DROP COLUMN position")) - - # Add 'is_multiline' column to text_fields table - logger.info("[Library][Migration][200] Adding is_multiline column to text_fields...") - stmt = text( - "ALTER TABLE text_fields ADD COLUMN is_multiline BOOLEAN NOT NULL DEFAULT 0" - ) - session.execute(stmt) - session.flush() - - # Move values from old `type_key` columns into new `name` columns - logger.info("[Library][Migration][200] Moving values from type_key columns to name...") - session.execute(text("UPDATE text_fields SET name = type_key")) - session.execute(text("UPDATE datetime_fields SET name = type_key")) - session.flush() - - # Change `name` values to title case - logger.info("[Library][Migration][200] Normalizing TextField names...") - for text_field in session.execute(select(TextField)).scalars(): - # NOTE: The only exception to the "Title Case" conversion is the "URL" field. - text_field.name = text_field.name.title().replace("Url", "URL").replace("_", " ") - logger.info("[Library][Migration][200] Normalizing DatetimeField names...") - for datetime_field in session.execute(select(DatetimeField)).scalars(): - datetime_field.name = datetime_field.name.title().replace("_", " ") - session.flush() - - # Add correct `is_multiline` values to text_fields table - logger.info("[Library][Migration][200] Updating is_multiline for legacy TEXT_BOXes...") - text_boxes = [ - x.get("name") for x in LEGACY_FIELD_MAP.values() if x.get("is_multiline") is True - ] - update_stmt = ( - update(TextField).where(TextField.name.in_(text_boxes)).values(is_multiline=True) - ) - session.execute(update_stmt) - session.flush() - - # Repair legacy "Description" fields to use is_multiline = True - logger.info("[Library][Migration][200] Repairing legacy Description fields...") - desc_stmt = ( - update(TextField) - .where(TextField.name == "Description" and TextField.is_multiline == False) # noqa: E712 - .values(is_multiline=True) - ) - session.execute(desc_stmt) - - # Repair legacy "Comments" fields to use is_multiline = True - logger.info("[Library][Migration][200] Repairing legacy Comment fields...") - comm_stmt = ( - update(TextField) - .where(TextField.name == "Comments" and TextField.is_multiline == False) # noqa: E712 - .values(is_multiline=True) - ) - session.execute(comm_stmt) + # Drop unused 'boolean_fields' and 'value_type' tables + logger.info("[Library][Migration][200] Dropping boolean_fields and value_type tables...") + session.execute(text("DROP TABLE boolean_fields")) + session.execute(text("DROP TABLE value_type")) + + # Add 'name' column to text_fields and datetime_fields tables + logger.info("[Library][Migration][200] Adding name columns to field tables...") + stmt = text('ALTER TABLE text_fields ADD COLUMN name VARCHAR DEFAULT ""') + session.execute(stmt) + stmt = text('ALTER TABLE datetime_fields ADD COLUMN name VARCHAR DEFAULT ""') + session.execute(stmt) + + # Drop unnecessary 'position' columns + logger.info("[Library][Migration][200] Dropping position columns to field tables...") + session.execute(text("ALTER TABLE datetime_fields DROP COLUMN position")) + session.execute(text("ALTER TABLE text_fields DROP COLUMN position")) + + # Add 'is_multiline' column to text_fields table + logger.info("[Library][Migration][200] Adding is_multiline column to text_fields...") + stmt = text("ALTER TABLE text_fields ADD COLUMN is_multiline BOOLEAN NOT NULL DEFAULT 0") + session.execute(stmt) + session.flush() + + # Move values from old `type_key` columns into new `name` columns + logger.info("[Library][Migration][200] Moving values from type_key columns to name...") + session.execute(text("UPDATE text_fields SET name = type_key")) + session.execute(text("UPDATE datetime_fields SET name = type_key")) + session.flush() + + # Change `name` values to title case + logger.info("[Library][Migration][200] Normalizing TextField names...") + for text_field in session.execute(select(TextField)).scalars(): + # NOTE: The only exception to the "Title Case" conversion is the "URL" field. + text_field.name = text_field.name.title().replace("Url", "URL").replace("_", " ") + logger.info("[Library][Migration][200] Normalizing DatetimeField names...") + for datetime_field in session.execute(select(DatetimeField)).scalars(): + datetime_field.name = datetime_field.name.title().replace("_", " ") + session.flush() + + # Add correct `is_multiline` values to text_fields table + logger.info("[Library][Migration][200] Updating is_multiline for legacy TEXT_BOXes...") + text_boxes = [ + x.get("name") for x in LEGACY_FIELD_MAP.values() if x.get("is_multiline") is True + ] + update_stmt = ( + update(TextField).where(TextField.name.in_(text_boxes)).values(is_multiline=True) + ) + session.execute(update_stmt) + session.flush() + + # Repair legacy "Description" fields to use is_multiline = True + logger.info("[Library][Migration][200] Repairing legacy Description fields...") + desc_stmt = ( + update(TextField) + .where(TextField.name == "Description" and TextField.is_multiline == False) # noqa: E712 + .values(is_multiline=True) + ) + session.execute(desc_stmt) + + # Repair legacy "Comments" fields to use is_multiline = True + logger.info("[Library][Migration][200] Repairing legacy Comment fields...") + comm_stmt = ( + update(TextField) + .where(TextField.name == "Comments" and TextField.is_multiline == False) # noqa: E712 + .values(is_multiline=True) + ) + session.execute(comm_stmt) - # Add default field templates - logger.info("[Library][Migration][200] Adding default field templates...") - for template in get_default_field_templates(): - try: - session.add(template) - session.flush() - except IntegrityError: - logger.error("[Library] FieldTemplate already exists", field_template=template) - session.rollback() + # Add default field templates + logger.info("[Library][Migration][200] Adding default field templates...") + for template in get_default_field_templates(): + try: + session.add(template) + session.flush() + except IntegrityError: + logger.error("[Library] FieldTemplate already exists", field_template=template) + session.rollback() - session.commit() + session.commit() - # DB indices for improved performance - session.execute( - text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") - ) - session.execute( - text( - "CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id ON tag_parents (child_id)" - ) - ) - session.execute( - text( - "CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)" - ) - ) + # DB indices for improved performance + session.execute( + text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") + ) + session.execute( + text("CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id ON tag_parents (child_id)") + ) + session.execute( + text("CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)") + ) def __apply_db201_migration(self, session: Session): """Migrate DB to DB_VERSION 201.""" - with session: - create_text_fields_table = text(""" - CREATE TABLE text_fields_new ( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - name VARCHAR NOT NULL, - entry_id INTEGER NOT NULL, - value VARCHAR, - is_multiline BOOLEAN NOT NULL, - FOREIGN KEY(entry_id) REFERENCES entries (id) - ) + create_text_fields_table = text(""" + CREATE TABLE text_fields_new ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR NOT NULL, + entry_id INTEGER NOT NULL, + value VARCHAR, + is_multiline BOOLEAN NOT NULL, + FOREIGN KEY(entry_id) REFERENCES entries (id) + ) + """) + create_datetime_fields_table = text(""" + CREATE TABLE datetime_fields_new ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR NOT NULL, + entry_id INTEGER NOT NULL, + value VARCHAR, + FOREIGN KEY(entry_id) REFERENCES entries (id) + ) + """) + + logger.info("[Library][Migration][201] Dropping type_key from text_fields table...") + session.execute(create_text_fields_table) + session.flush() + session.execute( + text(""" + INSERT INTO text_fields_new (id, name, entry_id, value, is_multiline) + SELECT id, name, entry_id, value, is_multiline + FROM text_fields """) - create_datetime_fields_table = text(""" - CREATE TABLE datetime_fields_new ( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - name VARCHAR NOT NULL, - entry_id INTEGER NOT NULL, - value VARCHAR, - FOREIGN KEY(entry_id) REFERENCES entries (id) - ) + ) + session.execute(text("DROP TABLE text_fields")) + session.execute(text("ALTER TABLE text_fields_new RENAME TO text_fields")) + + logger.info("[Library][Migration][201] Dropping type_key from datetime_fields table...") + session.execute(create_datetime_fields_table) + session.flush() + session.execute( + text(""" + INSERT INTO datetime_fields_new (id, name, entry_id, value) + SELECT id, name, entry_id, value + FROM datetime_fields """) + ) + session.execute(text("DROP TABLE datetime_fields")) + session.execute(text("ALTER TABLE datetime_fields_new RENAME TO datetime_fields")) - logger.info("[Library][Migration][201] Dropping type_key from text_fields table...") - session.execute(create_text_fields_table) - session.flush() - session.execute( - text(""" - INSERT INTO text_fields_new (id, name, entry_id, value, is_multiline) - SELECT id, name, entry_id, value, is_multiline - FROM text_fields - """) - ) - session.execute(text("DROP TABLE text_fields")) - session.execute(text("ALTER TABLE text_fields_new RENAME TO text_fields")) - - logger.info("[Library][Migration][201] Dropping type_key from datetime_fields table...") - session.execute(create_datetime_fields_table) - session.flush() - session.execute( - text(""" - INSERT INTO datetime_fields_new (id, name, entry_id, value) - SELECT id, name, entry_id, value - FROM datetime_fields - """) - ) - session.execute(text("DROP TABLE datetime_fields")) - session.execute(text("ALTER TABLE datetime_fields_new RENAME TO datetime_fields")) - - session.commit() + session.commit() def __apply_db202_migration(self, session: Session): """Migrate DB to DB_VERSION 202.""" - with session: - stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct())) - session.execute(stmt) - session.commit() - logger.info("[Library][Migration] Verified TagParent table data") + stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct())) + session.execute(stmt) + session.commit() + logger.info("[Library][Migration] Verified TagParent table data") @property def field_templates(self) -> Sequence[BaseFieldTemplate]: @@ -2204,9 +2190,8 @@ def set_version(self, session: Session, key: str, value: int) -> None: key(str): The key for the name of the version type to set. value(int): The version value to set. """ - with session: - # Insert if key has no value yet, otherwise update the value - session.merge(Version(key=key, value=value)) + # Insert if key has no value yet, otherwise update the value + session.merge(Version(key=key, value=value)) def mirror_entry_fields(self, entries: list[Entry]) -> None: """Mirror fields among multiple Entry items.""" From 577cf00453ba5c68f90e2d0f0c51088fb3ca310f Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 16:50:16 +0200 Subject: [PATCH 17/27] refactor: replace all commits in the migrations with flushes Also removes various try-except statements in the migrations. These were introduced to catch the case where the migration is run twice due to a later migration failing, this is not necessary anymore as every migration will only be executed at most once per library. --- src/tagstudio/core/library/alchemy/library.py | 142 ++++++------------ 1 file changed, 46 insertions(+), 96 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 6ee0f940c..e89ce3d34 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -566,9 +566,6 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: self.library_dir = None # migrate DB step by step from one version to the next - # TODO: every migration step should either complete sucessfully or be aborted fully - # - the migration steps seem to have try-except cases to catch a previously failed - # migration # (migration_method, db_version, initial_db_version) # TODO: log statements in migrations are inconsistent migrations = [ @@ -587,6 +584,7 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: for migration, v, iv in migrations: if loaded_db_version < v and (iv is None or initial_db_version < iv): with session: + # any error causes transaction to rollback migration(session) loaded_db_version = v self.set_version(session, DB_VERSION_CURRENT_KEY, v) @@ -624,24 +622,16 @@ def __apply_db7_migration(self, session: Session): .values(disambiguation_id=None) ) session.execute(disam_stmt) - session.commit() + session.flush() def __apply_db8_migration(self, session: Session): """Migrate DB from DB_VERSION 7 to 8.""" # Add the missing color_border column to the TagColorGroups table. - color_border_stmt = text( - "ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL" + session.execute( + text("ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL") ) - try: - session.execute(color_border_stmt) - session.commit() - logger.info("[Library][Migration] Added color_border column to tag_colors table") - except Exception as e: - logger.error( - "[Library][Migration] Could not create color_border column in tag_colors table!", - error=e, - ) - session.rollback() + session.flush() + logger.info("[Library][Migration] Added color_border column to tag_colors table") # collect new default tag colors tag_colors: list[TagColorGroup] = default_color_groups.standard() @@ -653,44 +643,34 @@ def __apply_db8_migration(self, session: Session): # Add any new default colors introduced in DB_VERSION 8 for color in tag_colors: - try: - session.add(color) - logger.info( - "[Library][Migration] Migrated tag color to DB_VERSION 8+", - color_name=color.name, - ) - session.commit() - except IntegrityError: - session.rollback() + session.add(color) + session.flush() + logger.info( + "[Library][Migration] Migrated tag colors to DB_VERSION 8+", + color_name=tag_colors, + ) # Update Neon colors to use the the color_border property for color in default_color_groups.neon(): - try: - neon_stmt = ( - update(TagColorGroup) - .where( - and_( - TagColorGroup.namespace == color.namespace, - TagColorGroup.slug == color.slug, - ) - ) - .values( - slug=color.slug, - namespace=color.namespace, - name=color.name, - primary=color.primary, - secondary=color.secondary, - color_border=color.color_border, + neon_stmt = ( + update(TagColorGroup) + .where( + and_( + TagColorGroup.namespace == color.namespace, + TagColorGroup.slug == color.slug, ) ) - session.execute(neon_stmt) - session.commit() - except IntegrityError as e: - logger.error( - "[Library] Could not migrate Neon colors to DB_VERSION 8+!", - error=e, + .values( + slug=color.slug, + namespace=color.namespace, + name=color.name, + primary=color.primary, + secondary=color.secondary, + color_border=color.color_border, ) - session.rollback() + ) + session.execute(neon_stmt) + session.flush() def __apply_db9_migration(self, session: Session): """Migrate DB from DB_VERSION 8 to 9.""" @@ -698,21 +678,14 @@ def __apply_db9_migration(self, session: Session): add_filename_column = text( "ALTER TABLE entries ADD COLUMN filename TEXT NOT NULL DEFAULT ''" ) - try: - session.execute(add_filename_column) - session.commit() - logger.info("[Library][Migration] Added filename column to entries table") - except Exception as e: - logger.error( - "[Library][Migration] Could not create filename column in entries table!", - error=e, - ) - session.rollback() + session.execute(add_filename_column) + session.flush() + logger.info("[Library][Migration] Added filename column to entries table") # Populate the new filename column. for entry in self.all_entries(): session.merge(entry).filename = entry.path.name - session.commit() + session.flush() logger.info("[Library][Migration] Populated filename column in entries table") def __apply_db100_migration(self, session: Session): @@ -723,57 +696,40 @@ def __apply_db100_migration(self, session: Session): child_id=TagParent.parent_id, ) session.execute(stmt) - session.commit() + session.flush() logger.info("[Library][Migration] Refactored TagParent table") def __apply_db101_migration(self, session: Session): """Migrate DB to DB_VERSION 101.""" # Ensure version rows are present session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) - session.commit() + session.flush() def __apply_db102_migration(self, session: Session): """Migrate DB to DB_VERSION 102.""" stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct())) session.execute(stmt) - session.commit() + session.flush() logger.info("[Library][Migration] Verified TagParent table data") def __apply_db103_migration(self, session: Session): """Migrate DB from DB_VERSION 102 to 103.""" # add the new hidden column for tags - add_is_hidden_column = text( - "ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0" - ) - try: - session.execute(add_is_hidden_column) - session.commit() - logger.info("[Library][Migration] Added is_hidden column to tags table") - except Exception as e: - logger.error( - "[Library][Migration] Could not create is_hidden column in tags table!", - error=e, - ) - session.rollback() + session.execute(text("ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0")) + session.flush() + logger.info("[Library][Migration] Added is_hidden column to tags table") # mark the "Archived" tag as hidden - try: - session.query(Tag).filter(Tag.id == TAG_ARCHIVED).update({"is_hidden": True}) - session.commit() - logger.info("[Library][Migration] Updated archived tag to be hidden") - except Exception as e: - logger.error( - "[Library][Migration] Could not update archived tag to be hidden!", - error=e, - ) - session.rollback() + session.query(Tag).filter(Tag.id == TAG_ARCHIVED).update({"is_hidden": True}) + session.flush() + logger.info("[Library][Migration] Updated archived tag to be hidden") def __apply_db104_migration(self, session: Session, library_dir: Path): """Migrate DB from DB_VERSION 103 to 104.""" # Convert file extension list to ts_ignore file, if a .ts_ignore file does not exist self.__migrate_sql_to_ts_ignore(session, library_dir) session.execute(text("DROP TABLE preferences")) - session.commit() + session.flush() def __migrate_sql_to_ts_ignore(self, session: Session, library_dir: Path): # Do not continue if existing '.ts_ignore' file is found @@ -865,14 +821,8 @@ def __apply_db200_migration(self, session: Session): # Add default field templates logger.info("[Library][Migration][200] Adding default field templates...") for template in get_default_field_templates(): - try: - session.add(template) - session.flush() - except IntegrityError: - logger.error("[Library] FieldTemplate already exists", field_template=template) - session.rollback() - - session.commit() + session.add(template) + session.flush() # DB indices for improved performance session.execute( @@ -933,13 +883,13 @@ def __apply_db201_migration(self, session: Session): session.execute(text("DROP TABLE datetime_fields")) session.execute(text("ALTER TABLE datetime_fields_new RENAME TO datetime_fields")) - session.commit() + session.flush() def __apply_db202_migration(self, session: Session): """Migrate DB to DB_VERSION 202.""" stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct())) session.execute(stmt) - session.commit() + session.flush() logger.info("[Library][Migration] Verified TagParent table data") @property From 20dc5022e9247597922938815778dc483dc3fbea Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 16:54:30 +0200 Subject: [PATCH 18/27] refactor: make sure the migration log statements are consistent --- src/tagstudio/core/library/alchemy/library.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index e89ce3d34..df4c5e8ef 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -567,7 +567,6 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: # migrate DB step by step from one version to the next # (migration_method, db_version, initial_db_version) - # TODO: log statements in migrations are inconsistent migrations = [ (self.__apply_db7_migration, 7, None), # changes: value_type, tags (self.__apply_db8_migration, 8, None), # changes: tag_colors @@ -613,7 +612,7 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: def __apply_db7_migration(self, session: Session): """Migrate DB from DB_VERSION 6 to 7.""" - logger.info("[Library][Migration] Applying patches to DB_VERSION: 6 library...") + logger.info("[Library][Migration][7] Applying patches to DB_VERSION: 6 library...") # Repair tags that may have a disambiguation_id pointing towards a deleted tag. all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all() disam_stmt = ( @@ -631,7 +630,7 @@ def __apply_db8_migration(self, session: Session): text("ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL") ) session.flush() - logger.info("[Library][Migration] Added color_border column to tag_colors table") + logger.info("[Library][Migration][8] Added color_border column to tag_colors table") # collect new default tag colors tag_colors: list[TagColorGroup] = default_color_groups.standard() @@ -646,7 +645,7 @@ def __apply_db8_migration(self, session: Session): session.add(color) session.flush() logger.info( - "[Library][Migration] Migrated tag colors to DB_VERSION 8+", + "[Library][Migration][8] Migrated tag colors to DB_VERSION 8+", color_name=tag_colors, ) @@ -680,13 +679,13 @@ def __apply_db9_migration(self, session: Session): ) session.execute(add_filename_column) session.flush() - logger.info("[Library][Migration] Added filename column to entries table") + logger.info("[Library][Migration][9] Added filename column to entries table") # Populate the new filename column. for entry in self.all_entries(): session.merge(entry).filename = entry.path.name session.flush() - logger.info("[Library][Migration] Populated filename column in entries table") + logger.info("[Library][Migration][9] Populated filename column in entries table") def __apply_db100_migration(self, session: Session): """Migrate DB to DB_VERSION 100.""" @@ -697,7 +696,7 @@ def __apply_db100_migration(self, session: Session): ) session.execute(stmt) session.flush() - logger.info("[Library][Migration] Refactored TagParent table") + logger.info("[Library][Migration][100] Refactored TagParent table") def __apply_db101_migration(self, session: Session): """Migrate DB to DB_VERSION 101.""" @@ -710,19 +709,19 @@ def __apply_db102_migration(self, session: Session): stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct())) session.execute(stmt) session.flush() - logger.info("[Library][Migration] Verified TagParent table data") + logger.info("[Library][Migration][102] Verified TagParent table data") def __apply_db103_migration(self, session: Session): """Migrate DB from DB_VERSION 102 to 103.""" # add the new hidden column for tags session.execute(text("ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0")) session.flush() - logger.info("[Library][Migration] Added is_hidden column to tags table") + logger.info("[Library][Migration][103] Added is_hidden column to tags table") # mark the "Archived" tag as hidden session.query(Tag).filter(Tag.id == TAG_ARCHIVED).update({"is_hidden": True}) session.flush() - logger.info("[Library][Migration] Updated archived tag to be hidden") + logger.info("[Library][Migration][103] Updated archived tag to be hidden") def __apply_db104_migration(self, session: Session, library_dir: Path): """Migrate DB from DB_VERSION 103 to 104.""" @@ -890,7 +889,7 @@ def __apply_db202_migration(self, session: Session): stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct())) session.execute(stmt) session.flush() - logger.info("[Library][Migration] Verified TagParent table data") + logger.info("[Library][Migration][202] Verified TagParent table data") @property def field_templates(self) -> Sequence[BaseFieldTemplate]: From c374843e917695b4f88fa808d93e1a9bf4ddac89 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 17:07:47 +0200 Subject: [PATCH 19/27] fix: pass library dir to migrations --- src/tagstudio/core/library/alchemy/library.py | 22 +++++++++---------- src/tagstudio/qt/mixed/migration_modal.py | 2 ++ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index df4c5e8ef..3684b606d 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -584,7 +584,7 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: if loaded_db_version < v and (iv is None or initial_db_version < iv): with session: # any error causes transaction to rollback - migration(session) + migration(session, library_dir) loaded_db_version = v self.set_version(session, DB_VERSION_CURRENT_KEY, v) session.commit() @@ -610,7 +610,7 @@ def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: self.library_dir = library_dir return LibraryStatus(success=True, library_path=library_dir) - def __apply_db7_migration(self, session: Session): + def __apply_db7_migration(self, session: Session, library_dir: Path): """Migrate DB from DB_VERSION 6 to 7.""" logger.info("[Library][Migration][7] Applying patches to DB_VERSION: 6 library...") # Repair tags that may have a disambiguation_id pointing towards a deleted tag. @@ -623,7 +623,7 @@ def __apply_db7_migration(self, session: Session): session.execute(disam_stmt) session.flush() - def __apply_db8_migration(self, session: Session): + def __apply_db8_migration(self, session: Session, library_dir: Path): """Migrate DB from DB_VERSION 7 to 8.""" # Add the missing color_border column to the TagColorGroups table. session.execute( @@ -671,7 +671,7 @@ def __apply_db8_migration(self, session: Session): session.execute(neon_stmt) session.flush() - def __apply_db9_migration(self, session: Session): + def __apply_db9_migration(self, session: Session, library_dir: Path): """Migrate DB from DB_VERSION 8 to 9.""" # Apply database schema changes add_filename_column = text( @@ -687,7 +687,7 @@ def __apply_db9_migration(self, session: Session): session.flush() logger.info("[Library][Migration][9] Populated filename column in entries table") - def __apply_db100_migration(self, session: Session): + def __apply_db100_migration(self, session: Session, library_dir: Path): """Migrate DB to DB_VERSION 100.""" # Repair parent-child tag relationships that are the wrong way around. stmt = update(TagParent).values( @@ -698,20 +698,20 @@ def __apply_db100_migration(self, session: Session): session.flush() logger.info("[Library][Migration][100] Refactored TagParent table") - def __apply_db101_migration(self, session: Session): + def __apply_db101_migration(self, session: Session, library_dir: Path): """Migrate DB to DB_VERSION 101.""" # Ensure version rows are present session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) session.flush() - def __apply_db102_migration(self, session: Session): + def __apply_db102_migration(self, session: Session, library_dir: Path): """Migrate DB to DB_VERSION 102.""" stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct())) session.execute(stmt) session.flush() logger.info("[Library][Migration][102] Verified TagParent table data") - def __apply_db103_migration(self, session: Session): + def __apply_db103_migration(self, session: Session, library_dir: Path): """Migrate DB from DB_VERSION 102 to 103.""" # add the new hidden column for tags session.execute(text("ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0")) @@ -747,7 +747,7 @@ def __migrate_sql_to_ts_ignore(self, session: Session, library_dir: Path): with open(ts_ignore, "w") as f: f.write(migrate_ext_list(extensions, is_exclude_list)) - def __apply_db200_migration(self, session: Session): + def __apply_db200_migration(self, session: Session, library_dir: Path): """Migrate DB to DB_VERSION 200.""" # Drop unused 'boolean_fields' and 'value_type' tables logger.info("[Library][Migration][200] Dropping boolean_fields and value_type tables...") @@ -834,7 +834,7 @@ def __apply_db200_migration(self, session: Session): text("CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)") ) - def __apply_db201_migration(self, session: Session): + def __apply_db201_migration(self, session: Session, library_dir: Path): """Migrate DB to DB_VERSION 201.""" create_text_fields_table = text(""" CREATE TABLE text_fields_new ( @@ -884,7 +884,7 @@ def __apply_db201_migration(self, session: Session): session.flush() - def __apply_db202_migration(self, session: Session): + def __apply_db202_migration(self, session: Session, library_dir: Path): """Migrate DB to DB_VERSION 202.""" stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct())) session.execute(stmt) diff --git a/src/tagstudio/qt/mixed/migration_modal.py b/src/tagstudio/qt/mixed/migration_modal.py index 93e983a13..79a26c26f 100644 --- a/src/tagstudio/qt/mixed/migration_modal.py +++ b/src/tagstudio/qt/mixed/migration_modal.py @@ -401,6 +401,8 @@ def migration_iterator(self): if self.temp_path.exists(): logger.info('Temporary migration file "temp_path" already exists. Removing...') self.temp_path.unlink() + # TODO: fix syntax error + # Is the usage of the temporary directory really necessary here? self.sql_lib.open_sqlite_library( self.json_lib.library_dir, is_new=True, storage_path=str(self.temp_path) ) From 660cc406b5cb48406251775e1ada126701e9cf4f Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 18:44:03 +0200 Subject: [PATCH 20/27] fix(db migration 8): only add colors that are actually new DB Migration 8 was previously adding all colors except for the neon ones, however, all but three of those have been around since DB version 4, meaning that they don't need to be added at all (which caused the tests to fail). --- src/tagstudio/core/library/alchemy/library.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 3684b606d..5513c26b1 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -633,12 +633,11 @@ def __apply_db8_migration(self, session: Session, library_dir: Path): logger.info("[Library][Migration][8] Added color_border column to tag_colors table") # collect new default tag colors - tag_colors: list[TagColorGroup] = default_color_groups.standard() - tag_colors += default_color_groups.pastels() - tag_colors += default_color_groups.shades() - tag_colors += default_color_groups.grayscale() - tag_colors += default_color_groups.earth_tones() - # tag_colors += default_color_groups.neon() # NOTE: Neon is handled separately + tag_colors: list[TagColorGroup] = [ + color + for color in default_color_groups.shades() + if color.slug in ["burgundy", "dark-teal", "dark_lavender"] + ] # Add any new default colors introduced in DB_VERSION 8 for color in tag_colors: From 6e64dc442751ada3cda9ef0c6933040869c5ee3f Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 19:05:48 +0200 Subject: [PATCH 21/27] fix: json migration used outdated interface --- src/tagstudio/core/library/alchemy/library.py | 28 +++++++++---------- src/tagstudio/qt/mixed/migration_modal.py | 13 ++++----- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 5513c26b1..32d22b671 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -380,7 +380,7 @@ def tag_display_name(self, tag: Tag | None) -> str: return tag.name def open_library(self, library_dir: Path, in_memory: bool = False) -> LibraryStatus: - """Wrapper for open_sqlite_library. + """Wrapper for open_sqlite_library and create_sqlite_library. Handles in-memory storage and checks whether a JSON-migration is necessary. """ @@ -399,21 +399,17 @@ def open_library(self, library_dir: Path, in_memory: bool = False) -> LibrarySta json_migration_req=True, ) - return self.open_sqlite_library(library_dir, is_new, in_memory) - - def open_sqlite_library( - self, library_dir: Path, is_new: bool, in_memory: bool - ) -> LibraryStatus: if is_new: - return self.new_lib(library_dir, in_memory) - return self.migrate_lib(library_dir, in_memory) + return self.create_sqlite_library(library_dir, in_memory) + + return self.open_sqlite_library(library_dir, in_memory) @staticmethod - def __get_engine(library_dir: Path, in_memory: bool): + def __get_engine(library_dir: Path, in_memory: bool, sql_filename: str): connection_string = URL.create( drivername="sqlite", database=( - ":memory:" if in_memory else str(library_dir / TS_FOLDER_NAME / SQL_FILENAME) + ":memory:" if in_memory else str(library_dir / TS_FOLDER_NAME / sql_filename) ), ) # NOTE: File-based databases should use NullPool to create new DB connection in order to @@ -432,8 +428,10 @@ def __get_engine(library_dir: Path, in_memory: bool): ) return create_engine(connection_string, poolclass=poolclass) - def new_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: - self.engine = self.__get_engine(library_dir, in_memory) + def create_sqlite_library( + self, library_dir: Path, in_memory: bool, sql_filename: str = SQL_FILENAME + ) -> LibraryStatus: + self.engine = self.__get_engine(library_dir, in_memory, sql_filename) loaded_db_version: int = 0 logger.info( @@ -514,8 +512,10 @@ def new_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: self.library_dir = library_dir return LibraryStatus(success=True, library_path=library_dir) - def migrate_lib(self, library_dir: Path, in_memory: bool) -> LibraryStatus: - self.engine = self.__get_engine(library_dir, in_memory) + def open_sqlite_library( + self, library_dir: Path, in_memory: bool, sql_filename: str = SQL_FILENAME + ) -> LibraryStatus: + self.engine = self.__get_engine(library_dir, in_memory, sql_filename) loaded_db_version: int = 0 initial_db_version: int = DB_VERSION diff --git a/src/tagstudio/qt/mixed/migration_modal.py b/src/tagstudio/qt/mixed/migration_modal.py index 79a26c26f..77850a729 100644 --- a/src/tagstudio/qt/mixed/migration_modal.py +++ b/src/tagstudio/qt/mixed/migration_modal.py @@ -395,16 +395,15 @@ def migration_iterator(self): # Convert JSON Library to SQLite yield Translations["json_migration.creating_database_tables"] self.sql_lib = SqliteLibrary() - self.temp_path: Path = ( - self.json_lib.library_dir / TS_FOLDER_NAME / "migration_ts_library.sqlite" - ) + temp_filename = "migration_ts_library.sqlite" + self.temp_path: Path = self.json_lib.library_dir / TS_FOLDER_NAME / temp_filename if self.temp_path.exists(): logger.info('Temporary migration file "temp_path" already exists. Removing...') self.temp_path.unlink() - # TODO: fix syntax error - # Is the usage of the temporary directory really necessary here? - self.sql_lib.open_sqlite_library( - self.json_lib.library_dir, is_new=True, storage_path=str(self.temp_path) + self.sql_lib.create_sqlite_library( + self.json_lib.library_dir, + in_memory=False, + sql_filename=temp_filename, ) yield Translations.format( "json_migration.migrating_files_entries", entries=len(self.json_lib.entries) From e5941b49424d622afc40d8d9d466c942ae9710f0 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 19:13:46 +0200 Subject: [PATCH 22/27] fix(open_library): create TS directory only if not opened in memory --- src/tagstudio/core/library/alchemy/library.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 32d22b671..108edd090 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -385,19 +385,20 @@ def open_library(self, library_dir: Path, in_memory: bool = False) -> LibrarySta Handles in-memory storage and checks whether a JSON-migration is necessary. """ assert isinstance(library_dir, Path) - self.verify_ts_folder(library_dir) # ensure .TagStudio directory exists sql_path = library_dir / TS_FOLDER_NAME / SQL_FILENAME json_path = library_dir / TS_FOLDER_NAME / JSON_FILENAME is_new = not sql_path.exists() - if not in_memory and is_new and json_path.exists(): - return LibraryStatus( - success=False, - library_path=library_dir, - message="[JSON] Legacy v9.4 library requires conversion to v9.5+", - json_migration_req=True, - ) + if not in_memory: + self.verify_ts_folder(library_dir) # ensure .TagStudio directory exists + if is_new and json_path.exists(): + return LibraryStatus( + success=False, + library_path=library_dir, + message="[JSON] Legacy v9.4 library requires conversion to v9.5+", + json_migration_req=True, + ) if is_new: return self.create_sqlite_library(library_dir, in_memory) From 48612a5277a8106a971585900192556493916c5c Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 20:35:59 +0200 Subject: [PATCH 23/27] fix: enable sane transaction behaviour By default in SQLAlchemy schema changes are automatically committed, even if auto-commit is turned off. This commit turns off auto-commit completely, which caused some schema to not be applied correctly, but those were fixed here as well. --- src/tagstudio/core/library/alchemy/db.py | 16 ++++-- src/tagstudio/core/library/alchemy/library.py | 55 +++++++++++-------- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/db.py b/src/tagstudio/core/library/alchemy/db.py index de119d0a5..d8d520d78 100644 --- a/src/tagstudio/core/library/alchemy/db.py +++ b/src/tagstudio/core/library/alchemy/db.py @@ -42,13 +42,17 @@ def make_engine(connection_string: str) -> Engine: def make_tables(engine: Engine) -> None: logger.info("[Library] Creating DB tables...") - Base.metadata.create_all(engine) - - # tag IDs < 1000 are reserved - # create tag and delete it to bump the autoincrement sequence - # TODO - find a better way - # is this the better way? with engine.connect() as conn: + # TODO: this should instead be migrations that create the exact tables that were added in + # the respective DB versions + Base.metadata.create_all(conn) + conn.commit() + + # TODO: this needs to be a migration + # tag IDs < 1000 are reserved + # create tag and delete it to bump the autoincrement sequence + # TODO - find a better way + # is this the better way? result = conn.execute(text("SELECT SEQ FROM sqlite_sequence WHERE name='tags'")) autoincrement_val = result.scalar() if not autoincrement_val or autoincrement_val <= RESERVED_TAG_END: diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 108edd090..6baaaf77b 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -427,7 +427,9 @@ def __get_engine(library_dir: Path, in_memory: bool, sql_filename: str): connection_string=connection_string, poolclass=poolclass, ) - return create_engine(connection_string, poolclass=poolclass) + return create_engine( + connection_string, poolclass=poolclass, connect_args={"autocommit": False} + ) def create_sqlite_library( self, library_dir: Path, in_memory: bool, sql_filename: str = SQL_FILENAME @@ -615,6 +617,7 @@ def __apply_db7_migration(self, session: Session, library_dir: Path): """Migrate DB from DB_VERSION 6 to 7.""" logger.info("[Library][Migration][7] Applying patches to DB_VERSION: 6 library...") # Repair tags that may have a disambiguation_id pointing towards a deleted tag. + # TODO: combine into single sql statement all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all() disam_stmt = ( update(Tag) @@ -682,7 +685,7 @@ def __apply_db9_migration(self, session: Session, library_dir: Path): logger.info("[Library][Migration][9] Added filename column to entries table") # Populate the new filename column. - for entry in self.all_entries(): + for entry in self.__all_entries(session): session.merge(entry).filename = entry.path.name session.flush() logger.info("[Library][Migration][9] Populated filename column in entries table") @@ -1038,32 +1041,36 @@ def entries_count(self) -> int: with Session(self.engine) as session: return unwrap(session.scalar(select(func.count(Entry.id)))) - def all_entries(self, with_joins: bool = False) -> Iterator[Entry]: + def __all_entries(self, session: Session, with_joins: bool = False) -> Iterator[Entry]: """Load entries without joins.""" - with Session(self.engine) as session: - stmt = select(Entry) - if with_joins: - # load Entry with all joins and all tags - stmt = ( - stmt.outerjoin(Entry.text_fields) - .outerjoin(Entry.datetime_fields) - .outerjoin(Entry.tags) - ) - stmt = stmt.options( - contains_eager(Entry.text_fields), - contains_eager(Entry.datetime_fields), - contains_eager(Entry.tags), - ) + stmt = select(Entry) + if with_joins: + # load Entry with all joins and all tags + stmt = ( + stmt.outerjoin(Entry.text_fields) + .outerjoin(Entry.datetime_fields) + .outerjoin(Entry.tags) + ) + stmt = stmt.options( + contains_eager(Entry.text_fields), + contains_eager(Entry.datetime_fields), + contains_eager(Entry.tags), + ) - stmt = stmt.distinct() + stmt = stmt.distinct() - entries = session.execute(stmt).scalars() - if with_joins: - entries = entries.unique() + entries = session.execute(stmt).scalars() + if with_joins: + entries = entries.unique() - for entry in entries: - yield entry - session.expunge(entry) + for entry in entries: + yield entry + session.expunge(entry) + + def all_entries(self, with_joins: bool = False) -> Iterator[Entry]: + """Load entries without joins.""" + with Session(self.engine) as session: + return self.__all_entries(session, with_joins) @property def tags(self) -> list[Tag]: From 87822f6aa7c1f8a5c2dbd296899c62f948fab399 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 21:19:44 +0200 Subject: [PATCH 24/27] refactor: hide 'argument is not accessed' notices --- src/tagstudio/core/library/alchemy/library.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 6baaaf77b..a4f1d2052 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -613,7 +613,7 @@ def open_sqlite_library( self.library_dir = library_dir return LibraryStatus(success=True, library_path=library_dir) - def __apply_db7_migration(self, session: Session, library_dir: Path): + def __apply_db7_migration(self, session: Session, __library_dir__: Path): """Migrate DB from DB_VERSION 6 to 7.""" logger.info("[Library][Migration][7] Applying patches to DB_VERSION: 6 library...") # Repair tags that may have a disambiguation_id pointing towards a deleted tag. @@ -627,7 +627,7 @@ def __apply_db7_migration(self, session: Session, library_dir: Path): session.execute(disam_stmt) session.flush() - def __apply_db8_migration(self, session: Session, library_dir: Path): + def __apply_db8_migration(self, session: Session, __library_dir__: Path): """Migrate DB from DB_VERSION 7 to 8.""" # Add the missing color_border column to the TagColorGroups table. session.execute( @@ -674,7 +674,7 @@ def __apply_db8_migration(self, session: Session, library_dir: Path): session.execute(neon_stmt) session.flush() - def __apply_db9_migration(self, session: Session, library_dir: Path): + def __apply_db9_migration(self, session: Session, __library_dir__: Path): """Migrate DB from DB_VERSION 8 to 9.""" # Apply database schema changes add_filename_column = text( @@ -690,7 +690,7 @@ def __apply_db9_migration(self, session: Session, library_dir: Path): session.flush() logger.info("[Library][Migration][9] Populated filename column in entries table") - def __apply_db100_migration(self, session: Session, library_dir: Path): + def __apply_db100_migration(self, session: Session, __library_dir__: Path): """Migrate DB to DB_VERSION 100.""" # Repair parent-child tag relationships that are the wrong way around. stmt = update(TagParent).values( @@ -701,20 +701,20 @@ def __apply_db100_migration(self, session: Session, library_dir: Path): session.flush() logger.info("[Library][Migration][100] Refactored TagParent table") - def __apply_db101_migration(self, session: Session, library_dir: Path): + def __apply_db101_migration(self, session: Session, __library_dir__: Path): """Migrate DB to DB_VERSION 101.""" # Ensure version rows are present session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) session.flush() - def __apply_db102_migration(self, session: Session, library_dir: Path): + def __apply_db102_migration(self, session: Session, __library_dir__: Path): """Migrate DB to DB_VERSION 102.""" stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct())) session.execute(stmt) session.flush() logger.info("[Library][Migration][102] Verified TagParent table data") - def __apply_db103_migration(self, session: Session, library_dir: Path): + def __apply_db103_migration(self, session: Session, __library_dir__: Path): """Migrate DB from DB_VERSION 102 to 103.""" # add the new hidden column for tags session.execute(text("ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0")) @@ -750,7 +750,7 @@ def __migrate_sql_to_ts_ignore(self, session: Session, library_dir: Path): with open(ts_ignore, "w") as f: f.write(migrate_ext_list(extensions, is_exclude_list)) - def __apply_db200_migration(self, session: Session, library_dir: Path): + def __apply_db200_migration(self, session: Session, __library_dir__: Path): """Migrate DB to DB_VERSION 200.""" # Drop unused 'boolean_fields' and 'value_type' tables logger.info("[Library][Migration][200] Dropping boolean_fields and value_type tables...") @@ -837,7 +837,7 @@ def __apply_db200_migration(self, session: Session, library_dir: Path): text("CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)") ) - def __apply_db201_migration(self, session: Session, library_dir: Path): + def __apply_db201_migration(self, session: Session, __library_dir__: Path): """Migrate DB to DB_VERSION 201.""" create_text_fields_table = text(""" CREATE TABLE text_fields_new ( @@ -887,7 +887,7 @@ def __apply_db201_migration(self, session: Session, library_dir: Path): session.flush() - def __apply_db202_migration(self, session: Session, library_dir: Path): + def __apply_db202_migration(self, session: Session, __library_dir__: Path): """Migrate DB to DB_VERSION 202.""" stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct())) session.execute(stmt) From 33bc77cc9d71e5b6ea6ab9c1e6f006d3569ce8b0 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 21:31:16 +0200 Subject: [PATCH 25/27] fix(db migration 9): filename property wasn't written correctly --- src/tagstudio/core/library/alchemy/library.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index a4f1d2052..068a09806 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -598,6 +598,8 @@ def open_sqlite_library( logger.info(f"[Library] Library migrated to DB version {DB_VERSION}") # check if folder matching current path exists already + # NOTE: this has been causing new Folders to be created when the library is moved, since + # its introduction self.folder = session.scalar(select(Folder).where(Folder.path == library_dir)) if not self.folder: folder = Folder( @@ -686,7 +688,8 @@ def __apply_db9_migration(self, session: Session, __library_dir__: Path): # Populate the new filename column. for entry in self.__all_entries(session): - session.merge(entry).filename = entry.path.name + entry.filename = entry.path.name + session.merge(entry) session.flush() logger.info("[Library][Migration][9] Populated filename column in entries table") From 81734c1373b3363289ded8d63bb7e81c71f53368 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 21:45:35 +0200 Subject: [PATCH 26/27] fix(db migration 104): include/exclude list was loaded incorrectly --- src/tagstudio/core/library/alchemy/library.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 068a09806..e42d43d8c 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -22,6 +22,7 @@ import sqlalchemy import structlog +import ujson from humanfriendly import format_timespan # pyright: ignore[reportUnknownVariableType] from sqlalchemy import ( URL, @@ -712,6 +713,7 @@ def __apply_db101_migration(self, session: Session, __library_dir__: Path): def __apply_db102_migration(self, session: Session, __library_dir__: Path): """Migrate DB to DB_VERSION 102.""" + # delete TagParents with a dangling parent reference stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct())) session.execute(stmt) session.flush() @@ -743,8 +745,10 @@ def __migrate_sql_to_ts_ignore(self, session: Session, library_dir: Path): return # Load legacy extension data - extensions: list[str] = unwrap( - session.scalar(text("SELECT value FROM preferences WHERE key = 'EXTENSION_LIST'")) + extensions: list[str] = ujson.loads( + unwrap( + session.scalar(text("SELECT value FROM preferences WHERE key = 'EXTENSION_LIST'")) + ) ) is_exclude_list: bool = unwrap( session.scalar(text("SELECT value FROM preferences WHERE key = 'IS_EXCLUDE_LIST'")) From 8369738c64b6dbb98709c2c4dc5650c128550ef1 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Mon, 6 Jul 2026 22:02:33 +0200 Subject: [PATCH 27/27] feat: log start and end of DB migrations --- src/tagstudio/core/library/alchemy/library.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index e42d43d8c..8732a7538 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -586,12 +586,14 @@ def open_sqlite_library( ] for migration, v, iv in migrations: if loaded_db_version < v and (iv is None or initial_db_version < iv): + logger.info(f"[Library][Migration][{v}] Starting DB Migration") with session: # any error causes transaction to rollback migration(session, library_dir) loaded_db_version = v self.set_version(session, DB_VERSION_CURRENT_KEY, v) session.commit() + logger.info(f"[Library][Migration][{v}] Completed DB Migration") assert loaded_db_version == DB_VERSION, ( "Ran all migrations, but the DB is still not on the newest version"