From 70f1a3d9752aefca0052adabe6b7a3e3e177d193 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Fri, 28 Aug 2026 14:31:36 +0200 Subject: [PATCH 01/21] refactor: sqlalchemy-independence for _get_version --- .../core/library/alchemy/migrations.py | 38 +++++++++---------- src/tagstudio/core/library/alchemy/utils.py | 11 ++++++ 2 files changed, 28 insertions(+), 21 deletions(-) create mode 100644 src/tagstudio/core/library/alchemy/utils.py diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 5acc31e8a..a20aea6aa 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -2,11 +2,11 @@ # SPDX-License-Identifier: MIT +import sqlite3 from collections.abc import Callable from pathlib import Path from typing import override -import sqlalchemy import structlog import ujson from sqlalchemy import Engine, and_, delete, select, text, update @@ -19,10 +19,12 @@ DB_VERSION_CURRENT_KEY, DB_VERSION_INITIAL_KEY, DEFAULT_FIELD_TEMPLATES, + SQL_FILENAME, ) from tagstudio.core.library.alchemy.fields import LEGACY_FIELD_MAP, DatetimeField, TextField from tagstudio.core.library.alchemy.joins import TagParent from tagstudio.core.library.alchemy.models import Entry, Tag, TagColorGroup, Version +from tagstudio.core.library.alchemy.utils import list_tables from tagstudio.core.library.ignore import migrate_ext_list from tagstudio.core.utils.types import unwrap from tagstudio.i18n.translations import Translations @@ -48,7 +50,10 @@ def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod) -> Non class DBMigrations: def __init__(self, library_dir: Path, engine: Engine) -> None: self.library_dir = library_dir - self.engine = engine + self.engine = engine # TODO: remove + self._connection = sqlite3.connect( + str(library_dir / TS_FOLDER_NAME / SQL_FILENAME), autocommit=False + ) # Don't check DB version when creating new library self.loaded_db_version = self._get_version(DB_VERSION_CURRENT_KEY) @@ -136,25 +141,16 @@ def run(self): ) def _get_version(self, key: str) -> int: - with Session(self.engine) as session: - inspector = sqlalchemy.inspect(self.engine) - try: - # "Version" table added in DB_VERSION 101 - if inspector and inspector.has_table("versions"): - version = session.scalar(select(Version).where(Version.key == key)) - assert version - return version.value - # "Preferences" table deprecated in TagStudio 9.5.4 - else: - return int( - unwrap( - session.scalar( - text("SELECT value FROM preferences WHERE key == 'DB_VERSION'") - ) - ) - ) - except Exception: - return 0 + cur = self._connection.cursor() + + # "Version" table added in DB_VERSION 101 + if "versions" in list_tables(self._connection): + query = ("SELECT value FROM versions WHERE key == ?", [key]) + # "Preferences" table deprecated in TagStudio 9.5.4 + else: + query = ("SELECT value FROM preferences WHERE key == 'DB_VERSION'", []) + + return int(unwrap(cur.execute(*query).fetchone())[0]) def _set_version(self, session: Session, key: str, value: int) -> None: """Set a version value to the DB. diff --git a/src/tagstudio/core/library/alchemy/utils.py b/src/tagstudio/core/library/alchemy/utils.py new file mode 100644 index 000000000..ae2c1f1cb --- /dev/null +++ b/src/tagstudio/core/library/alchemy/utils.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: (c) TagStudio Contributors +# SPDX-License-Identifier: MIT + + +from sqlite3 import Connection + + +def list_tables(con: Connection) -> list[str]: + cur = con.cursor() + res = cur.execute("SELECT name FROM sqlite_master WHERE type == 'table';") + return [row[0] for row in res.fetchall()] From e2b54ef3ff01b4efdbd70a1f960745837480dd98 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Fri, 28 Aug 2026 14:49:12 +0200 Subject: [PATCH 02/21] refactor: sqlalchemy-independence for _set_version --- src/tagstudio/core/library/alchemy/library.py | 2 +- .../core/library/alchemy/migrations.py | 25 +++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 473774b46..f9fb2dc2c 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -507,7 +507,7 @@ def open_sqlite_library( self.engine = self.__get_engine(library_dir, in_memory, sql_filename) try: - migrations = DBMigrations(library_dir, self.engine) + migrations = DBMigrations(library_dir, sql_filename, self.engine) # save backup if patches will be applied if migrations.required: diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index a20aea6aa..09f22e0b5 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -19,7 +19,6 @@ DB_VERSION_CURRENT_KEY, DB_VERSION_INITIAL_KEY, DEFAULT_FIELD_TEMPLATES, - SQL_FILENAME, ) from tagstudio.core.library.alchemy.fields import LEGACY_FIELD_MAP, DatetimeField, TextField from tagstudio.core.library.alchemy.joins import TagParent @@ -48,11 +47,11 @@ def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod) -> Non class DBMigrations: - def __init__(self, library_dir: Path, engine: Engine) -> None: + def __init__(self, library_dir: Path, sql_filename: str, engine: Engine) -> None: self.library_dir = library_dir self.engine = engine # TODO: remove self._connection = sqlite3.connect( - str(library_dir / TS_FOLDER_NAME / SQL_FILENAME), autocommit=False + str(library_dir / TS_FOLDER_NAME / sql_filename), autocommit=False ) # Don't check DB version when creating new library @@ -122,7 +121,7 @@ def run(self): ) self.loaded_db_version = migration.version try: - self._set_version(session, DB_VERSION_CURRENT_KEY, migration.version) + self._set_version(DB_VERSION_CURRENT_KEY, migration.version) logger.info( f"[Library][Migration][{migration.version}] Completed DB Migration" ) @@ -141,8 +140,11 @@ def run(self): ) def _get_version(self, key: str) -> int: - cur = self._connection.cursor() + """Get a version value from the DB. + Args: + key(str): The name of the version type to retrieve. + """ # "Version" table added in DB_VERSION 101 if "versions" in list_tables(self._connection): query = ("SELECT value FROM versions WHERE key == ?", [key]) @@ -150,18 +152,21 @@ def _get_version(self, key: str) -> int: else: query = ("SELECT value FROM preferences WHERE key == 'DB_VERSION'", []) - return int(unwrap(cur.execute(*query).fetchone())[0]) + return int(unwrap(self._connection.execute(*query).fetchone())[0]) - def _set_version(self, session: Session, key: str, value: int) -> None: + def _set_version(self, key: str, value: int) -> None: """Set a version value to the DB. Args: - session(Session): The SQLAlchemy DB Session to use. - key(str): The key for the name of the version type to set. + key(str): The the name of the version type to set. value(int): The version value to set. """ # Insert if key has no value yet, otherwise update the value - session.merge(Version(key=key, value=value)) + self._connection.execute( + "INSERT INTO versions (key, value) VALUES (?, ?)" + "ON CONFLICT(key) DO UPDATE SET value=excluded.value", + [key, value], + ) class MigrationTo7(DBMigration): From 27bd1622b7a9378ee6b228ba35af3d4cc24baddc Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Fri, 28 Aug 2026 14:59:05 +0200 Subject: [PATCH 03/21] refactor: remove engine; breaks migrations until individual refactor --- src/tagstudio/core/library/alchemy/library.py | 8 +-- .../core/library/alchemy/migrations.py | 55 +++++++++---------- src/tagstudio/core/library/alchemy/utils.py | 7 ++- 3 files changed, 33 insertions(+), 37 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index f9fb2dc2c..6316c1728 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -504,10 +504,9 @@ def open_sqlite_library( ) -> LibraryStatus: logger.info("[Library] Opening SQLite Library", library_dir=library_dir) - self.engine = self.__get_engine(library_dir, in_memory, sql_filename) - + # migrate if necessary try: - migrations = DBMigrations(library_dir, sql_filename, self.engine) + migrations = DBMigrations(library_dir, sql_filename) # save backup if patches will be applied if migrations.required: @@ -517,7 +516,8 @@ def open_sqlite_library( except MigrationError as e: return LibraryStatus(success=False, message=e.args[0]) - # everything is fine, set the library path + # open up-to-date library + self.engine = self.__get_engine(library_dir, in_memory, sql_filename) self.library_dir = library_dir return LibraryStatus(success=True, library_path=library_dir) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 09f22e0b5..16123c7d4 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -9,7 +9,7 @@ import structlog import ujson -from sqlalchemy import Engine, and_, delete, select, text, update +from sqlalchemy import and_, delete, select, text, update from sqlalchemy.orm import Session from tagstudio.core.constants import IGNORE_NAME, TAG_ARCHIVED, TS_FOLDER_NAME @@ -47,9 +47,8 @@ def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod) -> Non class DBMigrations: - def __init__(self, library_dir: Path, sql_filename: str, engine: Engine) -> None: + def __init__(self, library_dir: Path, sql_filename: str) -> None: self.library_dir = library_dir - self.engine = engine # TODO: remove self._connection = sqlite3.connect( str(library_dir / TS_FOLDER_NAME / sql_filename), autocommit=False ) @@ -106,34 +105,30 @@ def run(self): MigrationTo300, # changes: deletes folders MigrationTo400, # changes: add category_exclusions ] - with Session(self.engine) as session: - for migration in migrations: - if self.loaded_db_version < migration.version and ( - migration.initial_version is None - or self.initial_db_version < migration.initial_version - ): - logger.info(f"[Library][Migration][{migration.version}] Starting DB Migration") - # any error causes transaction to rollback - migration.run( - session, - self.library_dir, - lambda msg, v=migration.version: f"[Library][Migration][{v}] {msg}", + for migration in migrations: + if self.loaded_db_version < migration.version and ( + migration.initial_version is None + or self.initial_db_version < migration.initial_version + ): + logger.info(f"[Library][Migration][{migration.version}] Starting DB Migration") + # any error causes transaction to rollback + migration.run( + None, # TODO: remove session param once all Migrations have been updated + self.library_dir, + lambda msg, v=migration.version: f"[Library][Migration][{v}] {msg}", + ) + self.loaded_db_version = migration.version + try: + self._set_version(DB_VERSION_CURRENT_KEY, migration.version) + logger.info(f"[Library][Migration][{migration.version}] Completed DB Migration") + except Exception as e: + logger.info( + f"[Library][Migration][{migration.version}] " + "Couldn't update version, continuing without commit", + error=e, ) - self.loaded_db_version = migration.version - try: - self._set_version(DB_VERSION_CURRENT_KEY, migration.version) - logger.info( - f"[Library][Migration][{migration.version}] Completed DB Migration" - ) - except Exception as e: - logger.info( - f"[Library][Migration][{migration.version}] " - "Couldn't update version, continuing without commit", - error=e, - ) - session.flush() - else: - session.commit() + else: + self._connection.commit() assert self.loaded_db_version >= DB_VERSION, ( "Ran all migrations, but the DB is still not on the newest version" diff --git a/src/tagstudio/core/library/alchemy/utils.py b/src/tagstudio/core/library/alchemy/utils.py index ae2c1f1cb..3ee769082 100644 --- a/src/tagstudio/core/library/alchemy/utils.py +++ b/src/tagstudio/core/library/alchemy/utils.py @@ -6,6 +6,7 @@ def list_tables(con: Connection) -> list[str]: - cur = con.cursor() - res = cur.execute("SELECT name FROM sqlite_master WHERE type == 'table';") - return [row[0] for row in res.fetchall()] + return [ + row[0] + for row in con.execute("SELECT name FROM sqlite_master WHERE type == 'table'").fetchall() + ] From 50fded86827f042bee9de421fdf7c57d58ae94b0 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 19:47:08 +0200 Subject: [PATCH 04/21] refactor: sqlalchemy-independence for the DB 7 migration --- .../core/library/alchemy/migrations.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 16123c7d4..7de3dcacc 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -5,6 +5,7 @@ import sqlite3 from collections.abc import Callable from pathlib import Path +from sqlite3 import Connection from typing import override import structlog @@ -42,7 +43,7 @@ class DBMigration: initial_version: int | None = None @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod) -> None: # pyright: ignore[reportUnusedParameter] + def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod) -> None: # pyright: ignore[reportUnusedParameter] raise NotImplementedError @@ -113,7 +114,7 @@ def run(self): logger.info(f"[Library][Migration][{migration.version}] Starting DB Migration") # any error causes transaction to rollback migration.run( - None, # TODO: remove session param once all Migrations have been updated + self._connection, self.library_dir, lambda msg, v=migration.version: f"[Library][Migration][{v}] {msg}", ) @@ -169,19 +170,17 @@ class MigrationTo7(DBMigration): @override @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): + def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): """Migrate DB from DB_VERSION 6 to 7.""" logger.info(fmt_log("Applying patches to DB_VERSION: 6 library...")) # Repair tags that may have a disambiguation_id pointing towards a deleted tag. - # TODO: combine into single sql statement - all_tag_ids = session.scalars(text("SELECT DISTINCT id FROM tags")).all() - disam_stmt = ( - update(Tag) - .where(Tag.disambiguation_id.not_in(all_tag_ids)) - .values(disambiguation_id=None) - ) - session.execute(disam_stmt) - session.flush() + conn.execute( + "UPDATE tags " + "SET disambiguation_id = null " + "WHERE NOT disambiguation_id IN (" + "SELECT id FROM tags" + ")" + ) class MigrationTo8(DBMigration): From df75731a2033d16213223d697794bb2caef20692 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 19:59:48 +0200 Subject: [PATCH 05/21] refactor: sqlalchemy-independence for the DB 8 migration --- .../core/library/alchemy/migrations.py | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 7de3dcacc..cc8b06951 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -10,7 +10,7 @@ import structlog import ujson -from sqlalchemy import and_, delete, select, text, update +from sqlalchemy import delete, select, text, update from sqlalchemy.orm import Session from tagstudio.core.constants import IGNORE_NAME, TAG_ARCHIVED, TS_FOLDER_NAME @@ -188,13 +188,12 @@ class MigrationTo8(DBMigration): @override @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): + def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): """Migrate DB from DB_VERSION 7 to 8.""" # Add the missing color_border column to the TagColorGroups table. - session.execute( - text("ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL") + conn.execute( + "ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL" ) - session.flush() logger.info(fmt_log("Added color_border column to tag_colors table")) # collect new default tag colors @@ -206,8 +205,11 @@ def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): # Add any new default colors introduced in DB_VERSION 8 for color in tag_colors: - session.add(color) - session.flush() + conn.execute( + 'INSERT INTO tag_colors (slug, namespace, name, "primary", secondary) ' + "VALUES (?, ?, ?, ?, ?)", + [color.slug, color.namespace, color.name, color.primary, color.secondary], + ) logger.info( fmt_log("Migrated tag colors to DB_VERSION 8+"), color_name=tag_colors, @@ -215,25 +217,22 @@ def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): # Update Neon colors to use the the color_border property for color in default_color_groups.neon(): - neon_stmt = ( - update(TagColorGroup) - .where( - and_( - TagColorGroup.namespace == color.namespace, - TagColorGroup.slug == color.slug, - ) - ) - .values( - slug=color.slug, - namespace=color.namespace, - name=color.name, - primary=color.primary, - secondary=color.secondary, - color_border=color.color_border, - ) + conn.execute( + "UPDATE tag_colors" + "SET slug = ?, namespace = ?, name = ?, " + '"primary" = ?, secondary = ?, color_border = ? ' + "WHERE namespace == ? AND slug = ?", + [ + color.slug, + color.namespace, + color.name, + color.primary, + color.secondary, + color.color_border, + color.namespace, + color.slug, + ], ) - session.execute(neon_stmt) - session.flush() class MigrationTo9(DBMigration): From 5a447a5b83f5271667a3a6939ed30206a8bc7ddd Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 20:05:56 +0200 Subject: [PATCH 06/21] refactor: sqlalchemy-independence for the DB 9 migration --- .../core/library/alchemy/migrations.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index cc8b06951..be5ae9c04 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -23,7 +23,7 @@ ) from tagstudio.core.library.alchemy.fields import LEGACY_FIELD_MAP, DatetimeField, TextField from tagstudio.core.library.alchemy.joins import TagParent -from tagstudio.core.library.alchemy.models import Entry, Tag, TagColorGroup, Version +from tagstudio.core.library.alchemy.models import Tag, TagColorGroup, Version from tagstudio.core.library.alchemy.utils import list_tables from tagstudio.core.library.ignore import migrate_ext_list from tagstudio.core.utils.types import unwrap @@ -191,6 +191,9 @@ class MigrationTo8(DBMigration): def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): """Migrate DB from DB_VERSION 7 to 8.""" # Add the missing color_border column to the TagColorGroups table. + # TODO: as before, this migration uses the current default colors, while it should really be + # using the default colors as they were in that specific version. + # FUTURE CHANGES TO THE DEFAULT COLORS WILL BREAK THIS conn.execute( "ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL" ) @@ -240,23 +243,19 @@ class MigrationTo9(DBMigration): @override @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): + def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): """Migrate DB from DB_VERSION 8 to 9.""" # Apply database schema changes - add_filename_column = text( - "ALTER TABLE entries ADD COLUMN filename TEXT NOT NULL DEFAULT ''" - ) - session.execute(add_filename_column) - session.flush() + conn.execute("ALTER TABLE entries ADD COLUMN filename TEXT NOT NULL DEFAULT ''") logger.info(fmt_log("Added filename column to entries table")) # Populate the new filename column. - # TODO: this could still break in the future through changes to the definition of Entry - entries = session.execute(select(Entry).distinct()).scalars() - for entry in entries: - entry.filename = entry.path.name - session.merge(entry) - session.flush() + paths = [ + (id, Path(path_str)) + for id, path_str in conn.execute("SELECT id, path FROM entries").fetchall() + ] + for eid, path in paths: + conn.execute("UPDATE entries SET filename = ? WHERE id = ?", [path.name, eid]) logger.info(fmt_log("Populated filename column in entries table")) From 5c207e879dacc331047bcc21e9c5951983b20db7 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 20:08:12 +0200 Subject: [PATCH 07/21] refactor: sqlalchemy-independence for the DB 100 migration --- src/tagstudio/core/library/alchemy/migrations.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index be5ae9c04..8d4b1cd10 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -264,15 +264,10 @@ class MigrationTo100(DBMigration): @override @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): + def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): """Migrate DB to DB_VERSION 100.""" # Repair parent-child tag relationships that are the wrong way around. - stmt = update(TagParent).values( - parent_id=TagParent.child_id, - child_id=TagParent.parent_id, - ) - session.execute(stmt) - session.flush() + conn.execute("UPDATE tag_parents SET parent_id = child_id, child_id = parent_id") logger.info(fmt_log("Refactored TagParent table")) From eed336b8fe87432ac2cdf94d14f613d5d9be1915 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 20:11:39 +0200 Subject: [PATCH 08/21] refactor: sqlalchemy-independence for the DB 101 migration --- .../core/library/alchemy/migrations.py | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 8d4b1cd10..b7f96a5e1 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -23,7 +23,7 @@ ) from tagstudio.core.library.alchemy.fields import LEGACY_FIELD_MAP, DatetimeField, TextField from tagstudio.core.library.alchemy.joins import TagParent -from tagstudio.core.library.alchemy.models import Tag, TagColorGroup, Version +from tagstudio.core.library.alchemy.models import Tag, TagColorGroup from tagstudio.core.library.alchemy.utils import list_tables from tagstudio.core.library.ignore import migrate_ext_list from tagstudio.core.utils.types import unwrap @@ -276,21 +276,19 @@ class MigrationTo101(DBMigration): @override @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): + def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): """Migrate DB to DB_VERSION 101.""" # Create versions table - session.execute( - text(""" - CREATE TABLE versions ( - "key" VARCHAR NOT NULL PRIMARY KEY, - value INTEGER NOT NULL - ) + conn.execute(""" + CREATE TABLE versions ( + "key" VARCHAR NOT NULL PRIMARY KEY, + value INTEGER NOT NULL + ) """) - ) - session.flush() # Ensure version rows are present - session.add(Version(key=DB_VERSION_INITIAL_KEY, value=100)) - session.flush() + conn.execute( + 'INSERT INTO versions ("key", value) VALUES (?, ?)', [DB_VERSION_INITIAL_KEY, 100] + ) logger.info(fmt_log("Created versions table")) From 4496754f49ef37be54833beb03e2ce74542c032d Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 20:14:04 +0200 Subject: [PATCH 09/21] refactor: sqlalchemy-independence for the DB 102 migration --- src/tagstudio/core/library/alchemy/migrations.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index b7f96a5e1..3c977d550 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -297,12 +297,16 @@ class MigrationTo102(DBMigration): @override @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): + def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): """Migrate DB to DB_VERSION 102.""" # delete TagParents with a dangling parent reference - stmt = delete(TagParent).where(TagParent.parent_id.not_in(select(Tag.id).distinct())) - session.execute(stmt) - session.flush() + conn.execute(""" + DELETE FROM tag_parents + WHERE NOT parent_id IN ( + SELECT id + FROM tags + ) + """) logger.info(fmt_log("Verified TagParent table data")) From 98ae38f65693047cb29a82686e2776d85cd156a5 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 20:15:54 +0200 Subject: [PATCH 10/21] refactor: sqlalchemy-independence for the DB 103 migration --- src/tagstudio/core/library/alchemy/migrations.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 3c977d550..9c20e8363 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -315,16 +315,14 @@ class MigrationTo103(DBMigration): @override @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): + def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): """Migrate DB from DB_VERSION 102 to 103.""" # add the new hidden column for tags - session.execute(text("ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0")) - session.flush() + conn.execute("ALTER TABLE tags ADD COLUMN is_hidden BOOLEAN NOT NULL DEFAULT 0") logger.info(fmt_log("Added is_hidden column to tags table")) # mark the "Archived" tag as hidden - session.query(Tag).filter(Tag.id == TAG_ARCHIVED).update({"is_hidden": True}) - session.flush() + conn.execute("UPDATE tags SET is_hidden = true WHERE id = ?", [TAG_ARCHIVED]) logger.info(fmt_log("Updated archived tag to be hidden")) From 549f0363419202871d7ff0f0d43de6061678689b Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 20:21:25 +0200 Subject: [PATCH 11/21] refactor: sqlalchemy-independence for the DB 104 migration --- .../core/library/alchemy/migrations.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 9c20e8363..0241345d1 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -331,15 +331,14 @@ class MigrationTo104(DBMigration): @override @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): + def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): """Migrate DB from DB_VERSION 103 to 104.""" # Convert file extension list to ts_ignore file, if a .ts_ignore file does not exist - cls.__migrate_sql_to_ts_ignore(session, library_dir) - session.execute(text("DROP TABLE preferences")) - session.flush() + cls.__migrate_sql_to_ts_ignore(conn, library_dir) + conn.execute("DROP TABLE preferences") @classmethod - def __migrate_sql_to_ts_ignore(cls, session: Session, library_dir: Path): + def __migrate_sql_to_ts_ignore(cls, conn: Connection, library_dir: Path): # Do not continue if existing '.ts_ignore' file is found ts_ignore = library_dir / TS_FOLDER_NAME / IGNORE_NAME if Path(ts_ignore).exists(): @@ -348,11 +347,17 @@ def __migrate_sql_to_ts_ignore(cls, session: Session, library_dir: Path): # Load legacy extension data extensions: list[str] = ujson.loads( unwrap( - session.scalar(text("SELECT value FROM preferences WHERE key = 'EXTENSION_LIST'")) - ) + conn.execute( + "SELECT value FROM preferences WHERE key = 'EXTENSION_LIST'" + ).fetchone() + )[0] ) is_exclude_list: bool = unwrap( - session.scalar(text("SELECT value FROM preferences WHERE key = 'IS_EXCLUDE_LIST'")) + conn.execute(""" + SELECT value + FROM preferences + WHERE key = 'IS_EXCLUDE_LIST' + """).fetchone()[0] ) with open(ts_ignore, "w") as f: From c29a3a797865d9c7a73610a93d4acd5c832f7faa Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 20:47:43 +0200 Subject: [PATCH 12/21] refactor: sqlalchemy-independence for the DB 200 migration --- .../core/library/alchemy/constants.py | 7 +- .../core/library/alchemy/migrations.py | 132 ++++++++---------- 2 files changed, 65 insertions(+), 74 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/constants.py b/src/tagstudio/core/library/alchemy/constants.py index aba48a278..7cc8d0445 100644 --- a/src/tagstudio/core/library/alchemy/constants.py +++ b/src/tagstudio/core/library/alchemy/constants.py @@ -39,7 +39,7 @@ """) -DEFAULT_FIELD_TEMPLATES = ( +DEFAULT_TEXT_FIELD_TEMPLATES = ( TextFieldTemplate(name="Title"), TextFieldTemplate(name="Author"), TextFieldTemplate(name="Artist"), @@ -47,5 +47,8 @@ TextFieldTemplate(name="Description", is_multiline=True), TextFieldTemplate(name="Notes", is_multiline=True), TextFieldTemplate(name="Comments", is_multiline=True), - DatetimeFieldTemplate(name="Date"), ) + +DEFAULT_DATETIME_FIELD_TEMPLATES = (DatetimeFieldTemplate(name="Date"),) + +DEFAULT_FIELD_TEMPLATES = DEFAULT_TEXT_FIELD_TEMPLATES + DEFAULT_DATETIME_FIELD_TEMPLATES diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 0241345d1..ee15159bf 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -10,7 +10,7 @@ import structlog import ujson -from sqlalchemy import delete, select, text, update +from sqlalchemy import delete, select, text from sqlalchemy.orm import Session from tagstudio.core.constants import IGNORE_NAME, TAG_ARCHIVED, TS_FOLDER_NAME @@ -19,9 +19,10 @@ DB_VERSION, DB_VERSION_CURRENT_KEY, DB_VERSION_INITIAL_KEY, - DEFAULT_FIELD_TEMPLATES, + DEFAULT_DATETIME_FIELD_TEMPLATES, + DEFAULT_TEXT_FIELD_TEMPLATES, ) -from tagstudio.core.library.alchemy.fields import LEGACY_FIELD_MAP, DatetimeField, TextField +from tagstudio.core.library.alchemy.fields import LEGACY_FIELD_MAP from tagstudio.core.library.alchemy.joins import TagParent from tagstudio.core.library.alchemy.models import Tag, TagColorGroup from tagstudio.core.library.alchemy.utils import list_tables @@ -369,111 +370,98 @@ class MigrationTo200(DBMigration): @override @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): + def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): """Migrate DB to DB_VERSION 200.""" + # TODO: this migration uses default values of the most recent DB version, fix + # THIS WILL BREAK ONCE THESE DEFAULT VALUES ARE CHANGED! # Drop unused 'boolean_fields' and 'value_type' tables logger.info(fmt_log("Dropping boolean_fields and value_type tables...")) - session.execute(text("DROP TABLE boolean_fields")) - session.execute(text("DROP TABLE value_type")) + conn.execute("DROP TABLE boolean_fields") + conn.execute("DROP TABLE value_type") # Add 'name' column to text_fields and datetime_fields tables logger.info(fmt_log("Adding name columns to field tables...")) - stmt = text('ALTER TABLE text_fields ADD COLUMN name VARCHAR DEFAULT ""') - session.execute(stmt) - stmt = text('ALTER TABLE datetime_fields ADD COLUMN name VARCHAR DEFAULT ""') - session.execute(stmt) + conn.execute('ALTER TABLE text_fields ADD COLUMN name VARCHAR DEFAULT ""') + conn.execute('ALTER TABLE datetime_fields ADD COLUMN name VARCHAR DEFAULT ""') # Drop unnecessary 'position' columns logger.info(fmt_log("Dropping position columns to field tables...")) - session.execute(text("ALTER TABLE datetime_fields DROP COLUMN position")) - session.execute(text("ALTER TABLE text_fields DROP COLUMN position")) + conn.execute("ALTER TABLE datetime_fields DROP COLUMN position") + conn.execute("ALTER TABLE text_fields DROP COLUMN position") # Add 'is_multiline' column to text_fields table logger.info(fmt_log("Adding is_multiline column to text_fields...")) - stmt = text("ALTER TABLE text_fields ADD COLUMN is_multiline BOOLEAN NOT NULL DEFAULT 0") - session.execute(stmt) - session.flush() + conn.execute("ALTER TABLE text_fields ADD COLUMN is_multiline BOOLEAN NOT NULL DEFAULT 0") # Move values from old `type_key` columns into new `name` columns logger.info(fmt_log("Moving values from type_key columns to name...")) - session.execute(text("UPDATE text_fields SET name = type_key")) - session.execute(text("UPDATE datetime_fields SET name = type_key")) - session.flush() + conn.execute("UPDATE text_fields SET name = type_key") + conn.execute("UPDATE datetime_fields SET name = type_key") # Change `name` values to title case logger.info(fmt_log("Normalizing TextField names...")) - for text_field in session.execute(select(TextField)).scalars(): - # NOTE: The only exception to the "Title Case" conversion is the "URL" field. - text_field.name = text_field.name.title().replace("Url", "URL").replace("_", " ") + # NOTE: The only exception to the "Title Case" conversion is the "URL" field. + names = [ + (name.title().replace("Url", "URL").replace("_", " "), id) + for id, name in conn.execute("SELECT id, name FROM text_fields").fetchall() + ] + conn.executemany("UPDATE text_fields SET name = ? WHERE id = ?", names) + logger.info(fmt_log("Normalizing DatetimeField names...")) - for datetime_field in session.execute(select(DatetimeField)).scalars(): - datetime_field.name = datetime_field.name.title().replace("_", " ") - session.flush() + names = [ + (name.title().replace("_", " "), id) + for id, name in conn.execute("SELECT id, name FROM datetime_fields").fetchall() + ] + conn.executemany("UPDATE datetime_fields SET name = ? WHERE id = ?", names) # Add correct `is_multiline` values to text_fields table logger.info(fmt_log("Updating is_multiline for legacy TEXT_BOXes...")) text_boxes = [ x.get("name") for x in LEGACY_FIELD_MAP.values() if x.get("is_multiline") is True ] - update_stmt = ( - update(TextField).where(TextField.name.in_(text_boxes)).values(is_multiline=True) - ) - session.execute(update_stmt) - session.flush() + conn.execute("UPDATE text_fields SET is_multiline = true WHERE name in ?", text_boxes) - # Repair legacy "Description" fields to use is_multiline = True - logger.info(fmt_log("Repairing legacy Description fields...")) - desc_stmt = ( - update(TextField) - .where(TextField.name == "Description" and TextField.is_multiline == False) # noqa: E712 - .values(is_multiline=True) - ) - session.execute(desc_stmt) - - # Repair legacy "Comments" fields to use is_multiline = True - logger.info(fmt_log("Repairing legacy Comment fields...")) - comm_stmt = ( - update(TextField) - .where(TextField.name == "Comments" and TextField.is_multiline == False) # noqa: E712 - .values(is_multiline=True) - ) - session.execute(comm_stmt) + # Repair legacy "Description" and "Comments" fields to use is_multiline = True + logger.info(fmt_log("Repairing legacy Description and Comments fields...")) + conn.execute(""" + UPDATE text_fields + SET is_multiline = true + WHERE name in ('Description', 'Comments') AND is_multiline = false + """) # Add field templates tables - session.execute( - text(""" - CREATE TABLE text_field_templates ( - id INTEGER NOT NULL PRIMARY KEY, - is_multiline BOOLEAN NOT NULL, - name VARCHAR NOT NULL - ) + conn.execute(""" + CREATE TABLE text_field_templates ( + id INTEGER NOT NULL PRIMARY KEY, + is_multiline BOOLEAN NOT NULL, + name VARCHAR NOT NULL + ) """) - ) - session.execute( - text(""" - CREATE TABLE datetime_field_templates ( - id INTEGER NOT NULL PRIMARY KEY, - name VARCHAR NOT NULL - ) + conn.execute(""" + CREATE TABLE datetime_field_templates ( + id INTEGER NOT NULL PRIMARY KEY, + name VARCHAR NOT NULL + ) """) - ) - session.flush() # Add default field templates logger.info(fmt_log("Adding default field templates...")) - for template in DEFAULT_FIELD_TEMPLATES: - session.add(template) - session.flush() + conn.executemany( + "INSERT INTO text_field_templates (id, name, is_multiline) VALUES (?, ?, ?)", + [(t.id, t.name, t.is_multiline) for t in DEFAULT_TEXT_FIELD_TEMPLATES], + ) + conn.executemany( + "INSERT INTO datetime_field_templates (id, name) VALUES (?, ?)", + [(t.id, t.name) for t in DEFAULT_DATETIME_FIELD_TEMPLATES], + ) # DB indices for improved performance - session.execute( - text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") - ) - session.execute( - text("CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id ON tag_parents (child_id)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)") + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_tag_parents_child_id ON tag_parents (child_id)" ) - session.execute( - text("CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)") + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_tag_entries_entry_id ON tag_entries (entry_id)" ) From 890f1becf6d606bee639e09613cbc66ccb335902 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 20:50:56 +0200 Subject: [PATCH 13/21] refactor: sqlalchemy-independence for the DB 201 migration --- .../core/library/alchemy/migrations.py | 77 ++++++++----------- 1 file changed, 33 insertions(+), 44 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index ee15159bf..a61857240 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -471,55 +471,44 @@ class MigrationTo201(DBMigration): @override @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): + def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): """Migrate DB to DB_VERSION 201.""" - create_text_fields_table = text(""" - CREATE TABLE text_fields_new ( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - name VARCHAR NOT NULL, - entry_id INTEGER NOT NULL, - value VARCHAR, - is_multiline BOOLEAN NOT NULL, - FOREIGN KEY(entry_id) REFERENCES entries (id) - ) + logger.info(fmt_log("Dropping type_key from text_fields table...")) + conn.execute(""" + CREATE TABLE text_fields_new ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR NOT NULL, + entry_id INTEGER NOT NULL, + value VARCHAR, + is_multiline BOOLEAN NOT NULL, + FOREIGN KEY(entry_id) REFERENCES entries (id) + ) """) - create_datetime_fields_table = text(""" - CREATE TABLE datetime_fields_new ( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - name VARCHAR NOT NULL, - entry_id INTEGER NOT NULL, - value VARCHAR, - FOREIGN KEY(entry_id) REFERENCES entries (id) - ) + conn.execute(""" + INSERT INTO text_fields_new (id, name, entry_id, value, is_multiline) + SELECT id, name, entry_id, value, is_multiline + FROM text_fields """) - - logger.info(fmt_log("Dropping type_key from text_fields table...")) - session.execute(create_text_fields_table) - session.flush() - session.execute( - text(""" - INSERT INTO text_fields_new (id, name, entry_id, value, is_multiline) - SELECT id, name, entry_id, value, is_multiline - FROM text_fields - """) - ) - session.execute(text("DROP TABLE text_fields")) - session.execute(text("ALTER TABLE text_fields_new RENAME TO text_fields")) + conn.execute("DROP TABLE text_fields") + conn.execute("ALTER TABLE text_fields_new RENAME TO text_fields") logger.info(fmt_log("Dropping type_key from datetime_fields table...")) - session.execute(create_datetime_fields_table) - session.flush() - session.execute( - text(""" - INSERT INTO datetime_fields_new (id, name, entry_id, value) - SELECT id, name, entry_id, value - FROM datetime_fields - """) - ) - session.execute(text("DROP TABLE datetime_fields")) - session.execute(text("ALTER TABLE datetime_fields_new RENAME TO datetime_fields")) - - session.flush() + conn.execute(""" + CREATE TABLE datetime_fields_new ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR NOT NULL, + entry_id INTEGER NOT NULL, + value VARCHAR, + FOREIGN KEY(entry_id) REFERENCES entries (id) + ) + """) + conn.execute(""" + INSERT INTO datetime_fields_new (id, name, entry_id, value) + SELECT id, name, entry_id, value + FROM datetime_fields + """) + conn.execute("DROP TABLE datetime_fields") + conn.execute("ALTER TABLE datetime_fields_new RENAME TO datetime_fields") class MigrationTo202(DBMigration): From 83797bc44b9b626b23ea9f5c1ccf8d22d174d8f1 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 20:53:48 +0200 Subject: [PATCH 14/21] refactor: sqlalchemy-independence for the DB 202 migration --- .../core/library/alchemy/migrations.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index a61857240..325cc78cb 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -10,7 +10,7 @@ import structlog import ujson -from sqlalchemy import delete, select, text +from sqlalchemy import text from sqlalchemy.orm import Session from tagstudio.core.constants import IGNORE_NAME, TAG_ARCHIVED, TS_FOLDER_NAME @@ -23,8 +23,7 @@ DEFAULT_TEXT_FIELD_TEMPLATES, ) from tagstudio.core.library.alchemy.fields import LEGACY_FIELD_MAP -from tagstudio.core.library.alchemy.joins import TagParent -from tagstudio.core.library.alchemy.models import Tag, TagColorGroup +from tagstudio.core.library.alchemy.models import TagColorGroup from tagstudio.core.library.alchemy.utils import list_tables from tagstudio.core.library.ignore import migrate_ext_list from tagstudio.core.utils.types import unwrap @@ -516,11 +515,15 @@ class MigrationTo202(DBMigration): @override @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): + def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): """Migrate DB to DB_VERSION 202.""" - stmt = delete(TagParent).where(TagParent.child_id.not_in(select(Tag.id).distinct())) - session.execute(stmt) - session.flush() + conn.execute(""" + DELETE FROM tag_parents + WHERE NOT child_id IN ( + SELECT id + FROM tags + ) + """) logger.info(fmt_log("Verified TagParent table data")) From cc84df0cad5a90dcc0e6301c592937ad111040f7 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 20:56:01 +0200 Subject: [PATCH 15/21] refactor: sqlalchemy-independence for the DB 300 migration --- .../core/library/alchemy/migrations.py | 56 +++++++++---------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 325cc78cb..0241d02bd 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -532,43 +532,39 @@ class MigrationTo300(DBMigration): @override @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod): - ## remove folder_id column from entries table - # create new table in the desired scheme (without folder_id column) - session.execute( - text(""" - CREATE TABLE entries_new ( - id INTEGER NOT NULL, - path VARCHAR NOT NULL, - suffix VARCHAR NOT NULL, - date_created DATETIME, - date_modified DATETIME, - date_added DATETIME, - filename TEXT NOT NULL DEFAULT '', - PRIMARY KEY (id), - UNIQUE (path) - ) + def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): + # remove folder_id column from entries table + ## create new table in the desired scheme (without folder_id column) + conn.execute(""" + CREATE TABLE entries_new ( + id INTEGER NOT NULL, + path VARCHAR NOT NULL, + suffix VARCHAR NOT NULL, + date_created DATETIME, + date_modified DATETIME, + date_added DATETIME, + filename TEXT NOT NULL DEFAULT '', + PRIMARY KEY (id), + UNIQUE (path) + ) """) - ) - session.flush() - # transfer data to new table - session.execute( - text(""" + + ## transfer data to new table + conn.execute(""" INSERT INTO entries_new (id, path, suffix, date_created, date_modified, date_added, filename) SELECT id, path, suffix, date_created, date_modified, date_added, filename FROM entries """) - ) - # delete old table - session.execute(text("DROP TABLE entries")) - # rename new table to old table - session.execute(text("ALTER TABLE entries_new RENAME TO entries")) - session.flush() - ## drop table "folders" - session.execute(text("DROP TABLE folders")) - session.flush() + ## delete old table + conn.execute("DROP TABLE entries") + + ## rename new table to old table + conn.execute("ALTER TABLE entries_new RENAME TO entries") + + # drop table "folders" + conn.execute("DROP TABLE folders") class MigrationTo400(DBMigration): From 93c460d7f2960bcd43a7cea5c89cdf077ecbe8ee Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 20:57:41 +0200 Subject: [PATCH 16/21] refactor: sqlalchemy-independence for the DB 400 migration --- .../core/library/alchemy/migrations.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 0241d02bd..94169d331 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -10,8 +10,6 @@ import structlog import ujson -from sqlalchemy import text -from sqlalchemy.orm import Session from tagstudio.core.constants import IGNORE_NAME, TAG_ARCHIVED, TS_FOLDER_NAME from tagstudio.core.library.alchemy import default_color_groups @@ -572,16 +570,13 @@ class MigrationTo400(DBMigration): @override @classmethod - def run(cls, session: Session, library_dir: Path, fmt_log): + def run(cls, conn: Connection, library_dir: Path, fmt_log): logger.info(fmt_log("Creating category_exclusions table...")) - session.execute( - text(""" - CREATE TABLE category_exclusions ( - tag_id INTEGER NOT NULL REFERENCES tags(id), - category_id INTEGER NOT NULL REFERENCES tags(id), + conn.execute(""" + CREATE TABLE category_exclusions ( + tag_id INTEGER NOT NULL REFERENCES tags(id), + category_id INTEGER NOT NULL REFERENCES tags(id), - PRIMARY KEY (tag_id, category_id) - ) + PRIMARY KEY (tag_id, category_id) + ) """) - ) - session.flush() From 9e11b86171a5cab2dcc03c0e5c5a92ca54f06716 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 21:01:34 +0200 Subject: [PATCH 17/21] refactor: use executemany instead of loop in DB 8 migration --- .../core/library/alchemy/migrations.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 94169d331..979c9fe37 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -21,7 +21,6 @@ DEFAULT_TEXT_FIELD_TEMPLATES, ) from tagstudio.core.library.alchemy.fields import LEGACY_FIELD_MAP -from tagstudio.core.library.alchemy.models import TagColorGroup from tagstudio.core.library.alchemy.utils import list_tables from tagstudio.core.library.ignore import migrate_ext_list from tagstudio.core.utils.types import unwrap @@ -198,19 +197,20 @@ def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): logger.info(fmt_log("Added color_border column to tag_colors table")) # collect new default tag colors - tag_colors: list[TagColorGroup] = [ - color - for color in default_color_groups.shades() - if color.slug in ["burgundy", "dark-teal", "dark_lavender"] + tag_colors: list[tuple] = [ + (c.slug, c.namespace, c.name, c.primary, c.secondary) + for c in default_color_groups.shades() + if c.slug in ["burgundy", "dark-teal", "dark_lavender"] ] # Add any new default colors introduced in DB_VERSION 8 - for color in tag_colors: - conn.execute( - 'INSERT INTO tag_colors (slug, namespace, name, "primary", secondary) ' - "VALUES (?, ?, ?, ?, ?)", - [color.slug, color.namespace, color.name, color.primary, color.secondary], - ) + conn.executemany( + """ + INSERT INTO tag_colors (slug, namespace, name, \"primary\", secondary) + VALUES (?, ?, ?, ?, ?) + """, + tag_colors, + ) logger.info( fmt_log("Migrated tag colors to DB_VERSION 8+"), color_name=tag_colors, From 2cdf6c310c265abe91d79766b3f4b177b555ebbd Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 21:05:52 +0200 Subject: [PATCH 18/21] refactor: use multiline strings consistenly --- .../core/library/alchemy/migrations.py | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index 979c9fe37..a5f2ab511 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -156,8 +156,10 @@ def _set_version(self, key: str, value: int) -> None: """ # Insert if key has no value yet, otherwise update the value self._connection.execute( - "INSERT INTO versions (key, value) VALUES (?, ?)" - "ON CONFLICT(key) DO UPDATE SET value=excluded.value", + """ + INSERT INTO versions (key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value=excluded.value + """, [key, value], ) @@ -171,13 +173,14 @@ def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): """Migrate DB from DB_VERSION 6 to 7.""" logger.info(fmt_log("Applying patches to DB_VERSION: 6 library...")) # Repair tags that may have a disambiguation_id pointing towards a deleted tag. - conn.execute( - "UPDATE tags " - "SET disambiguation_id = null " - "WHERE NOT disambiguation_id IN (" - "SELECT id FROM tags" - ")" - ) + conn.execute(""" + UPDATE tags + SET disambiguation_id = null + WHERE NOT disambiguation_id IN ( + SELECT id + FROM tags + ) + """) class MigrationTo8(DBMigration): @@ -191,9 +194,10 @@ def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): # TODO: as before, this migration uses the current default colors, while it should really be # using the default colors as they were in that specific version. # FUTURE CHANGES TO THE DEFAULT COLORS WILL BREAK THIS - conn.execute( - "ALTER TABLE tag_colors ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL" - ) + conn.execute(""" + ALTER TABLE tag_colors + ADD COLUMN color_border BOOLEAN DEFAULT FALSE NOT NULL + """) logger.info(fmt_log("Added color_border column to tag_colors table")) # collect new default tag colors @@ -219,10 +223,12 @@ def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): # Update Neon colors to use the the color_border property for color in default_color_groups.neon(): conn.execute( - "UPDATE tag_colors" - "SET slug = ?, namespace = ?, name = ?, " - '"primary" = ?, secondary = ?, color_border = ? ' - "WHERE namespace == ? AND slug = ?", + """ + UPDATE tag_colors + SET slug = ?, namespace = ?, name = ?, + \"primary\" = ?, secondary = ?, color_border = ? + WHERE namespace == ? AND slug = ? + """, [ color.slug, color.namespace, From 0848d7c141ba76d623600da3e5aaedf4efbb4a95 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 21:34:14 +0200 Subject: [PATCH 19/21] fix: various small synatactic sql errors --- .../core/library/alchemy/constants.py | 20 ++++++++++--------- .../core/library/alchemy/migrations.py | 17 +++++++++++----- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/constants.py b/src/tagstudio/core/library/alchemy/constants.py index 7cc8d0445..bfc469e9a 100644 --- a/src/tagstudio/core/library/alchemy/constants.py +++ b/src/tagstudio/core/library/alchemy/constants.py @@ -40,15 +40,17 @@ DEFAULT_TEXT_FIELD_TEMPLATES = ( - TextFieldTemplate(name="Title"), - TextFieldTemplate(name="Author"), - TextFieldTemplate(name="Artist"), - TextFieldTemplate(name="URL"), - TextFieldTemplate(name="Description", is_multiline=True), - TextFieldTemplate(name="Notes", is_multiline=True), - TextFieldTemplate(name="Comments", is_multiline=True), + {"name": "Title", "is_multiline": False}, + {"name": "Author", "is_multiline": False}, + {"name": "Artist", "is_multiline": False}, + {"name": "URL", "is_multiline": False}, + {"name": "Description", "is_multiline": True}, + {"name": "Notes", "is_multiline": True}, + {"name": "Comments", "is_multiline": True}, ) -DEFAULT_DATETIME_FIELD_TEMPLATES = (DatetimeFieldTemplate(name="Date"),) +DEFAULT_DATETIME_FIELD_TEMPLATES = ({"name": "Date"},) -DEFAULT_FIELD_TEMPLATES = DEFAULT_TEXT_FIELD_TEMPLATES + DEFAULT_DATETIME_FIELD_TEMPLATES +DEFAULT_FIELD_TEMPLATES = [TextFieldTemplate(**p) for p in DEFAULT_TEXT_FIELD_TEMPLATES] + [ + DatetimeFieldTemplate(**p) for p in DEFAULT_DATETIME_FIELD_TEMPLATES +] diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index a5f2ab511..aa0179e5f 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -422,7 +422,14 @@ def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): text_boxes = [ x.get("name") for x in LEGACY_FIELD_MAP.values() if x.get("is_multiline") is True ] - conn.execute("UPDATE text_fields SET is_multiline = true WHERE name in ?", text_boxes) + conn.execute( + f""" + UPDATE text_fields + SET is_multiline = true + WHERE name IN ({",".join(["?"] * len(text_boxes))}) + """, + text_boxes, + ) # Repair legacy "Description" and "Comments" fields to use is_multiline = True logger.info(fmt_log("Repairing legacy Description and Comments fields...")) @@ -450,12 +457,12 @@ def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): # Add default field templates logger.info(fmt_log("Adding default field templates...")) conn.executemany( - "INSERT INTO text_field_templates (id, name, is_multiline) VALUES (?, ?, ?)", - [(t.id, t.name, t.is_multiline) for t in DEFAULT_TEXT_FIELD_TEMPLATES], + "INSERT INTO text_field_templates (name, is_multiline) VALUES (:name, :is_multiline)", + DEFAULT_TEXT_FIELD_TEMPLATES, ) conn.executemany( - "INSERT INTO datetime_field_templates (id, name) VALUES (?, ?)", - [(t.id, t.name) for t in DEFAULT_DATETIME_FIELD_TEMPLATES], + "INSERT INTO datetime_field_templates (name) VALUES (:name)", + DEFAULT_DATETIME_FIELD_TEMPLATES, ) # DB indices for improved performance From b11d4fe3ee1fed4559e606259ddadabbc7989f48 Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Thu, 3 Sep 2026 21:50:02 +0200 Subject: [PATCH 20/21] refactor: introduce named placeholders and use executemany where it makes sense --- .../core/library/alchemy/migrations.py | 43 +++++++------------ src/tagstudio/core/library/alchemy/utils.py | 9 ++++ 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index aa0179e5f..ecec5a931 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -21,7 +21,7 @@ DEFAULT_TEXT_FIELD_TEMPLATES, ) from tagstudio.core.library.alchemy.fields import LEGACY_FIELD_MAP -from tagstudio.core.library.alchemy.utils import list_tables +from tagstudio.core.library.alchemy.utils import list_tables, sqlqlchemy_to_dict from tagstudio.core.library.ignore import migrate_ext_list from tagstudio.core.utils.types import unwrap from tagstudio.i18n.translations import Translations @@ -201,8 +201,8 @@ def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): logger.info(fmt_log("Added color_border column to tag_colors table")) # collect new default tag colors - tag_colors: list[tuple] = [ - (c.slug, c.namespace, c.name, c.primary, c.secondary) + tag_colors: list[dict] = [ + sqlqlchemy_to_dict(c) for c in default_color_groups.shades() if c.slug in ["burgundy", "dark-teal", "dark_lavender"] ] @@ -211,7 +211,7 @@ def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): conn.executemany( """ INSERT INTO tag_colors (slug, namespace, name, \"primary\", secondary) - VALUES (?, ?, ?, ?, ?) + VALUES (:slug, :namespace, :name, :primary, :secondary) """, tag_colors, ) @@ -221,25 +221,15 @@ def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): ) # Update Neon colors to use the the color_border property - for color in default_color_groups.neon(): - conn.execute( - """ - UPDATE tag_colors - SET slug = ?, namespace = ?, name = ?, - \"primary\" = ?, secondary = ?, color_border = ? - WHERE namespace == ? AND slug = ? - """, - [ - color.slug, - color.namespace, - color.name, - color.primary, - color.secondary, - color.color_border, - color.namespace, - color.slug, - ], - ) + conn.executemany( + """ + UPDATE tag_colors + SET slug = :slug, namespace = :namespace, name = :name, + \"primary\" = :primary, secondary = :secondary, color_border = :color_border + WHERE namespace == :namespace AND slug = :slug + """, + [sqlqlchemy_to_dict(c) for c in default_color_groups.neon()], + ) class MigrationTo9(DBMigration): @@ -254,12 +244,11 @@ def run(cls, conn: Connection, library_dir: Path, fmt_log: LoggingMethod): logger.info(fmt_log("Added filename column to entries table")) # Populate the new filename column. - paths = [ - (id, Path(path_str)) + filenames = [ + (Path(path_str).name, id) for id, path_str in conn.execute("SELECT id, path FROM entries").fetchall() ] - for eid, path in paths: - conn.execute("UPDATE entries SET filename = ? WHERE id = ?", [path.name, eid]) + conn.executemany("UPDATE entries SET filename = ? WHERE id = ?", filenames) logger.info(fmt_log("Populated filename column in entries table")) diff --git a/src/tagstudio/core/library/alchemy/utils.py b/src/tagstudio/core/library/alchemy/utils.py index 3ee769082..aad5b91db 100644 --- a/src/tagstudio/core/library/alchemy/utils.py +++ b/src/tagstudio/core/library/alchemy/utils.py @@ -4,9 +4,18 @@ from sqlite3 import Connection +from sqlalchemy import inspect + +from tagstudio.core.library.alchemy.db import Base + def list_tables(con: Connection) -> list[str]: return [ row[0] for row in con.execute("SELECT name FROM sqlite_master WHERE type == 'table'").fetchall() ] + + +def sqlqlchemy_to_dict(obj: Base) -> dict: + mapper = inspect(obj.__class__) + return {col.name: getattr(obj, col.name) for col in mapper.columns} From d9e419506cac5ac74affed26b4668841dcdcb17f Mon Sep 17 00:00:00 2001 From: Jann Stute Date: Fri, 4 Sep 2026 15:45:08 +0200 Subject: [PATCH 21/21] fix: make DBMigrations a context manager to correctly close connection --- src/tagstudio/core/library/alchemy/library.py | 11 +++++------ src/tagstudio/core/library/alchemy/migrations.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/tagstudio/core/library/alchemy/library.py b/src/tagstudio/core/library/alchemy/library.py index 6316c1728..28025fc74 100644 --- a/src/tagstudio/core/library/alchemy/library.py +++ b/src/tagstudio/core/library/alchemy/library.py @@ -506,13 +506,12 @@ def open_sqlite_library( # migrate if necessary try: - migrations = DBMigrations(library_dir, sql_filename) + with DBMigrations(library_dir, sql_filename) as migrations: + # save backup if patches will be applied + if migrations.required: + Library.save_library_backup_to_disk(library_dir) - # save backup if patches will be applied - if migrations.required: - Library.save_library_backup_to_disk(library_dir) - - migrations.run() + migrations.run() except MigrationError as e: return LibraryStatus(success=False, message=e.args[0]) diff --git a/src/tagstudio/core/library/alchemy/migrations.py b/src/tagstudio/core/library/alchemy/migrations.py index ecec5a931..c3cb0b437 100644 --- a/src/tagstudio/core/library/alchemy/migrations.py +++ b/src/tagstudio/core/library/alchemy/migrations.py @@ -78,11 +78,21 @@ def __init__(self, library_dir: Path, sql_filename: str) -> None: f"Opening Library with DB Version {self.loaded_db_version}/{DB_VERSION}" ) + self._exited = False + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self._connection.close() + self._exited = True + @property def required(self) -> bool: return self.loaded_db_version < DB_VERSION def run(self): + assert not self._exited if not self.required: return