refactor!: rework and expand refresh (sync) system

This commit is contained in:
Travis Abendshien
2026-09-09 23:05:02 -07:00
parent e2d150645e
commit 1c814d17ed
35 changed files with 2377 additions and 664 deletions
@@ -14,7 +14,7 @@ JSON_FILENAME: str = "ts_library.json"
DB_VERSION_CURRENT_KEY: str = "CURRENT"
DB_VERSION_INITIAL_KEY: str = "INITIAL"
DB_VERSION: int = 400
DB_VERSION: int = 500
TAG_CHILDREN_QUERY = text("""
WITH RECURSIVE ChildTags AS (
+3 -1
View File
@@ -9,6 +9,8 @@ import structlog
from sqlalchemy import Dialect, String, TypeDecorator
from sqlalchemy.orm import DeclarativeBase
from tagstudio.core.utils.normalization import norm_path
logger = structlog.getLogger(__name__)
@@ -19,7 +21,7 @@ class PathType(TypeDecorator):
@override
def process_bind_param(self, value: Path | None, dialect: Dialect):
if value is not None:
return Path(value).as_posix()
return norm_path(Path(value), case_sensitive=True).as_posix()
return None
@override
@@ -72,7 +72,10 @@ class ItemType(enum.Enum):
class SortingModeEnum(enum.Enum):
DATE_ADDED = "file.date_added"
DATE_CREATED = "file.date_created"
DATE_MODIFIED = "file.date_modified"
FILE_NAME = "generic.filename"
FILE_SIZE = "file.size"
PATH = "file.path"
RANDOM = "sorting.mode.random"
+167 -45
View File
@@ -12,7 +12,7 @@ from dataclasses import dataclass
from datetime import UTC, datetime
from os import makedirs
from pathlib import Path
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, NamedTuple
import structlog
from humanfriendly import format_timespan # pyright: ignore[reportUnknownVariableType]
@@ -28,7 +28,6 @@ from sqlalchemy import (
create_engine,
delete,
desc,
exists,
func,
inspect,
or_,
@@ -94,7 +93,8 @@ from tagstudio.core.library.alchemy.models import (
from tagstudio.core.library.alchemy.visitors import SQLBoolExpressionBuilder
from tagstudio.core.library.ignore import migrate_ext_list
from tagstudio.core.library.json.library import Library as JsonLibrary
from tagstudio.core.utils.stat import get_date_created, get_date_modified
from tagstudio.core.utils.normalization import norm_path
from tagstudio.core.utils.stat import get_date_created, get_date_modified, get_file_size
from tagstudio.core.utils.types import unwrap
if TYPE_CHECKING:
@@ -204,12 +204,22 @@ class LibraryStatus:
json_migration_req: bool = False
class FileStat(NamedTuple):
"""Cached stat() fields. Matches the ones stored in file entries."""
date_created: float | None
date_modified: float | None
file_size: int | None
class Library:
"""Class for the Library object, and all CRUD operations made upon it."""
library_dir: Path | None = None
engine: Engine | None = None
included_files: set[Path] = set()
path_cache: dict[Path, int] | None = None
duplicate_path_entry_ids: list[int] | None = None
is_case_sensitive_fs: bool | None = None
def __init__(self) -> None:
self.dupe_entries_count: int = -1 # NOTE: For internal management.
@@ -222,7 +232,9 @@ class Library:
self.engine.dispose()
self.library_dir = None
self.folder = None
self.included_files = set()
self.path_cache = None
self.duplicate_path_entry_ids = None
self.is_case_sensitive_fs = None
self.dupe_entries_count = -1
self.dupe_files_count = -1
@@ -651,54 +663,84 @@ class Library:
make_transient(entry)
return entry
def refresh_file_entry_stats(self, entry_id: int, path: Path | None):
"""Updates a file entry's associated stat() data."""
needs_update = False
def refresh_entries_stats(self, entries: list[tuple[int, Path]]) -> int:
"""Check and update os.stat() metadata for multiple file entries in bulk.
entry = self.get_entry_full(entry_id, with_fields=False, with_tags=False)
if not entry:
return
Only file entries that have differing stat data will be updated.
if not path:
full_path = unwrap(self.library_dir) / entry.path
else:
full_path = unwrap(self.library_dir) / path
Args:
entries (list[tuple[int, Path]]): A list of (ID, Path) tuples to check.
file_date_created = get_date_created(full_path)
file_date_modified = get_date_modified(full_path)
Returns:
int: The number of entries that were updated.
"""
if not entries:
return 0
# Log info
if entry.date_created != file_date_created:
logger.info(full_path)
logger.warning(f"Difference in date_created!: {entry.date_created}/{file_date_created}")
needs_update = True
# else:
# logger.info("No difference in date_created.")
library_dir = unwrap(self.library_dir)
entry_ids = [entry_id for entry_id, _ in entries]
if entry.date_modified != file_date_modified:
logger.info(full_path)
logger.warning(
f"Difference in date_modified!: {entry.date_modified}/{file_date_modified}"
stored: dict[int, FileStat] = {}
with Session(self.engine) as session:
for sub_list in [
entry_ids[i : i + MAX_SQL_VARIABLES]
for i in range(0, len(entry_ids), MAX_SQL_VARIABLES)
]:
stmt = select(
Entry.id, Entry.date_created, Entry.date_modified, Entry.file_size
).where(Entry.id.in_(sub_list))
for row in session.execute(stmt):
stored[row.id] = FileStat(row.date_created, row.date_modified, row.file_size)
updates: dict[int, FileStat] = {}
for entry_id, path in entries:
stored_stat = stored.get(entry_id, FileStat(None, None, None))
full_path = library_dir / path
try:
file_stat = full_path.stat()
except OSError as e:
logger.error(
"[Library] Could not stat file while refreshing entry metadata",
path=full_path,
error=e,
)
continue
current_stat = FileStat(
get_date_created(file_stat),
get_date_modified(file_stat),
get_file_size(file_stat),
)
needs_update = True
# else:
# logger.info("No difference in date_modified")
if not needs_update:
return
else:
logger.info(f"Updating entry file_metadata for {full_path}")
if stored_stat == current_stat:
continue
logger.info(
"[Library] Entry stat data changed",
path=full_path,
date_created=(stored_stat.date_created, current_stat.date_created),
date_modified=(stored_stat.date_modified, current_stat.date_modified),
file_size=(stored_stat.file_size, current_stat.file_size),
)
updates[entry_id] = current_stat
if not updates:
return 0
with Session(self.engine) as session:
stmt = update(Entry).where(Entry.id == entry_id)
if file_date_created:
stmt = stmt.values(date_created=file_date_created)
if file_date_modified:
stmt = stmt.values(date_modified=file_date_modified)
session.execute(stmt)
session.execute(
update(Entry),
[
{"id": entry_id, **entry_stat._asdict()}
for entry_id, entry_stat in updates.items()
],
)
session.commit()
logger.info(f"[Library] Refreshed stat data for {len(updates)} of {len(entries)} entries")
return len(updates)
def get_tag_entries(
self, tag_ids: Iterable[int], entry_ids: Iterable[int]
) -> dict[int, set[int]]:
@@ -777,8 +819,47 @@ class Library:
full_ts_path.mkdir(parents=True, exist_ok=True)
return False
def _path_cache_key(self, path: Path) -> Path:
case_sensitive = (
self.is_case_sensitive_fs if self.is_case_sensitive_fs is not None else True
)
return norm_path(path, case_sensitive=case_sensitive)
def _cache_add_path(self, entry_id: int, path: Path) -> None:
"""Keep the path cache consistent with a newly-added or relinked entry."""
if self.path_cache is None:
return
key = self._path_cache_key(path)
if key in self.path_cache:
displaced_id = self.path_cache[key]
logger.warning(
"[Library] Duplicate path discovered in path cache while normalizing path, "
"marking displaced entry as unlinked.",
path=path,
displaced_entry_id=displaced_id,
entry_id=entry_id,
)
if self.duplicate_path_entry_ids is None:
self.duplicate_path_entry_ids = []
self.duplicate_path_entry_ids.append(displaced_id)
self.path_cache[key] = entry_id
def _cache_remove_entries(self, entry_ids: Iterable[int]) -> None:
"""Keep the path cache consistent with removed entry ids."""
removed = set(entry_ids)
if not removed:
return
if self.path_cache is not None:
stale_keys = [key for key, eid in self.path_cache.items() if eid in removed]
for key in stale_keys:
del self.path_cache[key]
if self.duplicate_path_entry_ids:
self.duplicate_path_entry_ids = [
eid for eid in self.duplicate_path_entry_ids if eid not in removed
]
def add_entries(self, items: list[Entry]) -> list[int]:
"""Add multiple Entry records to the Library."""
"""Add multiple entries to the Library."""
assert items
with Session(self.engine) as session:
@@ -795,6 +876,9 @@ class Library:
new_ids = [item.id for item in items]
session.expunge_all()
for entry_id, item in zip(new_ids, items, strict=True):
self._cache_add_path(entry_id, item.path)
return new_ids
def remove_entries(self, entry_ids: list[int]) -> None:
@@ -806,12 +890,40 @@ class Library:
]:
session.query(Entry).where(Entry.id.in_(sub_list)).delete()
session.commit()
self._cache_remove_entries(entry_ids)
def get_entry_id_from_path(self, path: Path) -> int:
"""Attempt to return an Entry ID given a filepath, else return -1."""
with Session(self.engine) as session:
return session.scalar(select(Entry.id).where(Entry.path == path).limit(1)) or -1
def all_paths_with_ids(self) -> dict[int, Path]:
"""Bulk fetch every Entry's (id, path). Only used to init the path cache."""
with Session(self.engine) as session:
rows = session.execute(select(Entry.id, Entry.path)).all()
return {row.id: row.path for row in rows}
def get_or_build_path_cache(self) -> dict[Path, int]:
"""Return the dict cache of normalized paths -> entry IDs."""
if self.path_cache is None:
cache: dict[Path, int] = {}
duplicates: list[int] = []
for entry_id, path in self.all_paths_with_ids().items():
key = self._path_cache_key(path)
if key in cache:
logger.warning(
"[Library] Duplicate normalized path created while building cache, "
"marking the displaced entry as unlinked.",
path=path,
displaced_entry_id=cache[key],
entry_id=entry_id,
)
duplicates.append(cache[key])
cache[key] = entry_id
self.path_cache = cache
self.duplicate_path_entry_ids = duplicates
return self.path_cache
def get_paths(self, limit: int = -1) -> list[str]:
path_strings: list[str] = []
with Session(self.engine) as session:
@@ -829,7 +941,8 @@ class Library:
) -> SearchResult:
"""Filter library by search query.
:return: number of entries matching the query and one page of results.
Returns:
SearchResult: number of entries matching the query and one page of results.
"""
assert isinstance(search, BrowsingState)
assert self.library_dir
@@ -867,8 +980,14 @@ class Library:
match search.sorting_mode:
case SortingModeEnum.DATE_ADDED:
sort_on = Entry.id
case SortingModeEnum.DATE_CREATED:
sort_on = Entry.date_created
case SortingModeEnum.DATE_MODIFIED:
sort_on = Entry.date_modified
case SortingModeEnum.FILE_NAME:
sort_on = func.lower(Entry.filename)
case SortingModeEnum.FILE_SIZE:
sort_on = Entry.file_size
case SortingModeEnum.PATH:
sort_on = func.lower(Entry.path)
case SortingModeEnum.RANDOM:
@@ -1128,7 +1247,7 @@ class Library:
Returns True if the action succeeded and False if the path already exists.
"""
if self.get_entry_id_from_path(path):
if self.get_entry_id_from_path(path) >= 0:
return False
if isinstance(entry_id, Entry):
entry_id = entry_id.id
@@ -1146,6 +1265,9 @@ class Library:
session.execute(update_stmt)
session.commit()
self._cache_remove_entries([entry_id])
self._cache_add_path(entry_id, path)
return True
def remove_tag(self, tag_id: int) -> bool:
@@ -24,6 +24,7 @@ from tagstudio.core.library.alchemy.fields import LEGACY_FIELD_MAP, DatetimeFiel
from tagstudio.core.library.alchemy.joins import TagParent
from tagstudio.core.library.alchemy.models import Entry, Tag, TagColorGroup, Version
from tagstudio.core.library.ignore import migrate_ext_list
from tagstudio.core.utils.normalization import norm_path
from tagstudio.core.utils.types import unwrap
from tagstudio.i18n.translations import Translations
@@ -101,6 +102,7 @@ class DBMigrations:
MigrationTo202, # changes: tag_parents
MigrationTo300, # changes: deletes folders
MigrationTo400, # changes: add category_exclusions
MigrationTo500, # changes: entries
]
with Session(self.engine) as session:
for migration in migrations:
@@ -616,3 +618,52 @@ class MigrationTo400(DBMigration):
""")
)
session.flush()
class MigrationTo500(DBMigration):
version = 500
@override
@classmethod
def run(cls, session: Session, library_dir: Path, fmt_log: LoggingMethod):
"""Migrate DB to DB_VERSION 500."""
# Drop date columns that were string based to add new float ones, plus int file_size
logger.info(fmt_log("Dropping old entry columns..."))
session.execute(text("ALTER TABLE entries DROP COLUMN date_created"))
session.execute(text("ALTER TABLE entries DROP COLUMN date_modified"))
session.flush()
logger.info(fmt_log("Adding new entry columns..."))
session.execute(text("ALTER TABLE entries ADD COLUMN date_created REAL"))
session.execute(text("ALTER TABLE entries ADD COLUMN date_modified REAL"))
session.execute(text("ALTER TABLE entries ADD COLUMN file_size INTEGER"))
session.flush()
# Normalize entry paths to NFD
logger.info(fmt_log("Normalizing file entry paths..."))
rows = session.execute(text("SELECT id, path FROM entries")).all()
entries_by_key: dict[Path, list[tuple[int, str]]] = {}
for entry_id, path in rows:
nfd_path = norm_path(Path(path), case_sensitive=True)
entries_by_key.setdefault(nfd_path, []).append((entry_id, path))
for nfd_path, group in entries_by_key.items():
if len(group) > 1:
continue
entry_id, path = group[0]
if path == nfd_path.as_posix():
continue # Already normalized
session.execute(
text(
"UPDATE entries SET path = :path, filename = :filename, "
"suffix = :suffix WHERE id = :id"
),
{
"path": nfd_path.as_posix(),
"filename": nfd_path.name,
"suffix": nfd_path.suffix.lstrip(".").lower(),
"id": entry_id,
},
)
session.flush()
+8 -3
View File
@@ -17,6 +17,7 @@ from tagstudio.core.library.alchemy.fields import (
TextField,
)
from tagstudio.core.library.alchemy.joins import CategoryExclusion, TagParent
from tagstudio.core.utils.normalization import norm_path
class Namespace(Base):
@@ -204,6 +205,7 @@ class Entry(Base):
suffix: Mapped[str] = mapped_column()
date_created: Mapped[float | None]
date_modified: Mapped[float | None]
file_size: Mapped[int | None]
date_added: Mapped[dt | None]
tags: Mapped[set[Tag]] = relationship(secondary="tag_entries")
@@ -239,19 +241,22 @@ class Entry(Base):
id: int | None = None,
date_created: float | None = None,
date_modified: float | None = None,
file_size: int | None = None,
date_added: dt | None = None,
) -> None:
super().__init__()
self.path = path
self.id = id # pyright: ignore[reportAttributeAccessIssue]
self.filename = path.name
self.suffix = path.suffix.lstrip(".").lower()
self.path = norm_path(path, case_sensitive=True) # NFD is enforced
self.filename = self.path.name
self.suffix = self.path.suffix.lstrip(".").lower()
# The date the file associated with this entry was created.
# st_birthtime on Windows and Mac, st_ctime on Linux.
self.date_created = date_created
# The date the file associated with this entry was last modified: st_mtime.
self.date_modified = date_modified
# The size of the file associated with this entry, in bytes: st_size.
self.file_size = file_size
# The date this entry was added to the library.
self.date_added = date_added
@@ -1,98 +0,0 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: GPL-3.0-only
from collections.abc import Iterator
from dataclasses import dataclass, field
from pathlib import Path
import structlog
from wcmatch import glob, pathlib
from tagstudio.core.library.alchemy.library import Library
from tagstudio.core.library.alchemy.models import Entry
from tagstudio.core.library.ignore import PATH_GLOB_FLAGS, Ignore, ignore_to_glob
from tagstudio.core.utils.types import unwrap
logger = structlog.get_logger()
@dataclass
class UnlinkedRegistry:
"""State tracker for unlinked entries."""
lib: Library
files_fixed_count: int = 0
unlinked_entries: list[Entry] = field(default_factory=list)
@property
def unlinked_entries_count(self) -> int:
return len(self.unlinked_entries)
def reset(self):
self.unlinked_entries.clear()
def refresh_unlinked_files(self) -> Iterator[int]:
"""Track the number of entries that point to an invalid filepath."""
logger.info("[UnlinkedRegistry] Refreshing unlinked files...")
self.unlinked_entries = []
for i, entry in enumerate(self.lib.all_entries()):
yield i
full_path = unwrap(self.lib.library_dir) / entry.path
if not full_path.exists() or not full_path.is_file():
self.unlinked_entries.append(entry)
def match_unlinked_file_entry(self, match_entry: Entry) -> list[Path]:
"""Try and match unlinked file entries with matching results in the library directory.
Works if files were just moved to different subfolders and don't have duplicate names.
"""
library_dir = unwrap(self.lib.library_dir)
matches: list[Path] = []
# NOTE: ignore_to_glob() is needed for wcmatch, not ripgrep.
ignore_patterns = ignore_to_glob(Ignore.get_patterns(library_dir))
for path in pathlib.Path(str(library_dir)).glob(
patterns=f"***/{glob.escape(match_entry.path.name)}",
flags=PATH_GLOB_FLAGS,
exclude=ignore_patterns,
):
if path.is_dir():
continue
if path.name == match_entry.path.name:
new_path = Path(path).relative_to(library_dir)
matches.append(new_path)
logger.info("[UnlinkedRegistry] Matches", matches=matches)
return matches
def fix_unlinked_entries(self) -> Iterator[int]:
"""Attempt to fix unlinked file entries by finding a match in the library directory."""
self.files_fixed_count = 0
matched_entries: list[Entry] = []
for i, entry in enumerate(self.unlinked_entries):
yield i
item_matches = self.match_unlinked_file_entry(entry)
if len(item_matches) == 1:
logger.info(
"[UnlinkedRegistry]",
entry=entry.path.as_posix(),
item_matches=item_matches[0].as_posix(),
)
if not self.lib.update_entry_path(entry.id, item_matches[0]):
try:
match = unwrap(self.lib.get_entry_full_by_path(item_matches[0]))
entry_full = unwrap(self.lib.get_entry_full(entry.id))
self.lib.merge_entries(entry_full, match)
except AttributeError:
continue
self.files_fixed_count += 1
matched_entries.append(entry)
for entry in matched_entries:
self.unlinked_entries.remove(entry)
def remove_unlinked_entries(self) -> None:
self.lib.remove_entries(list(map(lambda unlinked: unlinked.id, self.unlinked_entries)))
self.unlinked_entries = []
+2 -2
View File
@@ -1,5 +1,5 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: GPL-3.0-only
# SPDX-License-Identifier: MIT
from copy import deepcopy
@@ -14,7 +14,7 @@ from tagstudio.core.utils.singleton import Singleton
logger = structlog.get_logger()
PATH_GLOB_FLAGS = glob.GLOBSTARLONG | glob.DOTGLOB | glob.NEGATE | pathlib.MATCHBASE
PATH_GLOB_FLAGS: int = glob.GLOBSTARLONG | glob.DOTGLOB | glob.NEGATE | pathlib.MATCHBASE
GLOBAL_IGNORE = [
-232
View File
@@ -1,232 +0,0 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: GPL-3.0-only
import shutil
from collections.abc import Iterator
from dataclasses import dataclass, field
from datetime import datetime as dt
from pathlib import Path
from time import time
import structlog
from wcmatch import pathlib
from tagstudio.core.library.alchemy.library import Library
from tagstudio.core.library.alchemy.models import Entry
from tagstudio.core.library.ignore import PATH_GLOB_FLAGS, Ignore, ignore_to_glob
from tagstudio.core.utils.silent_subprocess import silent_run # pyright: ignore
from tagstudio.core.utils.stat import get_date_created, get_date_modified
from tagstudio.core.utils.types import unwrap
logger = structlog.get_logger(__name__)
@dataclass
class RefreshTracker:
library: Library
files_not_in_library: list[Path] = field(default_factory=list)
@property
def files_count(self) -> int:
return len(self.files_not_in_library)
def save_new_files(self) -> Iterator[int]:
"""Save the list of files that are not in the library."""
batch_size = 200
index = 0
while index < len(self.files_not_in_library):
yield index
end = min(len(self.files_not_in_library), index + batch_size)
entries = [
Entry(
path=entry_path,
fields=[],
date_created=get_date_created(unwrap(self.library.library_dir) / entry_path),
date_modified=get_date_modified(unwrap(self.library.library_dir) / entry_path),
date_added=dt.now(),
)
for entry_path in self.files_not_in_library[index:end]
]
self.library.add_entries(entries)
index = end
self.files_not_in_library = []
def refresh_dir(self, library_dir: Path, force_internal_tools: bool = False) -> Iterator[int]:
"""Scan a directory for files, and add those relative filenames to internal variables.
Args:
library_dir (Path): The library directory.
force_internal_tools (bool): Option to force the use of internal tools for scanning
(i.e. wcmatch) instead of using tools found on the system (i.e. ripgrep).
"""
if self.library.library_dir is None:
raise ValueError("No library directory set.")
ignore_patterns = Ignore.get_patterns(library_dir)
if force_internal_tools:
return self.__wc_add(library_dir, ignore_to_glob(ignore_patterns))
dir_list: list[str] | None = self.__get_dir_list(library_dir, ignore_patterns)
# Use ripgrep if it was found and working, else fallback to wcmatch.
if dir_list is not None:
return self.__rg_add(library_dir, dir_list)
else:
return self.__wc_add(library_dir, ignore_to_glob(ignore_patterns))
def __get_dir_list(self, library_dir: Path, ignore_patterns: list[str]) -> list[str] | None:
"""Use ripgrep to return a list of matched directories and files.
Return `None` if ripgrep not found on system.
"""
rg_path = shutil.which("rg")
# Use ripgrep if found on system
if rg_path is not None:
logger.info("[Refresh: Using ripgrep for scanning]")
compiled_ignore_path = library_dir / ".TagStudio" / ".compiled_ignore"
# Write compiled ignore patterns (built-in + user) to a temp file to pass to ripgrep
with open(compiled_ignore_path, "w") as pattern_file:
pattern_file.write("\n".join(ignore_patterns))
result = silent_run(
" ".join(
[
"rg",
"--files",
"--follow",
"--hidden",
"--ignore-file",
f'"{str(compiled_ignore_path)}"',
]
),
cwd=library_dir,
capture_output=True,
shell=True,
encoding="UTF-8",
)
try:
compiled_ignore_path.unlink()
except Exception as e:
logger.error(
"[Refresh] Could not remove compiled ignore path",
path=compiled_ignore_path,
error=e,
)
if result.stderr:
logger.error(result.stderr)
return result.stdout.splitlines() # pyright: ignore [reportReturnType]
logger.warning("[Refresh: ripgrep not found on system]")
return None
def __rg_add(self, library_dir: Path, dir_list: list[str]) -> Iterator[int]:
start_time_total = time()
start_time_loop = time()
dir_file_count = 0
self.files_not_in_library = []
for r in dir_list:
f = pathlib.Path(r)
end_time_loop = time()
# Yield output every 1/30 of a second
if (end_time_loop - start_time_loop) > 0.034:
yield dir_file_count
start_time_loop = time()
# Skip if the file/path is already mapped in the Library
if f in self.library.included_files:
dir_file_count += 1
entry_id = self.library.get_entry_id_from_path(f)
self.library.refresh_file_entry_stats(entry_id, path=f)
continue
# Ignore if the file is a directory
if f.is_dir():
continue
dir_file_count += 1
self.library.included_files.add(f)
# if not self.library.has_entry_with_path(f):
# self.files_not_in_library.append(f)
entry_id = self.library.get_entry_id_from_path(f)
if entry_id < 0:
self.files_not_in_library.append(f)
else:
self.library.refresh_file_entry_stats(entry_id, path=f)
end_time_total = time()
yield dir_file_count
logger.info(
"[Refresh]: Directory scan time",
path=library_dir,
duration=(end_time_total - start_time_total),
files_scanned=dir_file_count,
tool_used="ripgrep (system)",
)
def __wc_add(self, library_dir: Path, ignore_patterns: list[str]) -> Iterator[int]:
start_time_total = time()
start_time_loop = time()
dir_file_count = 0
self.files_not_in_library = []
logger.info("[Refresh]: Falling back to wcmatch for scanning")
try:
for f in pathlib.Path(str(library_dir)).glob(
"***/*", flags=PATH_GLOB_FLAGS, exclude=ignore_patterns
):
end_time_loop = time()
# Yield output every 1/30 of a second
if (end_time_loop - start_time_loop) > 0.034:
yield dir_file_count
start_time_loop = time()
# Skip if the file/path is already mapped in the Library
if f in self.library.included_files:
dir_file_count += 1
relative_path = f.relative_to(library_dir)
entry_id = self.library.get_entry_id_from_path(relative_path)
self.library.refresh_file_entry_stats(entry_id, path=relative_path)
continue
# Ignore if the file is a directory
if f.is_dir():
continue
dir_file_count += 1
self.library.included_files.add(f)
relative_path = f.relative_to(library_dir)
# if not self.library.has_entry_with_path(relative_path):
# self.files_not_in_library.append(relative_path)
entry_id = self.library.get_entry_id_from_path(relative_path)
if entry_id < 0:
self.files_not_in_library.append(relative_path)
else:
self.library.refresh_file_entry_stats(entry_id, path=relative_path)
except ValueError:
logger.info("[Refresh]: ValueError when refreshing directory with wcmatch!")
end_time_total = time()
yield dir_file_count
logger.info(
"[Refresh]: Directory scan time",
path=library_dir,
duration=(end_time_total - start_time_total),
files_scanned=dir_file_count,
tool_used="wcmatch (internal)",
)
+101
View File
@@ -0,0 +1,101 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: MIT
import subprocess
from collections.abc import Iterator
from pathlib import Path
import structlog
from wcmatch import pathlib
from tagstudio.core.constants import TS_FOLDER_NAME
from tagstudio.core.library.ignore import PATH_GLOB_FLAGS, ignore_to_glob
from tagstudio.core.utils.ripgrep_status import RipgrepStatus
from tagstudio.core.utils.silent_subprocess import silent_popen # pyright: ignore
logger = structlog.get_logger(__name__)
def scan_paths(
scan_dir: Path, ignore_patterns: list[str], force_internal_scanner: bool = False
) -> Iterator[Path]:
"""Scan `scan_dir` for files, yielding each match's path relative to `scan_dir`.
Uses ripgrep if present on the system, falling back to the internal (wcmatch) scanner
otherwise or if `force_internal_scanner` is set.
"""
if not force_internal_scanner and RipgrepStatus.which() is not None:
yield from _scan_with_ripgrep(scan_dir, ignore_patterns)
return
yield from _scan_with_internal_scanner(scan_dir, ignore_patterns)
def _scan_with_ripgrep(scan_dir: Path, ignore_patterns: list[str]) -> Iterator[Path]:
"""Scan for files with ripgrep."""
logger.info("[Scanners] Using ripgrep for scanning", path=scan_dir)
compiled_ignore_path = scan_dir / TS_FOLDER_NAME / ".compiled_ignore"
compiled_ignore_path.parent.mkdir(parents=True, exist_ok=True)
compiled_ignore_path.write_text("\n".join(ignore_patterns), encoding="utf-8")
proc: subprocess.Popen[str] | None = None
try:
proc = silent_popen(
["rg", "--files", "--follow", "--hidden", "--ignore-file", str(compiled_ignore_path)],
cwd=scan_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
encoding="UTF-8",
)
assert proc.stdout is not None
for line in proc.stdout:
line = line.rstrip("\n")
if not line:
continue
path = Path(line)
if (scan_dir / path).is_dir():
continue
yield path
proc.wait()
if proc.returncode not in (0, 1): # 1 == "no matches", still successful
logger.error(
"[Scanners] ripgrep exited with an error",
returncode=proc.returncode,
stderr=proc.stderr.read() if proc.stderr else "",
)
finally:
if proc is not None:
# Loop finished
if proc.stdout is not None:
proc.stdout.close()
# Still running, but cancelled mid-loop
if proc.poll() is None:
proc.terminate()
proc.wait()
try:
compiled_ignore_path.unlink(missing_ok=True)
except OSError as e:
logger.error(
"[Scanners] Could not remove compiled ignore path",
path=compiled_ignore_path,
error=e,
)
def _scan_with_internal_scanner(scan_dir: Path, ignore_patterns: list[str]) -> Iterator[Path]:
"""Scan for files with the internal glob-based scanner (wcmatch)."""
logger.info("[Scanners] Using internal scanner for scanning", path=scan_dir)
glob_patterns = ignore_to_glob(ignore_patterns)
try:
for f in pathlib.Path(str(scan_dir)).glob(
"***/*", flags=PATH_GLOB_FLAGS, exclude=glob_patterns
):
if f.is_dir():
continue
yield Path(f).relative_to(scan_dir)
except ValueError:
logger.error("[Scanners] ValueError while scanning directory with the internal scanner")
+391
View File
@@ -0,0 +1,391 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: MIT
from collections.abc import Iterator
from dataclasses import dataclass, field
from datetime import datetime as dt
from pathlib import Path
from time import time
import structlog
from wcmatch import glob, pathlib
from tagstudio.core.library.alchemy.library import Library
from tagstudio.core.library.alchemy.models import Entry
from tagstudio.core.library.ignore import PATH_GLOB_FLAGS, Ignore, ignore_to_glob
from tagstudio.core.library.scanners import scan_paths
from tagstudio.core.utils.filesystem import is_fs_case_sensitive
from tagstudio.core.utils.normalization import norm_path
from tagstudio.core.utils.stat import get_date_created, get_date_modified, get_file_size
from tagstudio.core.utils.types import unwrap
logger = structlog.get_logger(__name__)
# Yield progress this often during a loop to avoid overwhelming the UI.
# TODO: Look into whether or not this can be handled on the UI side.
YIELD_INTERVAL_SECONDS = 0.034
@dataclass
class LibrarySyncEngine:
"""Keeps a Library's entries in sync with its content directories on disk."""
library: Library
new_paths: list[Path] = field(default_factory=list)
paths_to_restat: list[tuple[int, Path]] = field(default_factory=list)
unlinked_entries: list[Entry] = field(default_factory=list)
relinked_entries: list[Entry] = field(default_factory=list)
manual_relink_count: int = 0
cancelled: bool = False
_scanned_paths: list[Path] = field(default_factory=list, init=False, repr=False)
_filename_to_path_map: dict[Path, list[Path]] | None = field(
default=None, init=False, repr=False
)
@property
def new_file_count(self) -> int:
return len(self.new_paths)
@property
def restat_count(self) -> int:
return len(self.paths_to_restat)
@property
def unlinked_entries_count(self) -> int:
return len(self.unlinked_entries)
@property
def relinked_entries_count(self) -> int:
return len(self.relinked_entries)
def reset(self) -> None:
"""Clear this engine's scan results."""
self.new_paths = []
self.paths_to_restat = []
self.unlinked_entries = []
self.relinked_entries = []
self._scanned_paths = []
self._filename_to_path_map = None
def _get_case_sensitivity(self) -> bool:
if self.library.is_case_sensitive_fs is None:
self.library.is_case_sensitive_fs = is_fs_case_sensitive()
return self.library.is_case_sensitive_fs
def sync_dir(
self, library_dir: Path, force_internal_scanner: bool = False
) -> Iterator[tuple[int, int]]:
"""Scan library directory for files, then reconcile them against the Library's entries.
- Entries with no matching file on disk are marked as "unlinked"
- Automatically relink to appropriate new files on disk where possible
- Remaining new files on disk are added as new entires
- Remaining unlinked entries are tracked for manual review.
Yields (searched_count, found_count)
Args:
library_dir (Path): The library directory.
force_internal_scanner (bool): Option to force the use of the internal scanner
(i.e. wcmatch) instead of third-party tools found on the system (i.e. ripgrep).
"""
self.reset()
self.cancelled = False
case_sensitive = self._get_case_sensitivity()
cache = self.library.get_or_build_path_cache()
unvisited = set(cache.keys())
ignore_patterns = Ignore.get_patterns(library_dir)
start_time = time()
start_time_loop = time()
count = 0
for raw_path in scan_paths(library_dir, ignore_patterns, force_internal_scanner):
if self.cancelled:
break
count += 1
self._scanned_paths.append(raw_path)
key = norm_path(raw_path, case_sensitive=case_sensitive)
entry_id = cache.get(key)
if entry_id is not None:
unvisited.discard(key)
self.paths_to_restat.append((entry_id, raw_path))
else:
self.new_paths.append(raw_path)
if (time() - start_time_loop) > YIELD_INTERVAL_SECONDS:
yield count, len(self.new_paths)
start_time_loop = time()
if self.cancelled:
yield count, len(self.new_paths)
logger.info("[Sync] Directory scan cancelled", path=library_dir, files_scanned=count)
return
unlinked_ids = {cache[key] for key in unvisited}
if self.library.duplicate_path_entry_ids:
unlinked_ids.update(self.library.duplicate_path_entry_ids)
if unlinked_ids:
self.unlinked_entries = self.library.get_entries(list(unlinked_ids))
self._auto_relink_matched_entries(case_sensitive, cache)
yield count, len(self.new_paths)
logger.info(
"[Sync] Directory scan complete",
path=library_dir,
duration=(time() - start_time),
files_scanned=count,
new_files=len(self.new_paths),
unlinked_entries=len(self.unlinked_entries),
relinked_entries=len(self.relinked_entries),
)
def save_new_entries(self) -> Iterator[int]:
"""Save the paths found on disk that don't have a Library entry yet."""
batch_size = 200
library_dir = unwrap(self.library.library_dir)
index = 0
while index < len(self.new_paths):
if self.cancelled:
break
yield index
end = min(len(self.new_paths), index + batch_size)
batch = self.new_paths[index:end]
entries = []
for entry_path in batch:
file_stat = (library_dir / entry_path).stat()
entries.append(
Entry(
path=entry_path,
fields=[],
date_created=get_date_created(file_stat),
date_modified=get_date_modified(file_stat),
file_size=get_file_size(file_stat),
date_added=dt.now(),
)
)
self.library.add_entries(entries) # Path cache is updated in the library
index = end
self.new_paths = self.new_paths[index:] # Saved entries are removed from new_paths
def sync_entry_stats(self) -> Iterator[int]:
"""Refresh cached os.stat() metadata for entries already known to the Library."""
batch_size = 500
index = 0
while index < len(self.paths_to_restat):
if self.cancelled:
break
yield index
end = min(len(self.paths_to_restat), index + batch_size)
self.library.refresh_entries_stats(self.paths_to_restat[index:end])
index = end
self.paths_to_restat = self.paths_to_restat[index:]
def _build_filename_to_path_map(self, case_sensitive: bool) -> dict[Path, list[Path]]:
index: dict[Path, list[Path]] = {}
for path in self._scanned_paths:
key = norm_path(Path(path.name), case_sensitive=case_sensitive)
index.setdefault(key, []).append(path)
return index
def _glob_for_filename(self, filename: str, case_sensitive: bool) -> list[Path]:
"""Search the library directory for files matching `filename`.
Used only as a fallback when find_relink_candidates() is called without a prior
sync_dir() scan in this engine instance to reuse results from.
"""
library_dir = unwrap(self.library.library_dir)
ignore_patterns = ignore_to_glob(Ignore.get_patterns(library_dir))
target_path = norm_path(Path(filename), case_sensitive=case_sensitive)
flags = PATH_GLOB_FLAGS | (0 if case_sensitive else glob.IGNORECASE)
matches: list[Path] = []
for path in pathlib.Path(str(library_dir)).glob(
patterns=f"***/{glob.escape(filename)}",
flags=flags,
exclude=ignore_patterns,
):
if path.is_dir():
continue
candidate = Path(path).relative_to(library_dir)
if norm_path(Path(candidate.name), case_sensitive=case_sensitive) == target_path:
matches.append(candidate)
return matches
def find_relink_candidates(self, entry: Entry) -> list[Path]:
"""Try to find files in the library directory matching an unlinked entry's filename.
Comparisons are made using NFD normalization and the assumed filesystem's case sensitivity.
"""
case_sensitive = self._get_case_sensitivity()
target_key = norm_path(Path(entry.path.name), case_sensitive=case_sensitive)
if self._scanned_paths:
if self._filename_to_path_map is None:
self._filename_to_path_map = self._build_filename_to_path_map(case_sensitive)
matches = list(self._filename_to_path_map.get(target_key, []))
else:
matches = self._glob_for_filename(entry.path.name, case_sensitive)
logger.info("[Sync] Relink candidates", entry=entry.path.as_posix(), matches=matches)
return matches
def _apply_relink(
self, entry: Entry, new_path: Path, cache: dict[Path, int], case_sensitive: bool
) -> bool:
"""Assign `new_path` to the `entry`, merging into a single entry if entries exist for both.
Returns:
bool: True if the relink was successful.
"""
new_key = norm_path(new_path, case_sensitive=case_sensitive)
existing_id = cache.get(new_key)
if existing_id is not None and existing_id != entry.id:
# Merge both entries into one with the single path
target = unwrap(self.library.get_entry_full(existing_id))
source = unwrap(self.library.get_entry_full(entry.id))
return self.library.merge_entries(source, target)
return self.library.update_entry_path(entry.id, new_path)
def relink_unlinked_entries(self) -> Iterator[int]:
"""Attempt to fix unlinked entries by finding a single matching file in the library."""
self.manual_relink_count = 0
case_sensitive = self._get_case_sensitivity()
cache = self.library.get_or_build_path_cache()
matched: list[Entry] = []
for i, entry in enumerate(self.unlinked_entries):
yield i
candidates = self.find_relink_candidates(entry)
if len(candidates) != 1:
continue
new_path = candidates[0]
if not self._apply_relink(entry, new_path, cache, case_sensitive):
continue
self.manual_relink_count += 1
matched.append(entry)
logger.info(
"[Sync] Relinked entry",
entry=entry.path.as_posix(),
new_path=new_path.as_posix(),
)
for entry in matched:
self.unlinked_entries.remove(entry)
def _auto_relink_matched_entries(self, case_sensitive: bool, cache: dict[Path, int]) -> None:
"""Attempt to automatically relink unlinked entries under available conditions.
Auto-relink applies to:
- Files with the same filename but different paths
- Handles moves, moves + changes
- Files with different names and/or paths but the same date_modified and file_size
- Handles moves, renames + moves
Auto-relink DOES NOT apply to:
- Renames + Moves + Changes
- Deletions
- Ambiguous (more than one) matches
"""
if not self.new_paths or not self.unlinked_entries:
return
library_dir = unwrap(self.library.library_dir)
# Pass 1: Filename + metadata
by_name_and_stat: dict[tuple[Path, float | None, int | None], list[Entry]] = {}
for entry in self.unlinked_entries:
name_key = norm_path(Path(entry.path.name), case_sensitive=case_sensitive)
by_name_and_stat.setdefault(
(name_key, entry.date_modified, entry.file_size), []
).append(entry)
relinked: list[Entry] = []
remaining_new: list[Path] = []
stats_by_path: dict[Path, tuple[float | None, int | None]] = {}
for new_path in self.new_paths:
try:
file_stat = (library_dir / new_path).stat()
except OSError as e:
logger.error(
"[Sync] Could not stat file during auto-relink check",
path=new_path,
error=e,
)
remaining_new.append(new_path)
continue
mtime = get_date_modified(file_stat)
size = get_file_size(file_stat)
stats_by_path[new_path] = (mtime, size)
name_key = norm_path(Path(new_path.name), case_sensitive=case_sensitive)
key = (name_key, mtime, size)
candidates = by_name_and_stat.get(key, [])
if len(candidates) != 1:
remaining_new.append(new_path)
continue
entry = candidates[0]
if not self._apply_relink(entry, new_path, cache, case_sensitive):
remaining_new.append(new_path)
continue
by_name_and_stat[key] = [] # Don't match a second new_path here
logger.info(
"[Sync] Automatically relinked moved file",
old_path=entry.path.as_posix(),
new_path=new_path.as_posix(),
)
relinked.append(entry)
for entry in relinked:
self.unlinked_entries.remove(entry)
# Pass 2: Different filename checking for same metadata
by_stat_only: dict[tuple[float | None, int | None], list[Entry]] = {}
for entry in self.unlinked_entries:
by_stat_only.setdefault((entry.date_modified, entry.file_size), []).append(entry)
still_remaining: list[Path] = []
for new_path in remaining_new:
stat_key = stats_by_path.get(new_path)
if stat_key is None:
still_remaining.append(new_path)
continue
candidates = by_stat_only.get(stat_key, [])
if len(candidates) != 1:
still_remaining.append(new_path)
continue
entry = candidates[0]
if not self._apply_relink(entry, new_path, cache, case_sensitive):
still_remaining.append(new_path)
continue
by_stat_only[stat_key] = []
logger.info(
"[Sync] Automatically relinked renamed file (matched by size/date only)",
old_path=entry.path.as_posix(),
new_path=new_path.as_posix(),
)
relinked.append(entry)
self.unlinked_entries.remove(entry)
self.new_paths = still_remaining
self.relinked_entries = relinked
def remove_unlinked_entries(self) -> None:
"""Remove unlinked entries from the Library."""
# Path cache is updated in the library.
self.library.remove_entries([entry.id for entry in self.unlinked_entries])
self.unlinked_entries = []
+15
View File
@@ -0,0 +1,15 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: MIT
import platform
def is_fs_case_sensitive() -> bool:
"""Whether the filesystem is case sensitive.
NOTE: Not authoritative for OSes other than Windows.
"""
# TODO: Make this more robust instead of assuming Windows == NTFS/exFAT
# and other OS filesystems are automatically case sensitive.
return platform.system() != "Windows"
+16
View File
@@ -0,0 +1,16 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: MIT
import unicodedata
from pathlib import Path
def norm_path(path: Path | str, case_sensitive: bool) -> Path:
"""Return `path` normalized to Unicode Normalization Form D (NFD)."""
if isinstance(path, str):
path = Path(path)
normalized = unicodedata.normalize("NFD", path.as_posix())
if not case_sensitive:
normalized = normalized.casefold()
return Path(normalized)
+17 -5
View File
@@ -1,16 +1,28 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: MIT
import os
import platform
from pathlib import Path
def get_date_modified(path: Path) -> float:
return path.stat().st_mtime
def _resolve(path_or_stat: Path | os.stat_result) -> os.stat_result:
if isinstance(path_or_stat, os.stat_result):
return path_or_stat
return path_or_stat.stat()
def get_date_created(path: Path) -> float:
def get_date_modified(path_or_stat: Path | os.stat_result) -> float:
return _resolve(path_or_stat).st_mtime
def get_date_created(path_or_stat: Path | os.stat_result) -> float:
stat = _resolve(path_or_stat)
if platform.system() in {"Windows", "Darwin"}:
return path.stat().st_birthtime
return stat.st_birthtime
else:
return path.stat().st_ctime
return stat.st_ctime
def get_file_size(path_or_stat: Path | os.stat_result) -> int:
return _resolve(path_or_stat).st_size
+245
View File
@@ -0,0 +1,245 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: GPL-3.0-only
import time
from collections.abc import Callable
from typing import Literal, override
from PySide6.QtCore import (
QEasingCurve,
QPropertyAnimation,
QRectF,
Qt,
QTimer,
QVariantAnimation,
Signal,
)
from PySide6.QtGui import QColor, QPainter, QPainterPath, QPaintEvent
from PySide6.QtWidgets import QVBoxLayout, QWidget
from tagstudio.qt.views.banner_view import BannerView
from tagstudio.qt.views.styles.stylesheets import (
BANNER_CORNER_RADIUS,
banner_notice_bg_color,
banner_notice_style,
banner_progress_bg_color,
banner_progress_chunk_color,
banner_progress_style,
)
BannerMode = Literal["progress", "notice", "fleeting_notice"]
class _BannerBackground(QWidget):
"""The banner's background widget. Used for custom animations, like fading the color."""
def __init__(self, parent: QWidget | None = None) -> None:
super().__init__(parent)
self._bg_color = QColor(Qt.GlobalColor.transparent)
@property
def bg_color(self) -> QColor:
return self._bg_color
def set_bg_color(self, color: QColor) -> None:
self._bg_color = color
self.update()
@override
def paintEvent(self, event: QPaintEvent) -> None:
del event
painter = QPainter(self)
painter.setRenderHint(QPainter.RenderHint.Antialiasing)
path = QPainterPath()
path.addRoundedRect(QRectF(self.rect()), BANNER_CORNER_RADIUS, BANNER_CORNER_RADIUS)
painter.fillPath(path, self._bg_color)
painter.end()
class Banner(QWidget):
"""A notification banner with an optional progress bar, action button, and close button."""
CONTENT_HEIGHT = 36
GAP = 6
HEIGHT = CONTENT_HEIGHT + GAP
ANIMATION_MS = 250
COLOR_ANIMATION_MS = 250
MIN_VISIBLE_MS = 3000
STARTUP_EXTRA_HOLD_MS = 500 # Starting up may eat into time shown, so add extra time.
notice_action_clicked = Signal()
cancel_requested = Signal()
dismissed = Signal()
def __init__(self, parent: QWidget | None = None):
super().__init__(parent)
self.setMinimumHeight(0)
self.setMaximumHeight(0)
outer_layout = QVBoxLayout(self)
outer_layout.setContentsMargins(0, 0, 0, self.GAP)
outer_layout.setSpacing(0)
self._background = _BannerBackground(self)
self._background.setObjectName("banner")
self.view = BannerView()
self._background.setLayout(self.view)
outer_layout.addWidget(self._background)
self._mode: BannerMode = "progress"
self._progress_phase: object = None
self._notice_style = banner_notice_style()
self._progress_style = banner_progress_style()
self._notice_bg_color = banner_notice_bg_color()
self._progress_bg_color = banner_progress_bg_color()
self._background.setStyleSheet(self._notice_style)
self._background.set_bg_color(self._notice_bg_color)
self.view.progress_bar.set_corner_radius(BANNER_CORNER_RADIUS)
self.view.progress_bar.set_chunk_color(banner_progress_chunk_color())
self._card_color_anim = QVariantAnimation(self)
self._card_color_anim.setDuration(self.COLOR_ANIMATION_MS)
self._card_color_anim.setEasingCurve(QEasingCurve.Type.OutCubic)
self._card_color_anim.valueChanged.connect(self._background.set_bg_color)
self._connect_callbacks()
self._set_mode("notice")
self._height_anim = QPropertyAnimation(self, b"maximumHeight", self)
self._height_anim.setDuration(self.ANIMATION_MS)
self._height_anim.setEasingCurve(QEasingCurve.Type.OutCubic)
self._height_anim.valueChanged.connect(self.setMinimumHeight)
self._shown_at: float | None = None
self._extra_hold_ms = 0
self._hide_timer = QTimer(self)
self._hide_timer.setSingleShot(True)
self._hide_timer.timeout.connect(lambda: self._start_height_animation(0))
def request_extra_duration(self) -> None:
"""Add STARTUP_EXTRA_HOLD_MS to the next automatic hide's minimum-visible window."""
self._extra_hold_ms = self.STARTUP_EXTRA_HOLD_MS
def call_when_open(self, callback: Callable[[], None]) -> None:
"""Call `callback` if/when the banner is fully open."""
if self.maximumHeight() == self.HEIGHT and self._height_anim.state() != (
QPropertyAnimation.State.Running
):
callback()
return
def _on_finished() -> None:
self._height_anim.finished.disconnect(_on_finished)
callback()
self._height_anim.finished.connect(_on_finished)
def _connect_callbacks(self) -> None:
self.view.close_button.clicked.connect(self._on_dismiss)
self.view.action_button.clicked.connect(self._on_action_clicked)
def _on_action_clicked(self) -> None:
self.notice_action_clicked.emit()
def _start_height_animation(self, target_height: int) -> None:
if self.maximumHeight() == target_height and self._height_anim.state() != (
QPropertyAnimation.State.Running
):
return
self._height_anim.stop()
self._height_anim.setStartValue(self.maximumHeight())
self._height_anim.setEndValue(target_height)
self._height_anim.start()
def _animate_to(self, target_height: int, force: bool = False) -> None:
self._hide_timer.stop()
if target_height > 0:
self._shown_at = time.monotonic()
self._start_height_animation(target_height)
return
extra_hold_ms = self._extra_hold_ms
self._extra_hold_ms = 0
if not force and self._shown_at is not None:
elapsed_ms = (time.monotonic() - self._shown_at) * 1000
remaining_ms = (self.MIN_VISIBLE_MS + extra_hold_ms) - elapsed_ms
if remaining_ms > 0:
self._hide_timer.start(int(remaining_ms))
return
self._shown_at = None
self._start_height_animation(0)
def _set_mode(self, mode: BannerMode):
if mode == self._mode:
return
self._mode = mode
self._progress_phase = None
self.view.label.reset_width()
self.view.close_button.setVisible(mode != "fleeting_notice")
self.view.action_button.setVisible(mode == "notice")
self.view.progress_bar.setVisible(mode == "progress")
# The progress bar state gets a darkened background, while notices get the accent color.
is_progress = mode == "progress"
self._background.setStyleSheet(self._progress_style if is_progress else self._notice_style)
target_color = self._progress_bg_color if is_progress else self._notice_bg_color
self._card_color_anim.stop()
self._card_color_anim.setStartValue(self._background.bg_color)
self._card_color_anim.setEndValue(target_color)
self._card_color_anim.start()
def _present(self, mode: BannerMode, button_text: str, message: str) -> None:
"""Applies the banner mode and any label + button text, then animates the banner open."""
self._set_mode(mode)
self.view.action_button.setText(button_text)
self.view.label.setText(message)
self._animate_to(self.HEIGHT)
def _on_dismiss(self):
if self._mode == "progress":
self.cancel_requested.emit()
# Explicit dismiss, apply immediately
self._animate_to(0, force=True)
self.dismissed.emit()
def show_notice(self, message: str, action_text: str) -> None:
"""Show a dismissible notice with an action button, until dismissed or replaced."""
self._present("notice", action_text, message)
def show_fleeting_notice(self, message: str) -> None:
"""Show a brief notice with no action button that dismisses itself automatically."""
self._set_mode("fleeting_notice")
self.view.label.setText(message)
self._animate_to(self.HEIGHT)
# Deferred by the existing MIN_VISIBLE_MS guard, same as an unforced hide_banner().
self._animate_to(0)
def show_progress(self, text: str, value: int = 0, maximum: int = 0, phase: str | None = None):
"""Show the progress banner.
Args:
text (str): The status text shown in the banner body.
value (int): The current progress value.
maximum (int): The maximum progress value. If 0, shown as indeterminate.
phase (str | None): An identifier for the current sub-phase of progress.
Helps inform widgets that need to update between phases, like the StableLabel.
"""
self._set_mode("progress")
if phase != self._progress_phase:
self._progress_phase = phase
self.view.label.reset_width()
self.view.label.setText(text)
self.view.progress_bar.set_range(0, maximum)
self.view.progress_bar.set_value(value)
self._animate_to(self.HEIGHT)
def hide_banner(self, force: bool = False):
"""Hide the banner (if shown).
Args:
force (bool): Bypass the minimum visible duration and hide immediately.
"""
self._animate_to(0, force=force)
+4 -3
View File
@@ -158,7 +158,8 @@ class Inspector(QWidget):
if stats.duration is not None:
self._current_stats.duration = stats.duration
self.layout().file_attrs.update_stats(filepath, self._current_stats)
entry = unwrap(self._lib.get_entry(self._selected[0]))
self.layout().file_attrs.update_stats(filepath, self._current_stats, entry)
def _set_selection_callback(self) -> None:
with catch_warnings(record=True):
@@ -257,8 +258,8 @@ class Inspector(QWidget):
if update_preview:
stats: FileAttributeData = self.layout().preview_thumb.display_file(filepath)
self._current_stats = stats
self.layout().file_attrs.update_stats(filepath, stats)
self.layout().file_attrs.update_date_label(filepath)
self.layout().file_attrs.update_stats(filepath, stats, entry)
self.layout().file_attrs.update_date_label(entry)
self.layout().containers.update_from_entry(entry_id)
self._set_selection_callback()
+13 -7
View File
@@ -40,6 +40,7 @@ from tagstudio.core.enums import ShowFilepathOption
from tagstudio.core.library.alchemy.enums import SortingModeEnum
from tagstudio.i18n.platform_strings import trash_term
from tagstudio.i18n.translations import Translations
from tagstudio.qt.controllers.banner import Banner
from tagstudio.qt.controllers.inspector import Inspector
from tagstudio.qt.helpers.mnemonics import assign_mnemonics
from tagstudio.qt.mixed.landing import LandingWidget
@@ -64,7 +65,7 @@ class MainMenuBar(QMenuBar):
save_library_backup_action: QAction
settings_action: QAction
open_on_start_action: QAction
refresh_dir_action: QAction
sync_library_action: QAction
close_library_action: QAction
edit_menu: QMenu
@@ -152,17 +153,17 @@ class MainMenuBar(QMenuBar):
self.file_menu.addSeparator()
# Refresh Directories
self.refresh_dir_action = QAction(Translations["menu.file.refresh_directories"], self)
self.refresh_dir_action.setShortcut(
# Sync Library
self.sync_library_action = QAction(Translations["menu.file.sync_library"], self)
self.sync_library_action.setShortcut(
QtCore.QKeyCombination(
QtCore.Qt.KeyboardModifier(QtCore.Qt.KeyboardModifier.ControlModifier),
QtCore.Qt.Key.Key_R,
)
)
self.refresh_dir_action.setStatusTip("Ctrl+R")
self.refresh_dir_action.setEnabled(False)
self.file_menu.addAction(self.refresh_dir_action)
self.sync_library_action.setStatusTip("Ctrl+R")
self.sync_library_action.setEnabled(False)
self.file_menu.addAction(self.sync_library_action)
self.file_menu.addSeparator()
@@ -485,6 +486,7 @@ class MainWindow(QMainWindow):
# initialized in setup_entry_list
self.entry_list_container: QWidget
self.entry_list_layout: QVBoxLayout
self.banner: Banner
self.entry_scroll_area: QScrollArea
self.thumb_grid: QWidget
self.thumb_layout: ThumbGridLayout
@@ -691,6 +693,9 @@ class MainWindow(QMainWindow):
self.thumb_grid.setLayout(self.thumb_layout)
self.entry_scroll_area.setWidget(self.thumb_grid)
self.banner = Banner()
self.entry_list_layout.addWidget(self.banner)
self.entry_list_layout.addWidget(self.entry_scroll_area)
self.landing_widget = LandingWidget(driver, self.devicePixelRatio())
@@ -698,6 +703,7 @@ class MainWindow(QMainWindow):
self.pagination = Pagination()
self.entry_list_layout.addWidget(self.pagination)
self.content_splitter.addWidget(self.entry_list_container)
def setup_preview_panel(self, driver: QtDriver):
@@ -4,7 +4,7 @@
from PySide6.QtCore import QObject, Signal
from tagstudio.core.library.alchemy.registries.unlinked_registry import UnlinkedRegistry
from tagstudio.core.library.sync import LibrarySyncEngine
from tagstudio.i18n.translations import Translations
from tagstudio.qt.controllers.progress_bar import ProgressWidget
@@ -12,7 +12,7 @@ from tagstudio.qt.controllers.progress_bar import ProgressWidget
class RelinkUnlinkedEntriesProgress(QObject):
done = Signal()
def __init__(self, tracker: UnlinkedRegistry):
def __init__(self, tracker: LibrarySyncEngine):
super().__init__()
self.tracker = tracker
@@ -22,7 +22,7 @@ class RelinkUnlinkedEntriesProgress(QObject):
"entries.unlinked.relink.attempting",
index=x,
unlinked_count=self.tracker.unlinked_entries_count,
fixed_count=self.tracker.files_fixed_count,
fixed_count=self.tracker.manual_relink_count,
)
pw = ProgressWidget(
@@ -32,4 +32,6 @@ class RelinkUnlinkedEntriesProgress(QObject):
maximum=self.tracker.unlinked_entries_count,
)
pw.setWindowTitle(Translations["entries.unlinked.relink.title"])
pw.from_iterable_function(self.tracker.fix_unlinked_entries, displayed_text, self.done.emit)
pw.from_iterable_function(
self.tracker.relink_unlinked_entries, displayed_text, self.done.emit
)
@@ -0,0 +1,236 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: GPL-3.0-only
import time
from typing import override
from PySide6.QtCore import QEasingCurve, QRectF, Qt, QTimer, QVariantAnimation
from PySide6.QtGui import (
QColor,
QHideEvent,
QLinearGradient,
QPainter,
QPainterPath,
QPaintEvent,
QResizeEvent,
QShowEvent,
)
from PySide6.QtWidgets import QWidget
class RoundedProgressBar(QWidget):
"""A custom stylized progress bar that supports smooth animations and rounded corners."""
MARQUEE_FRACTION = 0.5
MARQUEE_INTERVAL_MS = 8
MARQUEE_CYCLE_MS = 2000
VALUE_ANIMATION_MS = 150
FADE_MS = 2000
def __init__(self, parent: QWidget | None = None) -> None:
super().__init__(parent)
self._minimum = 0
self._maximum = 0
self._value = 0
self._displayed_value = 0.0
self._corner_radius = 0.0
self._chunk_color = QColor(Qt.GlobalColor.transparent)
self._marquee_start: float | None = None
self._cached_path: QPainterPath | None = None
self._opacity = 1.0
self._pending_range_change = False
self._marquee_timer = QTimer(self)
self._marquee_timer.setTimerType(Qt.TimerType.PreciseTimer)
self._marquee_timer.setInterval(self.MARQUEE_INTERVAL_MS)
self._marquee_timer.timeout.connect(self.update)
self._value_anim = QVariantAnimation(self)
self._value_anim.setDuration(self.VALUE_ANIMATION_MS)
self._value_anim.setEasingCurve(QEasingCurve.Type.OutCubic)
self._value_anim.valueChanged.connect(self._on_value_anim_changed)
self._value_anim.finished.connect(self._on_value_anim_finished)
self._fade_anim = QVariantAnimation(self)
self._fade_anim.setDuration(self.FADE_MS)
self._fade_anim.setEasingCurve(QEasingCurve.Type.OutCubic)
self._fade_anim.valueChanged.connect(self._on_fade_anim_changed)
def _on_value_anim_changed(self, value: float) -> None:
self._displayed_value = value
self.update()
def _on_value_anim_finished(self) -> None:
"""Start the fade-out once the progress bar has reached its maximum."""
if not self._is_indeterminate() and self._value >= self._maximum:
self._fade_anim.stop()
self._fade_anim.setStartValue(self._opacity)
self._fade_anim.setEndValue(0.0)
self._fade_anim.start()
def _on_fade_anim_changed(self, value: float) -> None:
"""Update the fill opacity for the fade animation."""
self._opacity = value
self.update()
def _reset_fade(self) -> None:
"""Stop any fading and reset opacity back to full."""
self._fade_anim.stop()
if self._opacity != 1.0:
self._opacity = 1.0
self.update()
def set_range(self, minimum: int, maximum: int) -> None:
"""Set the value range, switching to indeterminate mode if maximum <= minimum."""
if (minimum, maximum) == (self._minimum, self._maximum):
return
self._pending_range_change = True
self._minimum = minimum
self._maximum = maximum
if self._is_indeterminate():
self._reset_fade()
self._sync_marquee_timer()
self.update()
def set_value(self, value: int) -> None:
"""Animate the fill towards `value`."""
self._value = value
if self._is_indeterminate():
self._pending_range_change = False
return
if self._pending_range_change:
self._pending_range_change = False
if value >= self._maximum:
self._value_anim.stop()
self._displayed_value = float(self._maximum)
self.update()
self._on_value_anim_finished()
return
self._displayed_value = float(self._minimum)
self._reset_fade()
elif value < self._maximum:
self._reset_fade()
self._value_anim.stop()
self._value_anim.setStartValue(self._displayed_value)
self._value_anim.setEndValue(float(value))
self._value_anim.start()
def set_corner_radius(self, radius: float) -> None:
"""Set the bottom corner radius and invalidate the cached QPainterPath."""
self._corner_radius = radius
self._cached_path = None
self.update()
def set_chunk_color(self, color: QColor) -> None:
"""Set the fill color."""
self._chunk_color = color
self.update()
def _is_indeterminate(self) -> bool:
"""Whether the bar is in indeterminate (marquee) mode."""
return self._maximum <= self._minimum
def _sync_marquee_timer(self) -> None:
"""Start or stop the marquee timer to match the current mode/visibility."""
if self._is_indeterminate() and self.isVisible():
if self._marquee_start is None:
self._marquee_start = time.monotonic()
if not self._marquee_timer.isActive():
self._marquee_timer.start()
else:
self._marquee_timer.stop()
self._marquee_start = None
def _marquee_fraction(self) -> float:
"""Return the marquee's current position as a fraction of its cycle."""
if self._marquee_start is None:
return 0.0
elapsed_ms = (time.monotonic() - self._marquee_start) * 1000
return (elapsed_ms % self.MARQUEE_CYCLE_MS) / self.MARQUEE_CYCLE_MS
@override
def showEvent(self, event: QShowEvent) -> None:
"""Resume the marquee timer when the bar becomes visible."""
super().showEvent(event)
self._sync_marquee_timer()
@override
def hideEvent(self, event: QHideEvent) -> None:
"""Stop all animations and reset state when the bar is hidden."""
super().hideEvent(event)
self._marquee_timer.stop()
self._marquee_start = None
self._value_anim.stop()
self._reset_fade()
self._pending_range_change = False
@override
def resizeEvent(self, event: QResizeEvent) -> None:
super().resizeEvent(event)
self._cached_path = None # Invalidate the cached QPainterPath on resize
def _bottom_rounded_path(self, rect: QRectF) -> QPainterPath:
"""Return the bottom-rounded clip path for `rect`."""
if self._cached_path is not None:
return self._cached_path
# TODO: Currently these values are hardcoded for use with the Banner widget, but this
# could be made customizable to specify the exact rounding configuration.
# If you're reading this and want to make use of this progress bar, there you go.
radius = max(0.0, min(self._corner_radius, rect.width() / 2))
diam = radius * 2
path = QPainterPath()
path.moveTo(rect.left(), rect.top())
path.lineTo(rect.right(), rect.top())
path.lineTo(rect.right(), rect.bottom() - radius)
path.arcTo(QRectF(rect.right() - diam, rect.bottom() - diam, diam, diam), 0, -90)
path.lineTo(rect.left() + radius, rect.bottom())
path.arcTo(QRectF(rect.left(), rect.bottom() - diam, diam, diam), -90, -90)
path.closeSubpath()
self._cached_path = path
return path
def _marquee_gradient(self, full_chunk_rect: QRectF) -> QLinearGradient:
"""Gradient for the indeterminate marquee mode (Transparent -> Color -> Transparent)."""
gradient = QLinearGradient(full_chunk_rect.left(), 0, full_chunk_rect.right(), 0)
transparent = QColor(self._chunk_color)
transparent.setAlpha(0)
gradient.setColorAt(0.0, transparent)
gradient.setColorAt(0.5, self._chunk_color)
gradient.setColorAt(1.0, transparent)
return gradient
@override
def paintEvent(self, event: QPaintEvent) -> None:
del event
# Paint the marquee or normal chunk fill, clipped to the rounded bottom corners.
rect = QRectF(self.rect())
if self._is_indeterminate():
chunk_width = rect.width() * self.MARQUEE_FRACTION
travel = rect.width() * (1 + self.MARQUEE_FRACTION)
chunk_left = (self._marquee_fraction() * travel) - chunk_width
full_chunk_rect = QRectF(chunk_left, 0, chunk_width, rect.height())
chunk_rect = full_chunk_rect.intersected(rect)
if chunk_rect.isEmpty():
return
fill = self._marquee_gradient(full_chunk_rect)
else:
fraction = (self._displayed_value - self._minimum) / (self._maximum - self._minimum)
fraction = max(0.0, min(1.0, fraction))
chunk_rect = QRectF(0, 0, rect.width() * fraction, rect.height()).intersected(rect)
if chunk_rect.isEmpty():
return
fill = self._chunk_color
painter = QPainter(self)
painter.setRenderHint(QPainter.RenderHint.Antialiasing)
painter.setClipPath(self._bottom_rounded_path(rect))
painter.setOpacity(self._opacity)
painter.fillRect(chunk_rect, fill)
painter.end()
@@ -0,0 +1,38 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: GPL-3.0-only
from typing import override
from PySide6.QtCore import QSize, Qt
from PySide6.QtWidgets import QLabel, QWidget
class StableLabel(QLabel):
"""A QLabel that resists "jiggling" from rapidly changing text.
Holds its `sizeHint()` width at the widest shown since the last reset_width() call,
and is always kept left aligned of that, so a centering layout's box stops
growing/shrinking on every update (aka the "jiggle" effect).
"""
def __init__(self, text: str = "", parent: QWidget | None = None) -> None:
super().__init__(text, parent)
self.setAlignment(Qt.AlignmentFlag.AlignLeft | Qt.AlignmentFlag.AlignVCenter)
self._min_width = 0
def reset_width(self) -> None:
"""Let the tracked width shrink again, for new unrelated text."""
self._min_width = 0
self.updateGeometry()
@override
def setText(self, text: str) -> None:
super().setText(text)
self._min_width = max(self._min_width, super().sizeHint().width())
self.updateGeometry()
@override
def sizeHint(self) -> QSize:
hint = super().sizeHint()
return QSize(max(hint.width(), self._min_width), hint.height())
+1 -1
View File
@@ -178,7 +178,7 @@ class DropImportModal(QWidget):
pw.from_iterable_function(
self.copy_files,
displayed_text,
self.driver.add_new_files_callback,
self.driver.sync_library_callback,
self.deleteLater,
)
+33 -40
View File
@@ -3,7 +3,6 @@
import os
import platform
import typing
from dataclasses import dataclass
from datetime import datetime as dt
@@ -17,6 +16,7 @@ from PySide6.QtWidgets import QLabel, QVBoxLayout, QWidget
from tagstudio.core.enums import ShowFilepathOption
from tagstudio.core.library.alchemy.library import Library
from tagstudio.core.library.alchemy.models import Entry
from tagstudio.core.library.ignore import Ignore
from tagstudio.core.media_types import MediaTypes
from tagstudio.core.query_lang.file_groups import SEARCH
@@ -92,40 +92,35 @@ class FileAttributes(QWidget):
self.library = library
self.driver = driver
def update_date_label(self, filepath: Path | None = None) -> None:
def _format_date_or_na(self, timestamp: float | None) -> str:
if timestamp is None:
return "<i>N/A</i>"
return self.driver.settings.format_datetime(dt.fromtimestamp(timestamp))
def update_date_label(self, entry: Entry | None = None) -> None:
"""Update the "Date Created" and "Date Modified" file property labels."""
if filepath and filepath.is_file():
created: dt
if platform.system() == "Windows" or platform.system() == "Darwin":
# NOTE: Accessing stat().st_birthtime causes linter checks to fail on some systems.
created = dt.fromtimestamp(filepath.stat().st_birthtime) # type: ignore[attr-defined, unused-ignore]
else:
created = dt.fromtimestamp(filepath.stat().st_ctime)
modified: dt = dt.fromtimestamp(filepath.stat().st_mtime)
self.date_created_label.setText(
f"<b>{Translations['file.date_created']}:</b>"
+ f" {self.driver.settings.format_datetime(created)}"
)
self.date_modified_label.setText(
f"<b>{Translations['file.date_modified']}:</b> "
f"{self.driver.settings.format_datetime(modified)}"
)
self.date_created_label.setHidden(False)
self.date_modified_label.setHidden(False)
elif filepath:
self.date_created_label.setText(
f"<b>{Translations['file.date_created']}:</b> <i>N/A</i>"
)
self.date_modified_label.setText(
f"<b>{Translations['file.date_modified']}:</b> <i>N/A</i>"
)
self.date_created_label.setHidden(False)
self.date_modified_label.setHidden(False)
else:
if entry is None:
self.date_created_label.setHidden(True)
self.date_modified_label.setHidden(True)
return
def update_stats(self, filepath: Path | None = None, stats: FileAttributeData | None = None):
created_text = self._format_date_or_na(entry.date_created)
modified_text = self._format_date_or_na(entry.date_modified)
self.date_created_label.setText(
f"<b>{Translations['file.date_created']}:</b> {created_text}"
)
self.date_modified_label.setText(
f"<b>{Translations['file.date_modified']}:</b> {modified_text}"
)
self.date_created_label.setHidden(False)
self.date_modified_label.setHidden(False)
def update_stats(
self,
filepath: Path | None = None,
stats: FileAttributeData | None = None,
entry: Entry | None = None,
):
"""Render the panel widgets with the newest data from the Library."""
if not stats:
stats = FileAttributeData()
@@ -170,18 +165,15 @@ class FileAttributes(QWidget):
# Initialize the possible stat variables
stats_label_text = ""
ext_display: str = ""
file_size: str = ""
file_size: str = format_size(entry.file_size) if entry and entry.file_size else ""
font_family: str = ""
# Attempt to populate the stat variables
ext_display = ext.upper()[1:] or filepath.stem.upper()
if filepath and filepath.is_file():
if filepath and filepath.is_file() and MediaTypes.contains("font", ext, SEARCH):
try:
file_size = format_size(filepath.stat().st_size)
if MediaTypes.contains("font", ext, SEARCH):
font = ImageFont.truetype(filepath)
font_family = f"{font.getname()[0]} ({font.getname()[1]}) "
font = ImageFont.truetype(filepath)
font_family = f"{font.getname()[0]} ({font.getname()[1]}) "
except (FileNotFoundError, OSError) as e:
logger.error(
"[FileAttributes] Could not process file stats", filepath=filepath, error=e
@@ -206,14 +198,15 @@ class FileAttributes(QWidget):
f" • <span style='color:{orange}'>"
f"{Translations['preview.ignored'].upper()}</span>"
)
if file_size:
stats_label_text += f"{file_size}"
if not filepath.exists():
stats_label_text = (
f"{stats_label_text}"
f" • <span style='color:{red}'>"
f"{Translations['preview.unlinked'].upper()}</span>"
)
if file_size:
stats_label_text += f"{file_size}"
elif file_size:
stats_label_text += file_size
+33 -24
View File
@@ -4,12 +4,13 @@
from typing import TYPE_CHECKING, override
import structlog
from PySide6 import QtCore, QtGui
from PySide6.QtCore import Qt
from PySide6.QtWidgets import QHBoxLayout, QLabel, QPushButton, QVBoxLayout, QWidget
from tagstudio.core.library.alchemy.library import Library
from tagstudio.core.library.alchemy.registries.unlinked_registry import UnlinkedRegistry
from tagstudio.core.utils.types import unwrap
from tagstudio.i18n.translations import Translations
from tagstudio.qt.controllers.merge_dupe_entries_progress import MergeDuplicateEntriesProgress
from tagstudio.qt.controllers.progress_bar import ProgressWidget
@@ -17,19 +18,19 @@ from tagstudio.qt.controllers.relink_entries_progress import RelinkUnlinkedEntri
from tagstudio.qt.mixed.remove_unlinked_modal import RemoveUnlinkedEntriesModal
from tagstudio.qt.views.styles.stylesheets import header
# Only import for type checking/autocompletion, will not be imported at runtime.
if TYPE_CHECKING:
from tagstudio.qt.qt_driver import QtDriver
logger = structlog.get_logger(__name__)
# TODO: Split to use MVC guidelines.
# TODO: Split to use MVC guidelines, or completely redo.
class FixUnlinkedEntriesModal(QWidget):
def __init__(self, library: Library, driver: QtDriver):
super().__init__()
self.lib = library
self.driver = driver
self.tracker = UnlinkedRegistry(lib=self.lib)
self.sync_engine = driver.sync_engine
self.unlinked_count = -1
self.dupe_count = -1
@@ -56,14 +57,13 @@ class FixUnlinkedEntriesModal(QWidget):
self.refresh_unlinked_button.clicked.connect(self.refresh_unlinked)
self.merge_class = MergeDuplicateEntriesProgress(self.lib, self.driver)
self.relink_class = RelinkUnlinkedEntriesProgress(self.tracker)
self.relink_class = RelinkUnlinkedEntriesProgress(self.sync_engine)
self.search_button = QPushButton(Translations["entries.unlinked.search_and_relink"])
self.relink_class.done.connect(
# refresh the grid
lambda: (
self.driver.update_browsing_state(),
self.refresh_unlinked(),
self._sync_ui_from_tracker(),
)
)
self.search_button.clicked.connect(self.relink_class.repair_entries)
@@ -72,16 +72,16 @@ class FixUnlinkedEntriesModal(QWidget):
self.manual_button.setHidden(True)
self.remove_button = QPushButton(Translations["entries.unlinked.remove_alt"])
self.remove_modal = RemoveUnlinkedEntriesModal(self.driver, self.tracker)
self.remove_modal = RemoveUnlinkedEntriesModal(self.driver, self.sync_engine)
self.remove_modal.done.connect(
lambda: (
self.set_unlinked_count(),
# refresh the grid
self.driver.update_browsing_state(),
self.refresh_unlinked(),
self._sync_ui_from_tracker(),
)
)
self.remove_button.clicked.connect(self.remove_modal.show)
self.remove_button.clicked.connect(
lambda: (self.remove_modal.refresh_list(), self.remove_modal.show())
)
self.button_container = QWidget()
self.button_layout = QHBoxLayout(self.button_container)
@@ -106,6 +106,11 @@ class FixUnlinkedEntriesModal(QWidget):
self.update_unlinked_count()
def refresh_unlinked(self):
if self.driver.file_scan_lock:
logger.info("[FixUnlinkedEntries] Sync already in progress, ignoring refresh request")
return
self.driver.file_scan_lock = True
pw = ProgressWidget(
cancel_button_text=None,
minimum=0,
@@ -114,37 +119,41 @@ class FixUnlinkedEntriesModal(QWidget):
pw.setWindowTitle(Translations["library.scan_library.title"])
pw.update_label(Translations["entries.unlinked.scanning"])
def update_driver_widgets():
def finish():
self.driver.file_scan_lock = False
if (
hasattr(self.driver, "library_info_window")
and self.driver.library_info_window.isVisible()
):
self.driver.library_info_window.update_cleanup()
# Uses the Library's shared path cache
pw.from_iterable_function(
self.tracker.refresh_unlinked_files,
lambda: self.sync_engine.sync_dir(unwrap(self.lib.library_dir)),
None,
self.set_unlinked_count,
finish,
self.update_unlinked_count,
self.remove_modal.refresh_list,
update_driver_widgets,
)
def _sync_ui_from_tracker(self) -> None:
"""Refresh the UI from the tracker's current state, without rescanning the library."""
self.set_unlinked_count()
self.update_unlinked_count()
self.remove_modal.refresh_list()
def set_unlinked_count(self):
"""Sets the unlinked_entries_count in the Library to the tracker's value."""
self.lib.unlinked_entries_count = self.tracker.unlinked_entries_count
self.lib.unlinked_entries_count = self.sync_engine.unlinked_entries_count
def update_unlinked_count(self):
"""Updates the UI to reflect the Library's current unlinked_entries_count."""
# Indicates that the library is new compared to the last update.
# NOTE: Make sure set_unlinked_count() is called before this!
if self.tracker.unlinked_entries_count > 0 and self.lib.unlinked_entries_count < 0:
self.tracker.reset()
count: int = self.lib.unlinked_entries_count
syncing = self.driver.file_scan_lock # Disabled while a sync is running
self.search_button.setDisabled(count < 1)
self.remove_button.setDisabled(count < 1)
self.search_button.setDisabled(count < 1 or syncing)
self.remove_button.setDisabled(count < 1 or syncing)
count_text: str = Translations.format(
"entries.unlinked.unlinked_count", count=count if count >= 0 else ""
@@ -9,7 +9,7 @@ from PySide6.QtCore import Qt, QThreadPool, Signal
from PySide6.QtGui import QStandardItem, QStandardItemModel
from PySide6.QtWidgets import QHBoxLayout, QLabel, QListView, QPushButton, QVBoxLayout, QWidget
from tagstudio.core.library.alchemy.registries.unlinked_registry import UnlinkedRegistry
from tagstudio.core.library.sync import LibrarySyncEngine
from tagstudio.i18n.translations import Translations
from tagstudio.qt.controllers.progress_bar import ProgressWidget
from tagstudio.qt.utils.custom_runnable import CustomRunnable
@@ -18,11 +18,11 @@ if TYPE_CHECKING:
from tagstudio.qt.qt_driver import QtDriver
# TODO: Split to use MVC guidelines.
# TODO: Split to use MVC guidelines or completely redo.
class RemoveUnlinkedEntriesModal(QWidget):
done = Signal()
def __init__(self, driver: QtDriver, tracker: UnlinkedRegistry):
def __init__(self, driver: QtDriver, tracker: LibrarySyncEngine):
super().__init__()
self.driver = driver
self.tracker = tracker
+300 -89
View File
@@ -17,10 +17,11 @@ import sys
import time
from argparse import Namespace
from collections import OrderedDict
from collections.abc import Callable, Iterator
from functools import partial
from pathlib import Path
from queue import Queue
from typing import TypeVar
from typing import Literal, TypeVar
from warnings import catch_warnings
import structlog
@@ -47,7 +48,7 @@ from tagstudio.core.library.alchemy.enums import BrowsingState, SortingModeEnum
from tagstudio.core.library.alchemy.library import Library, LibraryStatus
from tagstudio.core.library.alchemy.models import Entry
from tagstudio.core.library.ignore import Ignore
from tagstudio.core.library.refresh import RefreshTracker
from tagstudio.core.library.sync import LibrarySyncEngine
from tagstudio.core.media_types import MediaTypes
from tagstudio.core.query_lang.file_groups import SEARCH
from tagstudio.core.query_lang.util import ParsingError
@@ -67,7 +68,6 @@ from tagstudio.qt.controllers.ignore_modal import IgnoreModal
from tagstudio.qt.controllers.library_info_window import LibraryInfoWindow
from tagstudio.qt.controllers.main_window import MainWindow
from tagstudio.qt.controllers.modal import Modal
from tagstudio.qt.controllers.progress_bar import ProgressWidget
from tagstudio.qt.controllers.splash import SplashScreen
from tagstudio.qt.controllers.tag_search_panel import TagSearchPanel
from tagstudio.qt.controllers.update_available_message_box import UpdateAvailableMessageBox
@@ -104,6 +104,9 @@ else:
from signal import SIGINT, SIGQUIT, SIGTERM, signal # pyright: ignore
logger = structlog.get_logger(__name__)
T = TypeVar("T")
# Used to track the context state of the banner widget.
_BannerContext = Literal["new_files", "unlinked", "relinked", "sync_disabled", "sync_finished"]
def clamp(value, lower_bound, upper_bound):
@@ -128,9 +131,6 @@ class Consumer(QThread):
pass
T = TypeVar("T")
# Ex. User visits | A ->[B] |
# | A B ->[C]|
# | A [B]<- C |
@@ -192,13 +192,17 @@ class QtDriver(DriverMixin, QObject):
def __init__(self, args: Namespace):
super().__init__()
# prevent recursive badges update when multiple items selected
self.badge_update_lock = False
self.lib = Library()
self.sync_engine = LibrarySyncEngine(self.lib)
self.rm: ResourceManager = ResourceManager()
self.args = args
self.frame_content: list[int] = [] # List of Entry IDs for the current query
self.badge_update_lock = False
self.file_scan_lock: bool = False # Prevent multiple file scanning operations at once
self._selected: OrderedDict[int, None] = OrderedDict()
self._sync_session_id: int = 0 # Prevent current sync from affecting subsequent libraries.
self._sync_disabled_notice_shown: bool = False
self._banner_context: _BannerContext | None = None
self.pages_count = 0
self.scrollbar_pos = 0
@@ -454,9 +458,9 @@ class QtDriver(DriverMixin, QObject):
set_open_last_loaded_on_startup
)
# Refresh Directories
self.main_window.menu_bar.refresh_dir_action.triggered.connect(
lambda: self.call_if_library_open(self.add_new_files_callback)
# Sync Library
self.main_window.menu_bar.sync_library_action.triggered.connect(
lambda: self.call_if_library_open(self.sync_library_callback)
)
# Close Library
@@ -552,13 +556,8 @@ class QtDriver(DriverMixin, QObject):
# region Tools Menu ===========================================================
def create_fix_unlinked_entries_modal():
if not hasattr(self, "unlinked_modal"):
self.unlinked_modal = FixUnlinkedEntriesModal(self.lib, self)
self.unlinked_modal.show()
self.main_window.menu_bar.fix_unlinked_entries_action.triggered.connect(
create_fix_unlinked_entries_modal
self.open_fix_unlinked_entries_modal
)
def create_ignored_entries_modal():
@@ -577,7 +576,7 @@ class QtDriver(DriverMixin, QObject):
self.main_window.menu_bar.fix_dupe_files_action.triggered.connect(create_dupe_files_modal)
# TODO: Move this to a settings screen.
# TODO: Make this accessible somewhere more sensible too, like "Library Information"
self.main_window.menu_bar.clear_thumb_cache_action.triggered.connect(
lambda: unwrap(self.cache_manager).clear_cache()
)
@@ -638,6 +637,7 @@ class QtDriver(DriverMixin, QObject):
self.init_library_window()
self.migration_modal: JsonMigrationModal | None = None
self.main_window.banner.request_extra_duration()
path_result = self.evaluate_path(str(self.args.open).lstrip().rstrip())
if path_result.success and path_result.library_path:
self.open_library(path_result.library_path)
@@ -678,6 +678,9 @@ class QtDriver(DriverMixin, QObject):
# adj_font_size = math.floor(12 * self.main_window.devicePixelRatio())
def _update_browsing_state():
# Clear any banner asking for a manual refresh of the view
if self._banner_context == "new_files":
self._clear_notice()
try:
self.update_browsing_state(
BrowsingState.from_search_query(self.main_window.search_field.text())
@@ -725,9 +728,11 @@ class QtDriver(DriverMixin, QObject):
self.main_window.back_button.clicked.connect(lambda: self.navigation_callback(-1))
self.main_window.forward_button.clicked.connect(lambda: self.navigation_callback(1))
# NOTE: Putting this early will result in a white non-responsive
# window until everything is loaded. Consider adding a splash screen
# or implementing some clever loading tricks.
# Banner
self.main_window.banner.notice_action_clicked.connect(self._on_notice_action_clicked)
self.main_window.banner.cancel_requested.connect(self._on_sync_cancel_requested)
# NOTE: Putting this too early will result in a non-responsive white window on start.
self.main_window.show()
self.main_window.activateWindow()
self.main_window.toggle_landing_page(enabled=True)
@@ -784,8 +789,14 @@ class QtDriver(DriverMixin, QObject):
if not self.lib.library_dir:
logger.info("No Library to Close")
return
logger.info("Closing Library...")
self.sync_engine.cancelled = True
self.file_scan_lock = False
self._new_sync_session() # Invalidate any sync still active for the old library
self._banner_context = None
self.main_window.banner.hide_banner(force=True)
self.main_window.status_bar.showMessage(Translations["status.library_closing"])
start_time = time.time()
@@ -800,6 +811,7 @@ class QtDriver(DriverMixin, QObject):
self.__reset_navigation()
self.lib.close()
self.sync_engine.reset()
self.cache_manager = None
self.thumb_job_queue.queue.clear()
@@ -827,7 +839,7 @@ class QtDriver(DriverMixin, QObject):
try:
self.main_window.menu_bar.save_library_backup_action.setEnabled(False)
self.main_window.menu_bar.close_library_action.setEnabled(False)
self.main_window.menu_bar.refresh_dir_action.setEnabled(False)
self.main_window.menu_bar.sync_library_action.setEnabled(False)
self.main_window.menu_bar.tag_manager_action.setEnabled(False)
self.main_window.menu_bar.color_manager_action.setEnabled(False)
self.main_window.menu_bar.field_template_manager_action.setEnabled(False)
@@ -1067,82 +1079,269 @@ class QtDriver(DriverMixin, QObject):
return msg.exec()
def add_new_files_callback(self):
"""Run when user initiates adding new files to the Library."""
tracker = RefreshTracker(self.lib)
def _run_sync_step(
self,
generator: Callable[[], Iterator[T]],
on_progress: Callable[[T], None],
on_done: Callable[[], None],
) -> None:
"""Run a generator function on a background thread with signals for progress and completion.
pw = ProgressWidget(
cancel_button_text=None,
minimum=0,
maximum=0,
Args:
generator (Callable[[], Iterator[T]]): Zero-argument callable returning the
generator to iterate.
on_progress (Callable[[T], None]): Called on the main thread with each yielded value.
on_done (Callable[[], None]): Called on the main thread once `generator` is finished.
"""
iterator = FunctionIterator(generator)
iterator.value.connect(on_progress)
runnable = CustomRunnable(iterator.run)
runnable.done.connect(on_done)
QThreadPool.globalInstance().start(runnable)
def sync_library_callback(self):
"""Run when syncing a Library is initiated."""
if self.file_scan_lock:
logger.info("[QtDriver] Sync already in progress, ignoring request")
return
self.file_scan_lock = True
session_id = self._new_sync_session()
# Disable the "Fix Unlinked Entries" modal's relink/remove actions during the sync
if hasattr(self, "unlinked_modal") and self.unlinked_modal.isVisible():
self.unlinked_modal.update_unlinked_count()
engine = self.sync_engine
library_dir = unwrap(self.lib.library_dir)
self.main_window.banner.show_progress(
Translations["library.sync.preparing"], phase="preparing"
)
pw.setWindowTitle(Translations["library.refresh.title"])
pw.update_label(Translations["library.refresh.scanning_preparing"])
pw.show()
iterator = FunctionIterator(lambda lib=self.lib.library_dir: tracker.refresh_dir(lib))
iterator.value.connect(
lambda x: (
pw.update_progress(x + 1),
pw.update_label(
Translations.format(
"library.refresh.scanning.plural"
if x + 1 != 1
else "library.refresh.scanning.singular",
searched_count=f"{x + 1:n}",
found_count=f"{tracker.files_count:n}",
)
def on_progress(progress: tuple[int, int]) -> None:
if engine.cancelled:
return
searched_count, found_count = progress
self.main_window.banner.show_progress(
Translations.format(
"library.sync.scanning",
searched_count=f"{searched_count + 1:n}",
found_count=f"{found_count:n}",
),
phase="scanning",
)
)
r = CustomRunnable(iterator.run)
r.done.connect(
lambda: (
pw.hide(),
pw.deleteLater(),
self.add_new_files_runnable(tracker),
)
)
QThreadPool.globalInstance().start(r)
def add_new_files_runnable(self, tracker: RefreshTracker):
def _start_scan() -> None:
self._run_sync_step(
lambda lib=library_dir: engine.sync_dir(lib),
on_progress,
lambda: self.save_new_entries_runnable(engine, session_id=session_id),
)
self.main_window.banner.call_when_open(_start_scan)
def _finish_sync(
self,
new_count: int = 0,
unlinked_count: int = 0,
relinked_count: int = 0,
session_id: int = 0,
):
"""Reset the banner once the sync is completed.
Args:
new_count (int): New files count.
unlinked_count (int): Unlinked entries count.
relinked_count (int): Automatically relinked files count.
session_id (int): The sync_session_id this sync started with.
"""
if self._is_sync_stale(session_id):
return
self.file_scan_lock = False
self.lib.unlinked_entries_count = unlinked_count
if hasattr(self, "unlinked_modal") and self.unlinked_modal.isVisible():
self.unlinked_modal.update_unlinked_count()
if self.sync_engine.cancelled:
return
# Show fleeting count of any new files added with button to refresh view
if new_count:
text = Translations.format(
"library.sync.new_files_banner.plural"
if new_count != 1
else "library.sync.new_files_banner.singular",
count=f"{new_count:n}",
)
text += self._count_suffix(relinked_count, "library.sync.relinked_suffix")
if relinked_count:
text += self._count_suffix(unlinked_count, "library.sync.remaining_unlinked_suffix")
self._show_notice("new_files", text, Translations["entries.generic.refresh_alt"])
# Show persistent count of any remaining unlinked files and button to manually review
elif unlinked_count:
text = Translations.format(
"library.sync.unlinked_banner.plural"
if unlinked_count != 1
else "library.sync.unlinked_banner.singular",
count=f"{unlinked_count:n}",
)
text += self._count_suffix(relinked_count, "library.sync.relinked_suffix")
self._show_notice("unlinked", text, Translations["entries.unlinked.review"])
# Show fleeting notice number of entries automatically relinked
elif relinked_count:
text = Translations.format(
"library.sync.relinked_banner.plural"
if relinked_count != 1
else "library.sync.relinked_banner.singular",
count=f"{relinked_count:n}",
)
text += self._count_suffix(unlinked_count, "library.sync.remaining_unlinked_suffix")
self._show_notice("relinked", text, Translations["entries.generic.refresh_alt"])
# Show a fleeting "Library Synced" message
else:
self._show_notice("sync_finished", Translations["library.sync.complete"])
def _count_suffix(self, count: int, key: str) -> str:
"""Build a count suffix suffix, or "" if count is 0."""
if not count:
return ""
return " " + Translations.format(key, count=f"{count:n}")
def _new_sync_session(self) -> int:
"""Increment the sync session, invalidating any active sync's callbacks."""
self._sync_session_id += 1
return self._sync_session_id
def _is_sync_stale(self, session_id: int) -> bool:
"""Whether `session_id` belongs to an older sync session and should be invalidated."""
return session_id != self._sync_session_id
def _show_notice(
self, context: _BannerContext, message: str, button_text: str | None = None
) -> None:
"""Show a "notice" banner and keep track of its context type.
Args:
context (_BannerContext): The subtype of banner notice.
Used to keep track of the context state currently used for the banner.
This could be for a startup message, sync progress, an entry relink prompt, etc.
message (str): The notice message text.
button_text (str): The action button text.
"""
self._banner_context = context
if button_text is not None:
self.main_window.banner.show_notice(message, button_text)
else:
self.main_window.banner.show_fleeting_notice(message)
def _clear_notice(self, force: bool = False) -> None:
self._banner_context = None
self.main_window.banner.hide_banner(force=force)
def _on_notice_action_clicked(self) -> None:
if self._banner_context == "unlinked":
self._on_unlinked_banner_review()
elif self._banner_context == "sync_disabled":
self._on_sync_disabled_open_settings()
else: # "new_files" or "relinked"
self._on_new_files_banner_refresh()
def _on_new_files_banner_refresh(self):
self.update_browsing_state()
# If there are still unlinked entries after the automatic relinking step, show a notice.
if self.lib.unlinked_entries_count > 0:
count = self.lib.unlinked_entries_count
text = Translations.format(
"library.sync.unlinked_banner.plural"
if count != 1
else "library.sync.unlinked_banner.singular",
count=f"{count:n}",
)
self._show_notice("unlinked", text, Translations["entries.unlinked.review"])
else:
self._clear_notice(force=True)
def _on_unlinked_banner_review(self):
self._clear_notice(force=True)
self.open_fix_unlinked_entries_modal()
def _on_sync_disabled_open_settings(self):
self._clear_notice(force=True)
self.open_settings_modal()
def _on_sync_cancel_requested(self):
"""Stop the in-progress sync at its next opportunity."""
self.sync_engine.cancelled = True
logger.info("[QtDriver] Sync cancelled")
def open_fix_unlinked_entries_modal(self):
if not hasattr(self, "unlinked_modal"):
self.unlinked_modal = FixUnlinkedEntriesModal(self.lib, self)
self.unlinked_modal.show()
def sync_entry_stats_runnable(
self,
engine: LibrarySyncEngine,
new_count: int = 0,
unlinked_count: int = 0,
relinked_count: int = 0,
session_id: int = 0,
):
"""Refresh cached stat() data for files already known to the library.
Threaded method.
"""
if self._is_sync_stale(session_id):
return
restat_count = engine.restat_count
def on_progress(idx: int) -> None:
if engine.cancelled:
return
self.main_window.banner.show_progress(
Translations.format(
"library.sync.updating.label", idx=f"{idx:n}", total=f"{restat_count:n}"
),
idx,
restat_count,
phase="updating",
)
on_progress(0)
self._run_sync_step(
engine.sync_entry_stats,
on_progress,
lambda: self._finish_sync(new_count, unlinked_count, relinked_count, session_id),
)
def save_new_entries_runnable(self, engine: LibrarySyncEngine, session_id: int = 0):
"""Adds any known new files to the library and run default macros on them.
Threaded method.
"""
files_count = tracker.files_count
if self._is_sync_stale(session_id):
return
new_count = engine.new_file_count
unlinked_count = engine.unlinked_entries_count
relinked_count = engine.relinked_entries_count
iterator = FunctionIterator(tracker.save_new_files)
pw = ProgressWidget(
cancel_button_text=None,
minimum=0,
maximum=0,
)
pw.setWindowTitle(Translations["entries.running.dialog.title"])
pw.update_label(
Translations.format("entries.running.dialog.new_entries", total=f"{files_count:n}")
)
pw.show()
def on_progress(idx: int) -> None:
if engine.cancelled:
return
self.main_window.banner.show_progress(
Translations.format("entries.running.dialog.new_entries", total=f"{new_count:n}"),
idx,
new_count,
phase="new_entries",
)
iterator.value.connect(
lambda _count: (
pw.update_label(
Translations.format(
"entries.running.dialog.new_entries", total=f"{files_count:n}"
)
),
)
on_progress(0)
self._run_sync_step(
engine.save_new_entries,
on_progress,
lambda: self.sync_entry_stats_runnable(
engine, new_count, unlinked_count, relinked_count, session_id
),
)
r = CustomRunnable(iterator.run)
r.done.connect(
lambda: (
pw.hide(),
pw.deleteLater(),
# refresh the library only when new items are added
files_count and self.update_browsing_state(),
)
)
QThreadPool.globalInstance().start(r)
def new_file_macros_runnable(self, new_ids):
"""Threaded method that runs macros on a set of Entry IDs."""
@@ -1640,7 +1839,7 @@ class QtDriver(DriverMixin, QObject):
f"[Config] Thumbnail Cache Size: {format_size(cache_size)}",
)
# Migration is required
# JSON Migration is required
if open_status.json_migration_req:
self.migration_modal = JsonMigrationModal(path)
self.migration_modal.migration_finished.connect(
@@ -1666,7 +1865,19 @@ class QtDriver(DriverMixin, QObject):
self.__reset_navigation()
if self.settings.scan_files_on_open:
self.add_new_files_callback()
self.sync_library_callback()
elif not self._sync_disabled_notice_shown:
self._sync_disabled_notice_shown = True
# Show that the setting for opening a library on start is turned off,
# with a prompt to open the settings to change that (encouraged but not required).
self._show_notice(
"sync_disabled",
Translations.format(
"library.sync.disabled_notice",
sync_setting=Translations["settings.scan_files_on_open"],
),
Translations["library.sync.open_settings"],
)
if self.settings.show_filepath == ShowFilepathOption.SHOW_FULL_PATHS:
library_dir_display = self.lib.library_dir
@@ -1688,7 +1899,7 @@ class QtDriver(DriverMixin, QObject):
self.set_select_actions_visibility()
self.main_window.menu_bar.save_library_backup_action.setEnabled(True)
self.main_window.menu_bar.close_library_action.setEnabled(True)
self.main_window.menu_bar.refresh_dir_action.setEnabled(True)
self.main_window.menu_bar.sync_library_action.setEnabled(True)
self.main_window.menu_bar.tag_manager_action.setEnabled(True)
self.main_window.menu_bar.color_manager_action.setEnabled(True)
self.main_window.menu_bar.field_template_manager_action.setEnabled(True)
+47
View File
@@ -0,0 +1,47 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: GPL-3.0-only
from PySide6.QtWidgets import QHBoxLayout, QPushButton, QVBoxLayout
from tagstudio.i18n.translations import Translations
from tagstudio.qt.controllers.rounded_progress_bar import RoundedProgressBar
from tagstudio.qt.controllers.stable_label import StableLabel
class BannerView(QVBoxLayout):
PROGRESS_BAR_HEIGHT = 4
def __init__(self) -> None:
super().__init__()
self.setContentsMargins(0, 0, 0, 0)
self.setSpacing(0)
content_row = QHBoxLayout()
content_row.setContentsMargins(6, 6, 6, 2)
content_row.setSpacing(8)
self.close_button = QPushButton("×")
self.close_button.setObjectName("bannerCloseButton")
self.close_button.setFixedSize(24, 24)
content_row.addWidget(self.close_button)
content_row.addStretch(1)
self.label = StableLabel()
content_row.addWidget(self.label)
self.action_button = QPushButton(Translations["entries.generic.refresh_alt"])
self.action_button.setObjectName("bannerActionButton")
content_row.addWidget(self.action_button)
content_row.addStretch(1)
self.addLayout(content_row, 1)
self.progress_bar = RoundedProgressBar()
self.progress_bar.setFixedHeight(self.PROGRESS_BAR_HEIGHT)
policy = self.progress_bar.sizePolicy()
policy.setRetainSizeWhenHidden(True)
self.progress_bar.setSizePolicy(policy)
self.addWidget(self.progress_bar)
@@ -57,8 +57,6 @@ class ThumbGridLayout(QLayout):
self._scroll_to = entry_id
def set_entries(self, entry_ids: list[int]):
self.scroll_area.verticalScrollBar().setValue(0)
self._entry_ids = entry_ids
self._entries.clear()
self._tag_entries.clear()
@@ -211,9 +209,10 @@ class ThumbGridLayout(QLayout):
pass
self._scroll_to = None
visible_rows = math.ceil((view_height + (offset % height_offset)) / height_offset)
offset = int(offset / height_offset)
start = offset * per_row
row_offset = offset
visible_rows = math.ceil((view_height + (row_offset % height_offset)) / height_offset)
row_offset = int(row_offset / height_offset)
start = row_offset * per_row
end = start + (visible_rows * per_row)
first_visible = self._entry_ids[start] if 0 <= start < len(self._entry_ids) else None
@@ -19,6 +19,9 @@ from tagstudio.qt.views.styles.palette import (
# TODO: There's plenty of good opportunities here to consolidate similar styles.
# Work should be done to more closely use Qt's theming systems rather than override them.
# Shared with RoundedProgressBar.set_corner_radius() so both use the exact same corner arc.
BANNER_CORNER_RADIUS = 6
def add_button_style() -> str:
"""Style used for tag-like "Add" buttons [+]."""
@@ -557,6 +560,142 @@ def preview_warning_style() -> str:
"""
def _is_dark_theme() -> bool:
return QGuiApplication.styleHints().colorScheme() is Qt.ColorScheme.Dark
def get_contrast_text_color(background_color: QColor) -> QColor:
"""Return plain black or white, whichever reads better against `background_color`."""
return QColor(0, 0, 0) if background_color.lightness() > 120 else QColor(255, 255, 255)
def _banner_button_hover_style(object_name: str) -> str:
"""Shared hover/pressed/focus feedback for the banner's accent-tinted buttons."""
hover = Palette.accent().darker(140)
pressed = Palette.accent().lighter(120)
return f"""
QPushButton#{object_name}::hover {{
background-color: rgba{hover.toTuple()};
}}
QPushButton#{object_name}::pressed {{
background-color: rgba{pressed.toTuple()};
}}
QPushButton#{object_name}::focus {{
outline: none;
}}
"""
def banner_close_button_style() -> str:
"""Style for the banner's close ("X") button in accent-colored notice modes."""
accent = Palette.accent().darker(180)
text_color = get_contrast_text_color(accent)
return f"""
QPushButton#bannerCloseButton {{
font-size: 24pt;
padding-bottom: 4px;
background: transparent;
color: rgba{text_color.toTuple()};
border: none;
border-radius: 3px;
}}
{_banner_button_hover_style("bannerCloseButton")}
"""
def banner_action_button_style() -> str:
"""Style for the banner's action button (Refresh / Review / Open Settings)."""
accent = Palette.accent().darker(160)
text_color = get_contrast_text_color(accent)
return f"""
QPushButton#bannerActionButton {{
background-color: rgba{accent.toTuple()};
color: rgba{text_color.toTuple()};
border: none;
border-radius: 3px;
padding: 4px 8px;
outline: none;
}}
{_banner_button_hover_style("bannerActionButton")}
"""
def banner_notice_bg_color() -> QColor:
"""Fill color for the banner card in "notice" mode."""
color = QColor(Palette.accent())
color.setAlpha(235)
return color
def banner_notice_style() -> str:
"""Label/button rules for the banner's accent-colored "notice" mode."""
accent = Palette.accent()
text_color = get_contrast_text_color(accent)
return f"""
#banner QLabel {{ color: rgba{text_color.toTuple()}; background: transparent; }}
{banner_close_button_style()}
{banner_action_button_style()}
"""
def banner_close_button_progress_style() -> str:
"""Close button style for the progress banner mode."""
is_dark = _is_dark_theme()
text_color = QColor(255, 255, 255) if is_dark else QColor(0, 0, 0)
hover = "rgba(255, 255, 255, 40)" if is_dark else "rgba(0, 0, 0, 40)"
pressed = "rgba(255, 255, 255, 70)" if is_dark else "rgba(0, 0, 0, 70)"
return f"""
QPushButton#bannerCloseButton {{
font-size: 24pt;
padding-bottom: 4px;
background: transparent;
color: rgba{text_color.toTuple()};
border: none;
border-radius: 3px;
}}
QPushButton#bannerCloseButton::hover {{
background-color: {hover};
}}
QPushButton#bannerCloseButton::pressed {{
background-color: {pressed};
}}
QPushButton#bannerCloseButton::focus {{
outline: none;
}}
"""
def banner_progress_bg_color() -> QColor:
"""Fill color for the banner card in "progress"/"fleeting_notice" mode."""
is_dark = _is_dark_theme()
return QColor(
ThemePalette.COLOR_BG_DARK.value if is_dark else ThemePalette.COLOR_BG_LIGHT.value
)
def banner_progress_style() -> str:
"""Label/button rules for the banner's neutral "progress" mode."""
is_dark = _is_dark_theme()
text_str = "white" if is_dark else "black"
return f"""
#banner QLabel {{ color: {text_str}; background: transparent; }}
{banner_close_button_progress_style()}
"""
def banner_progress_chunk_color() -> QColor:
"""Fill color for the banner's custom-painted progress bar chunk."""
is_dark = _is_dark_theme()
chunk = QColor(Palette.accent().lighter(130) if is_dark else Palette.accent().darker(115))
chunk.setAlpha(235)
return chunk
def header(string: str, level: int, color: str | None = None) -> str:
"""Wrap a string in HTML header tags.
+20 -9
View File
@@ -3,13 +3,13 @@
"about.config_path": "Config Path",
"about.description": "TagStudio is a photo and file organization application with an underlying tag-based system that focuses on giving freedom and flexibility to the user. No proprietary programs or formats, no sea of sidecar files, and no complete upheaval of your filesystem structure.",
"about.documentation": "Documentation",
"about.library_version": "Library Format",
"about.module.found": "Found",
"about.modules.title": "Optional Modules",
"about.title": "About TagStudio",
"about.version": "Version",
"about.version.latest": "{built_version} (Latest Release: {latest_version})",
"about.website": "Website",
"about.library_version": "Library Format",
"app.git": "Git Commit",
"app.nightly": "Nightly",
"app.pre_release": "Pre-Release",
@@ -62,7 +62,6 @@
"entries.remove.plural.confirm": "Are you sure you want to remove these <b>{count}</b> entries from your library? No files on disk will be deleted.",
"entries.remove.singular.confirm": "Are you sure you want to remove this entry from your library? No files on disk will be deleted.",
"entries.running.dialog.new_entries": "Adding {total} New File Entries…",
"entries.running.dialog.title": "Adding New File Entries",
"entries.tags": "Tags",
"entries.unlinked.description": "Each library entry is linked to a file in one of your directories. If a file linked to an entry is moved or deleted outside of TagStudio, it is then considered unlinked.<br><br>Unlinked entries may be automatically relinked via searching your directories or deleted if desired.",
"entries.unlinked.relink.attempting": "Attempting to Relink {index}/{unlinked_count} Entries, {fixed_count} Successfully Relinked",
@@ -70,6 +69,7 @@
"entries.unlinked.relink.title": "Relinking Entries",
"entries.unlinked.remove": "Remove Unlinked Entries",
"entries.unlinked.remove_alt": "Remo&ve Unlinked Entries",
"entries.unlinked.review": "Manual &Review",
"entries.unlinked.scanning": "Scanning Library for Unlinked Entries…",
"entries.unlinked.search_and_relink": "&Search && Relink",
"entries.unlinked.title": "Fix Unlinked Entries",
@@ -121,6 +121,7 @@
"file.open_location.mac": "Reveal in Finder",
"file.open_location.windows": "Show in File Explorer",
"file.path": "File Path",
"file.size": "File Size",
"folders_to_tags.close_all": "Close All",
"folders_to_tags.converting": "Converting folders to Tags",
"folders_to_tags.description": "Creates tags based on your folder structure and applies them to your entries.\n The structure below shows all the tags that will be created and what entries they will be applied to.",
@@ -253,11 +254,21 @@
"library_object.slug_required": "ID Slug (Required)",
"library.missing": "Library Location is Missing",
"library.name": "Library",
"library.refresh.scanning_preparing": "Scanning Directories for New Files…\nPreparing…",
"library.refresh.scanning.plural": "Scanning Directories for New Files…\n{searched_count} Files Searched, {found_count} New Files Found",
"library.refresh.scanning.singular": "Scanning Directories for New Files…\n{searched_count} File Searched, {found_count} New Files Found",
"library.refresh.title": "Refreshing Directories",
"library.scan_library.title": "Scanning Library",
"library.sync.complete": "Library Synced",
"library.sync.disabled_notice": "\"{sync_setting}\" Is Currently Turned Off",
"library.sync.new_files_banner.plural": "{count} New Files Found",
"library.sync.new_files_banner.singular": "{count} New File Found",
"library.sync.open_settings": "Open &Settings",
"library.sync.preparing": "Preparing to Sync…",
"library.sync.relinked_banner.plural": "{count} Files Automatically Relinked",
"library.sync.relinked_banner.singular": "{count} File Automatically Relinked",
"library.sync.relinked_suffix": "({count} Automatically Relinked)",
"library.sync.remaining_unlinked_suffix": "({count} Still Unlinked)",
"library.sync.scanning": "Discovering Files… ({found_count} New Out of {searched_count})",
"library.sync.unlinked_banner.plural": "{count} Unlinked Entries Found",
"library.sync.unlinked_banner.singular": "{count} Unlinked Entry Found",
"library.sync.updating.label": "Syncing {idx}/{total} Entries…",
"macros.running.dialog.new_entries": "Running Configured Macros on {count}/{total} New File Entries…",
"macros.running.dialog.title": "Running Macros on New Entries",
"media_player.autoplay": "Autoplay",
@@ -280,9 +291,9 @@
"menu.file.open_create_library": "&Open/Create Library",
"menu.file.open_library": "Open Library",
"menu.file.open_recent_library": "Open Recent",
"menu.file.refresh_directories": "&Refresh Directories",
"menu.file.save_backup": "&Save Library Backup",
"menu.file.save_library": "Save Library",
"menu.file.sync_library": "&Sync Library",
"menu.help": "&Help",
"menu.help.about": "About",
"menu.macros": "&Macros",
@@ -335,10 +346,10 @@
"settings.library": "Library Settings",
"settings.localization": "Localization",
"settings.media": "Media",
"settings.open_library_on_start": "Open Library on Start",
"settings.open_library_on_start": "Open Last Library on Start",
"settings.page_size": "Page Size",
"settings.restart_required": "Please restart TagStudio for changes to take effect.",
"settings.scan_files_on_open": "Automatically Load New Files",
"settings.scan_files_on_open": "Sync Library on Open",
"settings.show_filenames_in_grid": "Show Filenames in Grid",
"settings.show_recent_libraries": "Show Recent Libraries",
"settings.splash.label": "Splash Screen",
+73 -3
View File
@@ -9,7 +9,7 @@ from tempfile import TemporaryDirectory
import pytest
import structlog
from tagstudio.core.library.alchemy.enums import BrowsingState
from tagstudio.core.library.alchemy.enums import BrowsingState, SortingModeEnum
from tagstudio.core.library.alchemy.fields import (
DatetimeField,
TextField,
@@ -80,11 +80,52 @@ def test_library_add_file(library: Library):
fields=[TextField(name="Title", value="I'm a Test Title")],
)
assert not library.get_entry_id_from_path(entry.path)
assert library.get_entry_id_from_path(entry.path) == -1
assert library.add_entries([entry])
assert library.get_entry_id_from_path(entry.path)
def test_path_cache_untouched_when_not_yet_built(library: Library):
"""Only `get_or_build_path_cache()` may build the path cache."""
assert library.path_cache is None
entry = Entry(path=Path("before_any_cache.txt"), fields=[])
library.add_entries([entry])
assert library.path_cache is None
def test_path_cache_self_maintained_by_add_entries(library: Library):
"""`add_entries()` must keep an already-built path cache up to date on its own."""
library.is_case_sensitive_fs = True
cache = library.get_or_build_path_cache()
assert Path("added_directly.txt") not in cache
entry = Entry(path=Path("added_directly.txt"), fields=[])
new_ids = library.add_entries([entry])
assert cache.get(Path("added_directly.txt")) == new_ids[0]
def test_path_cache_self_maintained_by_remove_entries(library: Library):
library.is_case_sensitive_fs = True
cache = library.get_or_build_path_cache()
entry = Entry(path=Path("to_remove.txt"), fields=[])
entry_id = library.add_entries([entry])[0]
assert cache.get(Path("to_remove.txt")) == entry_id
library.remove_entries([entry_id])
assert Path("to_remove.txt") not in cache
def test_path_cache_self_maintained_by_update_entry_path(library: Library):
library.is_case_sensitive_fs = True
cache = library.get_or_build_path_cache()
entry = Entry(path=Path("old_location.txt"), fields=[])
entry_id = library.add_entries([entry])[0]
assert library.update_entry_path(entry_id, Path("new_location.txt"))
assert Path("old_location.txt") not in cache
assert cache.get(Path("new_location.txt")) == entry_id
def test_create_tag(library: Library, generate_tag: Callable[..., Tag]):
# tag already exists
assert library.add_tag(generate_tag("foo", id=1000)) is None
@@ -148,6 +189,35 @@ def test_entries_count(library: Library):
assert len(results) == 5
@pytest.mark.parametrize(
"sorting_mode",
[SortingModeEnum.DATE_CREATED, SortingModeEnum.DATE_MODIFIED, SortingModeEnum.FILE_SIZE],
)
def test_search_library_sorting(library: Library, sorting_mode: SortingModeEnum):
entries = [
Entry(
path=Path(f"sort_{i}.txt"),
fields=[],
date_created=float(i),
date_modified=float(i),
file_size=i,
)
for i in range(3)
]
new_ids = library.add_entries(entries)
assert len(new_ids) == 3
results = library.search_library(
BrowsingState.show_all()
.with_sorting_mode(sorting_mode)
.with_sorting_direction(ascending=True),
page_size=None,
)
sorted_new_ids = [entry_id for entry_id in results if entry_id in new_ids]
assert sorted_new_ids == new_ids
def test_parents_add(library: Library, generate_tag: Callable[..., Tag]):
# Given
tag: Tag = library.tags[0]
@@ -338,7 +408,7 @@ def test_merge_entries(library: Library):
entry_b_: Entry = unwrap(library.get_entry_full(entry_b_id))
assert library.merge_entries(entry_a_, entry_b_)
assert not library.get_entry_id_from_path(Path("a"))
assert library.get_entry_id_from_path(Path("a")) == -1
assert library.get_entry_id_from_path(Path("b"))
entry_b_merged = unwrap(library.get_entry_full(entry_b_id))
-50
View File
@@ -1,50 +0,0 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: GPL-3.0-only
from pathlib import Path
from tempfile import TemporaryDirectory
import pytest
from tagstudio.core.constants import IGNORE_NAME
from tagstudio.core.library.alchemy.library import Library
from tagstudio.core.library.refresh import RefreshTracker
from tagstudio.core.utils.types import unwrap
CWD = Path(__file__).parent
@pytest.mark.parametrize("exclude_mode", [True, False])
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_refresh_new_files(library: Library, exclude_mode: bool):
library_dir = unwrap(library.library_dir)
# Given
registry = RefreshTracker(library=library)
library.included_files.clear()
(library_dir / "FOO.MD").touch()
(library_dir / IGNORE_NAME).write_text("*.md" if exclude_mode else "*\n!*.md")
# Test if the single file was added
list(registry.refresh_dir(library_dir, force_internal_tools=True))
assert set(registry.files_not_in_library) == set([Path(IGNORE_NAME), Path("FOO.MD")])
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_refresh_multi_byte_filenames(library: Library):
library_dir = unwrap(library.library_dir)
# Given
registry = RefreshTracker(library=library)
library.included_files.clear()
(library_dir / ".TagStudio").mkdir()
(library_dir / "こんにちは.txt").touch()
(library_dir / "emdash.txt").touch()
(library_dir / "apostrophe.txt").touch()
(library_dir / "umlaute äöü.txt").touch()
# Test if all files were added with their correct names and without exceptions
list(registry.refresh_dir(library_dir))
assert Path("こんにちは.txt") in registry.files_not_in_library
assert Path("emdash.txt") in registry.files_not_in_library
assert Path("apostrophe.txt") in registry.files_not_in_library
assert Path("umlaute äöü.txt") in registry.files_not_in_library
+406
View File
@@ -0,0 +1,406 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: GPL-3.0-only
# pyright: reportPrivateUsage=false
import os
import unicodedata
from pathlib import Path
from tempfile import TemporaryDirectory
import pytest
from tagstudio.core.constants import IGNORE_NAME, TS_FOLDER_NAME
from tagstudio.core.library.alchemy.enums import BrowsingState
from tagstudio.core.library.alchemy.library import Library
from tagstudio.core.library.alchemy.models import Entry
from tagstudio.core.library.sync import LibrarySyncEngine
from tagstudio.core.utils.types import unwrap
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_new_files(library: Library):
"""New files that aren't excluded by an ignore pattern must be picked up as new."""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
(library_dir / "foo.md").touch()
(library_dir / "bar.txt").touch()
ts_ignore_path = library_dir / TS_FOLDER_NAME / IGNORE_NAME
ts_ignore_path.parent.mkdir(parents=True, exist_ok=True)
ts_ignore_path.write_text("*.md")
list(engine.sync_dir(library_dir, force_internal_scanner=True))
assert set(engine.new_paths) == {Path("bar.txt")}
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_multi_byte_filenames(library: Library):
"""Multi-byte and accented Unicode filenames must be scanned and added without errors."""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
(library_dir / ".TagStudio").mkdir()
(library_dir / "こんにちは.txt").touch()
(library_dir / "emdash.txt").touch()
(library_dir / "apostrophe.txt").touch()
(library_dir / "umlaute äöü.txt").touch()
list(engine.sync_dir(library_dir))
assert Path("こんにちは.txt") in engine.new_paths
assert Path("emdash.txt") in engine.new_paths
assert Path("apostrophe.txt") in engine.new_paths
assert Path("umlaute äöü.txt") in engine.new_paths
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_unlinked_entries(library: Library):
"""An unlinked entry with one matching file must be found by `relink_unlinked_entries()`."""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
# Touch the file "bar.md" but in the wrong location, to simulate a moved file
(library_dir / "bar.md").touch()
# Neither library entry ("foo.txt", "one/two/bar.md") exists on disk, so both are unlinked
list(engine.sync_dir(library_dir, force_internal_scanner=True))
assert engine.unlinked_entries_count == 2
# Relinking bar.md should match and relink to the entry that was at "one/two/bar.md"
list(engine.relink_unlinked_entries())
assert engine.manual_relink_count == 1
assert engine.unlinked_entries_count == 1
results = library.search_library(BrowsingState.from_path("bar.md"), page_size=500)
entries = library.get_entries(results.ids)
assert entries[0].path == Path("bar.md")
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_nfd_nfc_false_positive(library: Library):
"""A file reappearing in a different Unicode form must not look like a new/duplicate entry."""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
nfc_name = unicodedata.normalize("NFC", "SKÅL.txt")
nfd_name = unicodedata.normalize("NFD", "SKÅL.txt")
assert nfc_name != nfd_name
(library_dir / nfc_name).touch()
list(engine.sync_dir(library_dir, force_internal_scanner=True))
list(engine.save_new_entries())
# Simulate the file later showing up in NFD form, as if the filesystem was changed
(library_dir / nfc_name).unlink()
(library_dir / nfd_name).touch()
list(engine.sync_dir(library_dir, force_internal_scanner=True))
assert engine.new_paths == []
assert len(engine.paths_to_restat) == 1
assert engine.unlinked_entries_count == 2
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_case_sensitivity_aware_relink(library: Library):
"""`find_relink_candidates()` must respect the library's case-sensitivity setting."""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
(library_dir / "Other").mkdir()
(library_dir / "Other" / "name.txt").touch()
library.add_entries([Entry(path=Path("Folder/Name.txt"), fields=[])])
list(engine.sync_dir(library_dir, force_internal_scanner=True))
unlinked_entry = next(e for e in engine.unlinked_entries if e.path == Path("Folder/Name.txt"))
library.is_case_sensitive_fs = True
assert engine.find_relink_candidates(unlinked_entry) == []
library.is_case_sensitive_fs = False
engine._filename_to_path_map = None # Rebuild the index with the new case sensitivity
assert engine.find_relink_candidates(unlinked_entry) == [Path("Other/name.txt")]
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_cache_pruned_on_remove(library: Library):
"""Removing an unlinked entry must prune the path cache."""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
tracked_path = Path("tracked.txt")
(library_dir / tracked_path).touch()
list(engine.sync_dir(library_dir, force_internal_scanner=True))
list(engine.save_new_entries())
(library_dir / tracked_path).unlink()
list(engine.sync_dir(library_dir, force_internal_scanner=True))
tracked_entry = next(e for e in engine.unlinked_entries if e.path == tracked_path)
cache = library.get_or_build_path_cache()
assert tracked_path in cache
# Only an explicit removal should prune the cache
engine.unlinked_entries = [tracked_entry]
engine.remove_unlinked_entries()
assert tracked_path not in cache
# A different file at the same path afterwards should be treated as new
(library_dir / tracked_path).touch()
list(engine.sync_dir(library_dir, force_internal_scanner=True))
assert engine.new_paths == [tracked_path]
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_duplicate_case_collision_treated_as_unlinked(library: Library):
"""A duplicate entry displaced by a case-insensitive collision must be treated as unlinked."""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
library.is_case_sensitive_fs = False # Force a collision using a case difference
(library_dir / "Dupe").mkdir()
(library_dir / "Dupe" / "photo.jpg").touch()
entry_a_id, entry_b_id = library.add_entries(
[
Entry(path=Path("Dupe/photo.jpg"), fields=[]),
Entry(path=Path("Dupe/PHOTO.JPG"), fields=[]),
]
)
cache = library.get_or_build_path_cache()
path_key = Path("dupe/photo.jpg") # Case-insensitive + NFD
assert library.duplicate_path_entry_ids is not None
assert len(library.duplicate_path_entry_ids) == 1
dupe_id = library.duplicate_path_entry_ids[0]
original_id = entry_b_id if dupe_id == entry_a_id else entry_a_id
assert cache.get(path_key) == original_id
# The dupe ID should be marked as unlinked, and the original ID should not
list(engine.sync_dir(library_dir, force_internal_scanner=True))
unlinked_ids = {e.id for e in engine.unlinked_entries}
assert dupe_id in unlinked_ids
assert original_id not in unlinked_ids
# Removing the duplicate entry shouldn't remove the original from the cache
engine.unlinked_entries = [e for e in engine.unlinked_entries if e.id == dupe_id]
engine.remove_unlinked_entries()
assert cache.get(path_key) == original_id
assert library.duplicate_path_entry_ids == []
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_auto_relink_moved_file(library: Library):
"""A moved file (same filename + stats, different path) must auto-relink to its entry."""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
(library_dir / "moveme.txt").touch()
list(engine.sync_dir(library_dir, force_internal_scanner=True))
list(engine.save_new_entries())
original_id = library.get_entry_id_from_path(Path("moveme.txt"))
assert original_id >= 0
(library_dir / "sub").mkdir()
(library_dir / "moveme.txt").rename(library_dir / "sub" / "moveme.txt")
list(engine.sync_dir(library_dir, force_internal_scanner=True))
assert engine.new_paths == []
assert engine.relinked_entries_count == 1
assert engine.relinked_entries[0].id == original_id
assert Path("sub/moveme.txt") not in {e.path for e in engine.unlinked_entries}
assert library.get_entry_id_from_path(Path("sub/moveme.txt")) == original_id
# The original path should no longer be associated with an entry
assert library.get_entry_id_from_path(Path("moveme.txt")) == -1
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_auto_relink_moved_file_ambiguous(library: Library):
"""Two equally-matching candidates (same filename + stats) must not auto-relink.
NOTE: This may change in the future as capability expands.
"""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
(library_dir / "a").mkdir()
(library_dir / "b").mkdir()
(library_dir / "a" / "dupe.txt").touch()
(library_dir / "b" / "dupe.txt").touch()
st = (library_dir / "a" / "dupe.txt").stat()
os.utime(library_dir / "b" / "dupe.txt", (st.st_atime, st.st_mtime))
list(engine.sync_dir(library_dir, force_internal_scanner=True))
list(engine.save_new_entries())
(library_dir / "a" / "dupe.txt").unlink()
(library_dir / "b" / "dupe.txt").unlink()
(library_dir / "c").mkdir()
(library_dir / "c" / "dupe.txt").touch()
os.utime(library_dir / "c" / "dupe.txt", (st.st_atime, st.st_mtime))
list(engine.sync_dir(library_dir, force_internal_scanner=True))
assert engine.relinked_entries_count == 0
assert Path("c/dupe.txt") in engine.new_paths
unlinked_paths = {e.path for e in engine.unlinked_entries}
assert Path("a/dupe.txt") in unlinked_paths
assert Path("b/dupe.txt") in unlinked_paths
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_auto_relink_moved_file_stat_mismatch(library: Library):
"""A file with (same filename, different stats) must not auto-relink.
NOTE: This is a limitation of the current auto-relinking system, NOT a design principle.
"""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
(library_dir / "notmoved.txt").touch()
list(engine.sync_dir(library_dir, force_internal_scanner=True))
list(engine.save_new_entries())
(library_dir / "notmoved.txt").unlink()
(library_dir / "sub").mkdir()
(library_dir / "sub" / "notmoved.txt").touch()
list(engine.sync_dir(library_dir, force_internal_scanner=True))
assert engine.relinked_entries_count == 0
assert Path("sub/notmoved.txt") in engine.new_paths
assert Path("notmoved.txt") in {e.path for e in engine.unlinked_entries}
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_auto_relink_renamed_file(library: Library):
"""A renamed file (different filename, same stats) must auto-relink to its entry."""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
(library_dir / "original_name.txt").touch()
list(engine.sync_dir(library_dir, force_internal_scanner=True))
list(engine.save_new_entries())
original_id = library.get_entry_id_from_path(Path("original_name.txt"))
assert original_id >= 0
(library_dir / "original_name.txt").rename(library_dir / "renamed.txt")
list(engine.sync_dir(library_dir, force_internal_scanner=True))
assert engine.new_paths == []
assert engine.relinked_entries_count == 1
assert engine.relinked_entries[0].id == original_id
assert Path("renamed.txt") not in {e.path for e in engine.unlinked_entries}
assert library.get_entry_id_from_path(Path("renamed.txt")) == original_id
# The original path should no longer be associated with an entry
assert library.get_entry_id_from_path(Path("original_name.txt")) == -1
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_auto_relink_renamed_file_ambiguous(library: Library):
"""Two equally-matching candidates (different filenames, same stats) must not auto-relink.
NOTE: This may be automated in the future, but for now differs to the user's judgement.
"""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
(library_dir / "a").mkdir()
(library_dir / "b").mkdir()
(library_dir / "a" / "one.txt").touch()
(library_dir / "b" / "two.txt").touch()
st = (library_dir / "a" / "one.txt").stat()
os.utime(library_dir / "b" / "two.txt", (st.st_atime, st.st_mtime))
list(engine.sync_dir(library_dir, force_internal_scanner=True))
list(engine.save_new_entries())
(library_dir / "a" / "one.txt").unlink()
(library_dir / "b" / "two.txt").unlink()
(library_dir / "c").mkdir()
(library_dir / "c" / "renamed.txt").touch()
os.utime(library_dir / "c" / "renamed.txt", (st.st_atime, st.st_mtime))
list(engine.sync_dir(library_dir, force_internal_scanner=True))
assert engine.relinked_entries_count == 0
assert Path("c/renamed.txt") in engine.new_paths
unlinked_paths = {e.path for e in engine.unlinked_entries}
assert Path("a/one.txt") in unlinked_paths
assert Path("b/two.txt") in unlinked_paths
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_auto_relink_moved_and_renamed_together(library: Library):
"""A moved file and a renamed file in the same sync must each auto-relink via their own pass."""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
(library_dir / "moveme.txt").touch()
(library_dir / "rename_orig.txt").touch()
list(engine.sync_dir(library_dir, force_internal_scanner=True))
list(engine.save_new_entries())
moved_id = library.get_entry_id_from_path(Path("moveme.txt"))
renamed_id = library.get_entry_id_from_path(Path("rename_orig.txt"))
assert moved_id >= 0
assert renamed_id >= 0
(library_dir / "sub").mkdir()
(library_dir / "moveme.txt").rename(library_dir / "sub" / "moveme.txt")
(library_dir / "rename_orig.txt").rename(library_dir / "rename_new.txt")
list(engine.sync_dir(library_dir, force_internal_scanner=True))
assert engine.new_paths == []
assert engine.relinked_entries_count == 2
assert {e.id for e in engine.relinked_entries} == {moved_id, renamed_id}
assert library.get_entry_id_from_path(Path("sub/moveme.txt")) == moved_id
assert library.get_entry_id_from_path(Path("rename_new.txt")) == renamed_id
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_cancel_skips_unlinked_finalization(library: Library):
"""A cancelled `sync_dir()` call must not mark any entries as unlinked."""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
(library_dir / "a.txt").touch()
list(engine.sync_dir(library_dir, force_internal_scanner=True))
list(engine.save_new_entries())
baseline_unlinked_count = engine.unlinked_entries_count
# Case where sync is cancelled before this scan even starts
engine.cancelled = True
list(engine.sync_dir(library_dir, force_internal_scanner=True))
assert engine.unlinked_entries_count == baseline_unlinked_count
assert engine.new_paths == []
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_cancel_skips_save_new_entries(library: Library):
"""A cancelled `save_new_entries()` call must not save any new entries."""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
baseline_count = library.entries_count
(library_dir / "f0.txt").touch()
list(engine.sync_dir(library_dir, force_internal_scanner=True))
assert engine.new_paths == [Path("f0.txt")]
engine.cancelled = True
list(engine.save_new_entries())
assert library.entries_count == baseline_count
assert engine.new_paths == [Path("f0.txt")]
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_sync_cancel_skips_sync_entry_stats(library: Library):
"""A cancelled `sync_entry_stats()` call must not touch `paths_to_restat`."""
library_dir = unwrap(library.library_dir)
engine = LibrarySyncEngine(library=library)
(library_dir / "f0.txt").touch()
list(engine.sync_dir(library_dir, force_internal_scanner=True))
list(engine.save_new_entries())
# A second scan finds the same file again, marking it for a restat
list(engine.sync_dir(library_dir, force_internal_scanner=True))
assert len(engine.paths_to_restat) == 1
expected = list(engine.paths_to_restat)
engine.cancelled = True
list(engine.sync_entry_stats())
assert engine.paths_to_restat == expected
@@ -1,38 +0,0 @@
# SPDX-FileCopyrightText: (c) TagStudio Contributors
# SPDX-License-Identifier: GPL-3.0-only
from pathlib import Path
from tempfile import TemporaryDirectory
import pytest
from tagstudio.core.library.alchemy.enums import BrowsingState
from tagstudio.core.library.alchemy.library import Library
from tagstudio.core.library.alchemy.registries.unlinked_registry import UnlinkedRegistry
from tagstudio.core.utils.types import unwrap
CWD = Path(__file__).parent
# NOTE: Does this test actually work?
@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
def test_refresh_unlinked_entries(library: Library):
registry = UnlinkedRegistry(lib=library)
# touch the file `one/two/bar.md` but in wrong location to simulate a moved file
(unwrap(library.library_dir) / "bar.md").touch()
# no files actually exist, so it should return all entries
assert list(registry.refresh_unlinked_files()) == [0, 1]
# neither of the library entries exist
assert len(registry.unlinked_entries) == 2
# iterate through two files
assert list(registry.fix_unlinked_entries()) == [0, 1]
# `bar.md` should be relinked to new correct path
results = library.search_library(BrowsingState.from_path("bar.md"), page_size=500)
entries = library.get_entries(results.ids)
assert entries[0].path == Path("bar.md")
Binary file not shown.
+1 -1
View File
@@ -134,7 +134,7 @@ def test_title_update(
qt_driver.main_window.menu_bar.ignore_modal_action = QAction(menu_bar)
qt_driver.main_window.menu_bar.save_library_backup_action = QAction(menu_bar)
qt_driver.main_window.menu_bar.close_library_action = QAction(menu_bar)
qt_driver.main_window.menu_bar.refresh_dir_action = QAction(menu_bar)
qt_driver.main_window.menu_bar.sync_library_action = QAction(menu_bar)
qt_driver.main_window.menu_bar.tag_manager_action = QAction(menu_bar)
qt_driver.main_window.menu_bar.color_manager_action = QAction(menu_bar)
qt_driver.main_window.menu_bar.new_tag_action = QAction(menu_bar)