From 926dfffebe75f4420a029e57fc4a1a3abdba2f96 Mon Sep 17 00:00:00 2001 From: Theasacraft <91694323+Thesacraft@users.noreply.github.com> Date: Sat, 8 Jun 2024 03:02:28 +0200 Subject: [PATCH] Add option to use a allowed extensions instead of ignored extensions (#251) * Add option to use a whitelist instead of a blacklist * maybe fix mypy? * Fix Mypy and rename ignored_extensions * This should fix mypy * Update checkbox text * Update window title * shorten if statment and update text * update variable names * Fix Mypy * hopefully fix mypy * Fix mypy * deprecate ignored_extensions Co-authored-by: Jiri * polishing * polishing * Fix mypy * finishing touches Co-authored-by: Jiri * Fix boolean loading * UI/UX + ext list loading tweaks - Change extension list mode setting from Checkbox to ComboBox to help better convey its purpose - Change and simplify wording - Add type hints to extension variables and change loading to use `get()` with default values - Sanitize older extension lists that don't use extensions with a leading "." - Misc. code organization and docstrings --------- Co-authored-by: Jiri Co-authored-by: Travis Abendshien --- tagstudio/src/core/json_typing.py | 5 +- tagstudio/src/core/library.py | 66 ++++++++++++++++------- tagstudio/src/qt/modals/file_extension.py | 53 ++++++++++++++++-- tagstudio/src/qt/ts_qt.py | 8 +-- 4 files changed, 105 insertions(+), 27 deletions(-) diff --git a/tagstudio/src/core/json_typing.py b/tagstudio/src/core/json_typing.py index ae63a885..29ffdc35 100644 --- a/tagstudio/src/core/json_typing.py +++ b/tagstudio/src/core/json_typing.py @@ -1,4 +1,5 @@ from typing import TypedDict +from typing_extensions import NotRequired class JsonLibary(TypedDict("", {"ts-version": str})): @@ -8,7 +9,9 @@ class JsonLibary(TypedDict("", {"ts-version": str})): fields: list # TODO macros: "list[JsonMacro]" entries: "list[JsonEntry]" - ignored_extensions: list[str] + ext_list: list[str] + is_exclude_list: bool + ignored_extensions: NotRequired[list[str]] # deprecated class JsonBase(TypedDict): diff --git a/tagstudio/src/core/library.py b/tagstudio/src/core/library.py index 4a4b70f4..59f49401 100644 --- a/tagstudio/src/core/library.py +++ b/tagstudio/src/core/library.py @@ -5,6 +5,7 @@ """The Library object and related methods for TagStudio.""" import datetime +import json import logging import os import time @@ -341,8 +342,9 @@ class Library: # That filename can then be used to provide quick lookup to image metadata entries in the Library. self.filename_to_entry_id_map: dict[Path, int] = {} # A list of file extensions to be ignored by TagStudio. - self.default_ext_blacklist: list = [".json", ".xmp", ".aae"] - self.ignored_extensions: list = self.default_ext_blacklist + self.default_ext_exclude_list: list[str] = [".json", ".xmp", ".aae"] + self.ext_list: list[str] = [] + self.is_exclude_list: bool = True # Tags ================================================================= # List of every Tag object (ts-v8). @@ -499,11 +501,35 @@ class Library: self.verify_ts_folders() major, minor, patch = json_dump["ts-version"].split(".") - # Load Extension Blacklist --------------------------------- - if "ignored_extensions" in json_dump.keys(): - self.ignored_extensions = json_dump["ignored_extensions"] + # Load Extension List -------------------------------------- + start_time = time.time() + if "ignored_extensions" in json_dump: + self.ext_list = json_dump.get( + "ignored_extensions", self.default_ext_exclude_list + ) + else: + self.ext_list = json_dump.get( + "ext_list", self.default_ext_exclude_list + ) - # Parse Tags --------------------------------------------------- + # Sanitizes older lists (v9.2.1) that don't use leading periods. + # Without this, existing lists (including default lists) + # have to otherwise be updated by hand in order to restore + # previous functionality. + sanitized_list: list[str] = [] + for ext in self.ext_list: + if not ext.startswith("."): + ext = "." + ext + sanitized_list.append(ext) + self.ext_list = sanitized_list + + self.is_exclude_list = json_dump.get("is_exclude_list", True) + end_time = time.time() + logging.info( + f"[LIBRARY] Extension list loaded in {(end_time - start_time):.3f} seconds" + ) + + # Parse Tags ----------------------------------------------- if "tags" in json_dump.keys(): start_time = time.time() @@ -557,7 +583,7 @@ class Library: f"[LIBRARY] Tags loaded in {(end_time - start_time):.3f} seconds" ) - # Parse Entries ------------------------------------------------ + # Parse Entries -------------------------------------------- if entries := json_dump.get("entries"): start_time = time.time() for entry in entries: @@ -581,7 +607,7 @@ class Library: del f[list(f.keys())[0]] fields = entry["fields"] - # Look through fields for legacy Collation data -------- + # Look through fields for legacy Collation data ---- if int(major) >= 9 and int(minor) < 1: for f in fields: if self.get_field_attr(f, "type") == "collation": @@ -658,7 +684,7 @@ class Library: f"[LIBRARY] Entries loaded in {(end_time - start_time):.3f} seconds" ) - # Parse Collations --------------------------------------------------- + # Parse Collations ----------------------------------------- if "collations" in json_dump.keys(): start_time = time.time() for collation in json_dump["collations"]: @@ -735,7 +761,8 @@ class Library: file_to_save: JsonLibary = { "ts-version": VERSION, - "ignored_extensions": [], + "ext_list": [i for i in self.ext_list if i], + "is_exclude_list": self.is_exclude_list, "tags": [], "collations": [], "fields": [], @@ -745,8 +772,6 @@ class Library: print("[LIBRARY] Formatting Tags to JSON...") - file_to_save["ignored_extensions"] = [i for i in self.ignored_extensions if i] - for tag in self.tags: file_to_save["tags"].append(tag.compressed_dict()) @@ -834,7 +859,7 @@ class Library: self.missing_files.clear() self.fixed_files.clear() self.filename_to_entry_id_map: dict[Path, int] = {} - self.ignored_extensions = self.default_ext_blacklist + self.ext_list = self.default_ext_exclude_list self.tags.clear() self._next_tag_id = 1000 @@ -864,7 +889,12 @@ class Library: and "tagstudio_thumbs" not in f.parts and not f.is_dir() ): - if f.suffix not in self.ignored_extensions: + if f.suffix not in self.ext_list and self.is_exclude_list: + self.dir_file_count += 1 + file = f.relative_to(self.library_dir) + if file not in self.filename_to_entry_id_map: + self.files_not_in_library.append(file) + elif f.suffix in self.ext_list and not self.is_exclude_list: self.dir_file_count += 1 file = f.relative_to(self.library_dir) try: @@ -1352,12 +1382,12 @@ class Library: # non_entry_count = 0 # Iterate over all Entries ============================================================= for entry in self.entries: - allowed_ext: bool = entry.filename.suffix not in self.ignored_extensions + allowed_ext: bool = entry.filename.suffix not in self.ext_list # try: # entry: Entry = self.entries[self.file_to_library_index_map[self._source_filenames[i]]] # print(f'{entry}') - if allowed_ext: + if allowed_ext == self.is_exclude_list: # If the entry has tags of any kind, append them to this main tag list. entry_tags: list[int] = [] entry_authors: list[str] = [] @@ -1509,8 +1539,8 @@ class Library: else: for entry in self.entries: added = False - allowed_ext = entry.filename.suffix not in self.ignored_extensions - if allowed_ext: + allowed_ext = entry.filename.suffix not in self.ext_list + if allowed_ext == self.is_exclude_list: for f in entry.fields: if self.get_field_attr(f, "type") == "collation": if ( diff --git a/tagstudio/src/qt/modals/file_extension.py b/tagstudio/src/qt/modals/file_extension.py index e61368dc..00fc2607 100644 --- a/tagstudio/src/qt/modals/file_extension.py +++ b/tagstudio/src/qt/modals/file_extension.py @@ -6,11 +6,15 @@ from PySide6.QtCore import Signal, Qt from PySide6.QtWidgets import ( QVBoxLayout, + QHBoxLayout, + QWidget, QPushButton, QTableWidget, QTableWidgetItem, QStyledItemDelegate, QLineEdit, + QComboBox, + QLabel, ) from src.core.library import Library @@ -30,41 +34,80 @@ class FileExtensionModal(PanelWidget): def __init__(self, library: "Library"): super().__init__() + # Initialize Modal ===================================================== self.lib = library self.setWindowTitle("File Extensions") self.setWindowModality(Qt.WindowModality.ApplicationModal) - self.setMinimumSize(200, 400) + self.setMinimumSize(240, 400) self.root_layout = QVBoxLayout(self) self.root_layout.setContentsMargins(6, 6, 6, 6) - self.table = QTableWidget(len(self.lib.ignored_extensions), 1) + # Create Table Widget -------------------------------------------------- + self.table = QTableWidget(len(self.lib.ext_list), 1) self.table.horizontalHeader().setVisible(False) self.table.verticalHeader().setVisible(False) self.table.horizontalHeader().setStretchLastSection(True) self.table.setItemDelegate(FileExtensionItemDelegate()) + # Create "Add Button" Widget ------------------------------------------- self.add_button = QPushButton() self.add_button.setText("&Add Extension") self.add_button.clicked.connect(self.add_item) self.add_button.setDefault(True) self.add_button.setMinimumWidth(100) + # Create Mode Widgets -------------------------------------------------- + self.mode_widget = QWidget() + self.mode_layout = QHBoxLayout(self.mode_widget) + self.mode_layout.setContentsMargins(0, 0, 0, 0) + self.mode_layout.setSpacing(12) + self.mode_label = QLabel() + self.mode_label.setText("List Mode:") + self.mode_combobox = QComboBox() + self.mode_combobox.setEditable(False) + self.mode_combobox.addItem("Exclude") + self.mode_combobox.addItem("Include") + self.mode_combobox.setCurrentIndex(0 if self.lib.is_exclude_list else 1) + self.mode_combobox.currentIndexChanged.connect( + lambda i: self.update_list_mode(i) + ) + self.mode_layout.addWidget(self.mode_label) + self.mode_layout.addWidget(self.mode_combobox) + self.mode_layout.setStretch(1, 1) + + # Add Widgets To Layout ------------------------------------------------ + self.root_layout.addWidget(self.mode_widget) self.root_layout.addWidget(self.table) self.root_layout.addWidget( self.add_button, alignment=Qt.AlignmentFlag.AlignCenter ) + + # Finalize Modal ------------------------------------------------------- self.refresh_list() + def update_list_mode(self, mode: int): + """ + Update the mode of the extension list: "Exclude" or "Include". + + Args: + mode (int): The list mode, given by the index of the mode inside + the mode combobox. 0 for "Exclude", 1 for "Include". + """ + if mode == 0: + self.lib.is_exclude_list = True + elif mode == 1: + self.lib.is_exclude_list = False + def refresh_list(self): - for i, ext in enumerate(self.lib.ignored_extensions): + for i, ext in enumerate(self.lib.ext_list): self.table.setItem(i, 0, QTableWidgetItem(ext)) def add_item(self): self.table.insertRow(self.table.rowCount()) def save(self): - self.lib.ignored_extensions.clear() + self.lib.ext_list.clear() for i in range(self.table.rowCount()): ext = self.table.item(i, 0) if ext and ext.text(): - self.lib.ignored_extensions.append(ext.text()) + self.lib.ext_list.append(ext.text()) diff --git a/tagstudio/src/qt/ts_qt.py b/tagstudio/src/qt/ts_qt.py index c562c4cd..89eabfc5 100644 --- a/tagstudio/src/qt/ts_qt.py +++ b/tagstudio/src/qt/ts_qt.py @@ -403,7 +403,7 @@ class QtDriver(QObject): edit_menu.addSeparator() - manage_file_extensions_action = QAction("Ignored File Extensions", menu_bar) + manage_file_extensions_action = QAction("Manage File Extensions", menu_bar) manage_file_extensions_action.triggered.connect( lambda: self.show_file_extension_modal() ) @@ -734,10 +734,12 @@ class QtDriver(QObject): self.modal.show() def show_file_extension_modal(self): - # self.modal = FileExtensionModal(self.lib) panel = FileExtensionModal(self.lib) self.modal = PanelModal( - panel, "Ignored File Extensions", "Ignored File Extensions", has_save=True + panel, + "File Extensions", + "File Extensions", + has_save=True, ) self.modal.saved.connect(lambda: (panel.save(), self.filter_items(""))) self.modal.show()