diff --git a/tagstudio/src/qt/modals/build_color.py b/tagstudio/src/qt/modals/build_color.py
index a5c88b41..6914a0be 100644
--- a/tagstudio/src/qt/modals/build_color.py
+++ b/tagstudio/src/qt/modals/build_color.py
@@ -3,6 +3,8 @@
 # Created for TagStudio: https://github.com/CyanVoxel/TagStudio
 
 
+import contextlib
+
 import structlog
 from PySide6.QtCore import Qt, Signal
 from PySide6.QtGui import QColor
@@ -320,7 +322,8 @@ class BuildColorPanel(PanelWidget):
         groups = self.lib.tag_color_groups
         colors = groups.get(self.color_group.namespace, [])
         self.known_colors = {c.slug for c in colors}
-        self.known_colors = self.known_colors.difference(self.color_group.slug)
+        with contextlib.suppress(KeyError):
+            self.known_colors.remove(self.color_group.slug)
 
     def update_preview_text(self):
         self.preview_button.button.setText(
diff --git a/tagstudio/src/qt/modals/build_namespace.py b/tagstudio/src/qt/modals/build_namespace.py
index 555b7816..5b34c2ed 100644
--- a/tagstudio/src/qt/modals/build_namespace.py
+++ b/tagstudio/src/qt/modals/build_namespace.py
@@ -3,6 +3,7 @@
 # Created for TagStudio: https://github.com/CyanVoxel/TagStudio
 
 
+import contextlib
 from uuid import uuid4
 
 import structlog
@@ -110,7 +111,8 @@ class BuildNamespacePanel(PanelWidget):
         namespaces = self.lib.namespaces
         self.known_namespaces = {n.namespace for n in namespaces}
         if self.namespace:
-            self.known_namespaces = self.known_namespaces.difference(self.namespace.namespace)
+            with contextlib.suppress(KeyError):
+                self.known_namespaces.remove(self.namespace.namespace)
 
     def on_text_changed(self):
         slug = ""