🚧 Hacky, untested SGDB code

src/importer/importer.py

@@ -1,7 +1,12 @@
+import logging
+from pathlib import Path
 from threading import Lock, Thread
 
-from gi.repository import Adw, Gtk
 
+import requests
+from gi.repository import Adw, Gio, Gtk
+
+from .save_cover import resize_cover, save_cover
 from .steamgriddb import SGDBHelper
 
 
@@ -12,33 +17,38 @@ class Importer:
     import_dialog = None
     sources = None
 
+    source_threads = None
+    sgdb_threads = None
     progress_lock = None
-    counts = None
-
     games_lock = None
+    sgdb_threads_lock = None
+    counts = None
     games = None
 
     def __init__(self, win):
         self.games = set()
         self.sources = set()
-        self.counts = dict()
+        self.counts = {}
+        self.source_threads = []
+        self.sgdb_threads = []
         self.games_lock = Lock()
         self.progress_lock = Lock()
+        self.sgdb_threads_lock = Lock()
         self.win = win
 
     @property
     def progress(self):
         # Compute overall values
-        done = 0
+        overall = {"games": 0, "covers": 0, "total": 0}
-        total = 0
         with self.progress_lock:
             for source in self.sources:
-                done += self.counts[source.id]["done"]
+                for key in overall:
-                total += self.counts[source.id]["total"]
+                    overall[key] = self.counts[source.id][key]
         # Compute progress
+        try:
+            progress = 1 - (overall["games"] + overall["covers"]) / overall["total"] * 2
+        except ZeroDivisionError:
             progress = 1
-        if total > 0:
-            progress = 1 - done / total
         return progress
 
     def create_dialog(self):
@@ -66,22 +76,21 @@ class Importer:
 
     def add_source(self, source):
         self.sources.add(source)
-        self.counts[source.id] = {"done": 0, "total": 0}
+        self.counts[source.id] = {"games": 0, "covers": 0, "total": 0}
 
     def import_games(self):
         self.create_dialog()
 
         # Scan sources in threads
-        threads = []
         for source in self.sources:
             print(f"{source.full_name}, installed: {source.is_installed}")
             if not source.is_installed:
                 continue
             thread = Thread(target=self.__import_source__, args=tuple([source]))  # fmt: skip
-            threads.append(thread)
+            self.source_threads.append(thread)
             thread.start()
 
-        for thread in threads:
+        for thread in self.source_threads:
             thread.join()
 
         # Save games
@@ -93,20 +102,89 @@ class Importer:
                 continue
             game.save()
 
-        self.close_dialog()
+        # Wait for SGDB image import to finish
+        for thread in self.sgdb_threads:
+            thread.join()
+
+        self.import_dialog.close()
 
     def __import_source__(self, *args, **_kwargs):
         """Source import thread entry point"""
-        # TODO error handling in source iteration
-        # TODO add SGDB image (move to a game manager)
         source, *_rest = args
+
+        # Initialize source iteration
         iterator = source.__iter__()
         with self.progress_lock:
             self.counts[source.id]["total"] = len(iterator)
-        for game in iterator:
+
+        # Handle iteration exceptions
+        def wrapper(iterator):
+            while True:
+                try:
+                    yield next(iterator)
+                except StopIteration:
+                    break
+                except Exception as exception:  # pylint: disable=broad-exception-caught
+                    logging.exception(
+                        msg=f"Exception in source {iterator.source.id}",
+                        exc_info=exception,
+                    )
+                    continue
+
+        # Get games from source
+        for game in wrapper(iterator):
             with self.games_lock:
                 self.games.add(game)
             with self.progress_lock:
-                if not game.blacklisted:
+                self.counts[source.id]["games"] += 1
-                    self.counts[source.id]["done"] += 1
             self.update_progressbar()
+
+            # Start sgdb lookup for game
+            # HACK move to a game manager
+            sgdb_thread = Thread(target=self.__sgdb_lookup__, args=tuple([game]))
+            with self.sgdb_threads_lock:
+                self.sgdb_threads.append(sgdb_thread)
+            sgdb_thread.start()
+
+    def __sgdb_lookup__(self, *args, **_kwargs):
+        """SGDB lookup thread entry point"""
+        game, *_rest = args
+
+        def inner():
+            # Skip obvious ones
+            if game.blacklisted:
+                return
+            use_sgdb = self.win.schema.get_boolean("sgdb")
+            if not use_sgdb:
+                return
+            # Check if we should query SGDB
+            prefer_sgdb = self.win.schema.get_boolean("sgdb-prefer")
+            prefer_animated = self.win.schema.get_boolean("sgdb-animated")
+            image_trunk = self.win.covers_dir / game.game_id
+            still = image_trunk.with_suffix(".tiff")
+            animated = image_trunk.with_suffix(".gif")
+            # breaking down the condition
+            is_missing = not still.is_file() and not animated.is_file()
+            is_not_best = not animated.is_file() and prefer_animated
+            should_query = is_missing or is_not_best or prefer_sgdb
+            if not should_query:
+                return
+            # Add image from sgdb
+            game.set_loading(1)
+            sgdb = SGDBHelper(self.win)
+            uri = sgdb.get_game_image_uri(game, animated=prefer_animated)
+            response = requests.get(uri, timeout=5)
+            tmp_file = Gio.File.new_tmp()[0]
+            tmp_file_path = tmp_file.get_path()
+            Path(tmp_file_path).write_bytes(response.content)
+            save_cover(self.win, game.game_id, resize_cover(self.win, tmp_file_path))
+            game.set_loading(0)
+
+        try:
+            inner()
+        except Exception:  # pylint: disable=broad-exception-caught
+            # TODO for god's sake handle exceptions correctly
+            # TODO (talk about that with Kramo)
+            pass
+        with self.progress_lock:
+            self.counts[game.source]["covers"] += 1

src/importer/source.py

@@ -16,11 +16,13 @@ class SourceIterator(Iterator, Sized):
 
     @abstractmethod
     def __len__(self):
-        pass
+        """Get a rough estimate of the number of games produced by the source"""
 
     @abstractmethod
     def __next__(self):
-        pass
+        """Get the next generated game from the source.
+        Raises StopIteration when exhausted.
+        May raise any other exception signifying an error on this specific game."""
 
 
 class Source(Iterable):

src/utils/steamgriddb.py

@@ -13,18 +13,16 @@ class SGDBError(Exception):
 
 
 class SGDBHelper:
+    """Helper class to make queries to SteamGridDB"""
+
     base_url = "https://www.steamgriddb.com/api/v2/"
-
     win = None
-    importer = None
-    exception = None
 
-    def __init__(self, win, importer=None) -> None:
+    def __init__(self, win):
         self.win = win
-        self.importer = importer
 
     @property
-    def auth_header(self):
+    def auth_headers(self):
         key = self.win.schema.get_string("sgdb-key")
         headers = {"Authorization": f"Bearer {key}"}
         return headers
@@ -38,7 +36,7 @@ class SGDBHelper:
             "open_preferences",
             _("Preferences"),
         )
-        dialog.connect("response", self.response)
+        dialog.connect("response", self.on_exception_dialog_response)
 
     # TODO same as create_exception_dialog
     def on_exception_dialog_response(self, _widget, response):
@@ -64,15 +62,15 @@ class SGDBHelper:
         res_json = res.json()
         if "error" in tuple(res_json):
             raise SGDBError(res_json["errors"])
-        else:
         raise SGDBError(res.status_code)
 
-    def get_image_uri(self, game, animated=False):
+    def get_game_image_uri(self, game, animated=False):
         """Get the image for a game"""
-        uri = f"{self.base_url}grids/game/{self.get_game_id(game)}?dimensions=600x900"
+        game_id = self.get_game_id(game)
+        uri = f"{self.base_url}grids/game/{game_id}?dimensions=600x900"
         if animated:
             uri += "&types=animated"
-        grid = requests.get(uri, headers=self.auth_header, timeout=5)
+        grid = requests.get(uri, headers=self.auth_headers, timeout=5)
         image_uri = grid.json()["data"][0]["url"]
         return image_uri