diff --git a/src/importer/importer.py b/src/importer/importer.py index b0d94ae..25e2bc8 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -1,7 +1,12 @@ +import logging +from pathlib import Path from threading import Lock, Thread -from gi.repository import Adw, Gtk +import requests +from gi.repository import Adw, Gio, Gtk + +from .save_cover import resize_cover, save_cover from .steamgriddb import SGDBHelper @@ -12,33 +17,38 @@ class Importer: import_dialog = None sources = None + source_threads = None + sgdb_threads = None progress_lock = None - counts = None - games_lock = None + sgdb_threads_lock = None + counts = None games = None def __init__(self, win): self.games = set() self.sources = set() - self.counts = dict() + self.counts = {} + self.source_threads = [] + self.sgdb_threads = [] self.games_lock = Lock() self.progress_lock = Lock() + self.sgdb_threads_lock = Lock() self.win = win @property def progress(self): # Compute overall values - done = 0 - total = 0 + overall = {"games": 0, "covers": 0, "total": 0} with self.progress_lock: for source in self.sources: - done += self.counts[source.id]["done"] - total += self.counts[source.id]["total"] + for key in overall: + overall[key] = self.counts[source.id][key] # Compute progress - progress = 1 - if total > 0: - progress = 1 - done / total + try: + progress = 1 - (overall["games"] + overall["covers"]) / overall["total"] * 2 + except ZeroDivisionError: + progress = 1 return progress def create_dialog(self): @@ -66,22 +76,21 @@ class Importer: def add_source(self, source): self.sources.add(source) - self.counts[source.id] = {"done": 0, "total": 0} + self.counts[source.id] = {"games": 0, "covers": 0, "total": 0} def import_games(self): self.create_dialog() # Scan sources in threads - threads = [] for source in self.sources: print(f"{source.full_name}, installed: {source.is_installed}") if not source.is_installed: continue thread = Thread(target=self.__import_source__, args=tuple([source])) # fmt: skip - threads.append(thread) + self.source_threads.append(thread) thread.start() - for thread in threads: + for thread in self.source_threads: thread.join() # Save games @@ -93,20 +102,89 @@ class Importer: continue game.save() - self.close_dialog() + # Wait for SGDB image import to finish + for thread in self.sgdb_threads: + thread.join() + + self.import_dialog.close() def __import_source__(self, *args, **_kwargs): """Source import thread entry point""" - # TODO error handling in source iteration - # TODO add SGDB image (move to a game manager) source, *_rest = args + + # Initialize source iteration iterator = source.__iter__() with self.progress_lock: self.counts[source.id]["total"] = len(iterator) - for game in iterator: + + # Handle iteration exceptions + def wrapper(iterator): + while True: + try: + yield next(iterator) + except StopIteration: + break + except Exception as exception: # pylint: disable=broad-exception-caught + logging.exception( + msg=f"Exception in source {iterator.source.id}", + exc_info=exception, + ) + continue + + # Get games from source + for game in wrapper(iterator): with self.games_lock: self.games.add(game) with self.progress_lock: - if not game.blacklisted: - self.counts[source.id]["done"] += 1 + self.counts[source.id]["games"] += 1 self.update_progressbar() + + # Start sgdb lookup for game + # HACK move to a game manager + sgdb_thread = Thread(target=self.__sgdb_lookup__, args=tuple([game])) + with self.sgdb_threads_lock: + self.sgdb_threads.append(sgdb_thread) + sgdb_thread.start() + + def __sgdb_lookup__(self, *args, **_kwargs): + """SGDB lookup thread entry point""" + game, *_rest = args + + def inner(): + # Skip obvious ones + if game.blacklisted: + return + use_sgdb = self.win.schema.get_boolean("sgdb") + if not use_sgdb: + return + # Check if we should query SGDB + prefer_sgdb = self.win.schema.get_boolean("sgdb-prefer") + prefer_animated = self.win.schema.get_boolean("sgdb-animated") + image_trunk = self.win.covers_dir / game.game_id + still = image_trunk.with_suffix(".tiff") + animated = image_trunk.with_suffix(".gif") + # breaking down the condition + is_missing = not still.is_file() and not animated.is_file() + is_not_best = not animated.is_file() and prefer_animated + should_query = is_missing or is_not_best or prefer_sgdb + if not should_query: + return + # Add image from sgdb + game.set_loading(1) + sgdb = SGDBHelper(self.win) + uri = sgdb.get_game_image_uri(game, animated=prefer_animated) + response = requests.get(uri, timeout=5) + tmp_file = Gio.File.new_tmp()[0] + tmp_file_path = tmp_file.get_path() + Path(tmp_file_path).write_bytes(response.content) + save_cover(self.win, game.game_id, resize_cover(self.win, tmp_file_path)) + game.set_loading(0) + + try: + inner() + except Exception: # pylint: disable=broad-exception-caught + # TODO for god's sake handle exceptions correctly + # TODO (talk about that with Kramo) + pass + with self.progress_lock: + self.counts[game.source]["covers"] += 1 diff --git a/src/importer/source.py b/src/importer/source.py index 2576034..7a959ed 100644 --- a/src/importer/source.py +++ b/src/importer/source.py @@ -16,11 +16,13 @@ class SourceIterator(Iterator, Sized): @abstractmethod def __len__(self): - pass + """Get a rough estimate of the number of games produced by the source""" @abstractmethod def __next__(self): - pass + """Get the next generated game from the source. + Raises StopIteration when exhausted. + May raise any other exception signifying an error on this specific game.""" class Source(Iterable): diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index 3a791af..460a6eb 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -13,18 +13,16 @@ class SGDBError(Exception): class SGDBHelper: + """Helper class to make queries to SteamGridDB""" + base_url = "https://www.steamgriddb.com/api/v2/" - win = None - importer = None - exception = None - def __init__(self, win, importer=None) -> None: + def __init__(self, win): self.win = win - self.importer = importer @property - def auth_header(self): + def auth_headers(self): key = self.win.schema.get_string("sgdb-key") headers = {"Authorization": f"Bearer {key}"} return headers @@ -38,7 +36,7 @@ class SGDBHelper: "open_preferences", _("Preferences"), ) - dialog.connect("response", self.response) + dialog.connect("response", self.on_exception_dialog_response) # TODO same as create_exception_dialog def on_exception_dialog_response(self, _widget, response): @@ -64,15 +62,15 @@ class SGDBHelper: res_json = res.json() if "error" in tuple(res_json): raise SGDBError(res_json["errors"]) - else: - raise SGDBError(res.status_code) + raise SGDBError(res.status_code) - def get_image_uri(self, game, animated=False): + def get_game_image_uri(self, game, animated=False): """Get the image for a game""" - uri = f"{self.base_url}grids/game/{self.get_game_id(game)}?dimensions=600x900" + game_id = self.get_game_id(game) + uri = f"{self.base_url}grids/game/{game_id}?dimensions=600x900" if animated: uri += "&types=animated" - grid = requests.get(uri, headers=self.auth_header, timeout=5) + grid = requests.get(uri, headers=self.auth_headers, timeout=5) image_uri = grid.json()["data"][0]["url"] return image_uri