From 0b188136a208d8dd3cbf6730b2ab84c018966800 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 31 May 2023 15:22:08 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=A7=20Initial=20work=20on=20retryable?= =?UTF-8?q?=20managers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/source.py | 6 +--- src/store/managers/manager.py | 44 +++++++++++++++++++++---- src/store/managers/sgdb_manager.py | 7 ++-- src/store/managers/steam_api_manager.py | 14 ++++---- src/utils/steam.py | 3 +- src/utils/steamgriddb.py | 4 +-- 6 files changed, 52 insertions(+), 26 deletions(-) diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index a45667a..921a8c0 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -54,11 +54,7 @@ class Source(Iterable): @property def game_id_format(self) -> str: """The string format used to construct game IDs""" - format_ = self.name.lower() - if self.variant is not None: - format_ += f"_{self.variant.lower()}" - format_ += "_{game_id}" - return format_ + return self.name.lower() + "_{game_id}" @property @abstractmethod diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 9c67205..19b20c2 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -1,5 +1,6 @@ +import logging from abc import abstractmethod -from typing import Callable, Any +from typing import Any, Callable from src.game import Game @@ -10,11 +11,15 @@ class Manager: * May connect to signals on the game to handle them. * May cancel its running tasks on critical error, in that case a new cancellable must be generated for new tasks to run. + * May be retried on some specific error types """ run_after: set[type["Manager"]] = set() - errors: list[Exception] blocking: bool = True + retryable_on: set[type[Exception]] = set() + max_tries: int = 3 + + errors: list[Exception] @property def name(self): @@ -37,13 +42,38 @@ class Manager: @abstractmethod def final_run(self, game: Game) -> None: """ - Abstract method overriden by final child classes, called by the run method. + Manager specific logic triggered by the run method + * Implemented by final child classes + * Called by the run method, not used directly * May block its thread - * May not raise exceptions, as they will be silently ignored + * May raise retryable exceptions that will be be retried if possible + * May raise other exceptions that will be reported """ def run(self, game: Game, callback: Callable[["Manager"], Any]) -> None: - """Pass the game through the manager. - In charge of calling the final_run method.""" - self.final_run(game) + """ + Pass the game through the manager + * Public method called by a pipeline + * In charge of calling the final_run method and handling its errors + """ + + for remaining_tries in range(self.max_tries, -1, -1): + try: + self.final_run(game, self.max_tries) + except Exception as error: + if type(error) in self.retryable_on: + # Handle unretryable errors + logging.error("Unretryable error in %s", self.name, exc_info=error) + self.report_error(error) + break + elif remaining_tries == 0: + # Handle being out of retries + logging.error("Out of retries in %s", self.name, exc_info=error) + self.report_error(error) + break + else: + # Retry + logging.debug("Retrying %s (%s)", self.name, type(error).__name__) + continue + callback(self) diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index d01e3f3..d2c1384 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -2,14 +2,15 @@ from requests import HTTPError from src.game import Game from src.store.managers.async_manager import AsyncManager -from src.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper from src.store.managers.steam_api_manager import SteamAPIManager +from src.utils.steamgriddb import HTTPError, SGDBAuthError, SGDBError, SGDBHelper class SGDBManager(AsyncManager): """Manager in charge of downloading a game's cover from steamgriddb""" run_after = set((SteamAPIManager,)) + retryable_on = set((HTTPError,)) def final_run(self, game: Game) -> None: try: @@ -19,7 +20,3 @@ class SGDBManager(AsyncManager): # If invalid auth, cancel all SGDBManager tasks self.cancellable.cancel() self.report_error(error) - except (HTTPError, SGDBError) as error: - # On other error, just report it - self.report_error(error) - pass diff --git a/src/store/managers/steam_api_manager.py b/src/store/managers/steam_api_manager.py index 599580d..15ad35b 100644 --- a/src/store/managers/steam_api_manager.py +++ b/src/store/managers/steam_api_manager.py @@ -1,13 +1,18 @@ -from requests import HTTPError, JSONDecodeError - from src.game import Game from src.store.managers.async_manager import AsyncManager -from src.utils.steam import SteamGameNotFoundError, SteamHelper, SteamNotAGameError +from src.utils.steam import ( + HTTPError, + SteamGameNotFoundError, + SteamHelper, + SteamNotAGameError, +) class SteamAPIManager(AsyncManager): """Manager in charge of completing a game's data from the Steam API""" + retryable_on = set((HTTPError,)) + def final_run(self, game: Game) -> None: # Skip non-steam games if not game.source.startswith("steam_"): @@ -18,9 +23,6 @@ class SteamAPIManager(AsyncManager): steam = SteamHelper() try: online_data = steam.get_api_data(appid=appid) - except (HTTPError, JSONDecodeError) as error: - # On minor error, just report it - self.report_error(error) except (SteamNotAGameError, SteamGameNotFoundError): game.update_values({"blacklisted": True}) else: diff --git a/src/utils/steam.py b/src/utils/steam.py index f332456..ba5c649 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -75,7 +75,8 @@ class SteamHelper: logging.debug("Appid %s not found", appid) raise SteamGameNotFoundError() - if data["data"]["type"] != "game": + game_types = ("game", "demo") + if data["data"]["type"] not in game_types: logging.debug("Appid %s is not a game", appid) raise SteamNotAGameError() diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index ea3ecd9..3841e9b 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -96,7 +96,7 @@ class SGDBHelper: sgdb_id = self.get_game_id(game) except (HTTPError, SGDBError) as error: logging.warning( - "%s while getting SGDB ID for %s", error.__class__.__name__, game.name + "%s while getting SGDB ID for %s", type(error).__name__, game.name ) raise error @@ -120,7 +120,7 @@ class SGDBHelper: except (HTTPError, SGDBError) as error: logging.warning( "%s while getting image for %s kwargs=%s", - error.__class__.__name__, + type(error).__name__, game.name, str(uri_kwargs), )