🚧 Initial work on retryable managers

src/importer/sources/source.py

@@ -54,11 +54,7 @@ class Source(Iterable):
     @property
     def game_id_format(self) -> str:
         """The string format used to construct game IDs"""
-        format_ = self.name.lower()
+        return self.name.lower() + "_{game_id}"
-        if self.variant is not None:
-            format_ += f"_{self.variant.lower()}"
-        format_ += "_{game_id}"
-        return format_
 
     @property
     @abstractmethod

src/store/managers/manager.py

@@ -1,5 +1,6 @@
+import logging
 from abc import abstractmethod
-from typing import Callable, Any
+from typing import Any, Callable
 
 from src.game import Game
 
@@ -10,11 +11,15 @@ class Manager:
     * May connect to signals on the game to handle them.
     * May cancel its running tasks on critical error,
     in that case a new cancellable must be generated for new tasks to run.
+    * May be retried on some specific error types
     """
 
     run_after: set[type["Manager"]] = set()
-    errors: list[Exception]
     blocking: bool = True
+    retryable_on: set[type[Exception]] = set()
+    max_tries: int = 3
+
+    errors: list[Exception]
 
     @property
     def name(self):
@@ -37,13 +42,38 @@ class Manager:
     @abstractmethod
     def final_run(self, game: Game) -> None:
         """
-        Abstract method overriden by final child classes, called by the run method.
+        Manager specific logic triggered by the run method
+        * Implemented by final child classes
+        * Called by the run method, not used directly
         * May block its thread
-        * May not raise exceptions, as they will be silently ignored
+        * May raise retryable exceptions that will be be retried if possible
+        * May raise other exceptions that will be reported
         """
 
     def run(self, game: Game, callback: Callable[["Manager"], Any]) -> None:
-        """Pass the game through the manager.
+        """
-        In charge of calling the final_run method."""
+        Pass the game through the manager
-        self.final_run(game)
+        * Public method called by a pipeline
+        * In charge of calling the final_run method and handling its errors
+        """
+
+        for remaining_tries in range(self.max_tries, -1, -1):
+            try:
+                self.final_run(game, self.max_tries)
+            except Exception as error:
+                if type(error) in self.retryable_on:
+                    # Handle unretryable errors
+                    logging.error("Unretryable error in %s", self.name, exc_info=error)
+                    self.report_error(error)
+                    break
+                elif remaining_tries == 0:
+                    # Handle being out of retries
+                    logging.error("Out of retries in %s", self.name, exc_info=error)
+                    self.report_error(error)
+                    break
+                else:
+                    # Retry
+                    logging.debug("Retrying %s (%s)", self.name, type(error).__name__)
+                    continue
+
         callback(self)

src/store/managers/sgdb_manager.py

@@ -2,14 +2,15 @@ from requests import HTTPError
 
 from src.game import Game
 from src.store.managers.async_manager import AsyncManager
-from src.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper
 from src.store.managers.steam_api_manager import SteamAPIManager
+from src.utils.steamgriddb import HTTPError, SGDBAuthError, SGDBError, SGDBHelper
 
 
 class SGDBManager(AsyncManager):
     """Manager in charge of downloading a game's cover from steamgriddb"""
 
     run_after = set((SteamAPIManager,))
+    retryable_on = set((HTTPError,))
 
     def final_run(self, game: Game) -> None:
         try:
@@ -19,7 +20,3 @@ class SGDBManager(AsyncManager):
             # If invalid auth, cancel all SGDBManager tasks
             self.cancellable.cancel()
             self.report_error(error)
-        except (HTTPError, SGDBError) as error:
-            # On other error, just report it
-            self.report_error(error)
-            pass

src/store/managers/steam_api_manager.py

@@ -1,13 +1,18 @@
-from requests import HTTPError, JSONDecodeError
-
 from src.game import Game
 from src.store.managers.async_manager import AsyncManager
-from src.utils.steam import SteamGameNotFoundError, SteamHelper, SteamNotAGameError
+from src.utils.steam import (
+    HTTPError,
+    SteamGameNotFoundError,
+    SteamHelper,
+    SteamNotAGameError,
+)
 
 
 class SteamAPIManager(AsyncManager):
     """Manager in charge of completing a game's data from the Steam API"""
 
+    retryable_on = set((HTTPError,))
+
     def final_run(self, game: Game) -> None:
         # Skip non-steam games
         if not game.source.startswith("steam_"):
@@ -18,9 +23,6 @@ class SteamAPIManager(AsyncManager):
         steam = SteamHelper()
         try:
             online_data = steam.get_api_data(appid=appid)
-        except (HTTPError, JSONDecodeError) as error:
-            # On minor error, just report it
-            self.report_error(error)
         except (SteamNotAGameError, SteamGameNotFoundError):
             game.update_values({"blacklisted": True})
         else:

src/utils/steam.py

@@ -75,7 +75,8 @@ class SteamHelper:
             logging.debug("Appid %s not found", appid)
             raise SteamGameNotFoundError()
 
-        if data["data"]["type"] != "game":
+        game_types = ("game", "demo")
+        if data["data"]["type"] not in game_types:
             logging.debug("Appid %s is not a game", appid)
             raise SteamNotAGameError()
 

src/utils/steamgriddb.py

@@ -96,7 +96,7 @@ class SGDBHelper:
             sgdb_id = self.get_game_id(game)
         except (HTTPError, SGDBError) as error:
             logging.warning(
-                "%s while getting SGDB ID for %s", error.__class__.__name__, game.name
+                "%s while getting SGDB ID for %s", type(error).__name__, game.name
             )
             raise error
 
@@ -120,7 +120,7 @@ class SGDBHelper:
             except (HTTPError, SGDBError) as error:
                 logging.warning(
                     "%s while getting image for %s kwargs=%s",
-                    error.__class__.__name__,
+                    type(error).__name__,
                     game.name,
                     str(uri_kwargs),
                 )