From 1d2253ff9491f84cc33c3bb26e8925b8cad16374 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 24 May 2023 19:34:07 +0200 Subject: [PATCH] Various changes - Removed useless format manager - Moved pipeline to its own file - Fixed steam source next not returning game - Changed pipeline order --- src/importer/importer.py | 18 ++++- src/importer/sources/steam_source.py | 2 + src/main.py | 4 +- src/store/managers/file_manager.py | 4 +- src/store/managers/format_update_manager.py | 19 ----- src/store/managers/sgdb_manager.py | 3 + src/store/pipeline.py | 80 +++++++++++++++++++++ src/store/store.py | 79 +------------------- 8 files changed, 103 insertions(+), 106 deletions(-) delete mode 100644 src/store/managers/format_update_manager.py create mode 100644 src/store/pipeline.py diff --git a/src/importer/importer.py b/src/importer/importer.py index 84e1273..918d404 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -4,6 +4,7 @@ from gi.repository import Adw, Gio, Gtk from src import shared from src.utils.task import Task +from src.store.pipeline import Pipeline # pylint: disable=too-many-instance-attributes @@ -83,6 +84,7 @@ class Importer: if not source.is_installed: logging.info("Source %s skipped, not installed", source.id) return + logging.info("Scanning source %s", source.id) # Initialize source iteration iterator = iter(source) @@ -104,19 +106,29 @@ class Importer: continue # Register game - logging.info("Imported %s (%s)", game.name, game.game_id) - shared.store.add_game(game) - self.n_games_added += 1 + pipeline: Pipeline = shared.store.add_game(game) + if pipeline is not None: + logging.info("Imported %s (%s)", game.name, game.game_id) + pipeline.connect("manager-done", self.manager_done_callback) + self.n_games_added += 1 def source_task_callback(self, _obj, _result, data): """Source import callback""" source, *_rest = data logging.debug("Import done for source %s", source.id) self.n_source_tasks_done += 1 + # TODO remove, should be handled by manager_done_callback self.update_progressbar() if self.finished: self.import_callback() + def manager_done_callback(self, pipeline: Pipeline): + """Callback called when a pipeline for a game has advanced""" + # TODO (optional) update progress bar more precisely from here + # TODO get number of games really added here (eg. exlude blacklisted) + # TODO trigger import_callback only when all pipelines have finished + pass + def import_callback(self): """Callback called when importing has finished""" logging.info("Import done") diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 34abb42..dd407ce 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -91,6 +91,8 @@ class SteamSourceIterator(SourceIterator): if cover_path.is_file(): save_cover(game.game_id, resize_cover(cover_path)) + return game + class SteamSource(Source): name = "Steam" diff --git a/src/main.py b/src/main.py index a24af59..8e4b6c9 100644 --- a/src/main.py +++ b/src/main.py @@ -43,7 +43,6 @@ from src.importer.sources.steam_source import ( from src.preferences import PreferencesWindow from src.store.managers.display_manager import DisplayManager from src.store.managers.file_manager import FileManager -from src.store.managers.format_update_manager import FormatUpdateManager from src.store.managers.sgdb_manager import SGDBManager from src.store.managers.steam_api_manager import SteamAPIManager from src.store.store import Store @@ -80,7 +79,6 @@ class CartridgesApplication(Adw.Application): # Create the games store ready to load games from disk if not shared.store: shared.store = Store() - shared.store.add_manager(FormatUpdateManager()) shared.store.add_manager(DisplayManager()) # Load games from disk @@ -238,6 +236,6 @@ class CartridgesApplication(Adw.Application): def main(version): # pylint: disable=unused-argument log_level = os.environ.get("LOGLEVEL", "ERROR").upper() - logging.basicConfig(level="DEBUG") # TODO remove debug + logging.basicConfig(level="INFO") # TODO remove before release, use env app = CartridgesApplication() return app.run(sys.argv) diff --git a/src/store/managers/file_manager.py b/src/store/managers/file_manager.py index b5381a8..ffd2363 100644 --- a/src/store/managers/file_manager.py +++ b/src/store/managers/file_manager.py @@ -1,14 +1,12 @@ from src.game import Game -from src.store.managers.format_update_manager import FormatUpdateManager from src.store.managers.manager import Manager -from src.store.managers.sgdb_manager import SGDBManager from src.store.managers.steam_api_manager import SteamAPIManager class FileManager(Manager): """Manager in charge of saving a game to a file""" - run_after = set((SteamAPIManager, SGDBManager, FormatUpdateManager)) + run_after = set((SteamAPIManager,)) def run(self, game: Game) -> None: game.save() diff --git a/src/store/managers/format_update_manager.py b/src/store/managers/format_update_manager.py deleted file mode 100644 index fea4d63..0000000 --- a/src/store/managers/format_update_manager.py +++ /dev/null @@ -1,19 +0,0 @@ -from src.store.managers.manager import Manager -from src.game import Game - - -class FormatUpdateManager(Manager): - """Class in charge of migrating a game from an older format""" - - def v1_5_to_v2_0(self, game: Game) -> None: - """Convert a game from v1.5 format to v2.0 format""" - if game.blacklisted is None: - game.blacklisted = False - if game.removed is None: - game.removed = False - game.version = 2.0 - - def run(self, game: Game) -> None: - if game.version is None: - self.v1_5_to_v2_0(game) - game.save() diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index d68abf7..a87885d 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -3,11 +3,14 @@ from requests import HTTPError from src.game import Game from src.store.managers.manager import Manager from src.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper +from src.store.managers.steam_api_manager import SteamAPIManager class SGDBManager(Manager): """Manager in charge of downloading a game's cover from steamgriddb""" + run_after = set((SteamAPIManager,)) + def run(self, game: Game) -> None: try: sgdb = SGDBHelper() diff --git a/src/store/pipeline.py b/src/store/pipeline.py new file mode 100644 index 0000000..a58796a --- /dev/null +++ b/src/store/pipeline.py @@ -0,0 +1,80 @@ +from typing import Iterable + +from gi.repository import GObject + +from src.game import Game +from src.store.managers.manager import Manager +from src.utils.task import Task + + +class Pipeline(GObject.Object): + """Class representing a set of managers for a game""" + + game: Game + + waiting: set[Manager] + running: set[Manager] + done: set[Manager] + + def __init__(self, game: Game, managers: Iterable[Manager]) -> None: + super().__init__() + self.game = game + self.waiting = set(managers) + self.running = set() + self.done = set() + + @property + def not_done(self) -> set[Manager]: + """Get the managers that are not done yet""" + return self.waiting | self.running + + @property + def blocked(self) -> set[Manager]: + """Get the managers that cannot run because their dependencies aren't done""" + blocked = set() + for manager_a in self.waiting: + for manager_b in self.not_done: + if manager_a == manager_b: + continue + if type(manager_b) in manager_a.run_after: + blocked.add(manager_a) + return blocked + + @property + def ready(self) -> set[Manager]: + """Get the managers that can be run""" + return self.waiting - self.blocked + + def advance(self): + """Spawn tasks for managers that are able to run for a game""" + for manager in self.ready: + self.waiting.remove(manager) + self.running.add(manager) + data = (manager,) + task = Task.new(self, manager.cancellable, self.manager_task_callback, data) + task.set_task_data(data) + task.run_in_thread(self.manager_task_thread_func) + + @GObject.Signal(name="manager-started", arg_types=(object,)) + def manager_started(self, manager: Manager) -> None: + """Signal emitted when a manager is started""" + pass + + def manager_task_thread_func(self, _task, _source_object, data, cancellable): + """Thread function for manager tasks""" + manager, *_rest = data + self.emit("manager-started", manager) + manager.run(self.game) + + @GObject.Signal(name="manager-done", arg_types=(object,)) + def manager_done(self, manager: Manager) -> None: + """Signal emitted when a manager is done""" + pass + + def manager_task_callback(self, _source_object, _result, data): + """Callback function for manager tasks""" + manager, *_rest = data + self.running.remove(manager) + self.done.add(manager) + self.emit("manager-done", manager) + self.advance() diff --git a/src/store/store.py b/src/store/store.py index bbc09aa..ebed6ba 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -1,84 +1,7 @@ -from typing import Iterable - -from gi.repository import GObject - from src import shared from src.game import Game from src.store.managers.manager import Manager -from src.utils.task import Task - - -class Pipeline(GObject.Object): - """Class representing a set of managers for a game""" - - game: Game - - waiting: set[Manager] - running: set[Manager] - done: set[Manager] - - def __init__(self, game: Game, managers: Iterable[Manager]) -> None: - super().__init__() - self.game = game - self.waiting = set(managers) - self.running = set() - self.done = set() - - @property - def not_done(self) -> set[Manager]: - """Get the managers that are not done yet""" - return self.waiting | self.running - - @property - def blocked(self) -> set[Manager]: - """Get the managers that cannot run because their dependencies aren't done""" - blocked = set() - for manager_a in self.waiting: - for manager_b in self.not_done: - if manager_a == manager_b: - continue - if type(manager_b) in manager_a.run_after: - blocked.add(manager_a) - return blocked - - @property - def ready(self) -> set[Manager]: - """Get the managers that can be run""" - return self.waiting - self.blocked - - def advance(self): - """Spawn tasks for managers that are able to run for a game""" - for manager in self.ready: - self.waiting.remove(manager) - self.running.add(manager) - data = (manager,) - task = Task.new(self, manager.cancellable, self.manager_task_callback, data) - task.set_task_data(data) - task.run_in_thread(self.manager_task_thread_func) - - @GObject.Signal(name="manager-started", arg_types=(object,)) - def manager_started(self, manager: Manager) -> None: - """Signal emitted when a manager is started""" - pass - - def manager_task_thread_func(self, _task, _source_object, data, cancellable): - """Thread function for manager tasks""" - manager, *_rest = data - self.emit("manager-started", manager) - manager.run(self.game) - - @GObject.Signal(name="manager-done", arg_types=(object,)) - def manager_done(self, manager: Manager) -> None: - """Signal emitted when a manager is done""" - pass - - def manager_task_callback(self, _source_object, _result, data): - """Callback function for manager tasks""" - manager, *_rest = data - self.running.remove(manager) - self.done.add(manager) - self.emit("manager-done", manager) - self.advance() +from src.store.pipeline import Pipeline class Store: