From 524a56ea9ad53d96fb2dbebeac00c86526bfaf46 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 30 Apr 2023 00:49:41 +0200 Subject: [PATCH 001/173] WIP - Work on base classes --- src/game2.py | 30 +++++++ src/importer/__init__.py | 0 src/importer/importer.py | 80 ++++++++++++++++++ src/importer/location.py | 20 +++++ src/importer/source.py | 60 ++++++++++++++ src/importer/sources/__init__.py | 0 src/importer/sources/lutris_source.py | 115 ++++++++++++++++++++++++++ src/managers/__init__.py | 0 src/managers/game_manager.py | 14 ++++ 9 files changed, 319 insertions(+) create mode 100644 src/game2.py create mode 100644 src/importer/__init__.py create mode 100644 src/importer/importer.py create mode 100644 src/importer/location.py create mode 100644 src/importer/source.py create mode 100644 src/importer/sources/__init__.py create mode 100644 src/importer/sources/lutris_source.py create mode 100644 src/managers/__init__.py create mode 100644 src/managers/game_manager.py diff --git a/src/game2.py b/src/game2.py new file mode 100644 index 0000000..15b4f93 --- /dev/null +++ b/src/game2.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass, field +from time import time + + +@dataclass +class Game(): + """Simple game class that contains the necessary fields. + Saving, updating and removing is done by game manager classes.""" + + # State + removed : bool = field(default=False, init=False) + blacklisted: bool = field(default=False, init=False) + added : int = field(default=-1, init=False) + last_played: int = field(default=-1, init=False) + + # Metadata + source : str = None + name : str = None + game_id : str = None + developer : str = None + + # Launching + executable : str = None + + # Display + game_cover : str = None + hidden : bool = False + + def __post_init__(self): + self.added = int(time()) \ No newline at end of file diff --git a/src/importer/__init__.py b/src/importer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/importer/importer.py b/src/importer/importer.py new file mode 100644 index 0000000..7ca3b07 --- /dev/null +++ b/src/importer/importer.py @@ -0,0 +1,80 @@ +from gi.repository import Adw, Gtk, Gio + +class Importer(): + + # Display values + win = None + progressbar = None + import_statuspage = None + import_dialog = None + + # Importer values + count_total = 0 + count_done = 0 + sources = list() + + def __init__(self, win) -> None: + self.win = win + + def create_dialog(self): + """Create the import dialog""" + self.progressbar = Gtk.ProgressBar(margin_start=12, margin_end=12) + self.import_statuspage = Adw.StatusPage( + title=_("Importing Games…"), + child=self.progressbar, + ) + self.import_dialog = Adw.Window( + content=self.import_statuspage, + modal=True, + default_width=350, + default_height=-1, + transient_for=self.win, + deletable=False, + ) + self.import_dialog.present() + + def close_dialog(self): + """Close the import dialog""" + self.import_dialog.close() + + def get_progress(self): + """Get the current progression as a number between 0 and 1""" + progress = 1 + if self.total_queue > 0: + progress = 1 - self.queue / self.total_queue + return progress + + def update_progressbar(self): + """Update the progress bar""" + progress = self.get_progress() + self.progressbar.set_fraction(progress) + + def add_source(self, source): + """Add a source to import games from""" + self.sources.append(source) + + def import_games(self): + """Import games from the specified sources""" + self.create_dialog() + + # TODO make that async, you doofus + # Every source does its job on the side, informing of the amount of work and when a game is done. + # At the end of the task, it returns the games. + + # Idea 1 - Work stealing queue + # 1. Sources added to the queue + # 2. Worker A takes source X and advances it + # 3. Worker A puts back source X to the queue + # 4. Worker B takes source X, that has ended + # 5. Worker B doesn't add source X back to the queue + + # Idea 2 - Gio.Task + # 1. A task is created for every source + # 2. Source X finishes + # 3. Importer adds the games + + for source in self.sources: + for game in source: + game.save() + + self.close_dialog() \ No newline at end of file diff --git a/src/importer/location.py b/src/importer/location.py new file mode 100644 index 0000000..5aafb83 --- /dev/null +++ b/src/importer/location.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass +from functools import cached_property +from pathlib import Path + + +@dataclass +class Location(): + """Abstraction for a location that can be overriden by a schema key""" + + win = None + + default: str = None + key: str = None + + @cached_property + def path(self): + override = Path(self.win.schema.get_string(self.path_override_key)) + if override.exists(): + return override + return self.path_default \ No newline at end of file diff --git a/src/importer/source.py b/src/importer/source.py new file mode 100644 index 0000000..0b4517b --- /dev/null +++ b/src/importer/source.py @@ -0,0 +1,60 @@ +from collections.abc import Iterable, Iterator +from enum import IntEnum, auto + + +class SourceIterator(Iterator): + """Data producer for a source of games""" + + class States(IntEnum): + DEFAULT = auto() + READY = auto() + + state = States.DEFAULT + source = None + + def __init__(self, source) -> None: + super().__init__() + self.source = source + + def __iter__(self): + return self + + def __next__(self): + raise NotImplementedError() + + +class Source(Iterable): + """Source of games. Can be a program location on disk with a config file that points to game for example""" + + win = None + name: str = "GenericSource" + variant: str = None + + # Format to construct the executable command for a game. + # Available field names depend on the implementation + executable_format: str + + def __init__(self, win) -> None: + super().__init__() + self.win = win + + @property + def full_name(self): + """Get the source's full name""" + s = self.name + if self.variant is not None: + s += " (%s)" % self.variant + return s + + @property + def game_id_format(self): + """Get the string format used to construct game IDs""" + _format = self.name.lower() + if self.variant is not None: + _format += "_" + self.variant.lower() + _format += "_{game_id}_{game_internal_id}" + return _format + + def __iter__(self): + """Get the source's iterator, to use in for loops""" + raise NotImplementedError() \ No newline at end of file diff --git a/src/importer/sources/__init__.py b/src/importer/sources/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py new file mode 100644 index 0000000..fa6d7f6 --- /dev/null +++ b/src/importer/sources/lutris_source.py @@ -0,0 +1,115 @@ +from pathlib import Path +from functools import cached_property +from sqlite3 import connect + +from cartridges.game2 import Game +from cartridges.importer.source import Source, SourceIterator + + +class LutrisSourceIterator(SourceIterator): + + ignore_steam_games = False + + db_connection = None + db_cursor = None + db_location = None + db_request = None + + def __init__(self, ignore_steam_games) -> None: + super().__init__() + self.ignore_steam_games = ignore_steam_games + self.db_connection = None + self.db_cursor = None + self.db_location = self.source.location / "pga.db" + self.db_request = """ + SELECT + id, name, slug, runner, hidden + FROM + 'games' + WHERE + name IS NOT NULL + AND slug IS NOT NULL + AND configPath IS NOT NULL + AND installed IS TRUE + ; + """ + + def __next__(self): + """Produce games. Behaviour depends on the state of the iterator.""" + + # Get database contents iterator + if self.state == self.States.DEFAULT: + self.db_connection = connect(self.db_location) + self.db_cursor = self.db_connection.execute(self.db_request) + self.state = self.States.READY + + # Get next DB value + while True: + try: + row = self.db_cursor.__next__() + except StopIteration as e: + self.db_connection.close() + raise e + + # Ignore steam games if requested + if row[3] == "steam" and self.ignore_steam_games: + continue + + # Build basic game + game = Game( + name=row[1], + hidden=row[4], + source=self.source.full_name, + game_id=self.source.game_id_format.format(game_id=row[2]), + executable=self.source.executable_format.format(game_id=row[2]), + developer=None, # TODO get developer metadata on Lutris + ) + # TODO Add official image + # TODO Add SGDB image + return game + +class LutrisSource(Source): + + name = "Lutris" + executable_format = "xdg-open lutris:rungameid/{game_id}" + + location = None + cache_location = None + + def __iter__(self): + return LutrisSourceIterator(source=self) + + # TODO find a way to no duplicate this code + # Ideas: + # - Location class (verbose, set in __init__) + # - Schema key override decorator () + + # Lutris location property + @cached_property + def location(self): + ovr = Path(self.win.schema.get_string(self.location_key)) + if ovr.exists(): return ovr + return self.location_default + + # Lutris cache location property + @cached_property + def cache_location(self): + ovr = Path(self.win.schema.get_string(self.cache_location_key)) + if ovr.exists(): return ovr + return self.cache_location_default + + +class LutrisNativeSource(LutrisSource): + variant = "native" + location_key = "lutris-location" + location_default = Path("~/.local/share/lutris/").expanduser() + cache_location_key = "lutris-cache-location" + cache_location_default = location_default / "covers" + + +class LutrisFlatpakSource(LutrisSource): + variant = "flatpak" + location_key = "lutris-flatpak-location" + location_default = Path("~/.var/app/net.lutris.Lutris/data/lutris").expanduser() + cache_location_key = "lutris-flatpak-cache-location" + cache_location_default = location_default / "covers" \ No newline at end of file diff --git a/src/managers/__init__.py b/src/managers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/managers/game_manager.py b/src/managers/game_manager.py new file mode 100644 index 0000000..9af60b4 --- /dev/null +++ b/src/managers/game_manager.py @@ -0,0 +1,14 @@ +class GameManager(): + """Interface for systems that save, update and remove games""" + + def add(self, game): + """Add a game to the manager""" + raise NotImplementedError() + + def update(self, game): + """Update an existing game in the manager""" + raise NotImplementedError() + + def remove(self, game): + """Remove an existing game from the manager""" + raise NotImplementedError() \ No newline at end of file From 0abb7d3df9ce0850728d80b467c2ac39d626b7bb Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 1 May 2023 00:30:13 +0200 Subject: [PATCH 002/173] WIP location (to be discarded) --- src/importer/location.py | 31 +++++++++---- src/importer/source.py | 17 ++++++-- src/importer/sources/lutris_source.py | 63 +++++++++++++++------------ 3 files changed, 71 insertions(+), 40 deletions(-) diff --git a/src/importer/location.py b/src/importer/location.py index 5aafb83..ee664fe 100644 --- a/src/importer/location.py +++ b/src/importer/location.py @@ -1,20 +1,35 @@ from dataclasses import dataclass from functools import cached_property from pathlib import Path +from os import PathLike @dataclass class Location(): - """Abstraction for a location that can be overriden by a schema key""" + """Abstraction for a location that has multiple candidate paths""" - win = None + candidates: list[PathLike] = None - default: str = None - key: str = None + def __init__(self, *candidates): + self.candidates = list() + self.candidates.extend(candidates) + return self + + def add(self, canddiate): + """Add a candidate (last evaluated)""" + self.candidates.append(canddiate) + return self + + def add_override(self, candidate): + """Add a canddiate (first evaluated)""" + self.candidates.insert(0, candidate) + return self @cached_property def path(self): - override = Path(self.win.schema.get_string(self.path_override_key)) - if override.exists(): - return override - return self.path_default \ No newline at end of file + """Chosen path depending on availability on the disk.""" + for candidate in self.candidates: + p = Path(candidate).expanduser() + if p.exists: + return p + return None \ No newline at end of file diff --git a/src/importer/source.py b/src/importer/source.py index 0b4517b..d9685c1 100644 --- a/src/importer/source.py +++ b/src/importer/source.py @@ -27,16 +27,20 @@ class Source(Iterable): """Source of games. Can be a program location on disk with a config file that points to game for example""" win = None - name: str = "GenericSource" - variant: str = None + schema_keys: dict - # Format to construct the executable command for a game. - # Available field names depend on the implementation + name: str + variant: str executable_format: str def __init__(self, win) -> None: super().__init__() self.win = win + self.__init_schema_keys__() + + def __init_schema_keys__(self): + """Initialize schema keys needed by the source if necessary""" + raise NotImplementedError() @property def full_name(self): @@ -57,4 +61,9 @@ class Source(Iterable): def __iter__(self): """Get the source's iterator, to use in for loops""" + raise NotImplementedError() + + def __init_locations__(self): + """Initialize locations needed by the source. + Extended and called by **final** children.""" raise NotImplementedError() \ No newline at end of file diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index fa6d7f6..28e1f2a 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -4,6 +4,7 @@ from sqlite3 import connect from cartridges.game2 import Game from cartridges.importer.source import Source, SourceIterator +from cartridges.importer.location import Location class LutrisSourceIterator(SourceIterator): @@ -72,44 +73,50 @@ class LutrisSource(Source): name = "Lutris" executable_format = "xdg-open lutris:rungameid/{game_id}" + schema_keys = { + "location": None, + "cache_location": None + } - location = None - cache_location = None + def __init__(self, win) -> None: + super().__init__(win) + self.location = Location() + self.cache_location = Location() def __iter__(self): return LutrisSourceIterator(source=self) - # TODO find a way to no duplicate this code - # Ideas: - # - Location class (verbose, set in __init__) - # - Schema key override decorator () - - # Lutris location property - @cached_property - def location(self): - ovr = Path(self.win.schema.get_string(self.location_key)) - if ovr.exists(): return ovr - return self.location_default - - # Lutris cache location property - @cached_property - def cache_location(self): - ovr = Path(self.win.schema.get_string(self.cache_location_key)) - if ovr.exists(): return ovr - return self.cache_location_default + def __init_locations__(self): + super().__init_locations__() + self.location.add_override(self.win.schema.get_string(self.schema_keys["location"])) + self.cache_location.add_override(self.win.schema.get_string(self.schema_keys["cache_location"])) class LutrisNativeSource(LutrisSource): + """Class representing an installation of Lutris using native packaging""" + variant = "native" - location_key = "lutris-location" - location_default = Path("~/.local/share/lutris/").expanduser() - cache_location_key = "lutris-cache-location" - cache_location_default = location_default / "covers" + schema_keys = { + "location": "lutris-location", + "cache_location": "lutris-cache-location" + } + + def __init_locations__(self): + super().__init_locations__() + self.location.add("~/.local/share/lutris/") + self.cache_location.add("~/.local/share/lutris/covers") class LutrisFlatpakSource(LutrisSource): + """Class representing an installation of Lutris using flatpak""" + variant = "flatpak" - location_key = "lutris-flatpak-location" - location_default = Path("~/.var/app/net.lutris.Lutris/data/lutris").expanduser() - cache_location_key = "lutris-flatpak-cache-location" - cache_location_default = location_default / "covers" \ No newline at end of file + schema_keys = { + "location": "lutris-flatpak-location", + "cache_location": "lutris-flatpak-cache-location" + } + + def __init_locations__(self): + super().__init_locations__() + self.location.add("~/.var/app/net.lutris.Lutris/data/lutris") + self.cache_location.add("~/.var/app/net.lutris.Lutris/data/lutris/covers") \ No newline at end of file From 451fde8a918b0710d7066c5957202062c6d7a04d Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 1 May 2023 22:46:18 +0200 Subject: [PATCH 003/173] Locations with decorators --- src/importer/decorators.py | 41 ++++++++++++++++++++ src/importer/location.py | 35 ----------------- src/importer/source.py | 11 ------ src/importer/sources/lutris_source.py | 55 +++++++++++++-------------- 4 files changed, 67 insertions(+), 75 deletions(-) create mode 100644 src/importer/decorators.py delete mode 100644 src/importer/location.py diff --git a/src/importer/decorators.py b/src/importer/decorators.py new file mode 100644 index 0000000..5c36753 --- /dev/null +++ b/src/importer/decorators.py @@ -0,0 +1,41 @@ +""" +A decorator takes a callable A and returns a callable B that will override the name of A. +A decorator with arguments returns a closure decorator having access to the arguments. + +Example usage for the location decorators: + +class MyClass(): + @cached_property + @replaced_by_schema_key(key="source-location") + @replaced_by_path(override="/somewhere/that/doesnt/exist") + @replaced_by_path(override="/somewhere/that/exists") + def location(self): + return None +""" + +from pathlib import Path +from os import PathLike +from functools import wraps + +def replaced_by_path(path: PathLike): # Decorator builder + """Replace the method's returned path with the override if the override exists on disk""" + def decorator(original_function): # Built decorator (closure) + @wraps(original_function) + def wrapper(*args, **kwargs): # func's override + p = Path(path).expanduser() + if p.exists(): return p + else: return original_function(*args, **kwargs) + return wrapper + return decorator + +def replaced_by_schema_key(key: str): # Decorator builder + """Replace the method's returned path with the path pointed by the key if it exists on disk""" + def decorator(original_function): # Built decorator (closure) + @wraps(original_function) + def wrapper(*args, **kwargs): # func's override + schema = args[0].win.schema + try: override = schema.get_string(key) + except Exception: return original_function(*args, **kwargs) + else: return replaced_by_path(override)(*args, **kwargs) + return wrapper + return decorator \ No newline at end of file diff --git a/src/importer/location.py b/src/importer/location.py deleted file mode 100644 index ee664fe..0000000 --- a/src/importer/location.py +++ /dev/null @@ -1,35 +0,0 @@ -from dataclasses import dataclass -from functools import cached_property -from pathlib import Path -from os import PathLike - - -@dataclass -class Location(): - """Abstraction for a location that has multiple candidate paths""" - - candidates: list[PathLike] = None - - def __init__(self, *candidates): - self.candidates = list() - self.candidates.extend(candidates) - return self - - def add(self, canddiate): - """Add a candidate (last evaluated)""" - self.candidates.append(canddiate) - return self - - def add_override(self, candidate): - """Add a canddiate (first evaluated)""" - self.candidates.insert(0, candidate) - return self - - @cached_property - def path(self): - """Chosen path depending on availability on the disk.""" - for candidate in self.candidates: - p = Path(candidate).expanduser() - if p.exists: - return p - return None \ No newline at end of file diff --git a/src/importer/source.py b/src/importer/source.py index d9685c1..957fc45 100644 --- a/src/importer/source.py +++ b/src/importer/source.py @@ -27,7 +27,6 @@ class Source(Iterable): """Source of games. Can be a program location on disk with a config file that points to game for example""" win = None - schema_keys: dict name: str variant: str @@ -36,11 +35,6 @@ class Source(Iterable): def __init__(self, win) -> None: super().__init__() self.win = win - self.__init_schema_keys__() - - def __init_schema_keys__(self): - """Initialize schema keys needed by the source if necessary""" - raise NotImplementedError() @property def full_name(self): @@ -61,9 +55,4 @@ class Source(Iterable): def __iter__(self): """Get the source's iterator, to use in for loops""" - raise NotImplementedError() - - def __init_locations__(self): - """Initialize locations needed by the source. - Extended and called by **final** children.""" raise NotImplementedError() \ No newline at end of file diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 28e1f2a..a167f10 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,10 +1,9 @@ -from pathlib import Path from functools import cached_property from sqlite3 import connect from cartridges.game2 import Game from cartridges.importer.source import Source, SourceIterator -from cartridges.importer.location import Location +from cartridges.importer.decorators import replaced_by_schema_key, replaced_by_path class LutrisSourceIterator(SourceIterator): @@ -73,50 +72,48 @@ class LutrisSource(Source): name = "Lutris" executable_format = "xdg-open lutris:rungameid/{game_id}" - schema_keys = { - "location": None, - "cache_location": None - } + + location = None + cache_location = None def __init__(self, win) -> None: super().__init__(win) - self.location = Location() - self.cache_location = Location() def __iter__(self): return LutrisSourceIterator(source=self) - def __init_locations__(self): - super().__init_locations__() - self.location.add_override(self.win.schema.get_string(self.schema_keys["location"])) - self.cache_location.add_override(self.win.schema.get_string(self.schema_keys["cache_location"])) - class LutrisNativeSource(LutrisSource): """Class representing an installation of Lutris using native packaging""" variant = "native" - schema_keys = { - "location": "lutris-location", - "cache_location": "lutris-cache-location" - } - def __init_locations__(self): - super().__init_locations__() - self.location.add("~/.local/share/lutris/") - self.cache_location.add("~/.local/share/lutris/covers") + @cached_property + @replaced_by_schema_key("lutris-location") + @replaced_by_path("~/.local/share/lutris/") + def location(self): + raise FileNotFoundError() + + @cached_property + @replaced_by_schema_key("lutris-cache-location") + @replaced_by_path("~/.local/share/lutris/covers") + def cache_location(self): + raise FileNotFoundError() class LutrisFlatpakSource(LutrisSource): """Class representing an installation of Lutris using flatpak""" variant = "flatpak" - schema_keys = { - "location": "lutris-flatpak-location", - "cache_location": "lutris-flatpak-cache-location" - } - def __init_locations__(self): - super().__init_locations__() - self.location.add("~/.var/app/net.lutris.Lutris/data/lutris") - self.cache_location.add("~/.var/app/net.lutris.Lutris/data/lutris/covers") \ No newline at end of file + @cached_property + @replaced_by_schema_key("lutris-flatpak-location") + @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris") + def location(self): + raise FileNotFoundError() + + @cached_property + @replaced_by_schema_key("lutris-flatpak-cache-location") + @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/covers") + def cache_location(self): + raise FileNotFoundError() \ No newline at end of file From f432c41843665f716f6ff7858b44bd173b705241 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 3 May 2023 11:37:32 +0200 Subject: [PATCH 004/173] =?UTF-8?q?=F0=9F=94=A5=20Fo-cus.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/game2.py | 30 --------------------------- src/importer/source.py | 22 +++++++++++++------- src/importer/sources/lutris_source.py | 24 ++++++++++----------- src/managers/__init__.py | 0 src/managers/game_manager.py | 14 ------------- 5 files changed, 27 insertions(+), 63 deletions(-) delete mode 100644 src/game2.py delete mode 100644 src/managers/__init__.py delete mode 100644 src/managers/game_manager.py diff --git a/src/game2.py b/src/game2.py deleted file mode 100644 index 15b4f93..0000000 --- a/src/game2.py +++ /dev/null @@ -1,30 +0,0 @@ -from dataclasses import dataclass, field -from time import time - - -@dataclass -class Game(): - """Simple game class that contains the necessary fields. - Saving, updating and removing is done by game manager classes.""" - - # State - removed : bool = field(default=False, init=False) - blacklisted: bool = field(default=False, init=False) - added : int = field(default=-1, init=False) - last_played: int = field(default=-1, init=False) - - # Metadata - source : str = None - name : str = None - game_id : str = None - developer : str = None - - # Launching - executable : str = None - - # Display - game_cover : str = None - hidden : bool = False - - def __post_init__(self): - self.added = int(time()) \ No newline at end of file diff --git a/src/importer/source.py b/src/importer/source.py index 957fc45..55f9227 100644 --- a/src/importer/source.py +++ b/src/importer/source.py @@ -1,3 +1,4 @@ +from abc import abstractmethod from collections.abc import Iterable, Iterator from enum import IntEnum, auto @@ -19,18 +20,18 @@ class SourceIterator(Iterator): def __iter__(self): return self + @abstractmethod def __next__(self): - raise NotImplementedError() + pass class Source(Iterable): - """Source of games. Can be a program location on disk with a config file that points to game for example""" + """Source of games. E.g an installed app with a config file that lists game directories""" - win = None + win = None # TODO maybe not depend on that ? name: str variant: str - executable_format: str def __init__(self, win) -> None: super().__init__() @@ -38,7 +39,7 @@ class Source(Iterable): @property def full_name(self): - """Get the source's full name""" + """The source's full name""" s = self.name if self.variant is not None: s += " (%s)" % self.variant @@ -46,13 +47,20 @@ class Source(Iterable): @property def game_id_format(self): - """Get the string format used to construct game IDs""" + """The string format used to construct game IDs""" _format = self.name.lower() if self.variant is not None: _format += "_" + self.variant.lower() _format += "_{game_id}_{game_internal_id}" return _format + @property + @abstractmethod + def executable_format(self): + """The executable format used to construct game executables""" + pass + + @abstractmethod def __iter__(self): """Get the source's iterator, to use in for loops""" - raise NotImplementedError() \ No newline at end of file + pass \ No newline at end of file diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index a167f10..5f76f72 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,9 +1,9 @@ from functools import cached_property from sqlite3 import connect -from cartridges.game2 import Game -from cartridges.importer.source import Source, SourceIterator -from cartridges.importer.decorators import replaced_by_schema_key, replaced_by_path +from src.game2 import Game +from src.importer.source import Source, SourceIterator +from src.importer.decorators import replaced_by_schema_key, replaced_by_path class LutrisSourceIterator(SourceIterator): @@ -56,17 +56,17 @@ class LutrisSourceIterator(SourceIterator): continue # Build basic game - game = Game( - name=row[1], - hidden=row[4], - source=self.source.full_name, - game_id=self.source.game_id_format.format(game_id=row[2]), - executable=self.source.executable_format.format(game_id=row[2]), - developer=None, # TODO get developer metadata on Lutris - ) + values = { + "name" : row[1], + "hidden" : row[4], + "source" : self.source.full_name, + "game_id" : self.source.game_id_format.format(game_id=row[2]), + "executable": self.source.executable_format.format(game_id=row[2]), + "developer" : None, # TODO get developer metadata on Lutris + } # TODO Add official image # TODO Add SGDB image - return game + return values class LutrisSource(Source): diff --git a/src/managers/__init__.py b/src/managers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/managers/game_manager.py b/src/managers/game_manager.py deleted file mode 100644 index 9af60b4..0000000 --- a/src/managers/game_manager.py +++ /dev/null @@ -1,14 +0,0 @@ -class GameManager(): - """Interface for systems that save, update and remove games""" - - def add(self, game): - """Add a game to the manager""" - raise NotImplementedError() - - def update(self, game): - """Update an existing game in the manager""" - raise NotImplementedError() - - def remove(self, game): - """Remove an existing game from the manager""" - raise NotImplementedError() \ No newline at end of file From 8b7b23379f72dacaa593fce140b608d7ed16c356 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Fri, 5 May 2023 03:04:44 +0200 Subject: [PATCH 005/173] =?UTF-8?q?=F0=9F=8E=A8=20Black=20codestyle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/decorators.py | 37 +++++--- src/importer/importer.py | 129 +++++++++++++++++++------- src/importer/source.py | 26 ++++-- src/importer/sources/lutris_source.py | 47 ++++++---- 4 files changed, 167 insertions(+), 72 deletions(-) diff --git a/src/importer/decorators.py b/src/importer/decorators.py index 5c36753..75d5622 100644 --- a/src/importer/decorators.py +++ b/src/importer/decorators.py @@ -17,25 +17,38 @@ from pathlib import Path from os import PathLike from functools import wraps -def replaced_by_path(path: PathLike): # Decorator builder + +def replaced_by_path(path: PathLike): # Decorator builder """Replace the method's returned path with the override if the override exists on disk""" - def decorator(original_function): # Built decorator (closure) + + def decorator(original_function): # Built decorator (closure) @wraps(original_function) - def wrapper(*args, **kwargs): # func's override + def wrapper(*args, **kwargs): # func's override p = Path(path).expanduser() - if p.exists(): return p - else: return original_function(*args, **kwargs) + if p.exists(): + return p + else: + return original_function(*args, **kwargs) + return wrapper + return decorator -def replaced_by_schema_key(key: str): # Decorator builder + +def replaced_by_schema_key(key: str): # Decorator builder """Replace the method's returned path with the path pointed by the key if it exists on disk""" - def decorator(original_function): # Built decorator (closure) + + def decorator(original_function): # Built decorator (closure) @wraps(original_function) - def wrapper(*args, **kwargs): # func's override + def wrapper(*args, **kwargs): # func's override schema = args[0].win.schema - try: override = schema.get_string(key) - except Exception: return original_function(*args, **kwargs) - else: return replaced_by_path(override)(*args, **kwargs) + try: + override = schema.get_string(key) + except Exception: + return original_function(*args, **kwargs) + else: + return replaced_by_path(override)(*args, **kwargs) + return wrapper - return decorator \ No newline at end of file + + return decorator diff --git a/src/importer/importer.py b/src/importer/importer.py index 7ca3b07..7969d2c 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -1,21 +1,45 @@ +from threading import Thread, Lock from gi.repository import Adw, Gtk, Gio -class Importer(): +from .game import Game +from .steamgriddb import SGDBSave - # Display values + +class Importer: win = None progressbar = None import_statuspage = None import_dialog = None + sources = None - # Importer values - count_total = 0 - count_done = 0 - sources = list() + progress_lock = None + counts = None + + games_lock = None + games = None def __init__(self, win) -> None: + self.games = set() + self.sources = list() + self.counts = dict() + self.games_lock = Lock() + self.progress_lock = Lock() self.win = win + @property + def progress(self): + # Compute overall values + done = 0 + total = 0 + for source in self.sources: + done += self.counts[source.id]["done"] + total += self.counts[source.id]["total"] + # Compute progress + progress = 1 + if total > 0: + progress = 1 - done / total + return progress + def create_dialog(self): """Create the import dialog""" self.progressbar = Gtk.ProgressBar(margin_start=12, margin_end=12) @@ -37,44 +61,83 @@ class Importer(): """Close the import dialog""" self.import_dialog.close() - def get_progress(self): - """Get the current progression as a number between 0 and 1""" - progress = 1 - if self.total_queue > 0: - progress = 1 - self.queue / self.total_queue - return progress - def update_progressbar(self): """Update the progress bar""" - progress = self.get_progress() + progress = self.progress() self.progressbar.set_fraction(progress) def add_source(self, source): """Add a source to import games from""" self.sources.append(source) + self.counts[source.id] = {"done": 0, "total": 0} def import_games(self): """Import games from the specified sources""" + self.create_dialog() - # TODO make that async, you doofus - # Every source does its job on the side, informing of the amount of work and when a game is done. - # At the end of the task, it returns the games. - - # Idea 1 - Work stealing queue - # 1. Sources added to the queue - # 2. Worker A takes source X and advances it - # 3. Worker A puts back source X to the queue - # 4. Worker B takes source X, that has ended - # 5. Worker B doesn't add source X back to the queue - - # Idea 2 - Gio.Task - # 1. A task is created for every source - # 2. Source X finishes - # 3. Importer adds the games - + # Scan all sources + threads = [] for source in self.sources: - for game in source: - game.save() + t = Thread( + None, + self.__import_from_source, + args=tuple( + source, + ), + ) + threads.append(t) + t.start() - self.close_dialog() \ No newline at end of file + # Wait for all of them to finish + for t in threads: + t.join() + + self.close_dialog() + + def __import_from_source(self, *args, **kwargs): + """Source import thread entry point""" + # TODO just get Game objects from the sources + source, *rest = args + + iterator = source.__iter__() + for game_values in iterator: + game = Game(self.win, game_values) + + self.games_lock.acquire() + self.games.add(game) + self.games_lock.release() + + self.progress_lock.acquire() + self.counts[source.id]["total"] = len(iterator) + if not game.blacklisted: + self.counts[source.id]["done"] += 1 + self.update_progressbar() + self.progress_lock.release() + + # TODO remove after not needed + def save_game(self, values=None, cover_path=None): + if values: + game = Game(self.win, values) + + if save_cover: + save_cover(self.win, game.game_id, resize_cover(self.win, cover_path)) + + self.games.add(game) + + self.games_no += 1 + if game.blacklisted: + self.games_no -= 1 + + self.queue -= 1 + self.update_progressbar() + + if self.queue == 0 and not self.blocker: + if self.games: + self.total_queue = len(self.games) + self.queue = len(self.games) + self.import_statuspage.set_title(_("Importing Covers…")) + self.update_progressbar() + SGDBSave(self.win, self.games, self) + else: + self.done() diff --git a/src/importer/source.py b/src/importer/source.py index 55f9227..fd2b7c4 100644 --- a/src/importer/source.py +++ b/src/importer/source.py @@ -9,7 +9,7 @@ class SourceIterator(Iterator): class States(IntEnum): DEFAULT = auto() READY = auto() - + state = States.DEFAULT source = None @@ -28,7 +28,7 @@ class SourceIterator(Iterator): class Source(Iterable): """Source of games. E.g an installed app with a config file that lists game directories""" - win = None # TODO maybe not depend on that ? + win = None # TODO maybe not depend on that ? name: str variant: str @@ -42,17 +42,25 @@ class Source(Iterable): """The source's full name""" s = self.name if self.variant is not None: - s += " (%s)" % self.variant + s += f" ({self.variant})" + return s + + @property + def id(self): + """The source's identifier""" + s = self.name.lower() + if self.variant is not None: + s += f"_{self.variant.lower()}" return s @property def game_id_format(self): """The string format used to construct game IDs""" - _format = self.name.lower() - if self.variant is not None: - _format += "_" + self.variant.lower() - _format += "_{game_id}_{game_internal_id}" - return _format + f = self.name.lower() + if self.variant is not None: + f += f"_{self.variant.lower()}" + f += "_{game_id}" + return f @property @abstractmethod @@ -63,4 +71,4 @@ class Source(Iterable): @abstractmethod def __iter__(self): """Get the source's iterator, to use in for loops""" - pass \ No newline at end of file + pass diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 5f76f72..9a832d4 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,21 +1,20 @@ from functools import cached_property from sqlite3 import connect -from src.game2 import Game +from src.utils.save_cover import resize_cover, save_cover from src.importer.source import Source, SourceIterator from src.importer.decorators import replaced_by_schema_key, replaced_by_path class LutrisSourceIterator(SourceIterator): - - ignore_steam_games = False + ignore_steam_games = False # TODO get that value db_connection = None db_cursor = None db_location = None db_request = None - def __init__(self, ignore_steam_games) -> None: + def __init__(self, ignore_steam_games): super().__init__() self.ignore_steam_games = ignore_steam_games self.db_connection = None @@ -43,10 +42,10 @@ class LutrisSourceIterator(SourceIterator): self.db_cursor = self.db_connection.execute(self.db_request) self.state = self.States.READY - # Get next DB value while True: + # Get next DB value try: - row = self.db_cursor.__next__() + row = self.db_cursor.__next__() except StopIteration as e: self.db_connection.close() raise e @@ -54,29 +53,41 @@ class LutrisSourceIterator(SourceIterator): # Ignore steam games if requested if row[3] == "steam" and self.ignore_steam_games: continue - + # Build basic game values = { - "name" : row[1], - "hidden" : row[4], - "source" : self.source.full_name, - "game_id" : self.source.game_id_format.format(game_id=row[2]), + "hidden": row[4], + "name": row[1], + "source": f"{self.source.id}_{row[3]}", + "game_id": self.source.game_id_format.format( + game_id=row[2], game_internal_id=row[0] + ), "executable": self.source.executable_format.format(game_id=row[2]), - "developer" : None, # TODO get developer metadata on Lutris + "developer": None, # TODO get developer metadata on Lutris } - # TODO Add official image - # TODO Add SGDB image + + # Save official image + image_path = self.source.cache_location / "coverart" / f"{row[2]}.jpg" + if image_path.exists(): + resized = resize_cover(self.win, image_path) + save_cover(self.win, values["game_id"], resized) + + # Save SGDB + return values + class LutrisSource(Source): - name = "Lutris" executable_format = "xdg-open lutris:rungameid/{game_id}" - location = None cache_location = None - def __init__(self, win) -> None: + @property + def game_id_format(self): + return super().game_id_format + "_{game_internal_id}" + + def __init__(self, win): super().__init__(win) def __iter__(self): @@ -116,4 +127,4 @@ class LutrisFlatpakSource(LutrisSource): @replaced_by_schema_key("lutris-flatpak-cache-location") @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/covers") def cache_location(self): - raise FileNotFoundError() \ No newline at end of file + raise FileNotFoundError() From 5c6bfc8b3e03c3b6d8413d70867e36e10836f1f7 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Fri, 5 May 2023 03:07:52 +0200 Subject: [PATCH 006/173] =?UTF-8?q?=F0=9F=9A=A7=20Added=20todo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/lutris_source.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 9a832d4..9eea67b 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -69,10 +69,10 @@ class LutrisSourceIterator(SourceIterator): # Save official image image_path = self.source.cache_location / "coverart" / f"{row[2]}.jpg" if image_path.exists(): - resized = resize_cover(self.win, image_path) - save_cover(self.win, values["game_id"], resized) + resized = resize_cover(self.source.win, image_path) + save_cover(self.source.win, values["game_id"], resized) - # Save SGDB + # TODO Save SGDB return values From 0abe2836198f8fcb1b170f2c73d9e4be049d4bd5 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Fri, 5 May 2023 16:02:57 +0200 Subject: [PATCH 007/173] =?UTF-8?q?=F0=9F=9A=A7=20SGDB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 37 ++----------- src/importer/sources/lutris_source.py | 4 ++ src/utils/steamgriddb.py | 79 +++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 32 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 7969d2c..d4ad042 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -2,7 +2,7 @@ from threading import Thread, Lock from gi.repository import Adw, Gtk, Gio from .game import Game -from .steamgriddb import SGDBSave +from .steamgriddb import SGDBHelper class Importer: @@ -97,47 +97,20 @@ class Importer: def __import_from_source(self, *args, **kwargs): """Source import thread entry point""" - # TODO just get Game objects from the sources source, *rest = args iterator = source.__iter__() - for game_values in iterator: - game = Game(self.win, game_values) - + for game in iterator: self.games_lock.acquire() self.games.add(game) self.games_lock.release() + # TODO SGDB image + # Who's in charge of image adding ? + self.progress_lock.acquire() self.counts[source.id]["total"] = len(iterator) if not game.blacklisted: self.counts[source.id]["done"] += 1 self.update_progressbar() self.progress_lock.release() - - # TODO remove after not needed - def save_game(self, values=None, cover_path=None): - if values: - game = Game(self.win, values) - - if save_cover: - save_cover(self.win, game.game_id, resize_cover(self.win, cover_path)) - - self.games.add(game) - - self.games_no += 1 - if game.blacklisted: - self.games_no -= 1 - - self.queue -= 1 - self.update_progressbar() - - if self.queue == 0 and not self.blocker: - if self.games: - self.total_queue = len(self.games) - self.queue = len(self.games) - self.import_statuspage.set_title(_("Importing Covers…")) - self.update_progressbar() - SGDBSave(self.win, self.games, self) - else: - self.done() diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 9eea67b..eed7c15 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,6 +1,7 @@ from functools import cached_property from sqlite3 import connect +from src.game import Game from src.utils.save_cover import resize_cover, save_cover from src.importer.source import Source, SourceIterator from src.importer.decorators import replaced_by_schema_key, replaced_by_path @@ -55,6 +56,7 @@ class LutrisSourceIterator(SourceIterator): continue # Build basic game + # TODO decouple game creation from the window object (later) values = { "hidden": row[4], "name": row[1], @@ -65,6 +67,7 @@ class LutrisSourceIterator(SourceIterator): "executable": self.source.executable_format.format(game_id=row[2]), "developer": None, # TODO get developer metadata on Lutris } + game = Game(self.source.win, values) # Save official image image_path = self.source.cache_location / "coverart" / f"{row[2]}.jpg" @@ -73,6 +76,7 @@ class LutrisSourceIterator(SourceIterator): save_cover(self.source.win, values["game_id"], resized) # TODO Save SGDB + SGDBSave(self.win, self.games, self) return values diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index 095e913..3a791af 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -8,6 +8,85 @@ from .create_dialog import create_dialog from .save_cover import save_cover, resize_cover +class SGDBError(Exception): + pass + + +class SGDBHelper: + base_url = "https://www.steamgriddb.com/api/v2/" + + win = None + importer = None + exception = None + + def __init__(self, win, importer=None) -> None: + self.win = win + self.importer = importer + + @property + def auth_header(self): + key = self.win.schema.get_string("sgdb-key") + headers = {"Authorization": f"Bearer {key}"} + return headers + + # TODO delegate that to the app + def create_exception_dialog(self, exception): + dialog = create_dialog( + self.win, + _("Couldn't Connect to SteamGridDB"), + exception, + "open_preferences", + _("Preferences"), + ) + dialog.connect("response", self.response) + + # TODO same as create_exception_dialog + def on_exception_dialog_response(self, _widget, response): + if response == "open_preferences": + self.win.get_application().on_preferences_action(page_name="sgdb") + + def get_game_id(self, game): + """Get grid results for a game. Can raise an exception.""" + + # Request + res = requests.get( + f"{self.base_url}search/autocomplete/{game.name}", + headers=self.auth_headers, + timeout=5, + ) + if res.status_code == 200: + return res.json()["data"][0]["id"] + + # HTTP error + res.raise_for_status() + + # SGDB API error + res_json = res.json() + if "error" in tuple(res_json): + raise SGDBError(res_json["errors"]) + else: + raise SGDBError(res.status_code) + + def get_image_uri(self, game, animated=False): + """Get the image for a game""" + uri = f"{self.base_url}grids/game/{self.get_game_id(game)}?dimensions=600x900" + if animated: + uri += "&types=animated" + grid = requests.get(uri, headers=self.auth_header, timeout=5) + image_uri = grid.json()["data"][0]["url"] + return image_uri + + +# Current steps to save image for N games +# Create a task for every game +# Call update_cover +# If using sgdb and (prefer or no image) and not blacklisted +# Search for game +# Get image from sgdb (animated if preferred and found, or still) +# Exit task and enter task_done +# If error, create popup + + class SGDBSave: def __init__(self, games, importer=None): self.win = shared.win From 9a33660f9483ebcb513b256d10376ec295eef542 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 6 May 2023 23:28:29 +0200 Subject: [PATCH 008/173] =?UTF-8?q?=F0=9F=9A=A7=20More=20work=20on=20impor?= =?UTF-8?q?ter=20and=20source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 69 +++++++------- src/importer/source.py | 9 +- src/importer/sources/lutris_source.py | 125 +++++++++++++------------- 3 files changed, 102 insertions(+), 101 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index d4ad042..f98881e 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -20,7 +20,7 @@ class Importer: def __init__(self, win) -> None: self.games = set() - self.sources = list() + self.sources = set() self.counts = dict() self.games_lock = Lock() self.progress_lock = Lock() @@ -31,9 +31,10 @@ class Importer: # Compute overall values done = 0 total = 0 - for source in self.sources: - done += self.counts[source.id]["done"] - total += self.counts[source.id]["total"] + with self.progress_lock: + for source in self.sources: + done += self.counts[source.id]["done"] + total += self.counts[source.id]["total"] # Compute progress progress = 1 if total > 0: @@ -58,59 +59,55 @@ class Importer: self.import_dialog.present() def close_dialog(self): - """Close the import dialog""" self.import_dialog.close() def update_progressbar(self): - """Update the progress bar""" - progress = self.progress() - self.progressbar.set_fraction(progress) + self.progressbar.set_fraction(self.progress) def add_source(self, source): - """Add a source to import games from""" - self.sources.append(source) + self.sources.add(source) self.counts[source.id] = {"done": 0, "total": 0} def import_games(self): - """Import games from the specified sources""" - self.create_dialog() - # Scan all sources threads = [] + + # Scan all sources for source in self.sources: - t = Thread( - None, - self.__import_from_source, - args=tuple( - source, - ), - ) + t = Thread(target=self.__import_source, args=tuple(source,)) # fmt: skip threads.append(t) t.start() - - # Wait for all of them to finish for t in threads: t.join() + # Add SGDB images + # TODO isolate SGDB in a game manager + threads.clear() + for game in self.games: + t = Thread(target=self.__add_sgdb_image, args=tuple(game,)) # fmt: skip + threads.append(t) + t.start() + for t in threads: + t.join() self.close_dialog() - def __import_from_source(self, *args, **kwargs): + def __import_source(self, *args, **kwargs): """Source import thread entry point""" source, *rest = args - iterator = source.__iter__() - for game in iterator: - self.games_lock.acquire() - self.games.add(game) - self.games_lock.release() - - # TODO SGDB image - # Who's in charge of image adding ? - - self.progress_lock.acquire() + with self.progress_lock: self.counts[source.id]["total"] = len(iterator) - if not game.blacklisted: - self.counts[source.id]["done"] += 1 + for game in iterator: + with self.games_lock: + self.games.add(game) + with self.progress_lock: + if not game.blacklisted: + self.counts[source.id]["done"] += 1 self.update_progressbar() - self.progress_lock.release() + exit(0) + + def __add_sgdb_image(self, *args, **kwargs): + """SGDB import thread entry point""" + # TODO get id, then save image + exit(0) diff --git a/src/importer/source.py b/src/importer/source.py index fd2b7c4..f69957a 100644 --- a/src/importer/source.py +++ b/src/importer/source.py @@ -6,11 +6,6 @@ from enum import IntEnum, auto class SourceIterator(Iterator): """Data producer for a source of games""" - class States(IntEnum): - DEFAULT = auto() - READY = auto() - - state = States.DEFAULT source = None def __init__(self, source) -> None: @@ -20,6 +15,10 @@ class SourceIterator(Iterator): def __iter__(self): return self + @abstractmethod + def __len__(self): + pass + @abstractmethod def __next__(self): pass diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index eed7c15..a916468 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,4 +1,4 @@ -from functools import cached_property +from functools import cached_property, cache from sqlite3 import connect from src.game import Game @@ -8,77 +8,82 @@ from src.importer.decorators import replaced_by_schema_key, replaced_by_path class LutrisSourceIterator(SourceIterator): - ignore_steam_games = False # TODO get that value - + import_steam = False db_connection = None db_cursor = None db_location = None - db_request = None + db_len_request = """ + SELECT count(*) + FROM 'games' + WHERE + name IS NOT NULL + AND slug IS NOT NULL + AND configPath IS NOT NULL + AND installed + AND (runner IS NOT "steam" OR :import_steam) + ; + """ + db_games_request = """ + SELECT id, name, slug, runner, hidden + FROM 'games' + WHERE + name IS NOT NULL + AND slug IS NOT NULL + AND configPath IS NOT NULL + AND installed + AND (runner IS NOT "steam" OR :import_steam) + ; + """ + db_request_params = None - def __init__(self, ignore_steam_games): - super().__init__() - self.ignore_steam_games = ignore_steam_games - self.db_connection = None - self.db_cursor = None + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.import_steam = self.source.win.schema.get_boolean("lutris-import-steam") self.db_location = self.source.location / "pga.db" - self.db_request = """ - SELECT - id, name, slug, runner, hidden - FROM - 'games' - WHERE - name IS NOT NULL - AND slug IS NOT NULL - AND configPath IS NOT NULL - AND installed IS TRUE - ; - """ + self.db_connection = connect(self.db_location) + self.db_request_params = {"import_steam": self.import_steam} + self.__len__() # Init iterator length + self.db_cursor = self.db_connection.execute( + self.db_games_request, self.db_request_params + ) + + @cache + def __len__(self): + cursor = self.db_connection.execute(self.db_len_request, self.db_request_params) + return cursor.fetchone()[0] def __next__(self): """Produce games. Behaviour depends on the state of the iterator.""" + # TODO decouple game creation from the window object - # Get database contents iterator - if self.state == self.States.DEFAULT: - self.db_connection = connect(self.db_location) - self.db_cursor = self.db_connection.execute(self.db_request) - self.state = self.States.READY + row = None + try: + row = self.db_cursor.__next__() + except StopIteration as e: + self.db_connection.close() + raise e - while True: - # Get next DB value - try: - row = self.db_cursor.__next__() - except StopIteration as e: - self.db_connection.close() - raise e + # Create game + row = self.__next_row() + values = { + "hidden": row[4], + "name": row[1], + "source": f"{self.source.id}_{row[3]}", + "game_id": self.source.game_id_format.format( + game_id=row[2], game_internal_id=row[0] + ), + "executable": self.source.executable_format.format(game_id=row[2]), + "developer": None, # TODO get developer metadata on Lutris + } + game = Game(self.source.win, values) - # Ignore steam games if requested - if row[3] == "steam" and self.ignore_steam_games: - continue + # Save official image + image_path = self.source.cache_location / "coverart" / f"{row[2]}.jpg" + if image_path.exists(): + resized = resize_cover(self.source.win, image_path) + save_cover(self.source.win, values["game_id"], resized) - # Build basic game - # TODO decouple game creation from the window object (later) - values = { - "hidden": row[4], - "name": row[1], - "source": f"{self.source.id}_{row[3]}", - "game_id": self.source.game_id_format.format( - game_id=row[2], game_internal_id=row[0] - ), - "executable": self.source.executable_format.format(game_id=row[2]), - "developer": None, # TODO get developer metadata on Lutris - } - game = Game(self.source.win, values) - - # Save official image - image_path = self.source.cache_location / "coverart" / f"{row[2]}.jpg" - if image_path.exists(): - resized = resize_cover(self.source.win, image_path) - save_cover(self.source.win, values["game_id"], resized) - - # TODO Save SGDB - SGDBSave(self.win, self.games, self) - - return values + return game class LutrisSource(Source): From 718b6f30639b43347790932a01ff3b38d6961ce3 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 6 May 2023 23:37:28 +0200 Subject: [PATCH 009/173] Reorganized code --- src/importer/importer.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index f98881e..be85810 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -71,9 +71,8 @@ class Importer: def import_games(self): self.create_dialog() + # Scan sources in threads threads = [] - - # Scan all sources for source in self.sources: t = Thread(target=self.__import_source, args=tuple(source,)) # fmt: skip threads.append(t) @@ -81,15 +80,6 @@ class Importer: for t in threads: t.join() - # Add SGDB images - # TODO isolate SGDB in a game manager - threads.clear() - for game in self.games: - t = Thread(target=self.__add_sgdb_image, args=tuple(game,)) # fmt: skip - threads.append(t) - t.start() - for t in threads: - t.join() self.close_dialog() def __import_source(self, *args, **kwargs): @@ -105,9 +95,5 @@ class Importer: if not game.blacklisted: self.counts[source.id]["done"] += 1 self.update_progressbar() - exit(0) - - def __add_sgdb_image(self, *args, **kwargs): - """SGDB import thread entry point""" - # TODO get id, then save image + # TODO add SGDB image (move to a game manager) exit(0) From 855ab2c5e85d5487a93705d25020c09238ca0910 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 7 May 2023 16:11:41 +0200 Subject: [PATCH 010/173] =?UTF-8?q?=F0=9F=9A=A7=20Using=20the=20new=20impo?= =?UTF-8?q?rter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/lutris_source.py | 2 ++ src/main.py | 37 +++++---------------------- src/meson.build | 8 ++---- src/utils/check_install.py | 32 ----------------------- 4 files changed, 11 insertions(+), 68 deletions(-) delete mode 100644 src/utils/check_install.py diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index a916468..c656573 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,5 +1,6 @@ from functools import cached_property, cache from sqlite3 import connect +from time import time from src.game import Game from src.utils.save_cover import resize_cover, save_cover @@ -66,6 +67,7 @@ class LutrisSourceIterator(SourceIterator): # Create game row = self.__next_row() values = { + "added": int(time()), "hidden": row[4], "name": row[1], "source": f"{self.source.id}_{row[3]}", diff --git a/src/main.py b/src/main.py index d70e77c..60445b2 100644 --- a/src/main.py +++ b/src/main.py @@ -28,15 +28,11 @@ gi.require_version("Adw", "1") from gi.repository import Adw, Gio, GLib, Gtk from . import shared -from .bottles_importer import bottles_importer from .details_window import DetailsWindow -from .heroic_importer import heroic_importer -from .importer import Importer -from .itch_importer import itch_importer -from .lutris_importer import lutris_importer from .preferences import PreferencesWindow -from .steam_importer import steam_importer from .window import CartridgesWindow +from .importer import Importer +from .lutris_source import LutrisNativeSource, LutrisFlatpakSource class CartridgesApplication(Adw.Application): @@ -151,30 +147,11 @@ class CartridgesApplication(Adw.Application): DetailsWindow() def on_import_action(self, *_args): - shared.importer = Importer() - - shared.importer.blocker = True - - if shared.schema.get_boolean("steam"): - steam_importer() - - if shared.schema.get_boolean("lutris"): - lutris_importer() - - if shared.schema.get_boolean("heroic"): - heroic_importer() - - if shared.schema.get_boolean("bottles"): - bottles_importer() - - if shared.schema.get_boolean("itch"): - itch_importer() - - shared.importer.blocker = False - - if shared.importer.import_dialog.is_visible and shared.importer.queue == 0: - shared.importer.queue = 1 - shared.importer.save_game() + importer = Importer(self.win) + if self.win.schema.get_boolean("lutris"): + importer.add_source(LutrisNativeSource) + importer.add_source(LutrisFlatpakSource) + importer.import_games() def on_remove_game_action(self, *_args): self.win.active_game.remove_game() diff --git a/src/meson.build b/src/meson.build index 87099ba..617adf3 100644 --- a/src/meson.build +++ b/src/meson.build @@ -25,12 +25,8 @@ cartridges_sources = [ 'game.py', 'game_cover.py', 'shared.py', - 'importers/steam_importer.py', - 'importers/lutris_importer.py', - 'importers/heroic_importer.py', - 'importers/bottles_importer.py', - 'importers/itch_importer.py', - 'utils/importer.py', + 'importer/sources/lutris_source.py', + 'importer/importer.py', 'utils/steamgriddb.py', 'utils/save_cover.py', 'utils/create_dialog.py', diff --git a/src/utils/check_install.py b/src/utils/check_install.py deleted file mode 100644 index 5454be3..0000000 --- a/src/utils/check_install.py +++ /dev/null @@ -1,32 +0,0 @@ -# check_install.py -# -# Copyright 2022-2023 kramo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from pathlib import Path - - -def check_install(check, locations, setting=None, subdirs=(Path(),)): - for location in locations: - for subdir in (Path(),) + subdirs: - if (location / subdir / check).is_file() or ( - location / subdir / check - ).exists(): - if setting: - setting[0].set_string(setting[1], str(location / subdir)) - - return location / subdir From f3190e50b6930332484fdd36a820a99a5d7f4541 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 7 May 2023 23:20:39 +0200 Subject: [PATCH 011/173] Towards launching --- src/importer/importer.py | 1 + src/meson.build | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index be85810..91fcec9 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -84,6 +84,7 @@ class Importer: def __import_source(self, *args, **kwargs): """Source import thread entry point""" + # TODO error handling in source iteration source, *rest = args iterator = source.__iter__() with self.progress_lock: diff --git a/src/meson.build b/src/meson.build index 617adf3..e9bb74f 100644 --- a/src/meson.build +++ b/src/meson.build @@ -29,8 +29,7 @@ cartridges_sources = [ 'importer/importer.py', 'utils/steamgriddb.py', 'utils/save_cover.py', - 'utils/create_dialog.py', - 'utils/check_install.py' + 'utils/create_dialog.py' ] install_data(cartridges_sources, install_dir: moduledir) From cb337044ac6973666a52d654436524a68621ecb2 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 8 May 2023 12:37:33 +0200 Subject: [PATCH 012/173] =?UTF-8?q?=F0=9F=90=9B=20Reintroduced=20old=20imp?= =?UTF-8?q?orters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/meson.build | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/meson.build b/src/meson.build index e9bb74f..a5895c2 100644 --- a/src/meson.build +++ b/src/meson.build @@ -25,11 +25,18 @@ cartridges_sources = [ 'game.py', 'game_cover.py', 'shared.py', - 'importer/sources/lutris_source.py', - 'importer/importer.py', 'utils/steamgriddb.py', 'utils/save_cover.py', - 'utils/create_dialog.py' + 'utils/create_dialog.py', + + 'importer/sources/lutris_source.py', + 'importer/importer.py', + + 'importers/bottles_importer.py', + 'importers/heroic_importer.py', + 'importers/itch_importer.py', + 'importers/lutris_importer.py', + 'importers/steam_importer.py' ] install_data(cartridges_sources, install_dir: moduledir) From 74afc8b6ea15864db63cf5b31f9f8fa70a75dc1b Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 8 May 2023 12:39:31 +0200 Subject: [PATCH 013/173] =?UTF-8?q?=F0=9F=90=9B=20Reintroduced=20check=5Fi?= =?UTF-8?q?nstall?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/meson.build | 3 ++- src/utils/check_install.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 src/utils/check_install.py diff --git a/src/meson.build b/src/meson.build index a5895c2..45c0a4d 100644 --- a/src/meson.build +++ b/src/meson.build @@ -36,7 +36,8 @@ cartridges_sources = [ 'importers/heroic_importer.py', 'importers/itch_importer.py', 'importers/lutris_importer.py', - 'importers/steam_importer.py' + 'importers/steam_importer.py', + 'utils/check_install.py' ] install_data(cartridges_sources, install_dir: moduledir) diff --git a/src/utils/check_install.py b/src/utils/check_install.py new file mode 100644 index 0000000..9dfe4a0 --- /dev/null +++ b/src/utils/check_install.py @@ -0,0 +1,32 @@ +# check_install.py +# +# Copyright 2022-2023 kramo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + +from pathlib import Path + + +def check_install(check, locations, setting=None, subdirs=(Path(),)): + for location in locations: + for subdir in (Path(),) + subdirs: + if (location / subdir / check).is_file() or ( + location / subdir / check + ).exists(): + if setting: + setting[0].set_string(setting[1], str(location / subdir)) + + return location / subdir From 48ca1d938f70992dee5de28e0c740e3784a3a767 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 8 May 2023 13:48:51 +0200 Subject: [PATCH 014/173] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20some=20imports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/lutris_source.py | 8 ++++---- src/meson.build | 6 ++++++ src/{importer => utils}/decorators.py | 0 3 files changed, 10 insertions(+), 4 deletions(-) rename src/{importer => utils}/decorators.py (100%) diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index c656573..89423a9 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -2,10 +2,10 @@ from functools import cached_property, cache from sqlite3 import connect from time import time -from src.game import Game -from src.utils.save_cover import resize_cover, save_cover -from src.importer.source import Source, SourceIterator -from src.importer.decorators import replaced_by_schema_key, replaced_by_path +from .game import Game +from .save_cover import resize_cover, save_cover +from .source import Source, SourceIterator +from .decorators import replaced_by_schema_key, replaced_by_path class LutrisSourceIterator(SourceIterator): diff --git a/src/meson.build b/src/meson.build index 45c0a4d..53e6e5d 100644 --- a/src/meson.build +++ b/src/meson.build @@ -16,6 +16,8 @@ configure_file( install_dir: get_option('bindir') ) +# TODO move to absolute imports = keep real structure, do not flatten + cartridges_sources = [ '__init__.py', 'main.py', @@ -29,9 +31,13 @@ cartridges_sources = [ 'utils/save_cover.py', 'utils/create_dialog.py', + # Added 'importer/sources/lutris_source.py', 'importer/importer.py', + 'importer/source.py', + 'utils/decorators.py', + # TODO remove before merge 'importers/bottles_importer.py', 'importers/heroic_importer.py', 'importers/itch_importer.py', diff --git a/src/importer/decorators.py b/src/utils/decorators.py similarity index 100% rename from src/importer/decorators.py rename to src/utils/decorators.py From 78b91c0d5292e540be9fa17a7492496746fb69db Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Tue, 9 May 2023 11:25:15 +0200 Subject: [PATCH 015/173] =?UTF-8?q?=E2=9C=A8=20Initial=20importer=20work?= =?UTF-8?q?=20done?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data/hu.kramo.Cartridges.gschema.xml | 48 +++++++++++++++------------ src/importer/importer.py | 15 ++++++--- src/importer/source.py | 11 ++++-- src/importer/sources/lutris_source.py | 25 +++++++++----- src/main.py | 4 +-- src/utils/decorators.py | 8 +++-- 6 files changed, 70 insertions(+), 41 deletions(-) diff --git a/data/hu.kramo.Cartridges.gschema.xml b/data/hu.kramo.Cartridges.gschema.xml index 15a6ab6..f4a9368 100644 --- a/data/hu.kramo.Cartridges.gschema.xml +++ b/data/hu.kramo.Cartridges.gschema.xml @@ -1,9 +1,9 @@ - - - false - + + + false + false @@ -25,6 +25,12 @@ "~/.var/app/net.lutris.Lutris/cache/lutris" + + "~/.var/app/net.lutris.Lutris/data/lutris/" + + + "~/.var/app/net.lutris.Lutris/cache/lutris" + false @@ -34,19 +40,19 @@ "~/.var/app/com.heroicgameslauncher.hgl/config/heroic/" - - true - - - true - - - true - + + true + + + true + + + true + true - + "~/.var/app/com.usebottles.bottles/data/bottles/" @@ -67,7 +73,7 @@ false - + 1110 @@ -80,13 +86,13 @@ - - - - - + + + + + "a-z" - + \ No newline at end of file diff --git a/src/importer/importer.py b/src/importer/importer.py index 91fcec9..de6256a 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -74,21 +74,30 @@ class Importer: # Scan sources in threads threads = [] for source in self.sources: - t = Thread(target=self.__import_source, args=tuple(source,)) # fmt: skip + print(f"{source.full_name}, installed: {source.is_installed}") # ! DEBUG + if not source.is_installed: + continue + t = Thread(target=self.__import_source, args=tuple([source])) # fmt: skip threads.append(t) t.start() + for t in threads: t.join() + # Save games + for game in self.games: + game.save() + self.close_dialog() def __import_source(self, *args, **kwargs): """Source import thread entry point""" # TODO error handling in source iteration + # TODO add SGDB image (move to a game manager) source, *rest = args iterator = source.__iter__() with self.progress_lock: - self.counts[source.id]["total"] = len(iterator) + self.counts[source.id]["total"] = iterator.__len__() for game in iterator: with self.games_lock: self.games.add(game) @@ -96,5 +105,3 @@ class Importer: if not game.blacklisted: self.counts[source.id]["done"] += 1 self.update_progressbar() - # TODO add SGDB image (move to a game manager) - exit(0) diff --git a/src/importer/source.py b/src/importer/source.py index f69957a..c53687d 100644 --- a/src/importer/source.py +++ b/src/importer/source.py @@ -1,9 +1,8 @@ from abc import abstractmethod -from collections.abc import Iterable, Iterator -from enum import IntEnum, auto +from collections.abc import Iterable, Iterator, Sized -class SourceIterator(Iterator): +class SourceIterator(Iterator, Sized): """Data producer for a source of games""" source = None @@ -67,6 +66,12 @@ class Source(Iterable): """The executable format used to construct game executables""" pass + @property + @abstractmethod + def is_installed(self): + """Whether the source is detected as installed""" + pass + @abstractmethod def __iter__(self): """Get the source's iterator, to use in for loops""" diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 89423a9..c59a7ad 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,4 +1,4 @@ -from functools import cached_property, cache +from functools import cache from sqlite3 import connect from time import time @@ -65,7 +65,6 @@ class LutrisSourceIterator(SourceIterator): raise e # Create game - row = self.__next_row() values = { "added": int(time()), "hidden": row[4], @@ -98,8 +97,18 @@ class LutrisSource(Source): def game_id_format(self): return super().game_id_format + "_{game_internal_id}" - def __init__(self, win): - super().__init__(win) + @property + def is_installed(self): + try: + self.location + self.cache_location + except FileNotFoundError: + return False + else: + return True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) def __iter__(self): return LutrisSourceIterator(source=self) @@ -110,13 +119,13 @@ class LutrisNativeSource(LutrisSource): variant = "native" - @cached_property + @property @replaced_by_schema_key("lutris-location") @replaced_by_path("~/.local/share/lutris/") def location(self): raise FileNotFoundError() - @cached_property + @property @replaced_by_schema_key("lutris-cache-location") @replaced_by_path("~/.local/share/lutris/covers") def cache_location(self): @@ -128,13 +137,13 @@ class LutrisFlatpakSource(LutrisSource): variant = "flatpak" - @cached_property + @property @replaced_by_schema_key("lutris-flatpak-location") @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris") def location(self): raise FileNotFoundError() - @cached_property + @property @replaced_by_schema_key("lutris-flatpak-cache-location") @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/covers") def cache_location(self): diff --git a/src/main.py b/src/main.py index 60445b2..8a23105 100644 --- a/src/main.py +++ b/src/main.py @@ -149,8 +149,8 @@ class CartridgesApplication(Adw.Application): def on_import_action(self, *_args): importer = Importer(self.win) if self.win.schema.get_boolean("lutris"): - importer.add_source(LutrisNativeSource) - importer.add_source(LutrisFlatpakSource) + importer.add_source(LutrisNativeSource(self.win)) + importer.add_source(LutrisFlatpakSource(self.win)) importer.import_games() def on_remove_game_action(self, *_args): diff --git a/src/utils/decorators.py b/src/utils/decorators.py index 75d5622..e38a017 100644 --- a/src/utils/decorators.py +++ b/src/utils/decorators.py @@ -19,7 +19,8 @@ from functools import wraps def replaced_by_path(path: PathLike): # Decorator builder - """Replace the method's returned path with the override if the override exists on disk""" + """Replace the method's returned path with the override + if the override exists on disk""" def decorator(original_function): # Built decorator (closure) @wraps(original_function) @@ -36,7 +37,8 @@ def replaced_by_path(path: PathLike): # Decorator builder def replaced_by_schema_key(key: str): # Decorator builder - """Replace the method's returned path with the path pointed by the key if it exists on disk""" + """Replace the method's returned path with the path pointed by the key + if it exists on disk""" def decorator(original_function): # Built decorator (closure) @wraps(original_function) @@ -47,7 +49,7 @@ def replaced_by_schema_key(key: str): # Decorator builder except Exception: return original_function(*args, **kwargs) else: - return replaced_by_path(override)(*args, **kwargs) + return replaced_by_path(override)(original_function)(*args, **kwargs) return wrapper From c647ca1a3133069013a98d982516f09104ab34ed Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Tue, 9 May 2023 14:45:40 +0200 Subject: [PATCH 016/173] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 13 +++++++++---- src/importer/sources/lutris_source.py | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index de6256a..2015a04 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -74,10 +74,10 @@ class Importer: # Scan sources in threads threads = [] for source in self.sources: - print(f"{source.full_name}, installed: {source.is_installed}") # ! DEBUG + print(f"{source.full_name}, installed: {source.is_installed}") if not source.is_installed: continue - t = Thread(target=self.__import_source, args=tuple([source])) # fmt: skip + t = Thread(target=self.__import_source__, args=tuple([source])) # fmt: skip threads.append(t) t.start() @@ -86,18 +86,23 @@ class Importer: # Save games for game in self.games: + if ( + game.game_id in self.win.games + and not self.win.games[game.game_id].removed + ): + continue game.save() self.close_dialog() - def __import_source(self, *args, **kwargs): + def __import_source__(self, *args, **kwargs): """Source import thread entry point""" # TODO error handling in source iteration # TODO add SGDB image (move to a game manager) source, *rest = args iterator = source.__iter__() with self.progress_lock: - self.counts[source.id]["total"] = iterator.__len__() + self.counts[source.id]["total"] = len(iterator) for game in iterator: with self.games_lock: self.games.add(game) diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index c59a7ad..44e6bea 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -67,6 +67,7 @@ class LutrisSourceIterator(SourceIterator): # Create game values = { "added": int(time()), + "last_played": 0, "hidden": row[4], "name": row[1], "source": f"{self.source.id}_{row[3]}", From 8a0951c727c1179c9de1b1c974e79809703864f3 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 10 May 2023 00:53:36 +0200 Subject: [PATCH 017/173] =?UTF-8?q?=F0=9F=8E=A8=20Sorted=20imports,=20made?= =?UTF-8?q?=20pylint=20happy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 22 +++++++++++----------- src/importer/source.py | 25 +++++++++++-------------- src/importer/sources/lutris_source.py | 8 ++++---- src/utils/decorators.py | 16 +++++++--------- 4 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 2015a04..b0d94ae 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -1,7 +1,7 @@ -from threading import Thread, Lock -from gi.repository import Adw, Gtk, Gio +from threading import Lock, Thread + +from gi.repository import Adw, Gtk -from .game import Game from .steamgriddb import SGDBHelper @@ -18,7 +18,7 @@ class Importer: games_lock = None games = None - def __init__(self, win) -> None: + def __init__(self, win): self.games = set() self.sources = set() self.counts = dict() @@ -77,12 +77,12 @@ class Importer: print(f"{source.full_name}, installed: {source.is_installed}") if not source.is_installed: continue - t = Thread(target=self.__import_source__, args=tuple([source])) # fmt: skip - threads.append(t) - t.start() + thread = Thread(target=self.__import_source__, args=tuple([source])) # fmt: skip + threads.append(thread) + thread.start() - for t in threads: - t.join() + for thread in threads: + thread.join() # Save games for game in self.games: @@ -95,11 +95,11 @@ class Importer: self.close_dialog() - def __import_source__(self, *args, **kwargs): + def __import_source__(self, *args, **_kwargs): """Source import thread entry point""" # TODO error handling in source iteration # TODO add SGDB image (move to a game manager) - source, *rest = args + source, *_rest = args iterator = source.__iter__() with self.progress_lock: self.counts[source.id]["total"] = len(iterator) diff --git a/src/importer/source.py b/src/importer/source.py index c53687d..2576034 100644 --- a/src/importer/source.py +++ b/src/importer/source.py @@ -38,41 +38,38 @@ class Source(Iterable): @property def full_name(self): """The source's full name""" - s = self.name + full_name_ = self.name if self.variant is not None: - s += f" ({self.variant})" - return s + full_name_ += f" ({self.variant})" + return full_name_ @property - def id(self): + def id(self): # pylint: disable=invalid-name """The source's identifier""" - s = self.name.lower() + id_ = self.name.lower() if self.variant is not None: - s += f"_{self.variant.lower()}" - return s + id_ += f"_{self.variant.lower()}" + return id_ @property def game_id_format(self): """The string format used to construct game IDs""" - f = self.name.lower() + format_ = self.name.lower() if self.variant is not None: - f += f"_{self.variant.lower()}" - f += "_{game_id}" - return f + format_ += f"_{self.variant.lower()}" + format_ += "_{game_id}" + return format_ @property @abstractmethod def executable_format(self): """The executable format used to construct game executables""" - pass @property @abstractmethod def is_installed(self): """Whether the source is detected as installed""" - pass @abstractmethod def __iter__(self): """Get the source's iterator, to use in for loops""" - pass diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 44e6bea..b98c603 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -2,10 +2,10 @@ from functools import cache from sqlite3 import connect from time import time +from .decorators import replaced_by_path, replaced_by_schema_key from .game import Game from .save_cover import resize_cover, save_cover from .source import Source, SourceIterator -from .decorators import replaced_by_schema_key, replaced_by_path class LutrisSourceIterator(SourceIterator): @@ -128,7 +128,7 @@ class LutrisNativeSource(LutrisSource): @property @replaced_by_schema_key("lutris-cache-location") - @replaced_by_path("~/.local/share/lutris/covers") + @replaced_by_path("~/.local/share/lutris/covers/") def cache_location(self): raise FileNotFoundError() @@ -140,12 +140,12 @@ class LutrisFlatpakSource(LutrisSource): @property @replaced_by_schema_key("lutris-flatpak-location") - @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris") + @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/") def location(self): raise FileNotFoundError() @property @replaced_by_schema_key("lutris-flatpak-cache-location") - @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/covers") + @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/covers/") def cache_location(self): raise FileNotFoundError() diff --git a/src/utils/decorators.py b/src/utils/decorators.py index e38a017..0ec1ea7 100644 --- a/src/utils/decorators.py +++ b/src/utils/decorators.py @@ -18,18 +18,17 @@ from os import PathLike from functools import wraps -def replaced_by_path(path: PathLike): # Decorator builder +def replaced_by_path(override: PathLike): # Decorator builder """Replace the method's returned path with the override if the override exists on disk""" def decorator(original_function): # Built decorator (closure) @wraps(original_function) def wrapper(*args, **kwargs): # func's override - p = Path(path).expanduser() - if p.exists(): - return p - else: - return original_function(*args, **kwargs) + path = Path(override).expanduser() + if path.exists(): + return path + return original_function(*args, **kwargs) return wrapper @@ -46,10 +45,9 @@ def replaced_by_schema_key(key: str): # Decorator builder schema = args[0].win.schema try: override = schema.get_string(key) - except Exception: + except Exception: # pylint: disable=broad-exception-caught return original_function(*args, **kwargs) - else: - return replaced_by_path(override)(original_function)(*args, **kwargs) + return replaced_by_path(override)(original_function)(*args, **kwargs) return wrapper From 771c64acac23a4e8a548aa0eb6a4bd7f19f4ba93 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 10 May 2023 02:51:28 +0200 Subject: [PATCH 018/173] =?UTF-8?q?=F0=9F=9A=A7=20Hacky,=20untested=20SGDB?= =?UTF-8?q?=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 120 ++++++++++++++++++++++++++++++++------- src/importer/source.py | 6 +- src/utils/steamgriddb.py | 22 ++++--- 3 files changed, 113 insertions(+), 35 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index b0d94ae..25e2bc8 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -1,7 +1,12 @@ +import logging +from pathlib import Path from threading import Lock, Thread -from gi.repository import Adw, Gtk +import requests +from gi.repository import Adw, Gio, Gtk + +from .save_cover import resize_cover, save_cover from .steamgriddb import SGDBHelper @@ -12,33 +17,38 @@ class Importer: import_dialog = None sources = None + source_threads = None + sgdb_threads = None progress_lock = None - counts = None - games_lock = None + sgdb_threads_lock = None + counts = None games = None def __init__(self, win): self.games = set() self.sources = set() - self.counts = dict() + self.counts = {} + self.source_threads = [] + self.sgdb_threads = [] self.games_lock = Lock() self.progress_lock = Lock() + self.sgdb_threads_lock = Lock() self.win = win @property def progress(self): # Compute overall values - done = 0 - total = 0 + overall = {"games": 0, "covers": 0, "total": 0} with self.progress_lock: for source in self.sources: - done += self.counts[source.id]["done"] - total += self.counts[source.id]["total"] + for key in overall: + overall[key] = self.counts[source.id][key] # Compute progress - progress = 1 - if total > 0: - progress = 1 - done / total + try: + progress = 1 - (overall["games"] + overall["covers"]) / overall["total"] * 2 + except ZeroDivisionError: + progress = 1 return progress def create_dialog(self): @@ -66,22 +76,21 @@ class Importer: def add_source(self, source): self.sources.add(source) - self.counts[source.id] = {"done": 0, "total": 0} + self.counts[source.id] = {"games": 0, "covers": 0, "total": 0} def import_games(self): self.create_dialog() # Scan sources in threads - threads = [] for source in self.sources: print(f"{source.full_name}, installed: {source.is_installed}") if not source.is_installed: continue thread = Thread(target=self.__import_source__, args=tuple([source])) # fmt: skip - threads.append(thread) + self.source_threads.append(thread) thread.start() - for thread in threads: + for thread in self.source_threads: thread.join() # Save games @@ -93,20 +102,89 @@ class Importer: continue game.save() - self.close_dialog() + # Wait for SGDB image import to finish + for thread in self.sgdb_threads: + thread.join() + + self.import_dialog.close() def __import_source__(self, *args, **_kwargs): """Source import thread entry point""" - # TODO error handling in source iteration - # TODO add SGDB image (move to a game manager) source, *_rest = args + + # Initialize source iteration iterator = source.__iter__() with self.progress_lock: self.counts[source.id]["total"] = len(iterator) - for game in iterator: + + # Handle iteration exceptions + def wrapper(iterator): + while True: + try: + yield next(iterator) + except StopIteration: + break + except Exception as exception: # pylint: disable=broad-exception-caught + logging.exception( + msg=f"Exception in source {iterator.source.id}", + exc_info=exception, + ) + continue + + # Get games from source + for game in wrapper(iterator): with self.games_lock: self.games.add(game) with self.progress_lock: - if not game.blacklisted: - self.counts[source.id]["done"] += 1 + self.counts[source.id]["games"] += 1 self.update_progressbar() + + # Start sgdb lookup for game + # HACK move to a game manager + sgdb_thread = Thread(target=self.__sgdb_lookup__, args=tuple([game])) + with self.sgdb_threads_lock: + self.sgdb_threads.append(sgdb_thread) + sgdb_thread.start() + + def __sgdb_lookup__(self, *args, **_kwargs): + """SGDB lookup thread entry point""" + game, *_rest = args + + def inner(): + # Skip obvious ones + if game.blacklisted: + return + use_sgdb = self.win.schema.get_boolean("sgdb") + if not use_sgdb: + return + # Check if we should query SGDB + prefer_sgdb = self.win.schema.get_boolean("sgdb-prefer") + prefer_animated = self.win.schema.get_boolean("sgdb-animated") + image_trunk = self.win.covers_dir / game.game_id + still = image_trunk.with_suffix(".tiff") + animated = image_trunk.with_suffix(".gif") + # breaking down the condition + is_missing = not still.is_file() and not animated.is_file() + is_not_best = not animated.is_file() and prefer_animated + should_query = is_missing or is_not_best or prefer_sgdb + if not should_query: + return + # Add image from sgdb + game.set_loading(1) + sgdb = SGDBHelper(self.win) + uri = sgdb.get_game_image_uri(game, animated=prefer_animated) + response = requests.get(uri, timeout=5) + tmp_file = Gio.File.new_tmp()[0] + tmp_file_path = tmp_file.get_path() + Path(tmp_file_path).write_bytes(response.content) + save_cover(self.win, game.game_id, resize_cover(self.win, tmp_file_path)) + game.set_loading(0) + + try: + inner() + except Exception: # pylint: disable=broad-exception-caught + # TODO for god's sake handle exceptions correctly + # TODO (talk about that with Kramo) + pass + with self.progress_lock: + self.counts[game.source]["covers"] += 1 diff --git a/src/importer/source.py b/src/importer/source.py index 2576034..7a959ed 100644 --- a/src/importer/source.py +++ b/src/importer/source.py @@ -16,11 +16,13 @@ class SourceIterator(Iterator, Sized): @abstractmethod def __len__(self): - pass + """Get a rough estimate of the number of games produced by the source""" @abstractmethod def __next__(self): - pass + """Get the next generated game from the source. + Raises StopIteration when exhausted. + May raise any other exception signifying an error on this specific game.""" class Source(Iterable): diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index 3a791af..460a6eb 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -13,18 +13,16 @@ class SGDBError(Exception): class SGDBHelper: + """Helper class to make queries to SteamGridDB""" + base_url = "https://www.steamgriddb.com/api/v2/" - win = None - importer = None - exception = None - def __init__(self, win, importer=None) -> None: + def __init__(self, win): self.win = win - self.importer = importer @property - def auth_header(self): + def auth_headers(self): key = self.win.schema.get_string("sgdb-key") headers = {"Authorization": f"Bearer {key}"} return headers @@ -38,7 +36,7 @@ class SGDBHelper: "open_preferences", _("Preferences"), ) - dialog.connect("response", self.response) + dialog.connect("response", self.on_exception_dialog_response) # TODO same as create_exception_dialog def on_exception_dialog_response(self, _widget, response): @@ -64,15 +62,15 @@ class SGDBHelper: res_json = res.json() if "error" in tuple(res_json): raise SGDBError(res_json["errors"]) - else: - raise SGDBError(res.status_code) + raise SGDBError(res.status_code) - def get_image_uri(self, game, animated=False): + def get_game_image_uri(self, game, animated=False): """Get the image for a game""" - uri = f"{self.base_url}grids/game/{self.get_game_id(game)}?dimensions=600x900" + game_id = self.get_game_id(game) + uri = f"{self.base_url}grids/game/{game_id}?dimensions=600x900" if animated: uri += "&types=animated" - grid = requests.get(uri, headers=self.auth_header, timeout=5) + grid = requests.get(uri, headers=self.auth_headers, timeout=5) image_uri = grid.json()["data"][0]["url"] return image_uri From aa5252d7e89d43faab3b7ad6dd55db1cfea0c7eb Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 10 May 2023 02:53:22 +0200 Subject: [PATCH 019/173] removed unnecessary whitespace --- src/importer/importer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 25e2bc8..4a6af23 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -2,7 +2,6 @@ import logging from pathlib import Path from threading import Lock, Thread - import requests from gi.repository import Adw, Gio, Gtk From b706f140e759f87ed4b9113d147b24343d4166dc Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 10 May 2023 16:18:57 +0200 Subject: [PATCH 020/173] =?UTF-8?q?=F0=9F=8E=A8=20Better=20separation=20of?= =?UTF-8?q?=20threads=20in=20importer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 166 +++++++++++++++++++++++---------------- src/utils/steamgriddb.py | 5 +- 2 files changed, 100 insertions(+), 71 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 4a6af23..479083e 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -3,10 +3,11 @@ from pathlib import Path from threading import Lock, Thread import requests +from requests import HTTPError from gi.repository import Adw, Gio, Gtk from .save_cover import resize_cover, save_cover -from .steamgriddb import SGDBHelper +from .steamgriddb import SGDBHelper, SGDBError class Importer: @@ -85,7 +86,7 @@ class Importer: print(f"{source.full_name}, installed: {source.is_installed}") if not source.is_installed: continue - thread = Thread(target=self.__import_source__, args=tuple([source])) # fmt: skip + thread = SourceImportThread(self.win, source, self) self.source_threads.append(thread) thread.start() @@ -107,83 +108,112 @@ class Importer: self.import_dialog.close() - def __import_source__(self, *args, **_kwargs): - """Source import thread entry point""" - source, *_rest = args + +class SourceImportThread(Thread): + """Thread in charge of scanning a source for games""" + + win = None + source = None + importer = None + + def __init__(self, win, source, importer, *args, **kwargs): + super().__init__(*args, **kwargs) + self.win = win + self.source = source + self.importer = importer + + def run(self): + """Thread entry point""" # Initialize source iteration - iterator = source.__iter__() - with self.progress_lock: - self.counts[source.id]["total"] = len(iterator) - - # Handle iteration exceptions - def wrapper(iterator): - while True: - try: - yield next(iterator) - except StopIteration: - break - except Exception as exception: # pylint: disable=broad-exception-caught - logging.exception( - msg=f"Exception in source {iterator.source.id}", - exc_info=exception, - ) - continue + iterator = iter(self.source) + with self.importer.progress_lock: + self.importer.counts[self.source.id]["total"] = len(iterator) # Get games from source - for game in wrapper(iterator): - with self.games_lock: - self.games.add(game) - with self.progress_lock: - self.counts[source.id]["games"] += 1 - self.update_progressbar() + while True: + # Handle exceptions raised while iteration the source + try: + game = next(iterator) + except StopIteration: + break + except Exception as exception: # pylint: disable=broad-exception-caught + logging.exception( + msg=f"Exception in source {self.source.id}", + exc_info=exception, + ) + continue - # Start sgdb lookup for game + # Add game to importer + with self.importer.games_lock: + self.importer.games.add(game) + with self.importer.progress_lock: + self.importer.counts[self.source.id]["games"] += 1 + self.importer.update_progressbar() + + # Start sgdb lookup for game in another thread # HACK move to a game manager - sgdb_thread = Thread(target=self.__sgdb_lookup__, args=tuple([game])) - with self.sgdb_threads_lock: - self.sgdb_threads.append(sgdb_thread) + # Skip obvious cases + use_sgdb = self.win.schema.get_boolean("sgdb") + if not use_sgdb or game.blacklisted: + return + sgdb_thread = SGDBLookupThread(self.win, game, self.importer) + with self.importer.sgdb_threads_lock: + self.importer.sgdb_threads.append(sgdb_thread) sgdb_thread.start() - def __sgdb_lookup__(self, *args, **_kwargs): - """SGDB lookup thread entry point""" - game, *_rest = args - def inner(): - # Skip obvious ones - if game.blacklisted: - return - use_sgdb = self.win.schema.get_boolean("sgdb") - if not use_sgdb: - return - # Check if we should query SGDB - prefer_sgdb = self.win.schema.get_boolean("sgdb-prefer") - prefer_animated = self.win.schema.get_boolean("sgdb-animated") - image_trunk = self.win.covers_dir / game.game_id - still = image_trunk.with_suffix(".tiff") - animated = image_trunk.with_suffix(".gif") - # breaking down the condition - is_missing = not still.is_file() and not animated.is_file() - is_not_best = not animated.is_file() and prefer_animated - should_query = is_missing or is_not_best or prefer_sgdb - if not should_query: - return - # Add image from sgdb - game.set_loading(1) - sgdb = SGDBHelper(self.win) - uri = sgdb.get_game_image_uri(game, animated=prefer_animated) +class SGDBLookupThread(Thread): + """Thread in charge of querying SGDB for a game image""" + + win = None + game = None + importer = None + + def __init__(self, win, game, importer, *args, **kwargs): + super().__init__(*args, **kwargs) + self.win = win + self.game = game + self.importer = importer + + def run(self): + """Thread entry point""" + + # Check if we should query SGDB + prefer_sgdb = self.win.schema.get_boolean("sgdb-prefer") + prefer_animated = self.win.schema.get_boolean("sgdb-animated") + image_trunk = self.win.covers_dir / self.game.game_id + still = image_trunk.with_suffix(".tiff") + animated = image_trunk.with_suffix(".gif") + + # Breaking down the condition + is_missing = not still.is_file() and not animated.is_file() + is_not_best = not animated.is_file() and prefer_animated + if not (is_missing or is_not_best or prefer_sgdb): + return + + self.game.set_loading(1) + + # Add image from sgdb + sgdb = SGDBHelper(self.win) + try: + sgdb_id = sgdb.get_game_id(self.game) + uri = sgdb.get_game_image_uri(sgdb_id, animated=prefer_animated) response = requests.get(uri, timeout=5) + except HTTPError as _error: + # TODO handle http errors + pass + except SGDBError as _error: + # TODO handle SGDB API errors + pass + else: tmp_file = Gio.File.new_tmp()[0] tmp_file_path = tmp_file.get_path() Path(tmp_file_path).write_bytes(response.content) - save_cover(self.win, game.game_id, resize_cover(self.win, tmp_file_path)) - game.set_loading(0) + save_cover( + self.win, self.game.game_id, resize_cover(self.win, tmp_file_path) + ) - try: - inner() - except Exception: # pylint: disable=broad-exception-caught - # TODO for god's sake handle exceptions correctly - # TODO (talk about that with Kramo) - pass - with self.progress_lock: - self.counts[game.source]["covers"] += 1 + self.game.set_loading(0) + with self.importer.progress_lock: + self.importer.counts[self.game.source.id]["covers"] += 1 diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index 460a6eb..9e4453f 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -64,9 +64,8 @@ class SGDBHelper: raise SGDBError(res_json["errors"]) raise SGDBError(res.status_code) - def get_game_image_uri(self, game, animated=False): - """Get the image for a game""" - game_id = self.get_game_id(game) + def get_image_uri(self, game_id, animated=False): + """Get the image for a SGDB game id""" uri = f"{self.base_url}grids/game/{game_id}?dimensions=600x900" if animated: uri += "&types=animated" From b2ade45d766fe1286b2b3bca398ef64973ce7eaf Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 10 May 2023 16:43:44 +0200 Subject: [PATCH 021/173] =?UTF-8?q?=F0=9F=9A=A7=20Non-blocking=20importer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 74 ++++++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 479083e..42c4946 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -11,12 +11,18 @@ from .steamgriddb import SGDBHelper, SGDBError class Importer: - win = None + """A class in charge of scanning sources for games""" + + # Dialog widget progressbar = None import_statuspage = None import_dialog = None + + # Caller-set values + win = None sources = None + # Internal values source_threads = None sgdb_threads = None progress_lock = None @@ -78,23 +84,41 @@ class Importer: self.sources.add(source) self.counts[source.id] = {"games": 0, "covers": 0, "total": 0} - def import_games(self): - self.create_dialog() + def run_in_thread(self): + thread = ImporterThread(self, self.win) + thread.start() + + +class ImporterThread(Thread): + """Thread in charge of the import process""" + + importer = None + win = None + + def __init__(self, importer, win, *args, **kwargs): + super().__init__(*args, **kwargs) + self.importer = importer + self.win = win + + def run(self): + """Thread entry point""" + + self.importer.create_dialog() # Scan sources in threads - for source in self.sources: + for source in self.importer.sources: print(f"{source.full_name}, installed: {source.is_installed}") if not source.is_installed: continue - thread = SourceImportThread(self.win, source, self) - self.source_threads.append(thread) + thread = SourceThread(source, self.win, self.importer) + self.importer.source_threads.append(thread) thread.start() - for thread in self.source_threads: + for thread in self.importer.source_threads: thread.join() # Save games - for game in self.games: + for game in self.importer.games: if ( game.game_id in self.win.games and not self.win.games[game.game_id].removed @@ -102,24 +126,24 @@ class Importer: continue game.save() - # Wait for SGDB image import to finish - for thread in self.sgdb_threads: + # Wait for SGDB query threads to finish + for thread in self.importer.sgdb_threads: thread.join() - self.import_dialog.close() + self.importer.import_dialog.close() -class SourceImportThread(Thread): +class SourceThread(Thread): """Thread in charge of scanning a source for games""" - win = None source = None + win = None importer = None - def __init__(self, win, source, importer, *args, **kwargs): + def __init__(self, source, win, importer, *args, **kwargs): super().__init__(*args, **kwargs) - self.win = win self.source = source + self.win = win self.importer = importer def run(self): @@ -153,27 +177,25 @@ class SourceImportThread(Thread): # Start sgdb lookup for game in another thread # HACK move to a game manager - # Skip obvious cases use_sgdb = self.win.schema.get_boolean("sgdb") - if not use_sgdb or game.blacklisted: - return - sgdb_thread = SGDBLookupThread(self.win, game, self.importer) - with self.importer.sgdb_threads_lock: - self.importer.sgdb_threads.append(sgdb_thread) - sgdb_thread.start() + if use_sgdb and not game.blacklisted: + sgdb_thread = SGDBThread(game, self.win, self.importer) + with self.importer.sgdb_threads_lock: + self.importer.sgdb_threads.append(sgdb_thread) + sgdb_thread.start() -class SGDBLookupThread(Thread): +class SGDBThread(Thread): """Thread in charge of querying SGDB for a game image""" - win = None game = None + win = None importer = None - def __init__(self, win, game, importer, *args, **kwargs): + def __init__(self, game, win, importer, *args, **kwargs): super().__init__(*args, **kwargs) - self.win = win self.game = game + self.win = win self.importer = importer def run(self): From 211c5d670be75ffbd62b76d6b0cd303eee0af4b5 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 10 May 2023 16:45:58 +0200 Subject: [PATCH 022/173] Follow-up: changed how importer is run --- src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.py b/src/main.py index 8a23105..22858b6 100644 --- a/src/main.py +++ b/src/main.py @@ -151,7 +151,7 @@ class CartridgesApplication(Adw.Application): if self.win.schema.get_boolean("lutris"): importer.add_source(LutrisNativeSource(self.win)) importer.add_source(LutrisFlatpakSource(self.win)) - importer.import_games() + importer.run_in_thread() def on_remove_game_action(self, *_args): self.win.active_game.remove_game() From 2b92417674eb05e19b3becaad582cbbd9571b02b Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Thu, 11 May 2023 00:53:18 +0200 Subject: [PATCH 023/173] Work on after import dialogs --- src/importer/importer.py | 109 +++++++++++++++++++++++++++++---------- src/utils/steamgriddb.py | 70 ++++++++++++------------- 2 files changed, 116 insertions(+), 63 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 42c4946..de72958 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -3,11 +3,11 @@ from pathlib import Path from threading import Lock, Thread import requests -from requests import HTTPError from gi.repository import Adw, Gio, Gtk +from .create_dialog import create_dialog from .save_cover import resize_cover, save_cover -from .steamgriddb import SGDBHelper, SGDBError +from .steamgriddb import SGDBAuthError, SGDBHelper class Importer: @@ -23,6 +23,8 @@ class Importer: sources = None # Internal values + sgdb_error = None + sgdb_error_lock = None source_threads = None sgdb_threads = None progress_lock = None @@ -40,6 +42,7 @@ class Importer: self.games_lock = Lock() self.progress_lock = Lock() self.sgdb_threads_lock = Lock() + self.sgdb_error_lock = Lock() self.win = win @property @@ -74,8 +77,51 @@ class Importer: ) self.import_dialog.present() - def close_dialog(self): - self.import_dialog.close() + def create_sgdb_error_dialog(self): + create_dialog( + self.win, + _("Couldn't Connect to SteamGridDB"), + str(self.sgdb_error), + "open_preferences", + _("Preferences"), + ).connect("response", self.on_dialog_response, "sgdb") + + def create_import_done_dialog(self): + games_no = len(self.games) + if games_no == 0: + create_dialog( + self.win, + _("No Games Found"), + _("No new games were found on your system."), + "open_preferences", + _("Preferences"), + ).connect("response", self.on_dialog_response) + elif games_no == 1: + create_dialog( + self.win, + _("Game Imported"), + _("Successfully imported 1 game."), + ).connect("response", self.on_dialog_response) + elif games_no > 1: + create_dialog( + self.win, + _("Games Imported"), + # The variable is the number of games + _("Successfully imported {} games.").format(games_no), + ).connect("response", self.on_dialog_response) + + def on_dialog_response(self, _widget, response, *args): + if response == "open_preferences": + page, expander_row, *_rest = args + self.win.get_application().on_preferences_action( + page_name=page, expander_row=expander_row + ) + # HACK SGDB manager should be in charge of its error dialog + elif self.sgdb_error is not None: + self.create_sgdb_error_dialog() + self.sgdb_error = None + # TODO additional steam libraries tip + # (should be handled by the source somehow) def update_progressbar(self): self.progressbar.set_fraction(self.progress) @@ -131,6 +177,7 @@ class ImporterThread(Thread): thread.join() self.importer.import_dialog.close() + self.importer.create_import_done_dialog() class SourceThread(Thread): @@ -177,12 +224,10 @@ class SourceThread(Thread): # Start sgdb lookup for game in another thread # HACK move to a game manager - use_sgdb = self.win.schema.get_boolean("sgdb") - if use_sgdb and not game.blacklisted: - sgdb_thread = SGDBThread(game, self.win, self.importer) - with self.importer.sgdb_threads_lock: - self.importer.sgdb_threads.append(sgdb_thread) - sgdb_thread.start() + sgdb_thread = SGDBThread(game, self.win, self.importer) + with self.importer.sgdb_threads_lock: + self.importer.sgdb_threads.append(sgdb_thread) + sgdb_thread.start() class SGDBThread(Thread): @@ -198,8 +243,14 @@ class SGDBThread(Thread): self.win = win self.importer = importer - def run(self): - """Thread entry point""" + def conditionnaly_fetch_cover(self): + use_sgdb = self.win.schema.get_boolean("sgdb") + if ( + not use_sgdb + or self.game.blacklisted + or self.importer.sgdb_error is not None + ): + return # Check if we should query SGDB prefer_sgdb = self.win.schema.get_boolean("sgdb-prefer") @@ -216,26 +267,32 @@ class SGDBThread(Thread): self.game.set_loading(1) - # Add image from sgdb + # SGDB request sgdb = SGDBHelper(self.win) try: sgdb_id = sgdb.get_game_id(self.game) uri = sgdb.get_game_image_uri(sgdb_id, animated=prefer_animated) response = requests.get(uri, timeout=5) - except HTTPError as _error: - # TODO handle http errors - pass - except SGDBError as _error: - # TODO handle SGDB API errors - pass - else: - tmp_file = Gio.File.new_tmp()[0] - tmp_file_path = tmp_file.get_path() - Path(tmp_file_path).write_bytes(response.content) - save_cover( - self.win, self.game.game_id, resize_cover(self.win, tmp_file_path) - ) + except SGDBAuthError as error: + with self.importer.sgdb_error_lock: + if self.importer.sgdb_error is None: + self.importer.sgdb_error = error + logging.error("SGDB Auth error occured") + return + except Exception as error: # pylint: disable=broad-exception-caught + logging.warning("Non auth error in SGDB query", exc_info=error) + return + + # Image saving + tmp_file = Gio.File.new_tmp()[0] + tmp_file_path = tmp_file.get_path() + Path(tmp_file_path).write_bytes(response.content) + save_cover(self.win, self.game.game_id, resize_cover(self.win, tmp_file_path)) self.game.set_loading(0) + + def run(self): + """Thread entry point""" + self.conditionnaly_fetch_cover() with self.importer.progress_lock: self.importer.counts[self.game.source.id]["covers"] += 1 diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index 9e4453f..6eb27cc 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -12,6 +12,18 @@ class SGDBError(Exception): pass +class SGDBAuthError(SGDBError): + pass + + +class SGDBGameNotFoundError(SGDBError): + pass + + +class SGDBBadRequestError(SGDBError): + pass + + class SGDBHelper: """Helper class to make queries to SteamGridDB""" @@ -27,51 +39,35 @@ class SGDBHelper: headers = {"Authorization": f"Bearer {key}"} return headers - # TODO delegate that to the app - def create_exception_dialog(self, exception): - dialog = create_dialog( - self.win, - _("Couldn't Connect to SteamGridDB"), - exception, - "open_preferences", - _("Preferences"), - ) - dialog.connect("response", self.on_exception_dialog_response) - - # TODO same as create_exception_dialog - def on_exception_dialog_response(self, _widget, response): - if response == "open_preferences": - self.win.get_application().on_preferences_action(page_name="sgdb") - def get_game_id(self, game): """Get grid results for a game. Can raise an exception.""" - - # Request - res = requests.get( - f"{self.base_url}search/autocomplete/{game.name}", - headers=self.auth_headers, - timeout=5, - ) - if res.status_code == 200: - return res.json()["data"][0]["id"] - - # HTTP error - res.raise_for_status() - - # SGDB API error - res_json = res.json() - if "error" in tuple(res_json): - raise SGDBError(res_json["errors"]) - raise SGDBError(res.status_code) + uri = f"{self.base_url}search/autocomplete/{game.name}" + res = requests.get(uri, headers=self.auth_headers, timeout=5) + match res.status_code: + case 200: + return res.json()["data"][0]["id"] + case 401: + raise SGDBAuthError(res.json()["errors"][0]) + case 404: + raise SGDBGameNotFoundError(res.status_code) + case _: + res.raise_for_status() def get_image_uri(self, game_id, animated=False): """Get the image for a SGDB game id""" uri = f"{self.base_url}grids/game/{game_id}?dimensions=600x900" if animated: uri += "&types=animated" - grid = requests.get(uri, headers=self.auth_headers, timeout=5) - image_uri = grid.json()["data"][0]["url"] - return image_uri + res = requests.get(uri, headers=self.auth_headers, timeout=5) + match res.status_code: + case 200: + return res.json()["data"][0]["url"] + case 401: + raise SGDBAuthError(res.json()["errors"][0]) + case 404: + raise SGDBGameNotFoundError(res.status_code) + case _: + res.raise_for_status() # Current steps to save image for N games From d347c30dff5a7a51691e6510aa876dc577942232 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Thu, 11 May 2023 10:04:41 +0200 Subject: [PATCH 024/173] =?UTF-8?q?=F0=9F=9A=A7=20Switched=20back=20to=20t?= =?UTF-8?q?asks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 392 +++++++++++++++++---------------------- src/main.py | 6 +- 2 files changed, 178 insertions(+), 220 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index de72958..2371c7b 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -1,6 +1,5 @@ import logging from pathlib import Path -from threading import Lock, Thread import requests from gi.repository import Adw, Gio, Gtk @@ -13,53 +12,70 @@ from .steamgriddb import SGDBAuthError, SGDBHelper class Importer: """A class in charge of scanning sources for games""" - # Dialog widget progressbar = None import_statuspage = None import_dialog = None - # Caller-set values win = None sources = None - # Internal values + n_games_added = 0 + n_source_tasks_created = 0 + n_source_tasks_done = 0 + n_sgdb_tasks_created = 0 + n_sgdb_tasks_done = 0 + sgdb_cancellable = None sgdb_error = None - sgdb_error_lock = None - source_threads = None - sgdb_threads = None - progress_lock = None - games_lock = None - sgdb_threads_lock = None - counts = None - games = None def __init__(self, win): - self.games = set() - self.sources = set() - self.counts = {} - self.source_threads = [] - self.sgdb_threads = [] - self.games_lock = Lock() - self.progress_lock = Lock() - self.sgdb_threads_lock = Lock() - self.sgdb_error_lock = Lock() self.win = win + self.sources = set() + + @property + def n_tasks_created(self): + return self.n_source_tasks_created + self.n_sgdb_tasks_created + + @property + def n_tasks_done(self): + return self.n_source_tasks_done + self.n_sgdb_tasks_done @property def progress(self): - # Compute overall values - overall = {"games": 0, "covers": 0, "total": 0} - with self.progress_lock: - for source in self.sources: - for key in overall: - overall[key] = self.counts[source.id][key] - # Compute progress try: - progress = 1 - (overall["games"] + overall["covers"]) / overall["total"] * 2 + progress = 1 - self.n_tasks_created / self.n_tasks_done except ZeroDivisionError: progress = 1 return progress + @property + def finished(self): + return self.n_sgdb_tasks_created == self.n_tasks_done + + def add_source(self, source): + self.sources.add(source) + + def run(self): + """Use several Gio.Task to import games from added sources""" + + self.create_dialog() + + # Single SGDB cancellable shared by all its tasks + # (If SGDB auth is bad, cancel all SGDB tasks) + self.sgdb_cancellable = Gio.Cancellable() + + # Create a task for each source + tasks = set() + for source in self.sources: + self.n_source_tasks_created += 1 + logging.debug("Importing games from source %s", source.id) + task = Gio.Task(None, None, self.source_task_callback, (source,)) + task.set_task_data((source,)) + tasks.add(task) + + # Start all tasks + for task in tasks: + task.run_in_thread(self.source_task_thread_func) + def create_dialog(self): """Create the import dialog""" self.progressbar = Gtk.ProgressBar(margin_start=12, margin_end=12) @@ -77,40 +93,143 @@ class Importer: ) self.import_dialog.present() - def create_sgdb_error_dialog(self): - create_dialog( - self.win, - _("Couldn't Connect to SteamGridDB"), - str(self.sgdb_error), - "open_preferences", - _("Preferences"), - ).connect("response", self.on_dialog_response, "sgdb") + def update_progressbar(self): + self.progressbar.set_fraction(self.progress) + + def source_task_thread_func(self, _task, _obj, data, _cancellable): + """Source import task code""" + + source, *_rest = data + + # Early exit if not installed + if not source.is_installed: + logging.info("Source %s skipped, not installed", source.id) + return + + # Initialize source iteration + iterator = iter(source) + + # Get games from source + while True: + # Handle exceptions raised when iterating + try: + game = next(iterator) + except StopIteration: + break + except Exception as exception: # pylint: disable=broad-exception-caught + logging.exception( + msg=f"Exception in source {source.id}", + exc_info=exception, + ) + continue + + # TODO make sources return games AND avoid duplicates + game_id = game.game_id + if game.game_id in self.win.games and not self.win.games[game_id].removed: + continue + game.save() + self.n_games_added += 1 + + # Start sgdb lookup for game + # HACK move to its own manager + task = Gio.Task( + None, self.sgdb_cancellable, self.sgdb_task_callback, (game,) + ) + task.set_task_data((game,)) + task.run_in_thread(self.sgdb_task_thread_func) + + def source_task_callback(self, _obj, _result, data): + """Source import callback""" + _source, *_rest = data + self.n_source_tasks_done += 1 + self.update_progressbar() + if self.finished: + self.import_callback() + + def sgdb_task_thread_func(self, _task, _obj, data, cancellable): + """SGDB query code""" + + game, *_rest = data + + use_sgdb = self.win.schema.get_boolean("sgdb") + if not use_sgdb or game.blacklisted: + return + + # Check if we should query SGDB + prefer_sgdb = self.win.schema.get_boolean("sgdb-prefer") + prefer_animated = self.win.schema.get_boolean("sgdb-animated") + image_trunk = self.win.covers_dir / game.game_id + still = image_trunk.with_suffix(".tiff") + animated = image_trunk.with_suffix(".gif") + + # Breaking down the condition + is_missing = not still.is_file() and not animated.is_file() + is_not_best = not animated.is_file() and prefer_animated + if not (is_missing or is_not_best or prefer_sgdb): + return + + game.set_loading(1) + + # SGDB request + sgdb = SGDBHelper(self.win) + try: + sgdb_id = sgdb.get_game_id(game) + uri = sgdb.get_game_image_uri(sgdb_id, animated=prefer_animated) + response = requests.get(uri, timeout=5) + except SGDBAuthError as error: + # On auth error, cancel all present and future SGDB tasks for this import + self.sgdb_error = error + logging.error("SGDB Auth error occured", exc_info=error) + cancellable.cancel() + return + except Exception as error: # pylint: disable=broad-exception-caught + logging.warning("Non auth error in SGDB query", exc_info=error) + return + + # Image saving + tmp_file = Gio.File.new_tmp()[0] + tmp_file_path = tmp_file.get_path() + Path(tmp_file_path).write_bytes(response.content) + save_cover(self.win, game.game_id, resize_cover(self.win, tmp_file_path)) + + def sgdb_task_callback(self, _obj, _result, data): + """SGDB query callback""" + game, *_rest = data + game.set_loading(0) + self.n_sgdb_tasks_done += 1 + self.update_progressbar() + if self.finished: + self.import_callback() + + def import_callback(self): + """Callback called when importing has finished""" + self.import_dialog.close() + self.create_import_done_dialog() def create_import_done_dialog(self): - games_no = len(self.games) - if games_no == 0: + if self.n_games_added == 0: create_dialog( self.win, _("No Games Found"), _("No new games were found on your system."), "open_preferences", _("Preferences"), - ).connect("response", self.on_dialog_response) - elif games_no == 1: + ).connect("response", self.dialog_response_callback) + elif self.n_games_added == 1: create_dialog( self.win, _("Game Imported"), _("Successfully imported 1 game."), - ).connect("response", self.on_dialog_response) - elif games_no > 1: + ).connect("response", self.dialog_response_callback) + elif self.n_games_added > 1: create_dialog( self.win, _("Games Imported"), # The variable is the number of games - _("Successfully imported {} games.").format(games_no), - ).connect("response", self.on_dialog_response) + _("Successfully imported {} games.").format(self.n_games_added), + ).connect("response", self.dialog_response_callback) - def on_dialog_response(self, _widget, response, *args): + def dialog_response_callback(self, _widget, response, *args): if response == "open_preferences": page, expander_row, *_rest = args self.win.get_application().on_preferences_action( @@ -123,176 +242,11 @@ class Importer: # TODO additional steam libraries tip # (should be handled by the source somehow) - def update_progressbar(self): - self.progressbar.set_fraction(self.progress) - - def add_source(self, source): - self.sources.add(source) - self.counts[source.id] = {"games": 0, "covers": 0, "total": 0} - - def run_in_thread(self): - thread = ImporterThread(self, self.win) - thread.start() - - -class ImporterThread(Thread): - """Thread in charge of the import process""" - - importer = None - win = None - - def __init__(self, importer, win, *args, **kwargs): - super().__init__(*args, **kwargs) - self.importer = importer - self.win = win - - def run(self): - """Thread entry point""" - - self.importer.create_dialog() - - # Scan sources in threads - for source in self.importer.sources: - print(f"{source.full_name}, installed: {source.is_installed}") - if not source.is_installed: - continue - thread = SourceThread(source, self.win, self.importer) - self.importer.source_threads.append(thread) - thread.start() - - for thread in self.importer.source_threads: - thread.join() - - # Save games - for game in self.importer.games: - if ( - game.game_id in self.win.games - and not self.win.games[game.game_id].removed - ): - continue - game.save() - - # Wait for SGDB query threads to finish - for thread in self.importer.sgdb_threads: - thread.join() - - self.importer.import_dialog.close() - self.importer.create_import_done_dialog() - - -class SourceThread(Thread): - """Thread in charge of scanning a source for games""" - - source = None - win = None - importer = None - - def __init__(self, source, win, importer, *args, **kwargs): - super().__init__(*args, **kwargs) - self.source = source - self.win = win - self.importer = importer - - def run(self): - """Thread entry point""" - - # Initialize source iteration - iterator = iter(self.source) - with self.importer.progress_lock: - self.importer.counts[self.source.id]["total"] = len(iterator) - - # Get games from source - while True: - # Handle exceptions raised while iteration the source - try: - game = next(iterator) - except StopIteration: - break - except Exception as exception: # pylint: disable=broad-exception-caught - logging.exception( - msg=f"Exception in source {self.source.id}", - exc_info=exception, - ) - continue - - # Add game to importer - with self.importer.games_lock: - self.importer.games.add(game) - with self.importer.progress_lock: - self.importer.counts[self.source.id]["games"] += 1 - self.importer.update_progressbar() - - # Start sgdb lookup for game in another thread - # HACK move to a game manager - sgdb_thread = SGDBThread(game, self.win, self.importer) - with self.importer.sgdb_threads_lock: - self.importer.sgdb_threads.append(sgdb_thread) - sgdb_thread.start() - - -class SGDBThread(Thread): - """Thread in charge of querying SGDB for a game image""" - - game = None - win = None - importer = None - - def __init__(self, game, win, importer, *args, **kwargs): - super().__init__(*args, **kwargs) - self.game = game - self.win = win - self.importer = importer - - def conditionnaly_fetch_cover(self): - use_sgdb = self.win.schema.get_boolean("sgdb") - if ( - not use_sgdb - or self.game.blacklisted - or self.importer.sgdb_error is not None - ): - return - - # Check if we should query SGDB - prefer_sgdb = self.win.schema.get_boolean("sgdb-prefer") - prefer_animated = self.win.schema.get_boolean("sgdb-animated") - image_trunk = self.win.covers_dir / self.game.game_id - still = image_trunk.with_suffix(".tiff") - animated = image_trunk.with_suffix(".gif") - - # Breaking down the condition - is_missing = not still.is_file() and not animated.is_file() - is_not_best = not animated.is_file() and prefer_animated - if not (is_missing or is_not_best or prefer_sgdb): - return - - self.game.set_loading(1) - - # SGDB request - sgdb = SGDBHelper(self.win) - try: - sgdb_id = sgdb.get_game_id(self.game) - uri = sgdb.get_game_image_uri(sgdb_id, animated=prefer_animated) - response = requests.get(uri, timeout=5) - except SGDBAuthError as error: - with self.importer.sgdb_error_lock: - if self.importer.sgdb_error is None: - self.importer.sgdb_error = error - logging.error("SGDB Auth error occured") - return - except Exception as error: # pylint: disable=broad-exception-caught - logging.warning("Non auth error in SGDB query", exc_info=error) - return - - # Image saving - tmp_file = Gio.File.new_tmp()[0] - tmp_file_path = tmp_file.get_path() - Path(tmp_file_path).write_bytes(response.content) - save_cover(self.win, self.game.game_id, resize_cover(self.win, tmp_file_path)) - - self.game.set_loading(0) - - def run(self): - """Thread entry point""" - self.conditionnaly_fetch_cover() - with self.importer.progress_lock: - self.importer.counts[self.game.source.id]["covers"] += 1 + def create_sgdb_error_dialog(self): + create_dialog( + self.win, + _("Couldn't Connect to SteamGridDB"), + str(self.sgdb_error), + "open_preferences", + _("Preferences"), + ).connect("response", self.dialog_response_callback, "sgdb") diff --git a/src/main.py b/src/main.py index 22858b6..c45fbda 100644 --- a/src/main.py +++ b/src/main.py @@ -18,6 +18,8 @@ # SPDX-License-Identifier: GPL-3.0-or-later import sys +import os +import logging import gi @@ -151,7 +153,7 @@ class CartridgesApplication(Adw.Application): if self.win.schema.get_boolean("lutris"): importer.add_source(LutrisNativeSource(self.win)) importer.add_source(LutrisFlatpakSource(self.win)) - importer.run_in_thread() + importer.run() def on_remove_game_action(self, *_args): self.win.active_game.remove_game() @@ -198,5 +200,7 @@ class CartridgesApplication(Adw.Application): def main(version): # pylint: disable=unused-argument + log_level = os.environ.get("LOGLEVEL", "ERROR").upper() + logging.basicConfig(level="DEBUG") # TODO remove debug app = CartridgesApplication() return app.run(sys.argv) From e93c93e802b4ed1295e66f25aa388281673a2ce2 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Thu, 11 May 2023 13:08:25 +0200 Subject: [PATCH 025/173] Add version to game_id.json --- docs/game_id.json.md | 116 +++++++++++++++++++++++++++++++++++++++++++ src/window.py | 6 +++ 2 files changed, 122 insertions(+) create mode 100644 docs/game_id.json.md diff --git a/docs/game_id.json.md b/docs/game_id.json.md new file mode 100644 index 0000000..e768c48 --- /dev/null +++ b/docs/game_id.json.md @@ -0,0 +1,116 @@ +# [game_id].json specification +#### Version 2.0 + +Games are saved to disk in the form of [game_id].json files. These files contain all information about a game excluding its cover, which is handled separately. + +## Location + +The standard location for these files is `/cartridges/games/` under the data directory of the user (`XDG_DATA_HOME` on Linux). + +## Contents + +The following attributes are saved: + +- [added](#added) +- [executable](#executable) +- [game_id](#game_id) +- [source](#source) +- [hidden](#hidden) +- [last_played](#last_played) +- [name](#name) +- [developer](#developer) +- [removed](#removed) +- [blacklisted](#blacklisted) +- [version](#version) + +### added + +The date at which the game was added. + +Cartridges will set the value for itself. Don't touch it. + +Stored as a Unix time stamp. + +### executable + +The executable to run when launching a game. + +If the source has a URL handler, using that is preferred. In that case, the value should be `["xdg-open", "url://example/url"]` for Linux and `["start", "url://example/url"]` for Windows. + +Stored as an argument vector to be passed to the shell through [GLib.spawn_async](https://docs.gtk.org/glib/func.spawn_async.html). + +### game_id + +The unique ID of the game, prefixed with [`[source]_`](#source) to avoid clashes. + +If the game's source uses a consistent internal ID system, use the ID from there. If not, use a hash function that always returns the same hash for the same game, even if some of its attributes change inside of the source. + +Stored as a string. + +### source + +A unique ID for the source of the game in lowercase, without spaces. + +If a source provides multiple internal sources, these should be separately labeled, but share a common prefix. eg. `heoic_gog`, `heroic_epic`. + +Stored as a string. + +### hidden + +Whether or not a game is hidden. + +If the source provides a way of hiding games, take the value from there. Otherwise it should be set to false by default. + +Stored as a boolean. + +### last_played + +The date at which the game was last launched from Cartridges. + +Cartridges will set the value for itself. Don't touch it. + +Stored as a Unix time stamp. 0 if the game hasn't been played yet. + +### name + +The title of the game. + +Stored as a string. + +### developer + +The developer or publisher of the game. + +If there are multiple developers or publishers, they should be joined with a comma and a space (`, `) into one string. + +This is an optional attribute. If it can't be retrieved from the source, don't touch it. + +Stored as a string. + +### removed + +Whether or not a game has been removed. + +Cartridges will set the value for itself. Don't touch it. + +Stored as a boolean. + +### blacklisted + +Whether or not a game is blacklisted. Blacklisting a game means it is going to still be imported, but not displayed to the user. + +You should only blacklist a game based on information you pull from the web. This is to ensure that games which you would skip based on information online are still skipped even if the user loses their internet connection. If an entry is broken locally, just skip it. + +The only reason to blacklist a game is if you find out that the locally cached entry is not actually a game (eg. Proton) or is otherwise invalid. + +Unless the above criteria is met, don't touch the attribute. + +Stored as a boolean. + +### version + +The version number of the [game_id].json specification. + +Cartridges will set the value for itself. Don't touch it. + +Stored as a number. \ No newline at end of file diff --git a/src/window.py b/src/window.py index 3c99464..3d8855a 100644 --- a/src/window.py +++ b/src/window.py @@ -74,6 +74,9 @@ class CartridgesWindow(Adw.ApplicationWindow): details_view_game_cover = None sort_state = "a-z" + # The version of the game_id.json spec + spec_version = 2.0 + def __init__(self, **kwargs): super().__init__(**kwargs) @@ -111,6 +114,9 @@ class CartridgesWindow(Adw.ApplicationWindow): ): path.unlink(missing_ok=True) + elif game.get("version") > self.spec_version: + continue + else: Game(game).update() From 4553ab97e06878eefaa9ddb71c9f0a749c8c7287 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Thu, 11 May 2023 13:15:12 +0200 Subject: [PATCH 026/173] Fix version check logic --- src/window.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/window.py b/src/window.py index 3d8855a..45f618d 100644 --- a/src/window.py +++ b/src/window.py @@ -113,10 +113,6 @@ class CartridgesWindow(Adw.ApplicationWindow): shared.covers_dir / f"{game_id}.gif", ): path.unlink(missing_ok=True) - - elif game.get("version") > self.spec_version: - continue - else: Game(game).update() From 42e3b45ec639acf5fe76ed34ae76bf0e2a286fb0 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Thu, 11 May 2023 14:44:34 +0200 Subject: [PATCH 027/173] Make Game handle last_played --- src/game.py | 2 +- src/importer/sources/lutris_source.py | 1 - src/importers/bottles_importer.py | 1 - src/importers/heroic_importer.py | 3 --- src/importers/itch_importer.py | 1 - src/importers/lutris_importer.py | 1 - src/importers/steam_importer.py | 1 - src/window.py | 2 +- 8 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/game.py b/src/game.py index 8067e95..e46f81a 100644 --- a/src/game.py +++ b/src/game.py @@ -53,7 +53,7 @@ class Game(Gtk.Box): game_id = None source = None hidden = None - last_played = None + last_played = 0 name = None developer = None removed = None diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index b98c603..5a707ce 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -67,7 +67,6 @@ class LutrisSourceIterator(SourceIterator): # Create game values = { "added": int(time()), - "last_played": 0, "hidden": row[4], "name": row[1], "source": f"{self.source.id}_{row[3]}", diff --git a/src/importers/bottles_importer.py b/src/importers/bottles_importer.py index 463e192..707601e 100644 --- a/src/importers/bottles_importer.py +++ b/src/importers/bottles_importer.py @@ -84,7 +84,6 @@ def bottles_importer(): values["hidden"] = False values["source"] = "bottles" values["added"] = current_time - values["last_played"] = 0 importer.save_game( values, diff --git a/src/importers/heroic_importer.py b/src/importers/heroic_importer.py index 848f73a..e3fd503 100644 --- a/src/importers/heroic_importer.py +++ b/src/importers/heroic_importer.py @@ -98,7 +98,6 @@ def heroic_importer(): values["hidden"] = False values["source"] = "heroic_epic" values["added"] = current_time - values["last_played"] = 0 image_path = ( heroic_dir @@ -159,7 +158,6 @@ def heroic_importer(): values["hidden"] = False values["source"] = "heroic_gog" values["added"] = current_time - values["last_played"] = 0 importer.save_game(values, image_path if image_path.exists() else None) @@ -194,7 +192,6 @@ def heroic_importer(): values["hidden"] = False values["source"] = "heroic_sideload" values["added"] = current_time - values["last_played"] = 0 image_path = ( heroic_dir / "images-cache" diff --git a/src/importers/itch_importer.py b/src/importers/itch_importer.py index 26b1573..728670f 100644 --- a/src/importers/itch_importer.py +++ b/src/importers/itch_importer.py @@ -50,7 +50,6 @@ def get_game(task, current_time, row): else ["xdg-open", f"itch://caves/{row[4]}/launch"] ) values["hidden"] = False - values["last_played"] = 0 values["name"] = row[1] values["source"] = "itch" diff --git a/src/importers/lutris_importer.py b/src/importers/lutris_importer.py index ee2552b..40d038d 100644 --- a/src/importers/lutris_importer.py +++ b/src/importers/lutris_importer.py @@ -132,7 +132,6 @@ def lutris_importer(): values["added"] = current_time values["executable"] = ["xdg-open", f"lutris:rungameid/{row[0]}"] values["hidden"] = row[4] == 1 - values["last_played"] = 0 values["name"] = row[1] values["source"] = f"lutris_{row[3]}" diff --git a/src/importers/steam_importer.py b/src/importers/steam_importer.py index df50d5d..10b9540 100644 --- a/src/importers/steam_importer.py +++ b/src/importers/steam_importer.py @@ -72,7 +72,6 @@ def get_game(task, datatypes, current_time, appmanifest, steam_dir): values["hidden"] = False values["source"] = "steam" values["added"] = current_time - values["last_played"] = 0 image_path = ( steam_dir diff --git a/src/window.py b/src/window.py index 45f618d..ba98c34 100644 --- a/src/window.py +++ b/src/window.py @@ -230,7 +230,7 @@ class CartridgesWindow(Adw.ApplicationWindow): _("Added: {}").format(date) ) last_played_date = ( - self.get_time(game.last_played) if game.last_played != 0 else _("Never") + self.get_time(game.last_played) if game.last_played else _("Never") ) self.details_view_last_played.set_label( # The variable is the date when the game was last played From ee80c2c552211d52c6bf9f25e65e203fdd01a9b6 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Thu, 11 May 2023 15:45:54 +0200 Subject: [PATCH 028/173] Remove last_played from details_window --- src/details_window.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/details_window.py b/src/details_window.py index 57ad67e..23d6306 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -164,7 +164,6 @@ class DetailsWindow(Adw.Window): "hidden": False, "source": "imported", "added": int(time()), - "last_played": 0, }, ) From 0c6c0ea4674840ed7f5cb63d12388b0931f0d546 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Thu, 18 May 2023 16:18:31 +0200 Subject: [PATCH 029/173] Work on importer / SGDB integration --- src/game.py | 6 ++-- src/importer/importer.py | 59 +++++++++-------------------------- src/utils/steamgriddb.py | 66 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 48 deletions(-) diff --git a/src/game.py b/src/game.py index e46f81a..c568c2e 100644 --- a/src/game.py +++ b/src/game.py @@ -60,7 +60,7 @@ class Game(Gtk.Box): blacklisted = None game_cover = None - def __init__(self, data, **kwargs): + def __init__(self, win, data, allow_side_effects=False, **kwargs): super().__init__(**kwargs) self.win = shared.win @@ -69,7 +69,8 @@ class Game(Gtk.Box): self.update_values(data) - self.win.games[self.game_id] = self + if allow_side_effects: + self.win.games[self.game_id] = self self.set_play_icon() @@ -77,7 +78,6 @@ class Game(Gtk.Box): self.add_controller(self.event_contoller_motion) self.event_contoller_motion.connect("enter", self.toggle_play, False) self.event_contoller_motion.connect("leave", self.toggle_play, None, None) - self.cover_button.connect("clicked", self.main_button_clicked, False) self.play_button.connect("clicked", self.main_button_clicked, True) diff --git a/src/importer/importer.py b/src/importer/importer.py index 2371c7b..7868cdc 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -1,12 +1,10 @@ import logging -from pathlib import Path -import requests +from requests import HTTPError from gi.repository import Adw, Gio, Gtk from .create_dialog import create_dialog -from .save_cover import resize_cover, save_cover -from .steamgriddb import SGDBAuthError, SGDBHelper +from .steamgriddb import SGDBAuthError, SGDBError, SGDBHelper class Importer: @@ -25,11 +23,12 @@ class Importer: n_sgdb_tasks_created = 0 n_sgdb_tasks_done = 0 sgdb_cancellable = None - sgdb_error = None + errors = None def __init__(self, win): self.win = win self.sources = set() + self.errors = [] @property def n_tasks_created(self): @@ -123,10 +122,13 @@ class Importer: ) continue - # TODO make sources return games AND avoid duplicates - game_id = game.game_id - if game.game_id in self.win.games and not self.win.games[game_id].removed: + # Avoid duplicates + gid = game.game_id + if gid in self.win.games and not self.win.games[gid].removed: continue + + # Register game + self.win.games[gid] = game game.save() self.n_games_added += 1 @@ -148,49 +150,16 @@ class Importer: def sgdb_task_thread_func(self, _task, _obj, data, cancellable): """SGDB query code""" - game, *_rest = data - - use_sgdb = self.win.schema.get_boolean("sgdb") - if not use_sgdb or game.blacklisted: - return - - # Check if we should query SGDB - prefer_sgdb = self.win.schema.get_boolean("sgdb-prefer") - prefer_animated = self.win.schema.get_boolean("sgdb-animated") - image_trunk = self.win.covers_dir / game.game_id - still = image_trunk.with_suffix(".tiff") - animated = image_trunk.with_suffix(".gif") - - # Breaking down the condition - is_missing = not still.is_file() and not animated.is_file() - is_not_best = not animated.is_file() and prefer_animated - if not (is_missing or is_not_best or prefer_sgdb): - return - game.set_loading(1) - - # SGDB request sgdb = SGDBHelper(self.win) try: - sgdb_id = sgdb.get_game_id(game) - uri = sgdb.get_game_image_uri(sgdb_id, animated=prefer_animated) - response = requests.get(uri, timeout=5) + sgdb.conditionaly_update_cover(game) except SGDBAuthError as error: - # On auth error, cancel all present and future SGDB tasks for this import - self.sgdb_error = error - logging.error("SGDB Auth error occured", exc_info=error) cancellable.cancel() - return - except Exception as error: # pylint: disable=broad-exception-caught - logging.warning("Non auth error in SGDB query", exc_info=error) - return - - # Image saving - tmp_file = Gio.File.new_tmp()[0] - tmp_file_path = tmp_file.get_path() - Path(tmp_file_path).write_bytes(response.content) - save_cover(self.win, game.game_id, resize_cover(self.win, tmp_file_path)) + self.errors.append(error) + except (HTTPError, SGDBError) as error: + self.errors.append(error) def sgdb_task_callback(self, _obj, _result, data): """SGDB query callback""" diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index 6eb27cc..2a71f9b 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -1,6 +1,8 @@ from pathlib import Path +import logging import requests +from requests import HTTPError from gi.repository import Gio from . import shared @@ -24,6 +26,10 @@ class SGDBBadRequestError(SGDBError): pass +class SGDBNoImageFoundError(SGDBError): + pass + + class SGDBHelper: """Helper class to make queries to SteamGridDB""" @@ -69,6 +75,66 @@ class SGDBHelper: case _: res.raise_for_status() + def conditionaly_update_cover(self, game): + """Update the game's cover if appropriate""" + + # Obvious skips + use_sgdb = self.win.schema.get_boolean("sgdb") + if not use_sgdb or game.blacklisted: + return + + image_trunk = self.win.covers_dir / game.game_id + still = image_trunk.with_suffix(".tiff") + uri_kwargs = image_trunk.with_suffix(".gif") + prefer_sgdb = self.win.schema.get_boolean("sgdb-prefer") + + # Do nothing if file present and not prefer SGDB + if not prefer_sgdb and (still.is_file() or uri_kwargs.is_file()): + return + + # Get ID for the game + try: + sgdb_id = self.get_game_id(game) + except (HTTPError, SGDBError) as error: + logging.warning( + "Error while getting SGDB ID for %s", game.name, exc_info=error + ) + raise error + + # Build different SGDB options to try + image_uri_kwargs_sets = [{"animated": False}] + if self.win.schema.get_boolean("sgdb-animated"): + image_uri_kwargs_sets.insert(0, {"animated": True}) + + # Download covers + for uri_kwargs in image_uri_kwargs_sets: + try: + uri = self.get_game_image_uri(sgdb_id, **uri_kwargs) + response = requests.get(uri, timeout=5) + tmp_file = Gio.File.new_tmp()[0] + tmp_file_path = tmp_file.get_path() + Path(tmp_file_path).write_bytes(response.content) + save_cover( + self.win, game.game_id, resize_cover(self.win, tmp_file_path) + ) + except SGDBAuthError as error: + # Let caller handle auth errors + raise error + except (HTTPError, SGDBError) as error: + logging.warning("Error while getting image", exc_info=error) + continue + else: + # Stop as soon as one is finished + return + + # No image was added + logging.warning( + 'No matching image found for game "%s" (SGDB ID %d)', + game.name, + sgdb_id, + ) + raise SGDBNoImageFoundError() + # Current steps to save image for N games # Create a task for every game From 538b00bba5427344887dafca3605aabd34d60fef Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Thu, 18 May 2023 16:37:53 +0200 Subject: [PATCH 030/173] fixes --- src/importer/importer.py | 81 ++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 7868cdc..2646bf8 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -23,12 +23,11 @@ class Importer: n_sgdb_tasks_created = 0 n_sgdb_tasks_done = 0 sgdb_cancellable = None - errors = None + sgdb_error = None def __init__(self, win): self.win = win self.sources = set() - self.errors = [] @property def n_tasks_created(self): @@ -157,9 +156,10 @@ class Importer: sgdb.conditionaly_update_cover(game) except SGDBAuthError as error: cancellable.cancel() - self.errors.append(error) + self.sgdb_error = error except (HTTPError, SGDBError) as error: - self.errors.append(error) + # TODO handle other SGDB errors + pass def sgdb_task_callback(self, _obj, _result, data): """SGDB query callback""" @@ -173,45 +173,37 @@ class Importer: def import_callback(self): """Callback called when importing has finished""" self.import_dialog.close() - self.create_import_done_dialog() - - def create_import_done_dialog(self): - if self.n_games_added == 0: - create_dialog( - self.win, - _("No Games Found"), - _("No new games were found on your system."), - "open_preferences", - _("Preferences"), - ).connect("response", self.dialog_response_callback) - elif self.n_games_added == 1: - create_dialog( - self.win, - _("Game Imported"), - _("Successfully imported 1 game."), - ).connect("response", self.dialog_response_callback) - elif self.n_games_added > 1: - create_dialog( - self.win, - _("Games Imported"), - # The variable is the number of games - _("Successfully imported {} games.").format(self.n_games_added), - ).connect("response", self.dialog_response_callback) - - def dialog_response_callback(self, _widget, response, *args): - if response == "open_preferences": - page, expander_row, *_rest = args - self.win.get_application().on_preferences_action( - page_name=page, expander_row=expander_row - ) - # HACK SGDB manager should be in charge of its error dialog - elif self.sgdb_error is not None: + self.create_summary_toast() + if self.sgdb_error is not None: self.create_sgdb_error_dialog() - self.sgdb_error = None - # TODO additional steam libraries tip - # (should be handled by the source somehow) + + def create_summary_toast(self): + """N games imported toast""" + + toast = Adw.Toast() + toast.set_priority(Adw.ToastPriority.HIGH) + + if self.n_games_added == 0: + toast.set_title(_("No new games found")) + toast.set_button_label(_("Preferences")) + toast.connect( + "button-clicked", + self.dialog_response_callback, + "open_preferences", + "import", + ) + + elif self.n_games_added == 1: + toast.set_title(_("1 game imported")) + + elif self.n_games_added > 1: + # The variable is the number of games + toast.set_title(_("{} games imported").format(self.n_games_added)) + + self.win.toast_overlay.add_toast(toast) def create_sgdb_error_dialog(self): + """SGDB error dialog""" create_dialog( self.win, _("Couldn't Connect to SteamGridDB"), @@ -219,3 +211,12 @@ class Importer: "open_preferences", _("Preferences"), ).connect("response", self.dialog_response_callback, "sgdb") + + def dialog_response_callback(self, _widget, response, *args): + """Handle after-import dialogs callback""" + if response == "open_preferences": + page, expander_row, *_rest = args + self.win.get_application().on_preferences_action( + page_name=page, expander_row=expander_row + ) + # TODO handle steam libraries tip From d981f050951d2a23ec9f5e339a95368746895346 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Thu, 18 May 2023 18:56:23 +0200 Subject: [PATCH 031/173] =?UTF-8?q?=F0=9F=9A=A7=20using=20closures=20to=20?= =?UTF-8?q?pass=20data=20to=20Tasks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 2646bf8..bd44c7c 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -66,13 +66,14 @@ class Importer: for source in self.sources: self.n_source_tasks_created += 1 logging.debug("Importing games from source %s", source.id) - task = Gio.Task(None, None, self.source_task_callback, (source,)) - task.set_task_data((source,)) - tasks.add(task) - # Start all tasks - for task in tasks: - task.run_in_thread(self.source_task_thread_func) + def closure(task, obj, _data, cancellable): + self.source_task_thread_func(task, obj, (source,), cancellable) + + task = Gio.Task.new(None, None, self.source_task_callback, (source,)) + self.n_sgdb_tasks_created += 1 + task.run_in_thread(closure) + tasks.add(task) def create_dialog(self): """Create the import dialog""" @@ -133,11 +134,14 @@ class Importer: # Start sgdb lookup for game # HACK move to its own manager - task = Gio.Task( + + def closure(task, obj, _data, cancellable): + self.sgdb_task_thread_func(task, obj, (game,), cancellable) + + task = Gio.Task.new( None, self.sgdb_cancellable, self.sgdb_task_callback, (game,) ) - task.set_task_data((game,)) - task.run_in_thread(self.sgdb_task_thread_func) + task.run_in_thread(closure) def source_task_callback(self, _obj, _result, data): """Source import callback""" From 2abf671ea446093825a7678539f696e83e99221f Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Thu, 18 May 2023 23:42:58 +0200 Subject: [PATCH 032/173] =?UTF-8?q?=F0=9F=9A=A7=20Importer=20tasks=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 54 ++++++++++++++++++++++++++-------------- src/meson.build | 1 + src/utils/task.py | 7 ++++++ src/window.py | 2 +- 4 files changed, 45 insertions(+), 19 deletions(-) create mode 100644 src/utils/task.py diff --git a/src/importer/importer.py b/src/importer/importer.py index bd44c7c..1034979 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -3,6 +3,7 @@ import logging from requests import HTTPError from gi.repository import Adw, Gio, Gtk +from .task import make_task_thread_func_closure from .create_dialog import create_dialog from .steamgriddb import SGDBAuthError, SGDBError, SGDBHelper @@ -47,7 +48,7 @@ class Importer: @property def finished(self): - return self.n_sgdb_tasks_created == self.n_tasks_done + return self.n_tasks_created == self.n_tasks_done def add_source(self, source): self.sources.add(source) @@ -61,19 +62,30 @@ class Importer: # (If SGDB auth is bad, cancel all SGDB tasks) self.sgdb_cancellable = Gio.Cancellable() + """ # Create a task for each source - tasks = set() - for source in self.sources: - self.n_source_tasks_created += 1 - logging.debug("Importing games from source %s", source.id) - + def make_closure(source): def closure(task, obj, _data, cancellable): self.source_task_thread_func(task, obj, (source,), cancellable) - task = Gio.Task.new(None, None, self.source_task_callback, (source,)) - self.n_sgdb_tasks_created += 1 + return closure + + for source in self.sources: + self.n_source_tasks_created += 1 + logging.debug("Importing games from source %s", source.id) + task = Task.new(None, None, self.source_task_callback, (source,)) + closure = make_closure(source) + task.run_in_thread(closure) + """ + + for source in self.sources: + self.n_source_tasks_created += 1 + logging.debug("Importing games from source %s", source.id) + task = Gio.Task.new(None, None, self.source_task_callback, (source,)) + closure = make_task_thread_func_closure( + self.source_task_thread_func, (source,) + ) task.run_in_thread(closure) - tasks.add(task) def create_dialog(self): """Create the import dialog""" @@ -122,30 +134,34 @@ class Importer: ) continue + # TODO register in store instead of dict + # Avoid duplicates - gid = game.game_id - if gid in self.win.games and not self.win.games[gid].removed: + if ( + game.game_id in self.win.games + and not self.win.games[game.game_id].removed + ): continue # Register game - self.win.games[gid] = game + logging.info("New game registered %s (%s)", game.name, game.game_id) + self.win.games[game.game_id] = game game.save() self.n_games_added += 1 # Start sgdb lookup for game # HACK move to its own manager - - def closure(task, obj, _data, cancellable): - self.sgdb_task_thread_func(task, obj, (game,), cancellable) - task = Gio.Task.new( None, self.sgdb_cancellable, self.sgdb_task_callback, (game,) ) + closure = make_task_thread_func_closure(self.sgdb_task_thread_func, (game,)) + self.n_sgdb_tasks_created += 1 task.run_in_thread(closure) def source_task_callback(self, _obj, _result, data): """Source import callback""" - _source, *_rest = data + source, *_rest = data + logging.debug("Import done for source %s", source.id) self.n_source_tasks_done += 1 self.update_progressbar() if self.finished: @@ -168,7 +184,8 @@ class Importer: def sgdb_task_callback(self, _obj, _result, data): """SGDB query callback""" game, *_rest = data - game.set_loading(0) + logging.debug("SGDB import done for game %s", game.name) + game.set_loading(-1) self.n_sgdb_tasks_done += 1 self.update_progressbar() if self.finished: @@ -176,6 +193,7 @@ class Importer: def import_callback(self): """Callback called when importing has finished""" + logging.info("Import done") self.import_dialog.close() self.create_summary_toast() if self.sgdb_error is not None: diff --git a/src/meson.build b/src/meson.build index 53e6e5d..4476a3e 100644 --- a/src/meson.build +++ b/src/meson.build @@ -36,6 +36,7 @@ cartridges_sources = [ 'importer/importer.py', 'importer/source.py', 'utils/decorators.py', + 'utils/task.py', # TODO remove before merge 'importers/bottles_importer.py', diff --git a/src/utils/task.py b/src/utils/task.py new file mode 100644 index 0000000..356ce9a --- /dev/null +++ b/src/utils/task.py @@ -0,0 +1,7 @@ +def make_task_thread_func_closure(func, data): + """Prepare a Gio.TaskThreadFunc with its data bound in a closure""" + + def closure(task, obj, _data, cancellable): + func(task, obj, data, cancellable) + + return closure diff --git a/src/window.py b/src/window.py index ba98c34..b456b9e 100644 --- a/src/window.py +++ b/src/window.py @@ -114,7 +114,7 @@ class CartridgesWindow(Adw.ApplicationWindow): ): path.unlink(missing_ok=True) else: - Game(game).update() + Game(game, allow_side_effects=True).update() # Connect search entries self.search_bar.connect_entry(self.search_entry) From 46f30d289b3651ad055abe7577e889b10631a0bb Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Thu, 18 May 2023 23:49:09 +0200 Subject: [PATCH 033/173] =?UTF-8?q?=E2=9C=A8=20Fixed=20new=20importer=20SG?= =?UTF-8?q?DB=20cover=20import?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/steamgriddb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index 2a71f9b..ec62b5d 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -109,7 +109,7 @@ class SGDBHelper: # Download covers for uri_kwargs in image_uri_kwargs_sets: try: - uri = self.get_game_image_uri(sgdb_id, **uri_kwargs) + uri = self.get_image_uri(sgdb_id, **uri_kwargs) response = requests.get(uri, timeout=5) tmp_file = Gio.File.new_tmp()[0] tmp_file_path = tmp_file.get_path() From c0e275ac5cdbe986b81c490c452616795a664eb7 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Thu, 18 May 2023 23:57:43 +0200 Subject: [PATCH 034/173] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20still=20cover=20?= =?UTF-8?q?not=20got=20if=20animated=20failed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/steamgriddb.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index ec62b5d..172afd7 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -67,7 +67,10 @@ class SGDBHelper: res = requests.get(uri, headers=self.auth_headers, timeout=5) match res.status_code: case 200: - return res.json()["data"][0]["url"] + data = res.json()["data"] + if len(data) == 0: + raise SGDBNoImageFoundError() + return data[0]["url"] case 401: raise SGDBAuthError(res.json()["errors"][0]) case 404: From 505088e0535948419deba6669d756d0f260d03f9 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Fri, 19 May 2023 17:35:31 +0200 Subject: [PATCH 035/173] =?UTF-8?q?=F0=9F=90=9B=20Task=20fix=20+=20progres?= =?UTF-8?q?s=20bar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/game.py | 2 +- src/importer/importer.py | 36 +++++--------- src/importer/sources/lutris_source.py | 2 +- src/utils/task.py | 69 +++++++++++++++++++++++++-- src/window.py | 2 +- 5 files changed, 79 insertions(+), 32 deletions(-) diff --git a/src/game.py b/src/game.py index c568c2e..9f3952c 100644 --- a/src/game.py +++ b/src/game.py @@ -60,7 +60,7 @@ class Game(Gtk.Box): blacklisted = None game_cover = None - def __init__(self, win, data, allow_side_effects=False, **kwargs): + def __init__(self, win, data, allow_side_effects=True, **kwargs): super().__init__(**kwargs) self.win = shared.win diff --git a/src/importer/importer.py b/src/importer/importer.py index 1034979..dd517ec 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -3,7 +3,7 @@ import logging from requests import HTTPError from gi.repository import Adw, Gio, Gtk -from .task import make_task_thread_func_closure +from .task import Task from .create_dialog import create_dialog from .steamgriddb import SGDBAuthError, SGDBError, SGDBHelper @@ -41,7 +41,7 @@ class Importer: @property def progress(self): try: - progress = 1 - self.n_tasks_created / self.n_tasks_done + progress = self.n_tasks_done / self.n_tasks_created except ZeroDivisionError: progress = 1 return progress @@ -62,30 +62,12 @@ class Importer: # (If SGDB auth is bad, cancel all SGDB tasks) self.sgdb_cancellable = Gio.Cancellable() - """ - # Create a task for each source - def make_closure(source): - def closure(task, obj, _data, cancellable): - self.source_task_thread_func(task, obj, (source,), cancellable) - - return closure - for source in self.sources: - self.n_source_tasks_created += 1 logging.debug("Importing games from source %s", source.id) task = Task.new(None, None, self.source_task_callback, (source,)) - closure = make_closure(source) - task.run_in_thread(closure) - """ - - for source in self.sources: self.n_source_tasks_created += 1 - logging.debug("Importing games from source %s", source.id) - task = Gio.Task.new(None, None, self.source_task_callback, (source,)) - closure = make_task_thread_func_closure( - self.source_task_thread_func, (source,) - ) - task.run_in_thread(closure) + task.set_task_data((source,)) + task.run_in_thread(self.source_task_thread_func) def create_dialog(self): """Create the import dialog""" @@ -105,6 +87,9 @@ class Importer: self.import_dialog.present() def update_progressbar(self): + logging.debug( + "Progressbar updated (%f)", self.progress + ) # TODO why progress not workie? self.progressbar.set_fraction(self.progress) def source_task_thread_func(self, _task, _obj, data, _cancellable): @@ -151,12 +136,12 @@ class Importer: # Start sgdb lookup for game # HACK move to its own manager - task = Gio.Task.new( + task = Task.new( None, self.sgdb_cancellable, self.sgdb_task_callback, (game,) ) - closure = make_task_thread_func_closure(self.sgdb_task_thread_func, (game,)) self.n_sgdb_tasks_created += 1 - task.run_in_thread(closure) + task.set_task_data((game,)) + task.run_in_thread(self.sgdb_task_thread_func) def source_task_callback(self, _obj, _result, data): """Source import callback""" @@ -213,6 +198,7 @@ class Importer: self.dialog_response_callback, "open_preferences", "import", + None, ) elif self.n_games_added == 1: diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 5a707ce..4e755ba 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -76,7 +76,7 @@ class LutrisSourceIterator(SourceIterator): "executable": self.source.executable_format.format(game_id=row[2]), "developer": None, # TODO get developer metadata on Lutris } - game = Game(self.source.win, values) + game = Game(self.source.win, values, allow_side_effects=False) # Save official image image_path = self.source.cache_location / "coverart" / f"{row[2]}.jpg" diff --git a/src/utils/task.py b/src/utils/task.py index 356ce9a..a698e8b 100644 --- a/src/utils/task.py +++ b/src/utils/task.py @@ -1,7 +1,68 @@ -def make_task_thread_func_closure(func, data): - """Prepare a Gio.TaskThreadFunc with its data bound in a closure""" +from gi.repository import Gio +from functools import wraps - def closure(task, obj, _data, cancellable): - func(task, obj, data, cancellable) + +def create_task_thread_func_closure(func, data): + """Wrap a Gio.TaskThreadFunc with the given data in a closure""" + + def closure(task, source_object, _data, cancellable): + func(task, source_object, data, cancellable) return closure + + +def decorate_set_task_data(task): + """Decorate Gio.Task.set_task_data to replace it""" + + def decorator(original_method): + @wraps(original_method) + def new_method(task_data): + task.__task_data = task_data + pass + + return new_method + + return decorator + + +def decorate_run_in_thread(task): + """Decorate Gio.Task.run_in_thread to pass the task data correctly + Creates a closure around task_thread_func with the task data available.""" + + def decorator(original_method): + @wraps(original_method) + def new_method(task_thread_func): + closure = create_task_thread_func_closure( + task_thread_func, task.__task_data + ) + original_method(closure) + + return new_method + + return decorator + + +class Task: + """Wrapper around Gio.Task to patch task data not being passed""" + + @classmethod + def new(cls, source_object, cancellable, callback, callback_data): + """Create a new, monkey-patched Gio.Task. + The `set_task_data` and `run_in_thread` methods are decorated. + + As of 2023-05-19, pygobject does not work well with Gio.Task, so to pass data + the only viable way it to create a closure with the thread function and its data. + This class is supposed to make Gio.Task comply with its expected behaviour + per the docs: + + http://lazka.github.io/pgi-docs/#Gio-2.0/classes/Task.html#Gio.Task.set_task_data + + This code may break if pygobject overrides change in the future. + We need to manually pass `self` to the decorators since it's otherwise bound but + not accessible from Python's side. + """ + + task = Gio.Task.new(source_object, cancellable, callback, callback_data) + task.set_task_data = decorate_set_task_data(task)(task.set_task_data) + task.run_in_thread = decorate_run_in_thread(task)(task.run_in_thread) + return task diff --git a/src/window.py b/src/window.py index b456b9e..ba98c34 100644 --- a/src/window.py +++ b/src/window.py @@ -114,7 +114,7 @@ class CartridgesWindow(Adw.ApplicationWindow): ): path.unlink(missing_ok=True) else: - Game(game, allow_side_effects=True).update() + Game(game).update() # Connect search entries self.search_bar.connect_entry(self.search_entry) From 56c110ffa2aaa8424a166435967092cecf205451 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Fri, 19 May 2023 18:01:06 +0200 Subject: [PATCH 036/173] Improved SGDB logging messages --- src/utils/steamgriddb.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index 172afd7..cfb9dbb 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -100,7 +100,7 @@ class SGDBHelper: sgdb_id = self.get_game_id(game) except (HTTPError, SGDBError) as error: logging.warning( - "Error while getting SGDB ID for %s", game.name, exc_info=error + "%s while getting SGDB ID for %s", error.__class__.__name__, game.name ) raise error @@ -124,7 +124,12 @@ class SGDBHelper: # Let caller handle auth errors raise error except (HTTPError, SGDBError) as error: - logging.warning("Error while getting image", exc_info=error) + logging.warning( + "%s while getting image for %s kwargs=%s", + error.__class__.__name__, + game.name, + str(uri_kwargs), + ) continue else: # Stop as soon as one is finished From a176b33241efe1c6605093e5552ac3ddcb4c3029 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Fri, 19 May 2023 20:40:33 +0200 Subject: [PATCH 037/173] =?UTF-8?q?=F0=9F=8E=A8=20Using=20absolute=20impor?= =?UTF-8?q?ts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/__init__.py | 0 src/details_window.py | 11 ++++--- src/game.py | 2 +- src/importer/__init__.py | 0 src/importer/importer.py | 8 ++--- src/importer/sources/__init__.py | 0 src/importer/sources/lutris_source.py | 8 ++--- src/importers/bottles_importer.py | 2 +- src/importers/heroic_importer.py | 2 +- src/importers/itch_importer.py | 4 +-- src/importers/lutris_importer.py | 2 +- src/importers/steam_importer.py | 2 +- src/main.py | 17 ++++++---- src/meson.build | 47 +++++++++------------------ src/preferences.py | 13 ++++---- src/utils/check_install.py | 1 + src/utils/steamgriddb.py | 8 ++--- src/window.py | 2 +- 18 files changed, 59 insertions(+), 70 deletions(-) delete mode 100644 src/__init__.py delete mode 100644 src/importer/__init__.py delete mode 100644 src/importer/sources/__init__.py diff --git a/src/__init__.py b/src/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/details_window.py b/src/details_window.py index 23d6306..362f97b 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -24,12 +24,13 @@ from time import time from gi.repository import Adw, Gio, GLib, Gtk from PIL import Image +# TODO use SGDBHelper from . import shared -from .create_dialog import create_dialog -from .game import Game -from .game_cover import GameCover -from .save_cover import resize_cover, save_cover -from .steamgriddb import SGDBSave +from cartridges.game import Game +from cartridges.game_cover import GameCover +from cartridges.utils.create_dialog import create_dialog +from cartridges.utils.save_cover import resize_cover, save_cover +from cartridges.utils.steamgriddb import SGDBSave @Gtk.Template(resource_path="/hu/kramo/Cartridges/gtk/details_window.ui") diff --git a/src/game.py b/src/game.py index 9f3952c..218843e 100644 --- a/src/game.py +++ b/src/game.py @@ -27,7 +27,7 @@ from time import time from gi.repository import Adw, Gio, Gtk from . import shared -from .game_cover import GameCover +from cartridges.game_cover import GameCover @Gtk.Template(resource_path="/hu/kramo/Cartridges/gtk/game.ui") diff --git a/src/importer/__init__.py b/src/importer/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/importer/importer.py b/src/importer/importer.py index dd517ec..c6cfb4f 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -1,11 +1,11 @@ import logging -from requests import HTTPError from gi.repository import Adw, Gio, Gtk +from requests import HTTPError -from .task import Task -from .create_dialog import create_dialog -from .steamgriddb import SGDBAuthError, SGDBError, SGDBHelper +from cartridges.utils.create_dialog import create_dialog +from cartridges.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper +from cartridges.utils.task import Task class Importer: diff --git a/src/importer/sources/__init__.py b/src/importer/sources/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 4e755ba..16e9603 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -2,10 +2,10 @@ from functools import cache from sqlite3 import connect from time import time -from .decorators import replaced_by_path, replaced_by_schema_key -from .game import Game -from .save_cover import resize_cover, save_cover -from .source import Source, SourceIterator +from cartridges.game import Game +from cartridges.importer.source import Source, SourceIterator +from cartridges.utils.decorators import replaced_by_path, replaced_by_schema_key +from cartridges.utils.save_cover import resize_cover, save_cover class LutrisSourceIterator(SourceIterator): diff --git a/src/importers/bottles_importer.py b/src/importers/bottles_importer.py index 707601e..bd48047 100644 --- a/src/importers/bottles_importer.py +++ b/src/importers/bottles_importer.py @@ -23,7 +23,7 @@ from time import time import yaml from . import shared -from .check_install import check_install +from cartridges.utils.check_install import check_install def bottles_installed(path=None): diff --git a/src/importers/heroic_importer.py b/src/importers/heroic_importer.py index e3fd503..8ebdf6f 100644 --- a/src/importers/heroic_importer.py +++ b/src/importers/heroic_importer.py @@ -24,7 +24,7 @@ from pathlib import Path from time import time from . import shared -from .check_install import check_install +from cartridges.utils.check_install import check_install def heroic_installed(path=None): diff --git a/src/importers/itch_importer.py b/src/importers/itch_importer.py index 728670f..121ae3f 100644 --- a/src/importers/itch_importer.py +++ b/src/importers/itch_importer.py @@ -27,8 +27,8 @@ import requests from gi.repository import GdkPixbuf, Gio from . import shared -from .check_install import check_install -from .save_cover import resize_cover +from cartridges.utils.check_install import check_install +from cartridges.utils.save_cover import resize_cover def get_game(task, current_time, row): diff --git a/src/importers/lutris_importer.py b/src/importers/lutris_importer.py index 40d038d..a0bda89 100644 --- a/src/importers/lutris_importer.py +++ b/src/importers/lutris_importer.py @@ -23,7 +23,7 @@ from sqlite3 import connect from time import time from . import shared -from .check_install import check_install +from cartridges.utils.check_install import check_install def lutris_installed(path=None): diff --git a/src/importers/steam_importer.py b/src/importers/steam_importer.py index 10b9540..7824235 100644 --- a/src/importers/steam_importer.py +++ b/src/importers/steam_importer.py @@ -26,7 +26,7 @@ import requests from gi.repository import Gio from . import shared -from .check_install import check_install +from cartridges.utils.check_install import check_install def update_values_from_data(content, values): diff --git a/src/main.py b/src/main.py index c45fbda..fe98dd6 100644 --- a/src/main.py +++ b/src/main.py @@ -17,9 +17,9 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -import sys -import os import logging +import os +import sys import gi @@ -30,11 +30,14 @@ gi.require_version("Adw", "1") from gi.repository import Adw, Gio, GLib, Gtk from . import shared -from .details_window import DetailsWindow -from .preferences import PreferencesWindow -from .window import CartridgesWindow -from .importer import Importer -from .lutris_source import LutrisNativeSource, LutrisFlatpakSource +from cartridges.details_window import DetailsWindow +from cartridges.importer.importer import Importer +from cartridges.importer.sources.lutris_source import ( + LutrisFlatpakSource, + LutrisNativeSource, +) +from cartridges.preferences import PreferencesWindow +from cartridges.window import CartridgesWindow class CartridgesApplication(Adw.Application): diff --git a/src/meson.build b/src/meson.build index 4476a3e..8cf79f6 100644 --- a/src/meson.build +++ b/src/meson.build @@ -16,35 +16,18 @@ configure_file( install_dir: get_option('bindir') ) -# TODO move to absolute imports = keep real structure, do not flatten - -cartridges_sources = [ - '__init__.py', - 'main.py', - 'window.py', - 'preferences.py', - 'details_window.py', - 'game.py', - 'game_cover.py', - 'shared.py', - 'utils/steamgriddb.py', - 'utils/save_cover.py', - 'utils/create_dialog.py', - - # Added - 'importer/sources/lutris_source.py', - 'importer/importer.py', - 'importer/source.py', - 'utils/decorators.py', - 'utils/task.py', - - # TODO remove before merge - 'importers/bottles_importer.py', - 'importers/heroic_importer.py', - 'importers/itch_importer.py', - 'importers/lutris_importer.py', - 'importers/steam_importer.py', - 'utils/check_install.py' -] - -install_data(cartridges_sources, install_dir: moduledir) +install_subdir('importer', install_dir: moduledir) +install_subdir('importers', install_dir: moduledir) +install_subdir('utils', install_dir: moduledir) +install_data( + [ + 'main.py', + 'window.py', + 'preferences.py', + 'details_window.py', + 'game.py', + 'game_cover.py', + 'shared.py', + ], + install_dir: moduledir +) \ No newline at end of file diff --git a/src/preferences.py b/src/preferences.py index 35aaf85..4349560 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -25,12 +25,13 @@ from gi.repository import Adw, Gio, GLib, Gtk # pylint: disable=unused-import from . import shared -from .bottles_importer import bottles_installed -from .create_dialog import create_dialog -from .heroic_importer import heroic_installed -from .itch_importer import itch_installed -from .lutris_importer import lutris_cache_exists, lutris_installed -from .steam_importer import steam_installed +# TODO use the new sources +from cartridges.importers.bottles_importer import bottles_installed +from cartridges.importers.heroic_importer import heroic_installed +from cartridges.importers.itch_importer import itch_installed +from cartridges.importers.lutris_importer import lutris_cache_exists, lutris_installed +from cartridges.importers.steam_importer import steam_installed +from cartridges.utils.create_dialog import create_dialog @Gtk.Template(resource_path="/hu/kramo/Cartridges/gtk/preferences.ui") diff --git a/src/utils/check_install.py b/src/utils/check_install.py index 9dfe4a0..272df18 100644 --- a/src/utils/check_install.py +++ b/src/utils/check_install.py @@ -20,6 +20,7 @@ from pathlib import Path +# TODO delegate to the sources def check_install(check, locations, setting=None, subdirs=(Path(),)): for location in locations: for subdir in (Path(),) + subdirs: diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index cfb9dbb..15777b1 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -1,13 +1,13 @@ +import logging from pathlib import Path -import logging import requests -from requests import HTTPError from gi.repository import Gio +from requests import HTTPError from . import shared -from .create_dialog import create_dialog -from .save_cover import save_cover, resize_cover +from cartridges.utils.create_dialog import create_dialog +from cartridges.utils.save_cover import resize_cover, save_cover class SGDBError(Exception): diff --git a/src/window.py b/src/window.py index ba98c34..67766ba 100644 --- a/src/window.py +++ b/src/window.py @@ -23,7 +23,7 @@ from datetime import datetime from gi.repository import Adw, Gio, GLib, Gtk from . import shared -from .game import Game +from cartridges.game import Game @Gtk.Template(resource_path="/hu/kramo/Cartridges/gtk/window.ui") From 83d7aad0d19ed73dac3df6624286ae612d60da26 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Fri, 19 May 2023 21:34:47 +0200 Subject: [PATCH 038/173] =?UTF-8?q?=E2=9C=A8=20Made=20more=20linter-friend?= =?UTF-8?q?ly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/details_window.py | 10 +++++----- src/game.py | 2 +- src/importer/importer.py | 6 +++--- src/importer/sources/lutris_source.py | 8 ++++---- src/importers/bottles_importer.py | 2 +- src/importers/heroic_importer.py | 2 +- src/importers/itch_importer.py | 4 ++-- src/importers/lutris_importer.py | 2 +- src/importers/steam_importer.py | 2 +- src/main.py | 10 +++++----- src/meson.build | 2 +- src/preferences.py | 12 ++++++------ src/utils/steamgriddb.py | 4 ++-- src/window.py | 2 +- 14 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/details_window.py b/src/details_window.py index 362f97b..6d4adf6 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -26,11 +26,11 @@ from PIL import Image # TODO use SGDBHelper from . import shared -from cartridges.game import Game -from cartridges.game_cover import GameCover -from cartridges.utils.create_dialog import create_dialog -from cartridges.utils.save_cover import resize_cover, save_cover -from cartridges.utils.steamgriddb import SGDBSave +from src.game import Game +from src.game_cover import GameCover +from src.utils.create_dialog import create_dialog +from src.utils.save_cover import resize_cover, save_cover +from src.utils.steamgriddb import SGDBSave @Gtk.Template(resource_path="/hu/kramo/Cartridges/gtk/details_window.ui") diff --git a/src/game.py b/src/game.py index 218843e..5bab56e 100644 --- a/src/game.py +++ b/src/game.py @@ -27,7 +27,7 @@ from time import time from gi.repository import Adw, Gio, Gtk from . import shared -from cartridges.game_cover import GameCover +from src.game_cover import GameCover @Gtk.Template(resource_path="/hu/kramo/Cartridges/gtk/game.ui") diff --git a/src/importer/importer.py b/src/importer/importer.py index c6cfb4f..ca95a91 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -3,9 +3,9 @@ import logging from gi.repository import Adw, Gio, Gtk from requests import HTTPError -from cartridges.utils.create_dialog import create_dialog -from cartridges.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper -from cartridges.utils.task import Task +from src.utils.create_dialog import create_dialog +from src.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper +from src.utils.task import Task class Importer: diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 16e9603..25fe8aa 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -2,10 +2,10 @@ from functools import cache from sqlite3 import connect from time import time -from cartridges.game import Game -from cartridges.importer.source import Source, SourceIterator -from cartridges.utils.decorators import replaced_by_path, replaced_by_schema_key -from cartridges.utils.save_cover import resize_cover, save_cover +from src.game import Game +from src.importer.source import Source, SourceIterator +from src.utils.decorators import replaced_by_path, replaced_by_schema_key +from src.utils.save_cover import resize_cover, save_cover class LutrisSourceIterator(SourceIterator): diff --git a/src/importers/bottles_importer.py b/src/importers/bottles_importer.py index bd48047..38b8d87 100644 --- a/src/importers/bottles_importer.py +++ b/src/importers/bottles_importer.py @@ -23,7 +23,7 @@ from time import time import yaml from . import shared -from cartridges.utils.check_install import check_install +from src.utils.check_install import check_install def bottles_installed(path=None): diff --git a/src/importers/heroic_importer.py b/src/importers/heroic_importer.py index 8ebdf6f..5282cc1 100644 --- a/src/importers/heroic_importer.py +++ b/src/importers/heroic_importer.py @@ -24,7 +24,7 @@ from pathlib import Path from time import time from . import shared -from cartridges.utils.check_install import check_install +from src.utils.check_install import check_install def heroic_installed(path=None): diff --git a/src/importers/itch_importer.py b/src/importers/itch_importer.py index 121ae3f..af0b926 100644 --- a/src/importers/itch_importer.py +++ b/src/importers/itch_importer.py @@ -27,8 +27,8 @@ import requests from gi.repository import GdkPixbuf, Gio from . import shared -from cartridges.utils.check_install import check_install -from cartridges.utils.save_cover import resize_cover +from src.utils.check_install import check_install +from src.utils.save_cover import resize_cover def get_game(task, current_time, row): diff --git a/src/importers/lutris_importer.py b/src/importers/lutris_importer.py index a0bda89..04b92f7 100644 --- a/src/importers/lutris_importer.py +++ b/src/importers/lutris_importer.py @@ -23,7 +23,7 @@ from sqlite3 import connect from time import time from . import shared -from cartridges.utils.check_install import check_install +from src.utils.check_install import check_install def lutris_installed(path=None): diff --git a/src/importers/steam_importer.py b/src/importers/steam_importer.py index 7824235..37ea2b6 100644 --- a/src/importers/steam_importer.py +++ b/src/importers/steam_importer.py @@ -26,7 +26,7 @@ import requests from gi.repository import Gio from . import shared -from cartridges.utils.check_install import check_install +from src.utils.check_install import check_install def update_values_from_data(content, values): diff --git a/src/main.py b/src/main.py index fe98dd6..37ffe9a 100644 --- a/src/main.py +++ b/src/main.py @@ -30,14 +30,14 @@ gi.require_version("Adw", "1") from gi.repository import Adw, Gio, GLib, Gtk from . import shared -from cartridges.details_window import DetailsWindow -from cartridges.importer.importer import Importer -from cartridges.importer.sources.lutris_source import ( +from src.details_window import DetailsWindow +from src.importer.importer import Importer +from src.importer.sources.lutris_source import ( LutrisFlatpakSource, LutrisNativeSource, ) -from cartridges.preferences import PreferencesWindow -from cartridges.window import CartridgesWindow +from src.preferences import PreferencesWindow +from src.window import CartridgesWindow class CartridgesApplication(Adw.Application): diff --git a/src/meson.build b/src/meson.build index 8cf79f6..4a48b31 100644 --- a/src/meson.build +++ b/src/meson.build @@ -1,4 +1,4 @@ -moduledir = join_paths(pkgdatadir, 'cartridges') +moduledir = join_paths(pkgdatadir, 'src') python = import('python') diff --git a/src/preferences.py b/src/preferences.py index 4349560..204a02d 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -26,12 +26,12 @@ from gi.repository import Adw, Gio, GLib, Gtk # pylint: disable=unused-import from . import shared # TODO use the new sources -from cartridges.importers.bottles_importer import bottles_installed -from cartridges.importers.heroic_importer import heroic_installed -from cartridges.importers.itch_importer import itch_installed -from cartridges.importers.lutris_importer import lutris_cache_exists, lutris_installed -from cartridges.importers.steam_importer import steam_installed -from cartridges.utils.create_dialog import create_dialog +from src.importers.bottles_importer import bottles_installed +from src.importers.heroic_importer import heroic_installed +from src.importers.itch_importer import itch_installed +from src.importers.lutris_importer import lutris_cache_exists, lutris_installed +from src.importers.steam_importer import steam_installed +from src.utils.create_dialog import create_dialog @Gtk.Template(resource_path="/hu/kramo/Cartridges/gtk/preferences.ui") diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index 15777b1..e223dcd 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -6,8 +6,8 @@ from gi.repository import Gio from requests import HTTPError from . import shared -from cartridges.utils.create_dialog import create_dialog -from cartridges.utils.save_cover import resize_cover, save_cover +from src.utils.create_dialog import create_dialog +from src.utils.save_cover import resize_cover, save_cover class SGDBError(Exception): diff --git a/src/window.py b/src/window.py index 67766ba..69a4418 100644 --- a/src/window.py +++ b/src/window.py @@ -23,7 +23,7 @@ from datetime import datetime from gi.repository import Adw, Gio, GLib, Gtk from . import shared -from cartridges.game import Game +from src.game import Game @Gtk.Template(resource_path="/hu/kramo/Cartridges/gtk/window.ui") From 138dcb6271b569cf0820e85fb1fff46a2a37a79d Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 20 May 2023 01:30:46 +0200 Subject: [PATCH 039/173] =?UTF-8?q?=F0=9F=90=9B=20Added=20gettext's=20`=5F?= =?UTF-8?q?()`=20to=20builtin=20stubs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/__builtins__.pyi | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/__builtins__.pyi diff --git a/src/__builtins__.pyi b/src/__builtins__.pyi new file mode 100644 index 0000000..c051ccd --- /dev/null +++ b/src/__builtins__.pyi @@ -0,0 +1 @@ +def _(msg: str, /) -> str: ... From 7b97481b55e2d85c5c46c59040f3663a6677452b Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 20 May 2023 02:03:34 +0200 Subject: [PATCH 040/173] =?UTF-8?q?=F0=9F=9A=A8=20Fixed=20some=20pylint=20?= =?UTF-8?q?warnings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 6 ++---- src/importer/source.py | 2 +- src/importer/sources/lutris_source.py | 15 ++++++--------- src/utils/task.py | 11 +++++------ 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index ca95a91..2a1e7b7 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -8,6 +8,7 @@ from src.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper from src.utils.task import Task +# pylint: disable=too-many-instance-attributes class Importer: """A class in charge of scanning sources for games""" @@ -87,9 +88,6 @@ class Importer: self.import_dialog.present() def update_progressbar(self): - logging.debug( - "Progressbar updated (%f)", self.progress - ) # TODO why progress not workie? self.progressbar.set_fraction(self.progress) def source_task_thread_func(self, _task, _obj, data, _cancellable): @@ -162,7 +160,7 @@ class Importer: except SGDBAuthError as error: cancellable.cancel() self.sgdb_error = error - except (HTTPError, SGDBError) as error: + except (HTTPError, SGDBError) as _error: # TODO handle other SGDB errors pass diff --git a/src/importer/source.py b/src/importer/source.py index 7a959ed..a2fca0c 100644 --- a/src/importer/source.py +++ b/src/importer/source.py @@ -28,7 +28,7 @@ class SourceIterator(Iterator, Sized): class Source(Iterable): """Source of games. E.g an installed app with a config file that lists game directories""" - win = None # TODO maybe not depend on that ? + win = None name: str variant: str diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 25fe8aa..23c65b8 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,4 +1,4 @@ -from functools import cache +from functools import lru_cache from sqlite3 import connect from time import time @@ -48,7 +48,7 @@ class LutrisSourceIterator(SourceIterator): self.db_games_request, self.db_request_params ) - @cache + @lru_cache(maxsize=1) def __len__(self): cursor = self.db_connection.execute(self.db_len_request, self.db_request_params) return cursor.fetchone()[0] @@ -60,9 +60,9 @@ class LutrisSourceIterator(SourceIterator): row = None try: row = self.db_cursor.__next__() - except StopIteration as e: + except StopIteration as error: self.db_connection.close() - raise e + raise error # Create game values = { @@ -99,16 +99,13 @@ class LutrisSource(Source): @property def is_installed(self): + # pylint: disable=pointless-statement try: self.location self.cache_location except FileNotFoundError: return False - else: - return True - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + return True def __iter__(self): return LutrisSourceIterator(source=self) diff --git a/src/utils/task.py b/src/utils/task.py index a698e8b..c6b7076 100644 --- a/src/utils/task.py +++ b/src/utils/task.py @@ -1,6 +1,7 @@ -from gi.repository import Gio from functools import wraps +from gi.repository import Gio + def create_task_thread_func_closure(func, data): """Wrap a Gio.TaskThreadFunc with the given data in a closure""" @@ -17,8 +18,7 @@ def decorate_set_task_data(task): def decorator(original_method): @wraps(original_method) def new_method(task_data): - task.__task_data = task_data - pass + task.task_data = task_data return new_method @@ -32,9 +32,7 @@ def decorate_run_in_thread(task): def decorator(original_method): @wraps(original_method) def new_method(task_thread_func): - closure = create_task_thread_func_closure( - task_thread_func, task.__task_data - ) + closure = create_task_thread_func_closure(task_thread_func, task.task_data) original_method(closure) return new_method @@ -42,6 +40,7 @@ def decorate_run_in_thread(task): return decorator +# pylint: disable=too-few-public-methods class Task: """Wrapper around Gio.Task to patch task data not being passed""" From 7252655abaf2769ec122f96524488c09b790da63 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 20 May 2023 13:41:40 +0200 Subject: [PATCH 041/173] =?UTF-8?q?=F0=9F=90=9B=20fixed=20wrong=20module?= =?UTF-8?q?=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cartridges.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cartridges.in b/src/cartridges.in index a5be237..67e82e7 100755 --- a/src/cartridges.in +++ b/src/cartridges.in @@ -55,6 +55,6 @@ if __name__ == "__main__": resource = Gio.Resource.load(os.path.join(pkgdatadir, "cartridges.gresource")) resource._register() - from cartridges import main + from src import main sys.exit(main.main(VERSION)) From 08c4b53ccab0c2d3776c9c78a83c264b921d88d5 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 20 May 2023 14:04:38 +0200 Subject: [PATCH 042/173] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20importer=20popup?= =?UTF-8?q?=20callbacks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 2a1e7b7..42aac19 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -179,6 +179,7 @@ class Importer: logging.info("Import done") self.import_dialog.close() self.create_summary_toast() + # TODO create a summary of errors/warnings/tips popup (eg. SGDB, Steam libraries) if self.sgdb_error is not None: self.create_sgdb_error_dialog() @@ -196,7 +197,6 @@ class Importer: self.dialog_response_callback, "open_preferences", "import", - None, ) elif self.n_games_added == 1: @@ -218,11 +218,12 @@ class Importer: _("Preferences"), ).connect("response", self.dialog_response_callback, "sgdb") + def open_preferences(self, page=None, expander_row=None): + self.win.get_application().on_preferences_action( + page_name=page, expander_row=expander_row + ) + def dialog_response_callback(self, _widget, response, *args): """Handle after-import dialogs callback""" if response == "open_preferences": - page, expander_row, *_rest = args - self.win.get_application().on_preferences_action( - page_name=page, expander_row=expander_row - ) - # TODO handle steam libraries tip + self.open_preferences(*args) From 604bcfb2e92e20d01c9c6217da823b31d9112302 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 20 May 2023 19:48:03 +0200 Subject: [PATCH 043/173] =?UTF-8?q?=F0=9F=9A=A7=20Initial=20work=20on=20St?= =?UTF-8?q?eam=20source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data/hu.kramo.Cartridges.gschema.xml | 6 + src/importer/importer.py | 2 + src/importer/source.py | 31 ++--- src/importer/sources/steam_source.py | 180 +++++++++++++++++++++++++++ src/main.py | 9 ++ src/utils/decorators.py | 20 ++- 6 files changed, 233 insertions(+), 15 deletions(-) create mode 100644 src/importer/sources/steam_source.py diff --git a/data/hu.kramo.Cartridges.gschema.xml b/data/hu.kramo.Cartridges.gschema.xml index f4a9368..1230c42 100644 --- a/data/hu.kramo.Cartridges.gschema.xml +++ b/data/hu.kramo.Cartridges.gschema.xml @@ -16,6 +16,12 @@ "~/.steam/" + + ~/.var/app/com.valvesoftware.Steam/data/Steam/ + + + "C:\Program Files (x86)\Steam" + true diff --git a/src/importer/importer.py b/src/importer/importer.py index 42aac19..2f1e5ae 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -116,6 +116,8 @@ class Importer: exc_info=exception, ) continue + if game is None: + continue # TODO register in store instead of dict diff --git a/src/importer/source.py b/src/importer/source.py index a2fca0c..663b92e 100644 --- a/src/importer/source.py +++ b/src/importer/source.py @@ -1,35 +1,38 @@ from abc import abstractmethod from collections.abc import Iterable, Iterator, Sized +from src.game import Game +from src.window import CartridgesWindow + class SourceIterator(Iterator, Sized): """Data producer for a source of games""" - source = None + source: "Source" = None - def __init__(self, source) -> None: + def __init__(self, source: "Source") -> None: super().__init__() self.source = source - def __iter__(self): + def __iter__(self) -> "SourceIterator": return self @abstractmethod - def __len__(self): + def __len__(self) -> int: """Get a rough estimate of the number of games produced by the source""" @abstractmethod - def __next__(self): + def __next__(self) -> "Game" | None: """Get the next generated game from the source. Raises StopIteration when exhausted. - May raise any other exception signifying an error on this specific game.""" + May raise any other exception signifying an error on this specific game. + May return None when a game has been skipped without an error.""" class Source(Iterable): """Source of games. E.g an installed app with a config file that lists game directories""" - win = None - + win: "CartridgesWindow" = None name: str variant: str @@ -38,7 +41,7 @@ class Source(Iterable): self.win = win @property - def full_name(self): + def full_name(self) -> str: """The source's full name""" full_name_ = self.name if self.variant is not None: @@ -46,7 +49,7 @@ class Source(Iterable): return full_name_ @property - def id(self): # pylint: disable=invalid-name + def id(self) -> str: # pylint: disable=invalid-name """The source's identifier""" id_ = self.name.lower() if self.variant is not None: @@ -54,7 +57,7 @@ class Source(Iterable): return id_ @property - def game_id_format(self): + def game_id_format(self) -> str: """The string format used to construct game IDs""" format_ = self.name.lower() if self.variant is not None: @@ -64,14 +67,14 @@ class Source(Iterable): @property @abstractmethod - def executable_format(self): + def executable_format(self) -> str: """The executable format used to construct game executables""" @property @abstractmethod - def is_installed(self): + def is_installed(self) -> bool: """Whether the source is detected as installed""" @abstractmethod - def __iter__(self): + def __iter__(self) -> SourceIterator: """Get the source's iterator, to use in for loops""" diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py new file mode 100644 index 0000000..d548009 --- /dev/null +++ b/src/importer/sources/steam_source.py @@ -0,0 +1,180 @@ +import re +import logging +from time import time +from pathlib import Path + +import requests +from requests import HTTPError, JSONDecodeError + +from src.game import Game +from src.importer.source import Source, SourceIterator +from src.utils.decorators import ( + replaced_by_path, + replaced_by_schema_key, + replaced_by_env_path, +) +from src.utils.save_cover import resize_cover, save_cover + + +class SteamAPIError(Exception): + pass + + +class SteamSourceIterator(SourceIterator): + source: "SteamSource" + + manifests = None + manifests_iterator = None + + installed_state_mask = 4 + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.manifests = set() + + # Get dirs that contain steam app manifests + manifests_dirs = set() + libraryfolders_path = self.source.location / "steamapps" / "libraryfolders.vdf" + with open(libraryfolders_path, "r") as file: + for line in file.readlines(): + line = line.strip() + prefix = '"path"' + if not line.startswith(prefix): + continue + library_folder = Path(line[len(prefix) :].strip()[1:-1]) + manifests_dir = library_folder / "steamapps" + if not (manifests_dir).is_dir(): + continue + manifests_dirs.add(manifests_dir) + + # Get app manifests + for manifests_dir in manifests_dirs: + for child in manifests_dir.iterdir(): + if child.is_file() and "appmanifest" in child.name: + self.manifests.add(child) + + self.manifests_iterator = iter(self.manifests) + + def __len__(self): + return len(self.manifests) + + def __next__(self): + # Get metadata from manifest + # Ignore manifests that don't have a value for all keys + manifest = next(self.manifests_iterator) + manifest_data = {"name": None, "appid": None, "StateFlags": "0"} + try: + with open(manifest) as file: + contents = file.read() + for key in manifest_data: + regex = f'"{key}"\s+"(.*)"\n' + if (match := re.search(regex, contents)) is None: + return None + manifest_data[key] = match.group(1) + except OSError: + return None + + # Skip non installed games + if not int(manifest_data["StateFlags"]) & self.installed_state_mask: + return None + + # Build basic game + appid = manifest_data["appid"] + values = { + "added": int(time()), + "name": manifest_data["name"], + "hidden": False, + "source": self.source.id, + "game_id": self.source.game_id_format.format(game_id=appid), + "executable": self.source.executable_format.format(game_id=appid), + "blacklisted": False, + "developer": None, + } + game = Game(self.source.win, values, allow_side_effects=False) + + # Add official cover image + cover_path = ( + self.source.location + / "appcache" + / "librarycache" + / f"{appid}_library_600x900.jpg" + ) + if cover_path.is_file(): + save_cover(self.win, game.game_id, resize_cover(self.win, cover_path)) + + # Make Steam API call + try: + with requests.get( + "https://store.steampowered.com/api/appdetails?appids=%s" + % manifest_data["appid"], + timeout=5, + ) as response: + response.raise_for_status() + steam_api_data = response.json()[appid] + except (HTTPError, JSONDecodeError) as error: + logging.warning( + "Error while querying Steam API for %s (%s)", + manifest_data["name"], + manifest_data["appid"], + exc_info=error, + ) + return game + + # Fill out new values + if not steam_api_data["success"] or steam_api_data["data"]["type"] != "game": + values["blacklisted"] = True + else: + values["developer"] = ", ".join(steam_api_data["data"]["developers"]) + game.update_values(values) + return game + + +class SteamSource(Source): + name = "Steam" + executable_format = "xdg-open steam://rungameid/{game_id}" + location = None + + @property + def is_installed(self): + # pylint: disable=pointless-statement + try: + self.location + except FileNotFoundError: + return False + return True + + def __iter__(self): + return SteamSourceIterator(source=self) + + +class SteamNativeSource(SteamSource): + variant = "native" + + @property + @replaced_by_schema_key("steam-location") + @replaced_by_env_path("XDG_DATA_HOME", "Steam/") + @replaced_by_path("~/.steam/") + @replaced_by_path("~/.local/share/Steam/") + def location(self): + raise FileNotFoundError() + + +class SteamFlatpakSource(SteamSource): + variant = "flatpak" + + @property + @replaced_by_schema_key("steam-flatpak-location") + @replaced_by_path("~/.var/app/com.valvesoftware.Steam/data/Steam/") + def location(self): + raise FileNotFoundError() + + +class SteamWindowsSource(SteamSource): + variant = "windows" + + @property + @replaced_by_schema_key("steam-windows-location") + @replaced_by_env_path("programfiles(x86)", "Steam") + def location(self): + raise FileNotFoundError() diff --git a/src/main.py b/src/main.py index 37ffe9a..9d3a3b8 100644 --- a/src/main.py +++ b/src/main.py @@ -36,6 +36,11 @@ from src.importer.sources.lutris_source import ( LutrisFlatpakSource, LutrisNativeSource, ) +from src.importer.sources.steam_source import ( + SteamNativeSource, + SteamFlatpakSource, + SteamWindowsSource, +) from src.preferences import PreferencesWindow from src.window import CartridgesWindow @@ -156,6 +161,10 @@ class CartridgesApplication(Adw.Application): if self.win.schema.get_boolean("lutris"): importer.add_source(LutrisNativeSource(self.win)) importer.add_source(LutrisFlatpakSource(self.win)) + if self.win.schema.get_boolean("steam"): + importer.add_source(SteamNativeSource(self.win)) + importer.add_source(SteamFlatpakSource(self.win)) + importer.add_source(SteamWindowsSource(self.win)) importer.run() def on_remove_game_action(self, *_args): diff --git a/src/utils/decorators.py b/src/utils/decorators.py index 0ec1ea7..456c450 100644 --- a/src/utils/decorators.py +++ b/src/utils/decorators.py @@ -14,7 +14,7 @@ class MyClass(): """ from pathlib import Path -from os import PathLike +from os import PathLike, environ from functools import wraps @@ -52,3 +52,21 @@ def replaced_by_schema_key(key: str): # Decorator builder return wrapper return decorator + + +def replaced_by_env_path(env_var_name: str, suffix: PathLike | None = None): + """Replace the method's returned path with a path whose root is the env variable""" + + def decorator(original_function): # Built decorator (closure) + @wraps(original_function) + def wrapper(*args, **kwargs): # func's override + try: + env_var = environ[env_var_name] + except KeyError: + return original_function(*args, **kwargs) + override = Path(env_var) / suffix + return replaced_by_path(override)(original_function)(*args, **kwargs) + + return wrapper + + return decorator From 4bc35383ae923d09fc35ce3e902219f119042daa Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 20 May 2023 19:57:40 +0200 Subject: [PATCH 044/173] Changed source.__next__() type hint --- src/importer/source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/importer/source.py b/src/importer/source.py index 663b92e..25da61c 100644 --- a/src/importer/source.py +++ b/src/importer/source.py @@ -1,5 +1,6 @@ from abc import abstractmethod from collections.abc import Iterable, Iterator, Sized +from typing import Optional from src.game import Game from src.window import CartridgesWindow @@ -22,7 +23,7 @@ class SourceIterator(Iterator, Sized): """Get a rough estimate of the number of games produced by the source""" @abstractmethod - def __next__(self) -> "Game" | None: + def __next__(self) -> Optional[Game]: """Get the next generated game from the source. Raises StopIteration when exhausted. May raise any other exception signifying an error on this specific game. From 8587c8039418ef7e9ea89bf97b0dc2513ce7d436 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 21 May 2023 00:34:52 +0200 Subject: [PATCH 045/173] =?UTF-8?q?=F0=9F=9A=A7=20More=20work=20on=20Steam?= =?UTF-8?q?=20source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data/hu.kramo.Cartridges.gschema.xml | 2 +- src/importer/sources/steam_source.py | 94 ++++++++++++---------------- src/utils/steam.py | 78 +++++++++++++++++++++++ 3 files changed, 118 insertions(+), 56 deletions(-) create mode 100644 src/utils/steam.py diff --git a/data/hu.kramo.Cartridges.gschema.xml b/data/hu.kramo.Cartridges.gschema.xml index 1230c42..abb05bd 100644 --- a/data/hu.kramo.Cartridges.gschema.xml +++ b/data/hu.kramo.Cartridges.gschema.xml @@ -17,7 +17,7 @@ "~/.steam/" - ~/.var/app/com.valvesoftware.Steam/data/Steam/ + "~/.var/app/com.valvesoftware.Steam/data/Steam/" "C:\Program Files (x86)\Steam" diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index d548009..ed7edaf 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -1,32 +1,32 @@ -import re -import logging -from time import time +from abc import abstractmethod from pathlib import Path +from time import time +from typing import Iterator -import requests from requests import HTTPError, JSONDecodeError from src.game import Game from src.importer.source import Source, SourceIterator from src.utils.decorators import ( + replaced_by_env_path, replaced_by_path, replaced_by_schema_key, - replaced_by_env_path, ) from src.utils.save_cover import resize_cover, save_cover - - -class SteamAPIError(Exception): - pass +from src.utils.steam import ( + SteamGameNotFoundError, + SteamHelper, + SteamInvalidManifestError, + SteamNotAGameError, +) class SteamSourceIterator(SourceIterator): source: "SteamSource" - manifests = None - manifests_iterator = None - - installed_state_mask = 4 + manifests: set = None + manifests_iterator: Iterator[Path] = None + installed_state_mask: int = 4 def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -60,36 +60,28 @@ class SteamSourceIterator(SourceIterator): return len(self.manifests) def __next__(self): + """Produce games""" + # Get metadata from manifest - # Ignore manifests that don't have a value for all keys - manifest = next(self.manifests_iterator) - manifest_data = {"name": None, "appid": None, "StateFlags": "0"} + manifest_path = next(self.manifests_iterator) + steam = SteamHelper() try: - with open(manifest) as file: - contents = file.read() - for key in manifest_data: - regex = f'"{key}"\s+"(.*)"\n' - if (match := re.search(regex, contents)) is None: - return None - manifest_data[key] = match.group(1) - except OSError: + local_data = steam.get_manifest_data(manifest_path) + except (OSError, SteamInvalidManifestError): return None # Skip non installed games - if not int(manifest_data["StateFlags"]) & self.installed_state_mask: + if not int(local_data["StateFlags"]) & self.installed_state_mask: return None - # Build basic game - appid = manifest_data["appid"] + # Build game from local data + appid = local_data["appid"] values = { "added": int(time()), - "name": manifest_data["name"], - "hidden": False, + "name": local_data["name"], "source": self.source.id, "game_id": self.source.game_id_format.format(game_id=appid), "executable": self.source.executable_format.format(game_id=appid), - "blacklisted": False, - "developer": None, } game = Game(self.source.win, values, allow_side_effects=False) @@ -101,39 +93,31 @@ class SteamSourceIterator(SourceIterator): / f"{appid}_library_600x900.jpg" ) if cover_path.is_file(): - save_cover(self.win, game.game_id, resize_cover(self.win, cover_path)) - - # Make Steam API call - try: - with requests.get( - "https://store.steampowered.com/api/appdetails?appids=%s" - % manifest_data["appid"], - timeout=5, - ) as response: - response.raise_for_status() - steam_api_data = response.json()[appid] - except (HTTPError, JSONDecodeError) as error: - logging.warning( - "Error while querying Steam API for %s (%s)", - manifest_data["name"], - manifest_data["appid"], - exc_info=error, + save_cover( + self.source.win, game.game_id, resize_cover(self.source.win, cover_path) ) - return game - # Fill out new values - if not steam_api_data["success"] or steam_api_data["data"]["type"] != "game": - values["blacklisted"] = True + # Get online metadata + # TODO move to its own manager + try: + online_data = steam.get_api_data(appid=appid) + except (HTTPError, JSONDecodeError, SteamGameNotFoundError): + pass + except SteamNotAGameError: + game.update_values({"blacklisted": True}) else: - values["developer"] = ", ".join(steam_api_data["data"]["developers"]) - game.update_values(values) + game.update_values(online_data) return game class SteamSource(Source): name = "Steam" executable_format = "xdg-open steam://rungameid/{game_id}" - location = None + + @property + @abstractmethod + def location(self) -> Path: + pass @property def is_installed(self): diff --git a/src/utils/steam.py b/src/utils/steam.py new file mode 100644 index 0000000..5e5677c --- /dev/null +++ b/src/utils/steam.py @@ -0,0 +1,78 @@ +import re +import logging +from typing import TypedDict + +import requests +from requests import HTTPError, JSONDecodeError + + +class SteamError(Exception): + pass + + +class SteamGameNotFoundError(SteamError): + pass + + +class SteamNotAGameError(SteamError): + pass + + +class SteamInvalidManifestError(SteamError): + pass + + +class SteamManifestData(TypedDict): + name: str + appid: str + StateFlags: str + + +class SteamAPIData(TypedDict): + developers: str + + +class SteamHelper: + """Helper around the Steam API""" + + base_url = "https://store.steampowered.com/api" + + def get_manifest_data(self, manifest_path) -> SteamManifestData: + """Get local data for a game from its manifest""" + + with open(manifest_path) as file: + contents = file.read() + + data = {} + + for key in SteamManifestData.__required_keys__: + regex = f'"{key}"\s+"(.*)"\n' + if (match := re.search(regex, contents)) is None: + raise SteamInvalidManifestError() + data[key] = match.group(1) + + return SteamManifestData(**data) + + def get_api_data(self, appid) -> SteamAPIData: + """Get online data for a game from its appid""" + + try: + with requests.get( + f"{self.base_url}/appdetails?appids={appid}", timeout=5 + ) as response: + response.raise_for_status() + data = response.json()[appid] + + except (HTTPError, JSONDecodeError) as error: + logging.warning("Error while querying Steam API for %s", appid) + raise error + + if not data["success"]: + raise SteamGameNotFoundError() + + if data["data"]["type"] != "game": + raise SteamNotAGameError() + + # Return API values we're interested in + values = SteamAPIData(developers=", ".join(data["data"]["developers"])) + return values From 49f0b2d92d1489e39564f0c5ed06d7014afd1ab6 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 21 May 2023 00:35:18 +0200 Subject: [PATCH 046/173] =?UTF-8?q?=F0=9F=8E=A8=20Changes=20to=20Lutris=20?= =?UTF-8?q?source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/lutris_source.py | 33 +++++++++------------------ 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 23c65b8..03afa01 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,6 +1,8 @@ +from abc import abstractmethod from functools import lru_cache from sqlite3 import connect from time import time +from pathlib import Path from src.game import Game from src.importer.source import Source, SourceIterator @@ -54,8 +56,7 @@ class LutrisSourceIterator(SourceIterator): return cursor.fetchone()[0] def __next__(self): - """Produce games. Behaviour depends on the state of the iterator.""" - # TODO decouple game creation from the window object + """Produce games""" row = None try: @@ -79,7 +80,7 @@ class LutrisSourceIterator(SourceIterator): game = Game(self.source.win, values, allow_side_effects=False) # Save official image - image_path = self.source.cache_location / "coverart" / f"{row[2]}.jpg" + image_path = self.source.location / "covers" / "coverart" / f"{row[2]}.jpg" if image_path.exists(): resized = resize_cover(self.source.win, image_path) save_cover(self.source.win, values["game_id"], resized) @@ -88,10 +89,15 @@ class LutrisSourceIterator(SourceIterator): class LutrisSource(Source): + """Generic lutris source""" + name = "Lutris" executable_format = "xdg-open lutris:rungameid/{game_id}" - location = None - cache_location = None + + @property + @abstractmethod + def location(self) -> Path: + pass @property def game_id_format(self): @@ -102,7 +108,6 @@ class LutrisSource(Source): # pylint: disable=pointless-statement try: self.location - self.cache_location except FileNotFoundError: return False return True @@ -112,8 +117,6 @@ class LutrisSource(Source): class LutrisNativeSource(LutrisSource): - """Class representing an installation of Lutris using native packaging""" - variant = "native" @property @@ -122,16 +125,8 @@ class LutrisNativeSource(LutrisSource): def location(self): raise FileNotFoundError() - @property - @replaced_by_schema_key("lutris-cache-location") - @replaced_by_path("~/.local/share/lutris/covers/") - def cache_location(self): - raise FileNotFoundError() - class LutrisFlatpakSource(LutrisSource): - """Class representing an installation of Lutris using flatpak""" - variant = "flatpak" @property @@ -139,9 +134,3 @@ class LutrisFlatpakSource(LutrisSource): @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/") def location(self): raise FileNotFoundError() - - @property - @replaced_by_schema_key("lutris-flatpak-cache-location") - @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/covers/") - def cache_location(self): - raise FileNotFoundError() From 7fdf1641203f40ba56984fff3c174096487b28df Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 21 May 2023 00:56:34 +0200 Subject: [PATCH 047/173] Changes to Steam Windows source --- src/importer/sources/steam_source.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index ed7edaf..baeacac 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -156,9 +156,11 @@ class SteamFlatpakSource(SteamSource): class SteamWindowsSource(SteamSource): variant = "windows" + executable_format = "start steam://rungameid/{game_id}" @property @replaced_by_schema_key("steam-windows-location") @replaced_by_env_path("programfiles(x86)", "Steam") + @replaced_by_path("C:\\Program Files (x86)\\Steam") def location(self): raise FileNotFoundError() From dd37fda07bef3a0b3f44254ba62880373e49d225 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 21 May 2023 14:54:33 +0200 Subject: [PATCH 048/173] =?UTF-8?q?=F0=9F=8E=A8=20Better=20Steam=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/steam_source.py | 37 ++++++++++++++-------------- src/utils/steam.py | 10 ++++++-- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index baeacac..238c714 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -1,3 +1,4 @@ +import re from abc import abstractmethod from pathlib import Path from time import time @@ -34,25 +35,25 @@ class SteamSourceIterator(SourceIterator): self.manifests = set() # Get dirs that contain steam app manifests - manifests_dirs = set() libraryfolders_path = self.source.location / "steamapps" / "libraryfolders.vdf" with open(libraryfolders_path, "r") as file: - for line in file.readlines(): - line = line.strip() - prefix = '"path"' - if not line.startswith(prefix): - continue - library_folder = Path(line[len(prefix) :].strip()[1:-1]) - manifests_dir = library_folder / "steamapps" - if not (manifests_dir).is_dir(): - continue - manifests_dirs.add(manifests_dir) + contents = file.read() + steamapps_dirs = [ + Path(path) / "steamapps" + for path in re.findall('"path"\s+"(.*)"\n', contents, re.IGNORECASE) + ] # Get app manifests - for manifests_dir in manifests_dirs: - for child in manifests_dir.iterdir(): - if child.is_file() and "appmanifest" in child.name: - self.manifests.add(child) + for steamapps_dir in steamapps_dirs: + if not steamapps_dir.is_dir(): + continue + self.manifests.update( + [ + manifest + for manifest in steamapps_dir.glob("appmanifest_*.acf") + if manifest.is_file() + ] + ) self.manifests_iterator = iter(self.manifests) @@ -71,7 +72,7 @@ class SteamSourceIterator(SourceIterator): return None # Skip non installed games - if not int(local_data["StateFlags"]) & self.installed_state_mask: + if not int(local_data["stateflags"]) & self.installed_state_mask: return None # Build game from local data @@ -101,9 +102,9 @@ class SteamSourceIterator(SourceIterator): # TODO move to its own manager try: online_data = steam.get_api_data(appid=appid) - except (HTTPError, JSONDecodeError, SteamGameNotFoundError): + except (HTTPError, JSONDecodeError): pass - except SteamNotAGameError: + except (SteamNotAGameError, SteamGameNotFoundError): game.update_values({"blacklisted": True}) else: game.update_values(online_data) diff --git a/src/utils/steam.py b/src/utils/steam.py index 5e5677c..f332456 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -23,12 +23,16 @@ class SteamInvalidManifestError(SteamError): class SteamManifestData(TypedDict): + """Dict returned by SteamHelper.get_manifest_data""" + name: str appid: str - StateFlags: str + stateflags: str class SteamAPIData(TypedDict): + """Dict returned by SteamHelper.get_api_data""" + developers: str @@ -47,7 +51,7 @@ class SteamHelper: for key in SteamManifestData.__required_keys__: regex = f'"{key}"\s+"(.*)"\n' - if (match := re.search(regex, contents)) is None: + if (match := re.search(regex, contents, re.IGNORECASE)) is None: raise SteamInvalidManifestError() data[key] = match.group(1) @@ -68,9 +72,11 @@ class SteamHelper: raise error if not data["success"]: + logging.debug("Appid %s not found", appid) raise SteamGameNotFoundError() if data["data"]["type"] != "game": + logging.debug("Appid %s is not a game", appid) raise SteamNotAGameError() # Return API values we're interested in From f246a73b1932efc854a2d0e1ccf62be88a1d6ae7 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 21 May 2023 18:11:02 +0200 Subject: [PATCH 049/173] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Changed=20shared?= =?UTF-8?q?=20imports=20to=20absolute?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/details_window.py | 2 +- src/game.py | 2 +- src/importers/bottles_importer.py | 2 +- src/importers/heroic_importer.py | 2 +- src/importers/itch_importer.py | 2 +- src/importers/lutris_importer.py | 2 +- src/importers/steam_importer.py | 2 +- src/main.py | 2 +- src/preferences.py | 3 ++- src/utils/importer.py | 2 +- src/utils/save_cover.py | 2 +- src/utils/steamgriddb.py | 2 +- src/window.py | 2 +- 13 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/details_window.py b/src/details_window.py index 6d4adf6..c1ec7c3 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -25,7 +25,7 @@ from gi.repository import Adw, Gio, GLib, Gtk from PIL import Image # TODO use SGDBHelper -from . import shared +import src.shared from src.game import Game from src.game_cover import GameCover from src.utils.create_dialog import create_dialog diff --git a/src/game.py b/src/game.py index 5bab56e..b3ef0c7 100644 --- a/src/game.py +++ b/src/game.py @@ -26,7 +26,7 @@ from time import time from gi.repository import Adw, Gio, Gtk -from . import shared +import src.shared from src.game_cover import GameCover diff --git a/src/importers/bottles_importer.py b/src/importers/bottles_importer.py index 38b8d87..2af8a84 100644 --- a/src/importers/bottles_importer.py +++ b/src/importers/bottles_importer.py @@ -22,7 +22,7 @@ from time import time import yaml -from . import shared +import src.shared from src.utils.check_install import check_install diff --git a/src/importers/heroic_importer.py b/src/importers/heroic_importer.py index 5282cc1..77359e4 100644 --- a/src/importers/heroic_importer.py +++ b/src/importers/heroic_importer.py @@ -23,7 +23,7 @@ from hashlib import sha256 from pathlib import Path from time import time -from . import shared +import src.shared from src.utils.check_install import check_install diff --git a/src/importers/itch_importer.py b/src/importers/itch_importer.py index af0b926..4b89f6b 100644 --- a/src/importers/itch_importer.py +++ b/src/importers/itch_importer.py @@ -26,7 +26,7 @@ from time import time import requests from gi.repository import GdkPixbuf, Gio -from . import shared +import src.shared from src.utils.check_install import check_install from src.utils.save_cover import resize_cover diff --git a/src/importers/lutris_importer.py b/src/importers/lutris_importer.py index 04b92f7..d9032c3 100644 --- a/src/importers/lutris_importer.py +++ b/src/importers/lutris_importer.py @@ -22,7 +22,7 @@ from shutil import copyfile from sqlite3 import connect from time import time -from . import shared +import src.shared from src.utils.check_install import check_install diff --git a/src/importers/steam_importer.py b/src/importers/steam_importer.py index 37ea2b6..e0984b5 100644 --- a/src/importers/steam_importer.py +++ b/src/importers/steam_importer.py @@ -25,7 +25,7 @@ from time import time import requests from gi.repository import Gio -from . import shared +import src.shared from src.utils.check_install import check_install diff --git a/src/main.py b/src/main.py index 9d3a3b8..54e7445 100644 --- a/src/main.py +++ b/src/main.py @@ -29,7 +29,7 @@ gi.require_version("Adw", "1") # pylint: disable=wrong-import-position from gi.repository import Adw, Gio, GLib, Gtk -from . import shared +import src.shared from src.details_window import DetailsWindow from src.importer.importer import Importer from src.importer.sources.lutris_source import ( diff --git a/src/preferences.py b/src/preferences.py index 204a02d..81302db 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -24,7 +24,8 @@ from pathlib import Path from gi.repository import Adw, Gio, GLib, Gtk # pylint: disable=unused-import -from . import shared +import src.shared + # TODO use the new sources from src.importers.bottles_importer import bottles_installed from src.importers.heroic_importer import heroic_installed diff --git a/src/utils/importer.py b/src/utils/importer.py index 79ac849..ad18cc0 100644 --- a/src/utils/importer.py +++ b/src/utils/importer.py @@ -19,7 +19,7 @@ from gi.repository import Adw, GLib, Gtk -from . import shared +import src.shared from .create_dialog import create_dialog from .game import Game from .save_cover import resize_cover, save_cover diff --git a/src/utils/save_cover.py b/src/utils/save_cover.py index 620466b..ed5a14f 100644 --- a/src/utils/save_cover.py +++ b/src/utils/save_cover.py @@ -24,7 +24,7 @@ from shutil import copyfile from gi.repository import Gio from PIL import Image, ImageSequence -from . import shared +import src.shared def resize_cover(cover_path=None, pixbuf=None): diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index e223dcd..c2e36ee 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -5,7 +5,7 @@ import requests from gi.repository import Gio from requests import HTTPError -from . import shared +import src.shared from src.utils.create_dialog import create_dialog from src.utils.save_cover import resize_cover, save_cover diff --git a/src/window.py b/src/window.py index 69a4418..843a6fe 100644 --- a/src/window.py +++ b/src/window.py @@ -22,7 +22,7 @@ from datetime import datetime from gi.repository import Adw, Gio, GLib, Gtk -from . import shared +import src.shared from src.game import Game From e5d2657bb9b44e118603023b656744ae73b25789 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 21 May 2023 18:15:04 +0200 Subject: [PATCH 050/173] =?UTF-8?q?=F0=9F=9A=91=20More=20rebase=20conflict?= =?UTF-8?q?s=20resolved?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/details_window.py | 2 +- src/game.py | 4 ++-- src/importers/bottles_importer.py | 2 +- src/importers/heroic_importer.py | 2 +- src/importers/itch_importer.py | 2 +- src/importers/lutris_importer.py | 2 +- src/importers/steam_importer.py | 2 +- src/main.py | 2 +- src/preferences.py | 2 +- src/utils/importer.py | 2 +- src/utils/save_cover.py | 2 +- src/utils/steamgriddb.py | 2 +- src/window.py | 2 +- 13 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/details_window.py b/src/details_window.py index c1ec7c3..bfa89fc 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -25,7 +25,7 @@ from gi.repository import Adw, Gio, GLib, Gtk from PIL import Image # TODO use SGDBHelper -import src.shared +import src.shared as shared from src.game import Game from src.game_cover import GameCover from src.utils.create_dialog import create_dialog diff --git a/src/game.py b/src/game.py index b3ef0c7..d46dd05 100644 --- a/src/game.py +++ b/src/game.py @@ -26,7 +26,7 @@ from time import time from gi.repository import Adw, Gio, Gtk -import src.shared +import src.shared as shared from src.game_cover import GameCover @@ -60,7 +60,7 @@ class Game(Gtk.Box): blacklisted = None game_cover = None - def __init__(self, win, data, allow_side_effects=True, **kwargs): + def __init__(self, data, allow_side_effects=True, **kwargs): super().__init__(**kwargs) self.win = shared.win diff --git a/src/importers/bottles_importer.py b/src/importers/bottles_importer.py index 2af8a84..260543a 100644 --- a/src/importers/bottles_importer.py +++ b/src/importers/bottles_importer.py @@ -22,7 +22,7 @@ from time import time import yaml -import src.shared +import src.shared as shared from src.utils.check_install import check_install diff --git a/src/importers/heroic_importer.py b/src/importers/heroic_importer.py index 77359e4..a2c9412 100644 --- a/src/importers/heroic_importer.py +++ b/src/importers/heroic_importer.py @@ -23,7 +23,7 @@ from hashlib import sha256 from pathlib import Path from time import time -import src.shared +import src.shared as shared from src.utils.check_install import check_install diff --git a/src/importers/itch_importer.py b/src/importers/itch_importer.py index 4b89f6b..ee68384 100644 --- a/src/importers/itch_importer.py +++ b/src/importers/itch_importer.py @@ -26,7 +26,7 @@ from time import time import requests from gi.repository import GdkPixbuf, Gio -import src.shared +import src.shared as shared from src.utils.check_install import check_install from src.utils.save_cover import resize_cover diff --git a/src/importers/lutris_importer.py b/src/importers/lutris_importer.py index d9032c3..62d352a 100644 --- a/src/importers/lutris_importer.py +++ b/src/importers/lutris_importer.py @@ -22,7 +22,7 @@ from shutil import copyfile from sqlite3 import connect from time import time -import src.shared +import src.shared as shared from src.utils.check_install import check_install diff --git a/src/importers/steam_importer.py b/src/importers/steam_importer.py index e0984b5..4a6bc34 100644 --- a/src/importers/steam_importer.py +++ b/src/importers/steam_importer.py @@ -25,7 +25,7 @@ from time import time import requests from gi.repository import Gio -import src.shared +import src.shared as shared from src.utils.check_install import check_install diff --git a/src/main.py b/src/main.py index 54e7445..05e4c4e 100644 --- a/src/main.py +++ b/src/main.py @@ -29,7 +29,7 @@ gi.require_version("Adw", "1") # pylint: disable=wrong-import-position from gi.repository import Adw, Gio, GLib, Gtk -import src.shared +import src.shared as shared from src.details_window import DetailsWindow from src.importer.importer import Importer from src.importer.sources.lutris_source import ( diff --git a/src/preferences.py b/src/preferences.py index 81302db..9df90b5 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -24,7 +24,7 @@ from pathlib import Path from gi.repository import Adw, Gio, GLib, Gtk # pylint: disable=unused-import -import src.shared +import src.shared as shared # TODO use the new sources from src.importers.bottles_importer import bottles_installed diff --git a/src/utils/importer.py b/src/utils/importer.py index ad18cc0..9929757 100644 --- a/src/utils/importer.py +++ b/src/utils/importer.py @@ -19,7 +19,7 @@ from gi.repository import Adw, GLib, Gtk -import src.shared +import src.shared as shared from .create_dialog import create_dialog from .game import Game from .save_cover import resize_cover, save_cover diff --git a/src/utils/save_cover.py b/src/utils/save_cover.py index ed5a14f..5a2cef5 100644 --- a/src/utils/save_cover.py +++ b/src/utils/save_cover.py @@ -24,7 +24,7 @@ from shutil import copyfile from gi.repository import Gio from PIL import Image, ImageSequence -import src.shared +import src.shared as shared def resize_cover(cover_path=None, pixbuf=None): diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index c2e36ee..abf268c 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -5,7 +5,7 @@ import requests from gi.repository import Gio from requests import HTTPError -import src.shared +import src.shared as shared from src.utils.create_dialog import create_dialog from src.utils.save_cover import resize_cover, save_cover diff --git a/src/window.py b/src/window.py index 843a6fe..b97914e 100644 --- a/src/window.py +++ b/src/window.py @@ -22,7 +22,7 @@ from datetime import datetime from gi.repository import Adw, Gio, GLib, Gtk -import src.shared +import src.shared as shared from src.game import Game From 9fd58e6ba3391a2cbc376b8fcb4a4185a8742f87 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 21 May 2023 18:29:26 +0200 Subject: [PATCH 051/173] =?UTF-8?q?=F0=9F=9A=A7=20More=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 21 ++++++++++----------- src/importer/source.py | 6 ------ src/importer/sources/lutris_source.py | 8 ++++---- src/importer/sources/steam_source.py | 6 ++---- src/main.py | 16 ++++++++-------- src/utils/decorators.py | 5 +++-- src/utils/steamgriddb.py | 18 ++++++------------ 7 files changed, 33 insertions(+), 47 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 2f1e5ae..50f2bd5 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -3,6 +3,7 @@ import logging from gi.repository import Adw, Gio, Gtk from requests import HTTPError +import src.shared as shared from src.utils.create_dialog import create_dialog from src.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper from src.utils.task import Task @@ -16,7 +17,6 @@ class Importer: import_statuspage = None import_dialog = None - win = None sources = None n_games_added = 0 @@ -27,8 +27,7 @@ class Importer: sgdb_cancellable = None sgdb_error = None - def __init__(self, win): - self.win = win + def __init__(self): self.sources = set() @property @@ -82,7 +81,7 @@ class Importer: modal=True, default_width=350, default_height=-1, - transient_for=self.win, + transient_for=shared.win, deletable=False, ) self.import_dialog.present() @@ -123,14 +122,14 @@ class Importer: # Avoid duplicates if ( - game.game_id in self.win.games - and not self.win.games[game.game_id].removed + game.game_id in shared.win.games + and not shared.win.games[game.game_id].removed ): continue # Register game logging.info("New game registered %s (%s)", game.name, game.game_id) - self.win.games[game.game_id] = game + shared.win.games[game.game_id] = game game.save() self.n_games_added += 1 @@ -156,7 +155,7 @@ class Importer: """SGDB query code""" game, *_rest = data game.set_loading(1) - sgdb = SGDBHelper(self.win) + sgdb = SGDBHelper() try: sgdb.conditionaly_update_cover(game) except SGDBAuthError as error: @@ -208,12 +207,12 @@ class Importer: # The variable is the number of games toast.set_title(_("{} games imported").format(self.n_games_added)) - self.win.toast_overlay.add_toast(toast) + shared.win.toast_overlay.add_toast(toast) def create_sgdb_error_dialog(self): """SGDB error dialog""" create_dialog( - self.win, + shared.win, _("Couldn't Connect to SteamGridDB"), str(self.sgdb_error), "open_preferences", @@ -221,7 +220,7 @@ class Importer: ).connect("response", self.dialog_response_callback, "sgdb") def open_preferences(self, page=None, expander_row=None): - self.win.get_application().on_preferences_action( + shared.win.get_application().on_preferences_action( page_name=page, expander_row=expander_row ) diff --git a/src/importer/source.py b/src/importer/source.py index 25da61c..a45667a 100644 --- a/src/importer/source.py +++ b/src/importer/source.py @@ -3,7 +3,6 @@ from collections.abc import Iterable, Iterator, Sized from typing import Optional from src.game import Game -from src.window import CartridgesWindow class SourceIterator(Iterator, Sized): @@ -33,14 +32,9 @@ class SourceIterator(Iterator, Sized): class Source(Iterable): """Source of games. E.g an installed app with a config file that lists game directories""" - win: "CartridgesWindow" = None name: str variant: str - def __init__(self, win) -> None: - super().__init__() - self.win = win - @property def full_name(self) -> str: """The source's full name""" diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 03afa01..ccd3488 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -4,6 +4,7 @@ from sqlite3 import connect from time import time from pathlib import Path +import src.shared as shared from src.game import Game from src.importer.source import Source, SourceIterator from src.utils.decorators import replaced_by_path, replaced_by_schema_key @@ -41,7 +42,7 @@ class LutrisSourceIterator(SourceIterator): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.import_steam = self.source.win.schema.get_boolean("lutris-import-steam") + self.import_steam = shared.schema.get_boolean("lutris-import-steam") self.db_location = self.source.location / "pga.db" self.db_connection = connect(self.db_location) self.db_request_params = {"import_steam": self.import_steam} @@ -77,13 +78,12 @@ class LutrisSourceIterator(SourceIterator): "executable": self.source.executable_format.format(game_id=row[2]), "developer": None, # TODO get developer metadata on Lutris } - game = Game(self.source.win, values, allow_side_effects=False) + game = Game(values, allow_side_effects=False) # Save official image image_path = self.source.location / "covers" / "coverart" / f"{row[2]}.jpg" if image_path.exists(): - resized = resize_cover(self.source.win, image_path) - save_cover(self.source.win, values["game_id"], resized) + save_cover(values["game_id"], resize_cover(image_path)) return game diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 238c714..30ada4d 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -84,7 +84,7 @@ class SteamSourceIterator(SourceIterator): "game_id": self.source.game_id_format.format(game_id=appid), "executable": self.source.executable_format.format(game_id=appid), } - game = Game(self.source.win, values, allow_side_effects=False) + game = Game(values, allow_side_effects=False) # Add official cover image cover_path = ( @@ -94,9 +94,7 @@ class SteamSourceIterator(SourceIterator): / f"{appid}_library_600x900.jpg" ) if cover_path.is_file(): - save_cover( - self.source.win, game.game_id, resize_cover(self.source.win, cover_path) - ) + save_cover(game.game_id, resize_cover(cover_path)) # Get online metadata # TODO move to its own manager diff --git a/src/main.py b/src/main.py index 05e4c4e..647fb14 100644 --- a/src/main.py +++ b/src/main.py @@ -157,14 +157,14 @@ class CartridgesApplication(Adw.Application): DetailsWindow() def on_import_action(self, *_args): - importer = Importer(self.win) - if self.win.schema.get_boolean("lutris"): - importer.add_source(LutrisNativeSource(self.win)) - importer.add_source(LutrisFlatpakSource(self.win)) - if self.win.schema.get_boolean("steam"): - importer.add_source(SteamNativeSource(self.win)) - importer.add_source(SteamFlatpakSource(self.win)) - importer.add_source(SteamWindowsSource(self.win)) + importer = Importer() + if shared.schema.get_boolean("lutris"): + importer.add_source(LutrisNativeSource()) + importer.add_source(LutrisFlatpakSource()) + if shared.schema.get_boolean("steam"): + importer.add_source(SteamNativeSource()) + importer.add_source(SteamFlatpakSource()) + importer.add_source(SteamWindowsSource()) importer.run() def on_remove_game_action(self, *_args): diff --git a/src/utils/decorators.py b/src/utils/decorators.py index 456c450..d9a8433 100644 --- a/src/utils/decorators.py +++ b/src/utils/decorators.py @@ -17,6 +17,8 @@ from pathlib import Path from os import PathLike, environ from functools import wraps +import src.shared as shared + def replaced_by_path(override: PathLike): # Decorator builder """Replace the method's returned path with the override @@ -42,9 +44,8 @@ def replaced_by_schema_key(key: str): # Decorator builder def decorator(original_function): # Built decorator (closure) @wraps(original_function) def wrapper(*args, **kwargs): # func's override - schema = args[0].win.schema try: - override = schema.get_string(key) + override = shared.schema.get_string(key) except Exception: # pylint: disable=broad-exception-caught return original_function(*args, **kwargs) return replaced_by_path(override)(original_function)(*args, **kwargs) diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index abf268c..b685918 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -34,14 +34,10 @@ class SGDBHelper: """Helper class to make queries to SteamGridDB""" base_url = "https://www.steamgriddb.com/api/v2/" - win = None - - def __init__(self, win): - self.win = win @property def auth_headers(self): - key = self.win.schema.get_string("sgdb-key") + key = shared.schema.get_string("sgdb-key") headers = {"Authorization": f"Bearer {key}"} return headers @@ -82,14 +78,14 @@ class SGDBHelper: """Update the game's cover if appropriate""" # Obvious skips - use_sgdb = self.win.schema.get_boolean("sgdb") + use_sgdb = shared.schema.get_boolean("sgdb") if not use_sgdb or game.blacklisted: return - image_trunk = self.win.covers_dir / game.game_id + image_trunk = shared.covers_dir / game.game_id still = image_trunk.with_suffix(".tiff") uri_kwargs = image_trunk.with_suffix(".gif") - prefer_sgdb = self.win.schema.get_boolean("sgdb-prefer") + prefer_sgdb = shared.schema.get_boolean("sgdb-prefer") # Do nothing if file present and not prefer SGDB if not prefer_sgdb and (still.is_file() or uri_kwargs.is_file()): @@ -106,7 +102,7 @@ class SGDBHelper: # Build different SGDB options to try image_uri_kwargs_sets = [{"animated": False}] - if self.win.schema.get_boolean("sgdb-animated"): + if shared.schema.get_boolean("sgdb-animated"): image_uri_kwargs_sets.insert(0, {"animated": True}) # Download covers @@ -117,9 +113,7 @@ class SGDBHelper: tmp_file = Gio.File.new_tmp()[0] tmp_file_path = tmp_file.get_path() Path(tmp_file_path).write_bytes(response.content) - save_cover( - self.win, game.game_id, resize_cover(self.win, tmp_file_path) - ) + save_cover(game.game_id, resize_cover(tmp_file_path)) except SGDBAuthError as error: # Let caller handle auth errors raise error From 3df9380c2d5246236ef41875cca6a2693a19c199 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Tue, 23 May 2023 02:06:47 +0200 Subject: [PATCH 052/173] =?UTF-8?q?=F0=9F=9A=A7=20Base=20work=20for=20the?= =?UTF-8?q?=20Store/Manager=20architecture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/manager.py | 17 ++++++++++ src/store/store.py | 74 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 src/store/manager.py create mode 100644 src/store/store.py diff --git a/src/store/manager.py b/src/store/manager.py new file mode 100644 index 0000000..799851a --- /dev/null +++ b/src/store/manager.py @@ -0,0 +1,17 @@ +from abc import abstractmethod + +from src.game import Game + + +class Manager: + """Class in charge of handling a post creation action for games. + May connect to signals on the game to handle them.""" + + run_after: set[type["Manager"]] + + @abstractmethod + def run(self, game: Game) -> None: + """Pass the game through the manager. + May block its thread. + May not raise exceptions, as they will be silently ignored.""" + pass diff --git a/src/store/store.py b/src/store/store.py new file mode 100644 index 0000000..288b175 --- /dev/null +++ b/src/store/store.py @@ -0,0 +1,74 @@ +from src.game import Game +from src.store.manager import Manager +from src.utils.task import Task + + +class Pipeline(set): + """Class representing a set of Managers for a game""" + + @property + def blocked_managers(self) -> set(Manager): + """Get the managers that cannot run because their dependencies aren't done""" + blocked = set() + for manager_a in self: + for manager_b in self: + if manager_a == manager_b: + continue + if type(manager_b) in manager_a.run_after: + blocked.add(manager_a) + return blocked + + @property + def startable_managers(self) -> set(Manager): + """Get the managers that can be run""" + return self - self.blocked_managers + + +class Store: + """Class in charge of handling games being added to the app.""" + + managers: set[Manager] + pipelines: dict[str, Pipeline] + games: dict[str, Game] + + def __init__(self) -> None: + self.managers = set() + self.games = {} + self.pipelines = {} + + def add_manager(self, manager: Manager): + """Add a manager class that will run when games are added""" + self.managers.add(manager) + + def add_game(self, game: Game, replace=False): + """Add a game to the app if not already there + + :param replace bool: Replace the game if it already exists""" + if ( + game.game_id in self.games + and not self.games[game.game_id].removed + and not replace + ): + return + self.games[game.game_id] = game + self.pipelines[game.game_id] = Pipeline(self.managers) + self.advance_pipeline(game) + + def advance_pipeline(self, game: Game): + """Spawn tasks for managers that are able to run for a game""" + for manager in self.pipelines[game.game_id].startable_managers: + data = (manager, game) + task = Task.new(None, None, self.manager_task_callback, data) + task.set_task_data(data) + task.run_in_thread(self.manager_task_thread_func) + + def manager_task_thread_func(self, _task, _source_object, data, _cancellable): + """Thread function for manager tasks""" + manager, game, *_rest = data + manager.run(game) + + def manager_task_callback(self, _source_object, _result, data): + """Callback function for manager tasks""" + manager, game, *_rest = data + self.pipelines[game.game_id].remove(manager) + self.advance_pipeline(game) From 9ea2e9652d82e7cf69abc9d0c38dc1ac8c11f995 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Tue, 23 May 2023 02:31:00 +0200 Subject: [PATCH 053/173] =?UTF-8?q?=F0=9F=9A=A7=20More=20work=20on=20manag?= =?UTF-8?q?ers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 16 +++------------- src/main.py | 7 +++++++ src/shared.py | 1 + src/store/display_manager.py | 12 ++++++++++++ src/store/file_manager.py | 10 ++++++++++ src/store/sgdb_manager.py | 11 +++++++++++ src/store/steam_api_manager.py | 11 +++++++++++ src/store/store.py | 2 ++ 8 files changed, 57 insertions(+), 13 deletions(-) create mode 100644 src/store/display_manager.py create mode 100644 src/store/file_manager.py create mode 100644 src/store/sgdb_manager.py create mode 100644 src/store/steam_api_manager.py diff --git a/src/importer/importer.py b/src/importer/importer.py index 50f2bd5..cc26ca3 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -118,23 +118,13 @@ class Importer: if game is None: continue - # TODO register in store instead of dict - - # Avoid duplicates - if ( - game.game_id in shared.win.games - and not shared.win.games[game.game_id].removed - ): - continue - # Register game - logging.info("New game registered %s (%s)", game.name, game.game_id) - shared.win.games[game.game_id] = game - game.save() + logging.info("Imported %s (%s)", game.name, game.game_id) + shared.store.add_game(game) self.n_games_added += 1 # Start sgdb lookup for game - # HACK move to its own manager + # TODO move to its own manager task = Task.new( None, self.sgdb_cancellable, self.sgdb_task_callback, (game,) ) diff --git a/src/main.py b/src/main.py index 647fb14..0cbb3a7 100644 --- a/src/main.py +++ b/src/main.py @@ -30,6 +30,7 @@ gi.require_version("Adw", "1") from gi.repository import Adw, Gio, GLib, Gtk import src.shared as shared +from src.store.store import Store from src.details_window import DetailsWindow from src.importer.importer import Importer from src.importer.sources.lutris_source import ( @@ -47,6 +48,7 @@ from src.window import CartridgesWindow class CartridgesApplication(Adw.Application): win = None + store = None def __init__(self): super().__init__( @@ -54,6 +56,11 @@ class CartridgesApplication(Adw.Application): ) def do_activate(self): # pylint: disable=arguments-differ + # Create the games store + if not self.store: + # TODO add managers to the store + self.store = Store() + # Create the main window self.win = self.props.active_window # pylint: disable=no-member if not self.win: diff --git a/src/shared.py b/src/shared.py index 9cf54d5..0417d64 100644 --- a/src/shared.py +++ b/src/shared.py @@ -52,4 +52,5 @@ image_size = (200 * scale_factor, 300 * scale_factor) # pylint: disable=invalid-name win = None importer = None +store = None spec_version = 1.5 # The version of the game_id.json spec diff --git a/src/store/display_manager.py b/src/store/display_manager.py new file mode 100644 index 0000000..88537e7 --- /dev/null +++ b/src/store/display_manager.py @@ -0,0 +1,12 @@ +import src.shared as shared +from src.store.manager import Manager +from src.game import Game + + +class DisplayManager(Manager): + """Manager in charge of adding a game to the UI""" + + def run(self, game: Game) -> None: + # TODO decouple a game from its widget + shared.win.games[game.game_id] = game + game.update() diff --git a/src/store/file_manager.py b/src/store/file_manager.py new file mode 100644 index 0000000..75c8e58 --- /dev/null +++ b/src/store/file_manager.py @@ -0,0 +1,10 @@ +from src.store.manager import Manager +from src.game import Game + + +class FileManager(Manager): + """Manager in charge of saving a game to a file""" + + def run(self, game: Game) -> None: + # TODO make game.save (disk) not trigger game.update (UI) + game.save() diff --git a/src/store/sgdb_manager.py b/src/store/sgdb_manager.py new file mode 100644 index 0000000..22f5699 --- /dev/null +++ b/src/store/sgdb_manager.py @@ -0,0 +1,11 @@ +from src.store.manager import Manager +from src.game import Game +from src.utils.steamgriddb import SGDBHelper + + +class SGDBManager(Manager): + """Manager in charge of downloading a game's cover from steamgriddb""" + + def run(self, game: Game) -> None: + # TODO + pass diff --git a/src/store/steam_api_manager.py b/src/store/steam_api_manager.py new file mode 100644 index 0000000..30f3c42 --- /dev/null +++ b/src/store/steam_api_manager.py @@ -0,0 +1,11 @@ +from src.store.manager import Manager +from src.game import Game +from src.utils.steam import SteamHelper + + +class SteamAPIManager(Manager): + """Manager in charge of completing a game's data from the Steam API""" + + def run(self, game: Game) -> None: + # TODO + pass diff --git a/src/store/store.py b/src/store/store.py index 288b175..1c80032 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -1,3 +1,4 @@ +import src.shared as shared from src.game import Game from src.store.manager import Manager from src.utils.task import Task @@ -32,6 +33,7 @@ class Store: games: dict[str, Game] def __init__(self) -> None: + shared.store = self self.managers = set() self.games = {} self.pipelines = {} From abe41635fd71df22ef7e24ab9ef872633a33dbe5 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Tue, 23 May 2023 15:26:48 +0200 Subject: [PATCH 054/173] =?UTF-8?q?=F0=9F=9A=A7=20More=20work=20on=20manag?= =?UTF-8?q?ers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main.py | 20 +++++--- src/store/manager.py | 34 ++++++++++++- src/store/sgdb_manager.py | 19 ++++++-- src/store/store.py | 100 ++++++++++++++++++++++++++------------ 4 files changed, 130 insertions(+), 43 deletions(-) diff --git a/src/main.py b/src/main.py index 0cbb3a7..73b5f63 100644 --- a/src/main.py +++ b/src/main.py @@ -30,19 +30,20 @@ gi.require_version("Adw", "1") from gi.repository import Adw, Gio, GLib, Gtk import src.shared as shared -from src.store.store import Store from src.details_window import DetailsWindow from src.importer.importer import Importer -from src.importer.sources.lutris_source import ( - LutrisFlatpakSource, - LutrisNativeSource, -) +from src.importer.sources.lutris_source import LutrisFlatpakSource, LutrisNativeSource from src.importer.sources.steam_source import ( - SteamNativeSource, SteamFlatpakSource, + SteamNativeSource, SteamWindowsSource, ) from src.preferences import PreferencesWindow +from src.store.display_manager import DisplayManager +from src.store.file_manager import FileManager +from src.store.sgdb_manager import SGDBManager +from src.store.steam_api_manager import SteamAPIManager +from src.store.store import Store from src.window import CartridgesWindow @@ -56,10 +57,13 @@ class CartridgesApplication(Adw.Application): ) def do_activate(self): # pylint: disable=arguments-differ - # Create the games store + # Create the games store and its managers if not self.store: - # TODO add managers to the store self.store = Store() + self.store.add_manager(SteamAPIManager()) + self.store.add_manager(SGDBManager()) + self.store.add_manager(FileManager()) + self.store.add_manager(DisplayManager()) # Create the main window self.win = self.props.active_window # pylint: disable=no-member diff --git a/src/store/manager.py b/src/store/manager.py index 799851a..361f63a 100644 --- a/src/store/manager.py +++ b/src/store/manager.py @@ -1,14 +1,46 @@ from abc import abstractmethod +from gi.repository import Gio from src.game import Game class Manager: """Class in charge of handling a post creation action for games. - May connect to signals on the game to handle them.""" + + * May connect to signals on the game to handle them. + * May cancel its running tasks on critical error, + in that case a new cancellable must be generated for new tasks to run. + """ run_after: set[type["Manager"]] + cancellable: Gio.Cancellable + errors: list[Exception] + + def __init__(self) -> None: + super().__init__() + self.cancellable = Gio.Cancellable() + self.errors = list() + + def cancel_tasks(self): + """Cancel all tasks for this manager""" + self.cancellable.cancel() + + def reset_cancellable(self): + """Reset the cancellable for this manager. + Alreadyn scheduled Tasks will no longer be cancellable.""" + self.cancellable = Gio.Cancellable() + + def report_error(self, error: Exception): + """Report an error that happened in of run""" + self.errors.append(error) + + def collect_errors(self) -> list[Exception]: + """Get the errors produced by the manager and remove them from self.errors""" + errors = list(self.errors) + self.errors.clear() + return errors + @abstractmethod def run(self, game: Game) -> None: """Pass the game through the manager. diff --git a/src/store/sgdb_manager.py b/src/store/sgdb_manager.py index 22f5699..faec500 100644 --- a/src/store/sgdb_manager.py +++ b/src/store/sgdb_manager.py @@ -1,11 +1,22 @@ -from src.store.manager import Manager +from requests import HTTPError + from src.game import Game -from src.utils.steamgriddb import SGDBHelper +from src.store.manager import Manager +from src.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper class SGDBManager(Manager): """Manager in charge of downloading a game's cover from steamgriddb""" def run(self, game: Game) -> None: - # TODO - pass + try: + sgdb = SGDBHelper() + sgdb.conditionaly_update_cover(game) + except SGDBAuthError as error: + # If invalid auth, cancel all SGDBManager tasks + self.cancellable.cancel() + self.report_error(error) + except (HTTPError, SGDBError) as error: + # On other error, just report it + self.report_error(error) + pass diff --git a/src/store/store.py b/src/store/store.py index 1c80032..6aea015 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -1,18 +1,39 @@ +from typing import Iterable + +from gi.repository import GObject + import src.shared as shared from src.game import Game from src.store.manager import Manager from src.utils.task import Task -class Pipeline(set): - """Class representing a set of Managers for a game""" +class Pipeline(GObject.Object): + """Class representing a set of managers for a game""" + + game: Game + + waiting: set[Manager] + running: set[Manager] + done: set[Manager] + + def __init__(self, managers: Iterable[Manager]) -> None: + super().__init__() + self.waiting = set(managers) + self.running = set() + self.done = set() @property - def blocked_managers(self) -> set(Manager): + def not_done(self) -> set[Manager]: + """Get the managers that are not done yet""" + return self.waiting + self.running + + @property + def blocked(self) -> set[Manager]: """Get the managers that cannot run because their dependencies aren't done""" blocked = set() - for manager_a in self: - for manager_b in self: + for manager_a in self.waiting: + for manager_b in self.not_done: if manager_a == manager_b: continue if type(manager_b) in manager_a.run_after: @@ -20,9 +41,43 @@ class Pipeline(set): return blocked @property - def startable_managers(self) -> set(Manager): + def ready(self) -> set[Manager]: """Get the managers that can be run""" - return self - self.blocked_managers + return self.waiting - self.blocked + + def advance(self): + """Spawn tasks for managers that are able to run for a game""" + for manager in self.ready: + self.waiting.remove(manager) + self.running.add(manager) + data = (manager,) + task = Task.new(self, manager.cancellable, self.manager_task_callback, data) + task.set_task_data(data) + task.run_in_thread(self.manager_task_thread_func) + + @GObject.Signal(name="manager-started") + def manager_started(self, manager: Manager) -> None: + """Signal emitted when a manager is started""" + pass + + def manager_task_thread_func(self, _task, _source_object, data, cancellable): + """Thread function for manager tasks""" + manager, *_rest = data + self.emit("manager-started", manager) + manager.run(self.game, cancellable) + + @GObject.Signal(name="manager-done") + def manager_done(self, manager: Manager) -> None: + """Signal emitted when a manager is done""" + pass + + def manager_task_callback(self, _source_object, _result, data): + """Callback function for manager tasks""" + manager, *_rest = data + self.running.remove(manager) + self.done.add(manager) + self.emit("manager-done", manager) + self.advance() class Store: @@ -42,35 +97,20 @@ class Store: """Add a manager class that will run when games are added""" self.managers.add(manager) - def add_game(self, game: Game, replace=False): + def add_game(self, game: Game, replace=False) -> Pipeline: """Add a game to the app if not already there - :param replace bool: Replace the game if it already exists""" + :param replace bool: Replace the game if it already exists + :return: + """ if ( game.game_id in self.games and not self.games[game.game_id].removed and not replace ): return + pipeline = Pipeline(self.managers) self.games[game.game_id] = game - self.pipelines[game.game_id] = Pipeline(self.managers) - self.advance_pipeline(game) - - def advance_pipeline(self, game: Game): - """Spawn tasks for managers that are able to run for a game""" - for manager in self.pipelines[game.game_id].startable_managers: - data = (manager, game) - task = Task.new(None, None, self.manager_task_callback, data) - task.set_task_data(data) - task.run_in_thread(self.manager_task_thread_func) - - def manager_task_thread_func(self, _task, _source_object, data, _cancellable): - """Thread function for manager tasks""" - manager, game, *_rest = data - manager.run(game) - - def manager_task_callback(self, _source_object, _result, data): - """Callback function for manager tasks""" - manager, game, *_rest = data - self.pipelines[game.game_id].remove(manager) - self.advance_pipeline(game) + self.pipelines[game.game_id] = pipeline + pipeline.advance() + return pipeline From 95524563bbb91b8b929f6d4c2bcbd6b3ba699df0 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Tue, 23 May 2023 16:49:37 +0200 Subject: [PATCH 055/173] =?UTF-8?q?=F0=9F=8E=A8=20Moved=20things=20to=20ma?= =?UTF-8?q?nagers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 63 ++-------------------------- src/importer/sources/steam_source.py | 12 ------ src/store/display_manager.py | 6 ++- src/store/file_manager.py | 6 ++- src/store/steam_api_manager.py | 24 +++++++++-- 5 files changed, 34 insertions(+), 77 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index cc26ca3..8a6edda 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -22,33 +22,21 @@ class Importer: n_games_added = 0 n_source_tasks_created = 0 n_source_tasks_done = 0 - n_sgdb_tasks_created = 0 - n_sgdb_tasks_done = 0 - sgdb_cancellable = None - sgdb_error = None def __init__(self): self.sources = set() - @property - def n_tasks_created(self): - return self.n_source_tasks_created + self.n_sgdb_tasks_created - - @property - def n_tasks_done(self): - return self.n_source_tasks_done + self.n_sgdb_tasks_done - @property def progress(self): try: - progress = self.n_tasks_done / self.n_tasks_created + progress = self.n_source_tasks_done / self.n_source_tasks_created except ZeroDivisionError: progress = 1 return progress @property def finished(self): - return self.n_tasks_created == self.n_tasks_done + return self.n_source_tasks_created == self.n_source_tasks_done def add_source(self, source): self.sources.add(source) @@ -123,15 +111,6 @@ class Importer: shared.store.add_game(game) self.n_games_added += 1 - # Start sgdb lookup for game - # TODO move to its own manager - task = Task.new( - None, self.sgdb_cancellable, self.sgdb_task_callback, (game,) - ) - self.n_sgdb_tasks_created += 1 - task.set_task_data((game,)) - task.run_in_thread(self.sgdb_task_thread_func) - def source_task_callback(self, _obj, _result, data): """Source import callback""" source, *_rest = data @@ -141,38 +120,14 @@ class Importer: if self.finished: self.import_callback() - def sgdb_task_thread_func(self, _task, _obj, data, cancellable): - """SGDB query code""" - game, *_rest = data - game.set_loading(1) - sgdb = SGDBHelper() - try: - sgdb.conditionaly_update_cover(game) - except SGDBAuthError as error: - cancellable.cancel() - self.sgdb_error = error - except (HTTPError, SGDBError) as _error: - # TODO handle other SGDB errors - pass - - def sgdb_task_callback(self, _obj, _result, data): - """SGDB query callback""" - game, *_rest = data - logging.debug("SGDB import done for game %s", game.name) - game.set_loading(-1) - self.n_sgdb_tasks_done += 1 - self.update_progressbar() - if self.finished: - self.import_callback() - def import_callback(self): """Callback called when importing has finished""" logging.info("Import done") self.import_dialog.close() + # TODO replace by summary if necessary self.create_summary_toast() # TODO create a summary of errors/warnings/tips popup (eg. SGDB, Steam libraries) - if self.sgdb_error is not None: - self.create_sgdb_error_dialog() + # Get the error data from shared.store.managers) def create_summary_toast(self): """N games imported toast""" @@ -199,16 +154,6 @@ class Importer: shared.win.toast_overlay.add_toast(toast) - def create_sgdb_error_dialog(self): - """SGDB error dialog""" - create_dialog( - shared.win, - _("Couldn't Connect to SteamGridDB"), - str(self.sgdb_error), - "open_preferences", - _("Preferences"), - ).connect("response", self.dialog_response_callback, "sgdb") - def open_preferences(self, page=None, expander_row=None): shared.win.get_application().on_preferences_action( page_name=page, expander_row=expander_row diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 30ada4d..c1206fb 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -96,18 +96,6 @@ class SteamSourceIterator(SourceIterator): if cover_path.is_file(): save_cover(game.game_id, resize_cover(cover_path)) - # Get online metadata - # TODO move to its own manager - try: - online_data = steam.get_api_data(appid=appid) - except (HTTPError, JSONDecodeError): - pass - except (SteamNotAGameError, SteamGameNotFoundError): - game.update_values({"blacklisted": True}) - else: - game.update_values(online_data) - return game - class SteamSource(Source): name = "Steam" diff --git a/src/store/display_manager.py b/src/store/display_manager.py index 88537e7..97934df 100644 --- a/src/store/display_manager.py +++ b/src/store/display_manager.py @@ -1,11 +1,15 @@ import src.shared as shared -from src.store.manager import Manager from src.game import Game +from src.store.manager import Manager +from src.store.sgdb_manager import SGDBManager +from src.store.steam_api_manager import SteamAPIManager class DisplayManager(Manager): """Manager in charge of adding a game to the UI""" + run_after = set((SteamAPIManager, SGDBManager)) + def run(self, game: Game) -> None: # TODO decouple a game from its widget shared.win.games[game.game_id] = game diff --git a/src/store/file_manager.py b/src/store/file_manager.py index 75c8e58..b202157 100644 --- a/src/store/file_manager.py +++ b/src/store/file_manager.py @@ -1,10 +1,14 @@ -from src.store.manager import Manager from src.game import Game +from src.store.manager import Manager +from src.store.sgdb_manager import SGDBManager +from src.store.steam_api_manager import SteamAPIManager class FileManager(Manager): """Manager in charge of saving a game to a file""" + run_after = set((SteamAPIManager, SGDBManager)) + def run(self, game: Game) -> None: # TODO make game.save (disk) not trigger game.update (UI) game.save() diff --git a/src/store/steam_api_manager.py b/src/store/steam_api_manager.py index 30f3c42..de2643b 100644 --- a/src/store/steam_api_manager.py +++ b/src/store/steam_api_manager.py @@ -1,11 +1,27 @@ -from src.store.manager import Manager +from requests import HTTPError, JSONDecodeError + from src.game import Game -from src.utils.steam import SteamHelper +from src.store.manager import Manager +from src.utils.steam import SteamGameNotFoundError, SteamHelper, SteamNotAGameError class SteamAPIManager(Manager): """Manager in charge of completing a game's data from the Steam API""" def run(self, game: Game) -> None: - # TODO - pass + # Skip non-steam games + if not game.source.startswith("steam_"): + return + + # Get online metadata + appid = str(game.game_id).split("_")[-1] + steam = SteamHelper() + try: + online_data = steam.get_api_data(appid=appid) + except (HTTPError, JSONDecodeError) as error: + # On minor error, just report it + self.report_error(error) + except (SteamNotAGameError, SteamGameNotFoundError): + game.update_values({"blacklisted": True}) + else: + game.update_values(online_data) From a11569014d37bbbda25dba0f693df300f8d29118 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Tue, 23 May 2023 17:00:47 +0200 Subject: [PATCH 056/173] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Restructured=20sou?= =?UTF-8?q?rces=20and=20managers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/lutris_source.py | 4 ++-- src/importer/{ => sources}/source.py | 0 src/importer/sources/steam_source.py | 11 ++--------- src/main.py | 8 ++++---- src/store/{ => managers}/display_manager.py | 6 +++--- src/store/{ => managers}/file_manager.py | 6 +++--- src/store/{ => managers}/manager.py | 0 src/store/{ => managers}/sgdb_manager.py | 2 +- src/store/{ => managers}/steam_api_manager.py | 2 +- src/store/store.py | 2 +- 10 files changed, 17 insertions(+), 24 deletions(-) rename src/importer/{ => sources}/source.py (100%) rename src/store/{ => managers}/display_manager.py (67%) rename src/store/{ => managers}/file_manager.py (63%) rename src/store/{ => managers}/manager.py (100%) rename src/store/{ => managers}/sgdb_manager.py (93%) rename src/store/{ => managers}/steam_api_manager.py (95%) diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index ccd3488..be483db 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,12 +1,12 @@ from abc import abstractmethod from functools import lru_cache +from pathlib import Path from sqlite3 import connect from time import time -from pathlib import Path import src.shared as shared from src.game import Game -from src.importer.source import Source, SourceIterator +from src.importer.sources.source import Source, SourceIterator from src.utils.decorators import replaced_by_path, replaced_by_schema_key from src.utils.save_cover import resize_cover, save_cover diff --git a/src/importer/source.py b/src/importer/sources/source.py similarity index 100% rename from src/importer/source.py rename to src/importer/sources/source.py diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index c1206fb..69fc1b2 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -4,22 +4,15 @@ from pathlib import Path from time import time from typing import Iterator -from requests import HTTPError, JSONDecodeError - from src.game import Game -from src.importer.source import Source, SourceIterator +from src.importer.sources.source import Source, SourceIterator from src.utils.decorators import ( replaced_by_env_path, replaced_by_path, replaced_by_schema_key, ) from src.utils.save_cover import resize_cover, save_cover -from src.utils.steam import ( - SteamGameNotFoundError, - SteamHelper, - SteamInvalidManifestError, - SteamNotAGameError, -) +from src.utils.steam import SteamHelper, SteamInvalidManifestError class SteamSourceIterator(SourceIterator): diff --git a/src/main.py b/src/main.py index 73b5f63..c127fea 100644 --- a/src/main.py +++ b/src/main.py @@ -39,10 +39,10 @@ from src.importer.sources.steam_source import ( SteamWindowsSource, ) from src.preferences import PreferencesWindow -from src.store.display_manager import DisplayManager -from src.store.file_manager import FileManager -from src.store.sgdb_manager import SGDBManager -from src.store.steam_api_manager import SteamAPIManager +from src.store.managers.display_manager import DisplayManager +from src.store.managers.file_manager import FileManager +from src.store.managers.sgdb_manager import SGDBManager +from src.store.managers.steam_api_manager import SteamAPIManager from src.store.store import Store from src.window import CartridgesWindow diff --git a/src/store/display_manager.py b/src/store/managers/display_manager.py similarity index 67% rename from src/store/display_manager.py rename to src/store/managers/display_manager.py index 97934df..235ae62 100644 --- a/src/store/display_manager.py +++ b/src/store/managers/display_manager.py @@ -1,8 +1,8 @@ import src.shared as shared from src.game import Game -from src.store.manager import Manager -from src.store.sgdb_manager import SGDBManager -from src.store.steam_api_manager import SteamAPIManager +from src.store.managers.manager import Manager +from src.store.managers.sgdb_manager import SGDBManager +from src.store.managers.steam_api_manager import SteamAPIManager class DisplayManager(Manager): diff --git a/src/store/file_manager.py b/src/store/managers/file_manager.py similarity index 63% rename from src/store/file_manager.py rename to src/store/managers/file_manager.py index b202157..2a7b67e 100644 --- a/src/store/file_manager.py +++ b/src/store/managers/file_manager.py @@ -1,7 +1,7 @@ from src.game import Game -from src.store.manager import Manager -from src.store.sgdb_manager import SGDBManager -from src.store.steam_api_manager import SteamAPIManager +from src.store.managers.manager import Manager +from src.store.managers.sgdb_manager import SGDBManager +from src.store.managers.steam_api_manager import SteamAPIManager class FileManager(Manager): diff --git a/src/store/manager.py b/src/store/managers/manager.py similarity index 100% rename from src/store/manager.py rename to src/store/managers/manager.py diff --git a/src/store/sgdb_manager.py b/src/store/managers/sgdb_manager.py similarity index 93% rename from src/store/sgdb_manager.py rename to src/store/managers/sgdb_manager.py index faec500..d68abf7 100644 --- a/src/store/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -1,7 +1,7 @@ from requests import HTTPError from src.game import Game -from src.store.manager import Manager +from src.store.managers.manager import Manager from src.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper diff --git a/src/store/steam_api_manager.py b/src/store/managers/steam_api_manager.py similarity index 95% rename from src/store/steam_api_manager.py rename to src/store/managers/steam_api_manager.py index de2643b..93e4d42 100644 --- a/src/store/steam_api_manager.py +++ b/src/store/managers/steam_api_manager.py @@ -1,7 +1,7 @@ from requests import HTTPError, JSONDecodeError from src.game import Game -from src.store.manager import Manager +from src.store.managers.manager import Manager from src.utils.steam import SteamGameNotFoundError, SteamHelper, SteamNotAGameError diff --git a/src/store/store.py b/src/store/store.py index 6aea015..eb36374 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -4,8 +4,8 @@ from gi.repository import GObject import src.shared as shared from src.game import Game -from src.store.manager import Manager from src.utils.task import Task +from store.managers.manager import Manager class Pipeline(GObject.Object): From 2b8c594a50b948f900a76cb2e58e021cbde92fb3 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Tue, 23 May 2023 17:41:11 +0200 Subject: [PATCH 057/173] =?UTF-8?q?=F0=9F=90=9B=20Small=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/meson.build | 1 + src/store/store.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/meson.build b/src/meson.build index 4a48b31..5f47002 100644 --- a/src/meson.build +++ b/src/meson.build @@ -19,6 +19,7 @@ configure_file( install_subdir('importer', install_dir: moduledir) install_subdir('importers', install_dir: moduledir) install_subdir('utils', install_dir: moduledir) +install_subdir('store', install_dir: moduledir) install_data( [ 'main.py', diff --git a/src/store/store.py b/src/store/store.py index eb36374..b3ff3a9 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -4,13 +4,15 @@ from gi.repository import GObject import src.shared as shared from src.game import Game +from src.store.managers.manager import Manager from src.utils.task import Task -from store.managers.manager import Manager class Pipeline(GObject.Object): """Class representing a set of managers for a game""" + __gtype_name__ = "Pipeline" + game: Game waiting: set[Manager] From f06aed3ce0144d7dceb47aac5c162cbdaad49184 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Wed, 24 May 2023 13:57:52 +0200 Subject: [PATCH 058/173] Move spec_version to shared --- src/shared.py | 2 +- src/window.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/shared.py b/src/shared.py index 0417d64..2315994 100644 --- a/src/shared.py +++ b/src/shared.py @@ -53,4 +53,4 @@ image_size = (200 * scale_factor, 300 * scale_factor) win = None importer = None store = None -spec_version = 1.5 # The version of the game_id.json spec +spec_version = 2.0 # The version of the game_id.json spec diff --git a/src/window.py b/src/window.py index b97914e..a577e52 100644 --- a/src/window.py +++ b/src/window.py @@ -74,9 +74,6 @@ class CartridgesWindow(Adw.ApplicationWindow): details_view_game_cover = None sort_state = "a-z" - # The version of the game_id.json spec - spec_version = 2.0 - def __init__(self, **kwargs): super().__init__(**kwargs) From 36b6bc17bd2daace7fc1d141c0a40c5d90ae1f42 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Wed, 24 May 2023 14:09:16 +0200 Subject: [PATCH 059/173] Cleanups --- src/details_window.py | 2 +- src/game.py | 2 +- src/importer/importer.py | 2 +- src/importer/sources/lutris_source.py | 2 +- src/importers/bottles_importer.py | 2 +- src/importers/heroic_importer.py | 2 +- src/importers/itch_importer.py | 2 +- src/importers/lutris_importer.py | 2 +- src/importers/steam_importer.py | 2 +- src/main.py | 2 +- src/preferences.py | 4 ++-- src/store/managers/display_manager.py | 2 +- src/store/managers/manager.py | 2 +- src/store/store.py | 2 +- src/utils/decorators.py | 2 +- src/utils/importer.py | 2 +- src/utils/save_cover.py | 2 +- src/utils/steamgriddb.py | 2 +- src/window.py | 2 +- 19 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/details_window.py b/src/details_window.py index bfa89fc..d0cd7a3 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -25,7 +25,7 @@ from gi.repository import Adw, Gio, GLib, Gtk from PIL import Image # TODO use SGDBHelper -import src.shared as shared +from src import shared from src.game import Game from src.game_cover import GameCover from src.utils.create_dialog import create_dialog diff --git a/src/game.py b/src/game.py index d46dd05..2ca2f7d 100644 --- a/src/game.py +++ b/src/game.py @@ -26,7 +26,7 @@ from time import time from gi.repository import Adw, Gio, Gtk -import src.shared as shared +from src import shared from src.game_cover import GameCover diff --git a/src/importer/importer.py b/src/importer/importer.py index 8a6edda..811df36 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -3,7 +3,7 @@ import logging from gi.repository import Adw, Gio, Gtk from requests import HTTPError -import src.shared as shared +from src import shared from src.utils.create_dialog import create_dialog from src.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper from src.utils.task import Task diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index be483db..e245b95 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -4,7 +4,7 @@ from pathlib import Path from sqlite3 import connect from time import time -import src.shared as shared +from src import shared from src.game import Game from src.importer.sources.source import Source, SourceIterator from src.utils.decorators import replaced_by_path, replaced_by_schema_key diff --git a/src/importers/bottles_importer.py b/src/importers/bottles_importer.py index 260543a..04a9fe0 100644 --- a/src/importers/bottles_importer.py +++ b/src/importers/bottles_importer.py @@ -22,7 +22,7 @@ from time import time import yaml -import src.shared as shared +from src import shared from src.utils.check_install import check_install diff --git a/src/importers/heroic_importer.py b/src/importers/heroic_importer.py index a2c9412..85e84d9 100644 --- a/src/importers/heroic_importer.py +++ b/src/importers/heroic_importer.py @@ -23,7 +23,7 @@ from hashlib import sha256 from pathlib import Path from time import time -import src.shared as shared +from src import shared from src.utils.check_install import check_install diff --git a/src/importers/itch_importer.py b/src/importers/itch_importer.py index ee68384..2eb14c0 100644 --- a/src/importers/itch_importer.py +++ b/src/importers/itch_importer.py @@ -26,7 +26,7 @@ from time import time import requests from gi.repository import GdkPixbuf, Gio -import src.shared as shared +from src import shared from src.utils.check_install import check_install from src.utils.save_cover import resize_cover diff --git a/src/importers/lutris_importer.py b/src/importers/lutris_importer.py index 62d352a..d8a10b5 100644 --- a/src/importers/lutris_importer.py +++ b/src/importers/lutris_importer.py @@ -22,7 +22,7 @@ from shutil import copyfile from sqlite3 import connect from time import time -import src.shared as shared +from src import shared from src.utils.check_install import check_install diff --git a/src/importers/steam_importer.py b/src/importers/steam_importer.py index 4a6bc34..7ecaa83 100644 --- a/src/importers/steam_importer.py +++ b/src/importers/steam_importer.py @@ -25,7 +25,7 @@ from time import time import requests from gi.repository import Gio -import src.shared as shared +from src import shared from src.utils.check_install import check_install diff --git a/src/main.py b/src/main.py index c127fea..1c85754 100644 --- a/src/main.py +++ b/src/main.py @@ -29,7 +29,7 @@ gi.require_version("Adw", "1") # pylint: disable=wrong-import-position from gi.repository import Adw, Gio, GLib, Gtk -import src.shared as shared +from src import shared from src.details_window import DetailsWindow from src.importer.importer import Importer from src.importer.sources.lutris_source import LutrisFlatpakSource, LutrisNativeSource diff --git a/src/preferences.py b/src/preferences.py index 9df90b5..2bb2129 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -24,7 +24,7 @@ from pathlib import Path from gi.repository import Adw, Gio, GLib, Gtk # pylint: disable=unused-import -import src.shared as shared +from src import shared # TODO use the new sources from src.importers.bottles_importer import bottles_installed @@ -128,7 +128,7 @@ class PreferencesWindow(Adw.PreferencesWindow): if response == "choose_folder": self.choose_folder(widget, set_cache_dir) - if lutris_cache_exists(self.win, path): + if lutris_cache_exists(path): self.import_changed = True self.set_subtitle(self, "lutris-cache") diff --git a/src/store/managers/display_manager.py b/src/store/managers/display_manager.py index 235ae62..2fdfda9 100644 --- a/src/store/managers/display_manager.py +++ b/src/store/managers/display_manager.py @@ -1,4 +1,4 @@ -import src.shared as shared +from src import shared from src.game import Game from src.store.managers.manager import Manager from src.store.managers.sgdb_manager import SGDBManager diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 361f63a..1e4660e 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -20,7 +20,7 @@ class Manager: def __init__(self) -> None: super().__init__() self.cancellable = Gio.Cancellable() - self.errors = list() + self.errors = [] def cancel_tasks(self): """Cancel all tasks for this manager""" diff --git a/src/store/store.py b/src/store/store.py index b3ff3a9..b75d569 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -2,7 +2,7 @@ from typing import Iterable from gi.repository import GObject -import src.shared as shared +from src import shared from src.game import Game from src.store.managers.manager import Manager from src.utils.task import Task diff --git a/src/utils/decorators.py b/src/utils/decorators.py index d9a8433..cce9a09 100644 --- a/src/utils/decorators.py +++ b/src/utils/decorators.py @@ -17,7 +17,7 @@ from pathlib import Path from os import PathLike, environ from functools import wraps -import src.shared as shared +from src import shared def replaced_by_path(override: PathLike): # Decorator builder diff --git a/src/utils/importer.py b/src/utils/importer.py index 9929757..2d651ac 100644 --- a/src/utils/importer.py +++ b/src/utils/importer.py @@ -19,7 +19,7 @@ from gi.repository import Adw, GLib, Gtk -import src.shared as shared +from src import shared from .create_dialog import create_dialog from .game import Game from .save_cover import resize_cover, save_cover diff --git a/src/utils/save_cover.py b/src/utils/save_cover.py index 5a2cef5..a3e6c67 100644 --- a/src/utils/save_cover.py +++ b/src/utils/save_cover.py @@ -24,7 +24,7 @@ from shutil import copyfile from gi.repository import Gio from PIL import Image, ImageSequence -import src.shared as shared +from src import shared def resize_cover(cover_path=None, pixbuf=None): diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index b685918..ea3ecd9 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -5,7 +5,7 @@ import requests from gi.repository import Gio from requests import HTTPError -import src.shared as shared +from src import shared from src.utils.create_dialog import create_dialog from src.utils.save_cover import resize_cover, save_cover diff --git a/src/window.py b/src/window.py index a577e52..086055c 100644 --- a/src/window.py +++ b/src/window.py @@ -22,7 +22,7 @@ from datetime import datetime from gi.repository import Adw, Gio, GLib, Gtk -import src.shared as shared +from src import shared from src.game import Game From fda06e6c1a9893f5d221db173159b0b3a9c75a1d Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Wed, 24 May 2023 14:12:28 +0200 Subject: [PATCH 060/173] Update docs --- docs/game_id.json.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/game_id.json.md b/docs/game_id.json.md index e768c48..4b8b44c 100644 --- a/docs/game_id.json.md +++ b/docs/game_id.json.md @@ -37,7 +37,7 @@ The executable to run when launching a game. If the source has a URL handler, using that is preferred. In that case, the value should be `["xdg-open", "url://example/url"]` for Linux and `["start", "url://example/url"]` for Windows. -Stored as an argument vector to be passed to the shell through [GLib.spawn_async](https://docs.gtk.org/glib/func.spawn_async.html). +Stored as either a string or an argument vector to be passed to the shell through [subprocess.Popen](https://docs.python.org/3/library/subprocess.html#popen-constructor). ### game_id From 4943a9c7fd0c74c0c44a8f495aced49a60455760 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 24 May 2023 15:17:37 +0200 Subject: [PATCH 061/173] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20Pipeline=20GObje?= =?UTF-8?q?ct=20definition?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/store.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/store/store.py b/src/store/store.py index b75d569..bb850e9 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -11,8 +11,6 @@ from src.utils.task import Task class Pipeline(GObject.Object): """Class representing a set of managers for a game""" - __gtype_name__ = "Pipeline" - game: Game waiting: set[Manager] @@ -57,7 +55,7 @@ class Pipeline(GObject.Object): task.set_task_data(data) task.run_in_thread(self.manager_task_thread_func) - @GObject.Signal(name="manager-started") + @GObject.Signal(name="manager-started", arg_types=(object,)) def manager_started(self, manager: Manager) -> None: """Signal emitted when a manager is started""" pass @@ -68,7 +66,7 @@ class Pipeline(GObject.Object): self.emit("manager-started", manager) manager.run(self.game, cancellable) - @GObject.Signal(name="manager-done") + @GObject.Signal(name="manager-done", arg_types=(object,)) def manager_done(self, manager: Manager) -> None: """Signal emitted when a manager is done""" pass From 8026c41886cfdf8d9f248885b6b21495a20e7fcb Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 24 May 2023 16:32:13 +0200 Subject: [PATCH 062/173] =?UTF-8?q?=F0=9F=8E=A8=20Moved=20Initial=20game?= =?UTF-8?q?=20load=20to=20app.on=5Factivate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 3 --- src/main.py | 30 +++++++++++++++++++++--------- src/shared.py | 1 - src/store/store.py | 24 ++++++++++++++++++++---- src/window.py | 27 --------------------------- 5 files changed, 41 insertions(+), 44 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 811df36..84e1273 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -1,11 +1,8 @@ import logging from gi.repository import Adw, Gio, Gtk -from requests import HTTPError from src import shared -from src.utils.create_dialog import create_dialog -from src.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper from src.utils.task import Task diff --git a/src/main.py b/src/main.py index 1c85754..8a87446 100644 --- a/src/main.py +++ b/src/main.py @@ -17,6 +17,7 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +import json import logging import os import sys @@ -31,6 +32,7 @@ from gi.repository import Adw, Gio, GLib, Gtk from src import shared from src.details_window import DetailsWindow +from src.game import Game from src.importer.importer import Importer from src.importer.sources.lutris_source import LutrisFlatpakSource, LutrisNativeSource from src.importer.sources.steam_source import ( @@ -49,7 +51,6 @@ from src.window import CartridgesWindow class CartridgesApplication(Adw.Application): win = None - store = None def __init__(self): super().__init__( @@ -57,18 +58,12 @@ class CartridgesApplication(Adw.Application): ) def do_activate(self): # pylint: disable=arguments-differ - # Create the games store and its managers - if not self.store: - self.store = Store() - self.store.add_manager(SteamAPIManager()) - self.store.add_manager(SGDBManager()) - self.store.add_manager(FileManager()) - self.store.add_manager(DisplayManager()) + """Called on app creation""" # Create the main window self.win = self.props.active_window # pylint: disable=no-member if not self.win: - self.win = CartridgesWindow(application=self) + shared.win = self.win = CartridgesWindow(application=self) # Save window geometry shared.state_schema.bind( @@ -81,6 +76,23 @@ class CartridgesApplication(Adw.Application): "is-maximized", self.win, "maximized", Gio.SettingsBindFlags.DEFAULT ) + # Create the games store with bare minimum managers + if not shared.store: + shared.store = Store() + shared.store.add_manager(DisplayManager()) + + # Load games from disk + if shared.games_dir.exists(): + for game_file in shared.games_dir.iterdir(): + data = json.load(game_file.open()) + game = Game(data, allow_side_effects=False) + shared.store.add_game(game) + + # Add rest of the managers for game imports + shared.store.add_manager(SteamAPIManager()) + shared.store.add_manager(SGDBManager()) + shared.store.add_manager(FileManager()) + # Create actions self.create_actions( { diff --git a/src/shared.py b/src/shared.py index 2315994..71b900d 100644 --- a/src/shared.py +++ b/src/shared.py @@ -51,6 +51,5 @@ image_size = (200 * scale_factor, 300 * scale_factor) # pylint: disable=invalid-name win = None -importer = None store = None spec_version = 2.0 # The version of the game_id.json spec diff --git a/src/store/store.py b/src/store/store.py index bb850e9..c24ec66 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -88,7 +88,6 @@ class Store: games: dict[str, Game] def __init__(self) -> None: - shared.store = self self.managers = set() self.games = {} self.pipelines = {} @@ -97,18 +96,35 @@ class Store: """Add a manager class that will run when games are added""" self.managers.add(manager) - def add_game(self, game: Game, replace=False) -> Pipeline: + def add_game(self, game: Game, replace=False) -> Pipeline | None: """Add a game to the app if not already there :param replace bool: Replace the game if it already exists - :return: """ + + # Ignore games from a newer spec version + if (version := game.get("version")) and version > shared.spec_version: + return None + + # Ignore games that are already there if ( game.game_id in self.games and not self.games[game.game_id].removed and not replace ): - return + return None + + # Cleanup removed games + if game.get("removed"): + for path in ( + shared.games_dir / f"{game.game_id}.json", + shared.covers_dir / f"{game.game_id}.tiff", + shared.covers_dir / f"{game.game_id}.gif", + ): + path.unlink(missing_ok=True) + return None + + # Run the pipeline for the game pipeline = Pipeline(self.managers) self.games[game.game_id] = game self.pipelines[game.game_id] = pipeline diff --git a/src/window.py b/src/window.py index 086055c..5f1631b 100644 --- a/src/window.py +++ b/src/window.py @@ -17,14 +17,10 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -import json from datetime import datetime from gi.repository import Adw, Gio, GLib, Gtk -from src import shared -from src.game import Game - @Gtk.Template(resource_path="/hu/kramo/Cartridges/gtk/window.ui") class CartridgesWindow(Adw.ApplicationWindow): @@ -77,8 +73,6 @@ class CartridgesWindow(Adw.ApplicationWindow): def __init__(self, **kwargs): super().__init__(**kwargs) - shared.win = self - self.previous_page = self.library_view self.details_view.set_measure_overlay(self.details_view_box, True) @@ -92,27 +86,6 @@ class CartridgesWindow(Adw.ApplicationWindow): self.set_library_child() - games = {} - - if shared.games_dir.exists(): - for open_file in shared.games_dir.iterdir(): - data = json.load(open_file.open()) - games[data["game_id"]] = data - - for game_id, game in games.items(): - if (version := game.get("version")) and version > shared.spec_version: - continue - - if game.get("removed"): - for path in ( - shared.games_dir / f"{game_id}.json", - shared.covers_dir / f"{game_id}.tiff", - shared.covers_dir / f"{game_id}.gif", - ): - path.unlink(missing_ok=True) - else: - Game(game).update() - # Connect search entries self.search_bar.connect_entry(self.search_entry) self.hidden_search_bar.connect_entry(self.hidden_search_entry) From 722085229140c55370cf8967913c6d34233dbe59 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 24 May 2023 17:08:34 +0200 Subject: [PATCH 063/173] =?UTF-8?q?=F0=9F=8E=A8=20Reorganized=20game=20loa?= =?UTF-8?q?ding=20from=20disk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/game.py | 9 +-------- src/importer/sources/lutris_source.py | 1 + src/importer/sources/steam_source.py | 2 ++ src/main.py | 4 +++- src/store/managers/display_manager.py | 5 ++--- src/store/managers/file_manager.py | 3 ++- src/store/managers/format_update_manager.py | 20 ++++++++++++++++++++ src/store/store.py | 2 +- 8 files changed, 32 insertions(+), 14 deletions(-) create mode 100644 src/store/managers/format_update_manager.py diff --git a/src/game.py b/src/game.py index 2ca2f7d..a7845ba 100644 --- a/src/game.py +++ b/src/game.py @@ -59,13 +59,13 @@ class Game(Gtk.Box): removed = None blacklisted = None game_cover = None + version = None def __init__(self, data, allow_side_effects=True, **kwargs): super().__init__(**kwargs) self.win = shared.win self.app = self.win.get_application() - self.version = shared.spec_version self.update_values(data) @@ -145,13 +145,6 @@ class Game(Gtk.Box): "version", ) - # TODO: remove for 2.0 - attrs = list(attrs) - if not self.removed: - attrs.remove("removed") - if not self.blacklisted: - attrs.remove("blacklisted") - json.dump( {attr: getattr(self, attr) for attr in attrs if attr}, (shared.games_dir / f"{self.game_id}.json").open("w"), diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index e245b95..e80b278 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -68,6 +68,7 @@ class LutrisSourceIterator(SourceIterator): # Create game values = { + "version": shared.spec_version, "added": int(time()), "hidden": row[4], "name": row[1], diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 69fc1b2..34abb42 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -4,6 +4,7 @@ from pathlib import Path from time import time from typing import Iterator +from src import shared from src.game import Game from src.importer.sources.source import Source, SourceIterator from src.utils.decorators import ( @@ -71,6 +72,7 @@ class SteamSourceIterator(SourceIterator): # Build game from local data appid = local_data["appid"] values = { + "version": shared.spec_version, "added": int(time()), "name": local_data["name"], "source": self.source.id, diff --git a/src/main.py b/src/main.py index 8a87446..16f7362 100644 --- a/src/main.py +++ b/src/main.py @@ -43,6 +43,7 @@ from src.importer.sources.steam_source import ( from src.preferences import PreferencesWindow from src.store.managers.display_manager import DisplayManager from src.store.managers.file_manager import FileManager +from store.managers.format_update_manager import FormatUpdateManager from src.store.managers.sgdb_manager import SGDBManager from src.store.managers.steam_api_manager import SteamAPIManager from src.store.store import Store @@ -76,9 +77,10 @@ class CartridgesApplication(Adw.Application): "is-maximized", self.win, "maximized", Gio.SettingsBindFlags.DEFAULT ) - # Create the games store with bare minimum managers + # Create the games store ready to load games from disk if not shared.store: shared.store = Store() + shared.store.add_manager(FormatUpdateManager()) shared.store.add_manager(DisplayManager()) # Load games from disk diff --git a/src/store/managers/display_manager.py b/src/store/managers/display_manager.py index 2fdfda9..ba7ca39 100644 --- a/src/store/managers/display_manager.py +++ b/src/store/managers/display_manager.py @@ -1,14 +1,13 @@ from src import shared from src.game import Game +from src.store.managers.file_manager import FileManager from src.store.managers.manager import Manager -from src.store.managers.sgdb_manager import SGDBManager -from src.store.managers.steam_api_manager import SteamAPIManager class DisplayManager(Manager): """Manager in charge of adding a game to the UI""" - run_after = set((SteamAPIManager, SGDBManager)) + run_after = set((FileManager,)) def run(self, game: Game) -> None: # TODO decouple a game from its widget diff --git a/src/store/managers/file_manager.py b/src/store/managers/file_manager.py index 2a7b67e..936e305 100644 --- a/src/store/managers/file_manager.py +++ b/src/store/managers/file_manager.py @@ -1,4 +1,5 @@ from src.game import Game +from src.store.managers.format_update_manager import FormatUpdateManager from src.store.managers.manager import Manager from src.store.managers.sgdb_manager import SGDBManager from src.store.managers.steam_api_manager import SteamAPIManager @@ -7,7 +8,7 @@ from src.store.managers.steam_api_manager import SteamAPIManager class FileManager(Manager): """Manager in charge of saving a game to a file""" - run_after = set((SteamAPIManager, SGDBManager)) + run_after = set((SteamAPIManager, SGDBManager, FormatUpdateManager)) def run(self, game: Game) -> None: # TODO make game.save (disk) not trigger game.update (UI) diff --git a/src/store/managers/format_update_manager.py b/src/store/managers/format_update_manager.py new file mode 100644 index 0000000..e4521b5 --- /dev/null +++ b/src/store/managers/format_update_manager.py @@ -0,0 +1,20 @@ +from src.store.managers.manager import Manager +from src.game import Game + + +class FormatUpdateManager(Manager): + """Class in charge of migrating a game from an older format""" + + def v1_5_to_v2_0(self, game: Game) -> None: + """Convert a game from v1.5 format to v2.0 format""" + if game.blacklisted is None: + game.blacklisted = False + if game.removed is None: + game.removed = False + game.version = 2.0 + + def run(self, game: Game) -> None: + if game.version is None: + self.v1_5_to_v2_0(game) + # TODO make game.save (disk) not trigger game.update (UI) + game.save() diff --git a/src/store/store.py b/src/store/store.py index c24ec66..dee2716 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -103,7 +103,7 @@ class Store: """ # Ignore games from a newer spec version - if (version := game.get("version")) and version > shared.spec_version: + if game.version > shared.spec_version: return None # Ignore games that are already there From 8da7185d1708f4fd3403a420a04d2e0e48ac3c08 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 24 May 2023 17:30:19 +0200 Subject: [PATCH 064/173] Various fixes --- src/game.py | 2 -- src/main.py | 2 +- src/store/managers/file_manager.py | 1 - src/store/managers/format_update_manager.py | 1 - src/store/managers/manager.py | 4 ++-- src/store/store.py | 11 ++++++----- 6 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/game.py b/src/game.py index a7845ba..9435568 100644 --- a/src/game.py +++ b/src/game.py @@ -152,8 +152,6 @@ class Game(Gtk.Box): sort_keys=True, ) - self.update() - def create_toast(self, title, action=None): toast = Adw.Toast.new(title.format(self.name)) toast.set_priority(Adw.ToastPriority.HIGH) diff --git a/src/main.py b/src/main.py index 16f7362..a24af59 100644 --- a/src/main.py +++ b/src/main.py @@ -43,7 +43,7 @@ from src.importer.sources.steam_source import ( from src.preferences import PreferencesWindow from src.store.managers.display_manager import DisplayManager from src.store.managers.file_manager import FileManager -from store.managers.format_update_manager import FormatUpdateManager +from src.store.managers.format_update_manager import FormatUpdateManager from src.store.managers.sgdb_manager import SGDBManager from src.store.managers.steam_api_manager import SteamAPIManager from src.store.store import Store diff --git a/src/store/managers/file_manager.py b/src/store/managers/file_manager.py index 936e305..b5381a8 100644 --- a/src/store/managers/file_manager.py +++ b/src/store/managers/file_manager.py @@ -11,5 +11,4 @@ class FileManager(Manager): run_after = set((SteamAPIManager, SGDBManager, FormatUpdateManager)) def run(self, game: Game) -> None: - # TODO make game.save (disk) not trigger game.update (UI) game.save() diff --git a/src/store/managers/format_update_manager.py b/src/store/managers/format_update_manager.py index e4521b5..fea4d63 100644 --- a/src/store/managers/format_update_manager.py +++ b/src/store/managers/format_update_manager.py @@ -16,5 +16,4 @@ class FormatUpdateManager(Manager): def run(self, game: Game) -> None: if game.version is None: self.v1_5_to_v2_0(game) - # TODO make game.save (disk) not trigger game.update (UI) game.save() diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 1e4660e..376662f 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -12,7 +12,7 @@ class Manager: in that case a new cancellable must be generated for new tasks to run. """ - run_after: set[type["Manager"]] + run_after: set[type["Manager"]] = set() cancellable: Gio.Cancellable errors: list[Exception] @@ -42,7 +42,7 @@ class Manager: return errors @abstractmethod - def run(self, game: Game) -> None: + def run(self, game: Game, cancellable: Gio.Cancellable) -> None: """Pass the game through the manager. May block its thread. May not raise exceptions, as they will be silently ignored.""" diff --git a/src/store/store.py b/src/store/store.py index dee2716..bbc09aa 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -17,8 +17,9 @@ class Pipeline(GObject.Object): running: set[Manager] done: set[Manager] - def __init__(self, managers: Iterable[Manager]) -> None: + def __init__(self, game: Game, managers: Iterable[Manager]) -> None: super().__init__() + self.game = game self.waiting = set(managers) self.running = set() self.done = set() @@ -26,7 +27,7 @@ class Pipeline(GObject.Object): @property def not_done(self) -> set[Manager]: """Get the managers that are not done yet""" - return self.waiting + self.running + return self.waiting | self.running @property def blocked(self) -> set[Manager]: @@ -64,7 +65,7 @@ class Pipeline(GObject.Object): """Thread function for manager tasks""" manager, *_rest = data self.emit("manager-started", manager) - manager.run(self.game, cancellable) + manager.run(self.game) @GObject.Signal(name="manager-done", arg_types=(object,)) def manager_done(self, manager: Manager) -> None: @@ -115,7 +116,7 @@ class Store: return None # Cleanup removed games - if game.get("removed"): + if game.removed: for path in ( shared.games_dir / f"{game.game_id}.json", shared.covers_dir / f"{game.game_id}.tiff", @@ -125,7 +126,7 @@ class Store: return None # Run the pipeline for the game - pipeline = Pipeline(self.managers) + pipeline = Pipeline(game, self.managers) self.games[game.game_id] = game self.pipelines[game.game_id] = pipeline pipeline.advance() From e7e30c8ac5cf53138b37929520ca1b523c2b4055 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Wed, 24 May 2023 18:32:27 +0200 Subject: [PATCH 065/173] Set version to 0 by default --- src/game.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/game.py b/src/game.py index 9435568..d45dae2 100644 --- a/src/game.py +++ b/src/game.py @@ -59,7 +59,7 @@ class Game(Gtk.Box): removed = None blacklisted = None game_cover = None - version = None + version = 0 def __init__(self, data, allow_side_effects=True, **kwargs): super().__init__(**kwargs) From 1d2253ff9491f84cc33c3bb26e8925b8cad16374 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 24 May 2023 19:34:07 +0200 Subject: [PATCH 066/173] Various changes - Removed useless format manager - Moved pipeline to its own file - Fixed steam source next not returning game - Changed pipeline order --- src/importer/importer.py | 18 ++++- src/importer/sources/steam_source.py | 2 + src/main.py | 4 +- src/store/managers/file_manager.py | 4 +- src/store/managers/format_update_manager.py | 19 ----- src/store/managers/sgdb_manager.py | 3 + src/store/pipeline.py | 80 +++++++++++++++++++++ src/store/store.py | 79 +------------------- 8 files changed, 103 insertions(+), 106 deletions(-) delete mode 100644 src/store/managers/format_update_manager.py create mode 100644 src/store/pipeline.py diff --git a/src/importer/importer.py b/src/importer/importer.py index 84e1273..918d404 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -4,6 +4,7 @@ from gi.repository import Adw, Gio, Gtk from src import shared from src.utils.task import Task +from src.store.pipeline import Pipeline # pylint: disable=too-many-instance-attributes @@ -83,6 +84,7 @@ class Importer: if not source.is_installed: logging.info("Source %s skipped, not installed", source.id) return + logging.info("Scanning source %s", source.id) # Initialize source iteration iterator = iter(source) @@ -104,19 +106,29 @@ class Importer: continue # Register game - logging.info("Imported %s (%s)", game.name, game.game_id) - shared.store.add_game(game) - self.n_games_added += 1 + pipeline: Pipeline = shared.store.add_game(game) + if pipeline is not None: + logging.info("Imported %s (%s)", game.name, game.game_id) + pipeline.connect("manager-done", self.manager_done_callback) + self.n_games_added += 1 def source_task_callback(self, _obj, _result, data): """Source import callback""" source, *_rest = data logging.debug("Import done for source %s", source.id) self.n_source_tasks_done += 1 + # TODO remove, should be handled by manager_done_callback self.update_progressbar() if self.finished: self.import_callback() + def manager_done_callback(self, pipeline: Pipeline): + """Callback called when a pipeline for a game has advanced""" + # TODO (optional) update progress bar more precisely from here + # TODO get number of games really added here (eg. exlude blacklisted) + # TODO trigger import_callback only when all pipelines have finished + pass + def import_callback(self): """Callback called when importing has finished""" logging.info("Import done") diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 34abb42..dd407ce 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -91,6 +91,8 @@ class SteamSourceIterator(SourceIterator): if cover_path.is_file(): save_cover(game.game_id, resize_cover(cover_path)) + return game + class SteamSource(Source): name = "Steam" diff --git a/src/main.py b/src/main.py index a24af59..8e4b6c9 100644 --- a/src/main.py +++ b/src/main.py @@ -43,7 +43,6 @@ from src.importer.sources.steam_source import ( from src.preferences import PreferencesWindow from src.store.managers.display_manager import DisplayManager from src.store.managers.file_manager import FileManager -from src.store.managers.format_update_manager import FormatUpdateManager from src.store.managers.sgdb_manager import SGDBManager from src.store.managers.steam_api_manager import SteamAPIManager from src.store.store import Store @@ -80,7 +79,6 @@ class CartridgesApplication(Adw.Application): # Create the games store ready to load games from disk if not shared.store: shared.store = Store() - shared.store.add_manager(FormatUpdateManager()) shared.store.add_manager(DisplayManager()) # Load games from disk @@ -238,6 +236,6 @@ class CartridgesApplication(Adw.Application): def main(version): # pylint: disable=unused-argument log_level = os.environ.get("LOGLEVEL", "ERROR").upper() - logging.basicConfig(level="DEBUG") # TODO remove debug + logging.basicConfig(level="INFO") # TODO remove before release, use env app = CartridgesApplication() return app.run(sys.argv) diff --git a/src/store/managers/file_manager.py b/src/store/managers/file_manager.py index b5381a8..ffd2363 100644 --- a/src/store/managers/file_manager.py +++ b/src/store/managers/file_manager.py @@ -1,14 +1,12 @@ from src.game import Game -from src.store.managers.format_update_manager import FormatUpdateManager from src.store.managers.manager import Manager -from src.store.managers.sgdb_manager import SGDBManager from src.store.managers.steam_api_manager import SteamAPIManager class FileManager(Manager): """Manager in charge of saving a game to a file""" - run_after = set((SteamAPIManager, SGDBManager, FormatUpdateManager)) + run_after = set((SteamAPIManager,)) def run(self, game: Game) -> None: game.save() diff --git a/src/store/managers/format_update_manager.py b/src/store/managers/format_update_manager.py deleted file mode 100644 index fea4d63..0000000 --- a/src/store/managers/format_update_manager.py +++ /dev/null @@ -1,19 +0,0 @@ -from src.store.managers.manager import Manager -from src.game import Game - - -class FormatUpdateManager(Manager): - """Class in charge of migrating a game from an older format""" - - def v1_5_to_v2_0(self, game: Game) -> None: - """Convert a game from v1.5 format to v2.0 format""" - if game.blacklisted is None: - game.blacklisted = False - if game.removed is None: - game.removed = False - game.version = 2.0 - - def run(self, game: Game) -> None: - if game.version is None: - self.v1_5_to_v2_0(game) - game.save() diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index d68abf7..a87885d 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -3,11 +3,14 @@ from requests import HTTPError from src.game import Game from src.store.managers.manager import Manager from src.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper +from src.store.managers.steam_api_manager import SteamAPIManager class SGDBManager(Manager): """Manager in charge of downloading a game's cover from steamgriddb""" + run_after = set((SteamAPIManager,)) + def run(self, game: Game) -> None: try: sgdb = SGDBHelper() diff --git a/src/store/pipeline.py b/src/store/pipeline.py new file mode 100644 index 0000000..a58796a --- /dev/null +++ b/src/store/pipeline.py @@ -0,0 +1,80 @@ +from typing import Iterable + +from gi.repository import GObject + +from src.game import Game +from src.store.managers.manager import Manager +from src.utils.task import Task + + +class Pipeline(GObject.Object): + """Class representing a set of managers for a game""" + + game: Game + + waiting: set[Manager] + running: set[Manager] + done: set[Manager] + + def __init__(self, game: Game, managers: Iterable[Manager]) -> None: + super().__init__() + self.game = game + self.waiting = set(managers) + self.running = set() + self.done = set() + + @property + def not_done(self) -> set[Manager]: + """Get the managers that are not done yet""" + return self.waiting | self.running + + @property + def blocked(self) -> set[Manager]: + """Get the managers that cannot run because their dependencies aren't done""" + blocked = set() + for manager_a in self.waiting: + for manager_b in self.not_done: + if manager_a == manager_b: + continue + if type(manager_b) in manager_a.run_after: + blocked.add(manager_a) + return blocked + + @property + def ready(self) -> set[Manager]: + """Get the managers that can be run""" + return self.waiting - self.blocked + + def advance(self): + """Spawn tasks for managers that are able to run for a game""" + for manager in self.ready: + self.waiting.remove(manager) + self.running.add(manager) + data = (manager,) + task = Task.new(self, manager.cancellable, self.manager_task_callback, data) + task.set_task_data(data) + task.run_in_thread(self.manager_task_thread_func) + + @GObject.Signal(name="manager-started", arg_types=(object,)) + def manager_started(self, manager: Manager) -> None: + """Signal emitted when a manager is started""" + pass + + def manager_task_thread_func(self, _task, _source_object, data, cancellable): + """Thread function for manager tasks""" + manager, *_rest = data + self.emit("manager-started", manager) + manager.run(self.game) + + @GObject.Signal(name="manager-done", arg_types=(object,)) + def manager_done(self, manager: Manager) -> None: + """Signal emitted when a manager is done""" + pass + + def manager_task_callback(self, _source_object, _result, data): + """Callback function for manager tasks""" + manager, *_rest = data + self.running.remove(manager) + self.done.add(manager) + self.emit("manager-done", manager) + self.advance() diff --git a/src/store/store.py b/src/store/store.py index bbc09aa..ebed6ba 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -1,84 +1,7 @@ -from typing import Iterable - -from gi.repository import GObject - from src import shared from src.game import Game from src.store.managers.manager import Manager -from src.utils.task import Task - - -class Pipeline(GObject.Object): - """Class representing a set of managers for a game""" - - game: Game - - waiting: set[Manager] - running: set[Manager] - done: set[Manager] - - def __init__(self, game: Game, managers: Iterable[Manager]) -> None: - super().__init__() - self.game = game - self.waiting = set(managers) - self.running = set() - self.done = set() - - @property - def not_done(self) -> set[Manager]: - """Get the managers that are not done yet""" - return self.waiting | self.running - - @property - def blocked(self) -> set[Manager]: - """Get the managers that cannot run because their dependencies aren't done""" - blocked = set() - for manager_a in self.waiting: - for manager_b in self.not_done: - if manager_a == manager_b: - continue - if type(manager_b) in manager_a.run_after: - blocked.add(manager_a) - return blocked - - @property - def ready(self) -> set[Manager]: - """Get the managers that can be run""" - return self.waiting - self.blocked - - def advance(self): - """Spawn tasks for managers that are able to run for a game""" - for manager in self.ready: - self.waiting.remove(manager) - self.running.add(manager) - data = (manager,) - task = Task.new(self, manager.cancellable, self.manager_task_callback, data) - task.set_task_data(data) - task.run_in_thread(self.manager_task_thread_func) - - @GObject.Signal(name="manager-started", arg_types=(object,)) - def manager_started(self, manager: Manager) -> None: - """Signal emitted when a manager is started""" - pass - - def manager_task_thread_func(self, _task, _source_object, data, cancellable): - """Thread function for manager tasks""" - manager, *_rest = data - self.emit("manager-started", manager) - manager.run(self.game) - - @GObject.Signal(name="manager-done", arg_types=(object,)) - def manager_done(self, manager: Manager) -> None: - """Signal emitted when a manager is done""" - pass - - def manager_task_callback(self, _source_object, _result, data): - """Callback function for manager tasks""" - manager, *_rest = data - self.running.remove(manager) - self.done.add(manager) - self.emit("manager-done", manager) - self.advance() +from src.store.pipeline import Pipeline class Store: From 3202bd4332071dec539a6567bf48ca0dbafbc6d3 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 24 May 2023 19:38:26 +0200 Subject: [PATCH 067/173] changed importer.manager_done_callback stub --- src/importer/importer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 918d404..be1ed4d 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -5,6 +5,7 @@ from gi.repository import Adw, Gio, Gtk from src import shared from src.utils.task import Task from src.store.pipeline import Pipeline +from src.store.managers.manager import Manager # pylint: disable=too-many-instance-attributes @@ -122,7 +123,7 @@ class Importer: if self.finished: self.import_callback() - def manager_done_callback(self, pipeline: Pipeline): + def manager_done_callback(self, pipeline: Pipeline, manager: Manager): """Callback called when a pipeline for a game has advanced""" # TODO (optional) update progress bar more precisely from here # TODO get number of games really added here (eg. exlude blacklisted) From 39b7b35c1b9e6ed84ad8477d8add9e81d7b527f6 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 27 May 2023 18:24:46 +0200 Subject: [PATCH 068/173] =?UTF-8?q?=F0=9F=9A=A7=20WIP=20import=20progress?= =?UTF-8?q?=20based=20on=20game=20pipelines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 56 ++++++++++++++++++++++++---------------- src/store/pipeline.py | 4 +++ 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index be1ed4d..62fc0da 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -6,6 +6,7 @@ from src import shared from src.utils.task import Task from src.store.pipeline import Pipeline from src.store.managers.manager import Manager +from src.importer.sources.source import Source # pylint: disable=too-many-instance-attributes @@ -16,26 +17,40 @@ class Importer: import_statuspage = None import_dialog = None - sources = None + sources: set[Source] = None - n_games_added = 0 - n_source_tasks_created = 0 - n_source_tasks_done = 0 + n_source_tasks_created: int = 0 + n_source_tasks_done: int = 0 + n_pipelines_done: int = 0 + game_pipelines: set[Pipeline] = None def __init__(self): + self.game_pipelines = set() self.sources = set() @property - def progress(self): + def n_games_added(self): + return sum( + [ + 1 if not (pipeline.game.blacklisted or pipeline.game.removed) else 0 + for pipeline in self.game_pipelines + ] + ) + + @property + def pipelines_progress(self): try: - progress = self.n_source_tasks_done / self.n_source_tasks_created + progress = self.n_pipelines_done / len(self.game_pipelines) except ZeroDivisionError: progress = 1 return progress @property def finished(self): - return self.n_source_tasks_created == self.n_source_tasks_done + return ( + self.n_source_tasks_created == self.n_source_tasks_done + and len(self.game_pipelines) == self.n_pipelines_done + ) def add_source(self, source): self.sources.add(source) @@ -51,7 +66,7 @@ class Importer: for source in self.sources: logging.debug("Importing games from source %s", source.id) - task = Task.new(None, None, self.source_task_callback, (source,)) + task = Task.new(None, None, self.source_callback, (source,)) self.n_source_tasks_created += 1 task.set_task_data((source,)) task.run_in_thread(self.source_task_thread_func) @@ -73,9 +88,6 @@ class Importer: ) self.import_dialog.present() - def update_progressbar(self): - self.progressbar.set_fraction(self.progress) - def source_task_thread_func(self, _task, _obj, data, _cancellable): """Source import task code""" @@ -111,24 +123,24 @@ class Importer: if pipeline is not None: logging.info("Imported %s (%s)", game.name, game.game_id) pipeline.connect("manager-done", self.manager_done_callback) - self.n_games_added += 1 - def source_task_callback(self, _obj, _result, data): - """Source import callback""" + def update_progressbar(self): + """Update the progressbar to show the percentage of game pipelines done""" + self.progressbar.set_fraction(self.pipelines_progress) + + def source_callback(self, _obj, _result, data): + """Callback executed when a source is fully scanned""" source, *_rest = data logging.debug("Import done for source %s", source.id) self.n_source_tasks_done += 1 - # TODO remove, should be handled by manager_done_callback - self.update_progressbar() - if self.finished: - self.import_callback() def manager_done_callback(self, pipeline: Pipeline, manager: Manager): """Callback called when a pipeline for a game has advanced""" - # TODO (optional) update progress bar more precisely from here - # TODO get number of games really added here (eg. exlude blacklisted) - # TODO trigger import_callback only when all pipelines have finished - pass + if pipeline.is_done: + self.n_pipelines_done += 1 + self.update_progressbar() + if self.finished: + self.import_callback() def import_callback(self): """Callback called when importing has finished""" diff --git a/src/store/pipeline.py b/src/store/pipeline.py index a58796a..feea237 100644 --- a/src/store/pipeline.py +++ b/src/store/pipeline.py @@ -28,6 +28,10 @@ class Pipeline(GObject.Object): """Get the managers that are not done yet""" return self.waiting | self.running + @property + def is_done(self) -> bool: + return len(self.not_done) == 0 + @property def blocked(self) -> set[Manager]: """Get the managers that cannot run because their dependencies aren't done""" From 12ad5c598e966df67a738e252fa8065ab1130221 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Sat, 27 May 2023 19:36:49 +0200 Subject: [PATCH 069/173] Create shared.PROFILE --- meson.build | 1 + src/shared.py.in | 1 + src/window.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/meson.build b/meson.build index 2a65b03..4697e16 100644 --- a/meson.build +++ b/meson.build @@ -24,6 +24,7 @@ conf.set('PYTHON', python.find_installation('python3').full_path()) conf.set('APP_ID', app_id) conf.set('PREFIX', prefix) conf.set('VERSION', meson.project_version()) +conf.set('PROFILE', profile) conf.set('localedir', join_paths(get_option('prefix'), get_option('localedir'))) conf.set('pkgdatadir', pkgdatadir) diff --git a/src/shared.py.in b/src/shared.py.in index a55a0e9..6f08a29 100644 --- a/src/shared.py.in +++ b/src/shared.py.in @@ -25,6 +25,7 @@ from gi.repository import Gdk, Gio APP_ID = "@APP_ID@" VERSION = "@VERSION@" PREFIX = "@PREFIX@" +PROFILE = "@PROFILE@" SPEC_VERSION = 1.5 # The version of the game_id.json spec schema = Gio.Settings.new(APP_ID) diff --git a/src/window.py b/src/window.py index 4fc7ccf..d2b8ddf 100644 --- a/src/window.py +++ b/src/window.py @@ -90,7 +90,7 @@ class CartridgesWindow(Adw.ApplicationWindow): self.notice_empty.set_icon_name(shared.APP_ID + "-symbolic") - if "Devel" in shared.APP_ID: + if shared.PROFILE == "development": self.add_css_class("devel") # Connect search entries From e7fa20e4d4b1fc33f9bfc6006aa008c5859973a4 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 27 May 2023 19:39:52 +0200 Subject: [PATCH 070/173] =?UTF-8?q?=F0=9F=8E=A8=20Better=20logging=20init?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main.py b/src/main.py index 1511815..e8d68e7 100644 --- a/src/main.py +++ b/src/main.py @@ -238,7 +238,10 @@ class CartridgesApplication(Adw.Application): def main(version): # pylint: disable=unused-argument - log_level = os.environ.get("LOGLEVEL", "ERROR").upper() - logging.basicConfig(level="INFO") # TODO remove before release, use env + # Initiate logger + default_log_level = "DEBUG" if shared.PROFILE == "development" else "WARNING" + log_level = os.environ.get("LOGLEVEL", default_log_level).upper() + logging.basicConfig(level=log_level) + # Start app app = CartridgesApplication() return app.run(sys.argv) From aeab1de4a963f7a4e16159b0145a1a32837f866a Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 28 May 2023 22:19:43 +0200 Subject: [PATCH 071/173] =?UTF-8?q?=F0=9F=8E=A8=20Improved=20structure=20/?= =?UTF-8?q?=20added=20debug=20info?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 11 +++++++++++ src/main.py | 14 ++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 62fc0da..a64fdfa 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -123,6 +123,8 @@ class Importer: if pipeline is not None: logging.info("Imported %s (%s)", game.name, game.game_id) pipeline.connect("manager-done", self.manager_done_callback) + pipeline.connect("manager-started", self.manager_started_callback) + self.game_pipelines.add(pipeline) def update_progressbar(self): """Update the progressbar to show the percentage of game pipelines done""" @@ -134,8 +136,17 @@ class Importer: logging.debug("Import done for source %s", source.id) self.n_source_tasks_done += 1 + def manager_started_callback(self, pipeline: Pipeline, manager: Manager): + """Callback called when a game manager has started""" + logging.debug( + "Manager %s for %s started", manager.__class__.__name__, pipeline.game.name + ) + def manager_done_callback(self, pipeline: Pipeline, manager: Manager): """Callback called when a pipeline for a game has advanced""" + logging.debug( + "Manager %s for %s done", manager.__class__.__name__, pipeline.game.name + ) if pipeline.is_done: self.n_pipelines_done += 1 self.update_progressbar() diff --git a/src/main.py b/src/main.py index e8d68e7..bb135dc 100644 --- a/src/main.py +++ b/src/main.py @@ -84,12 +84,7 @@ class CartridgesApplication(Adw.Application): shared.store = Store() shared.store.add_manager(DisplayManager()) - # Load games from disk - if shared.games_dir.exists(): - for game_file in shared.games_dir.iterdir(): - data = json.load(game_file.open()) - game = Game(data, allow_side_effects=False) - shared.store.add_game(game) + self.load_games_from_disk() # Add rest of the managers for game imports shared.store.add_manager(SteamAPIManager()) @@ -135,6 +130,13 @@ class CartridgesApplication(Adw.Application): self.win.present() + def load_games_from_disk(self): + if shared.games_dir.exists(): + for game_file in shared.games_dir.iterdir(): + data = json.load(game_file.open()) + game = Game(data, allow_side_effects=False) + shared.store.add_game(game) + def on_about_action(self, *_args): about = Adw.AboutWindow( transient_for=self.win, From b99c058cd7273cfd348fe2a10727740bf1a51921 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 29 May 2023 00:05:44 +0200 Subject: [PATCH 072/173] =?UTF-8?q?=E2=9C=A8=20Added=20blocking/async=20ma?= =?UTF-8?q?nagers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/managers/async_manager.py | 41 +++++++++++++++++++++++ src/store/managers/display_manager.py | 3 +- src/store/managers/file_manager.py | 6 ++-- src/store/managers/manager.py | 44 +++++++++++++++---------- src/store/managers/sgdb_manager.py | 6 ++-- src/store/managers/steam_api_manager.py | 6 ++-- src/store/pipeline.py | 31 ++++++++--------- 7 files changed, 91 insertions(+), 46 deletions(-) create mode 100644 src/store/managers/async_manager.py diff --git a/src/store/managers/async_manager.py b/src/store/managers/async_manager.py new file mode 100644 index 0000000..046bb6e --- /dev/null +++ b/src/store/managers/async_manager.py @@ -0,0 +1,41 @@ +from gi.repository import Gio + +from src.game import Game +from src.store.managers.manager import Manager +from src.utils.task import Task + + +class AsyncManager(Manager): + """Manager that can run asynchronously""" + + blocking = False + cancellable: Gio.Cancellable = None + + def __init__(self) -> None: + super().__init__() + self.cancellable = Gio.Cancellable() + + def cancel_tasks(self): + """Cancel all tasks for this manager""" + self.cancellable.cancel() + + def reset_cancellable(self): + """Reset the cancellable for this manager. + Already scheduled Tasks will no longer be cancellable.""" + self.cancellable = Gio.Cancellable() + + def run(self, game: Game) -> None: + data = (game,) + task = Task.new(self, self.cancellable, self._task_callback, data) + task.set_task_data(data) + task.run_in_thread(self._task_thread_func) + + def _task_thread_func(self, _task, _source_object, data, cancellable): + """Task thread entry point""" + game, *_rest = data + self.emit("started") + self.final_run(game) + + def _task_callback(self, _source_object, _result, _data): + """Method run after the async task is done""" + self.emit("done") diff --git a/src/store/managers/display_manager.py b/src/store/managers/display_manager.py index ba7ca39..81a0f0b 100644 --- a/src/store/managers/display_manager.py +++ b/src/store/managers/display_manager.py @@ -9,7 +9,8 @@ class DisplayManager(Manager): run_after = set((FileManager,)) - def run(self, game: Game) -> None: + def final_run(self, game: Game) -> None: # TODO decouple a game from its widget + # TODO make the display manager async shared.win.games[game.game_id] = game game.update() diff --git a/src/store/managers/file_manager.py b/src/store/managers/file_manager.py index ffd2363..12884ed 100644 --- a/src/store/managers/file_manager.py +++ b/src/store/managers/file_manager.py @@ -1,12 +1,12 @@ from src.game import Game -from src.store.managers.manager import Manager +from src.store.managers.async_manager import AsyncManager from src.store.managers.steam_api_manager import SteamAPIManager -class FileManager(Manager): +class FileManager(AsyncManager): """Manager in charge of saving a game to a file""" run_after = set((SteamAPIManager,)) - def run(self, game: Game) -> None: + def final_run(self, game: Game) -> None: game.save() diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 376662f..8d73999 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -1,10 +1,11 @@ from abc import abstractmethod -from gi.repository import Gio + +from gi.repository import GObject from src.game import Game -class Manager: +class Manager(GObject.Object): """Class in charge of handling a post creation action for games. * May connect to signals on the game to handle them. @@ -13,26 +14,15 @@ class Manager: """ run_after: set[type["Manager"]] = set() - - cancellable: Gio.Cancellable errors: list[Exception] + blocking: bool = True def __init__(self) -> None: super().__init__() - self.cancellable = Gio.Cancellable() self.errors = [] - def cancel_tasks(self): - """Cancel all tasks for this manager""" - self.cancellable.cancel() - - def reset_cancellable(self): - """Reset the cancellable for this manager. - Alreadyn scheduled Tasks will no longer be cancellable.""" - self.cancellable = Gio.Cancellable() - def report_error(self, error: Exception): - """Report an error that happened in of run""" + """Report an error that happened in Manager.run""" self.errors.append(error) def collect_errors(self) -> list[Exception]: @@ -42,8 +32,26 @@ class Manager: return errors @abstractmethod - def run(self, game: Game, cancellable: Gio.Cancellable) -> None: + def final_run(self, game: Game) -> None: + """ + Abstract method overriden by final child classes, called by the run method. + * May block its thread + * May not raise exceptions, as they will be silently ignored + """ + + def run(self, game: Game) -> None: """Pass the game through the manager. - May block its thread. - May not raise exceptions, as they will be silently ignored.""" + In charge of calling the final_run method.""" + self.emit("started") + self.final_run(game) + self.emit("done") + + @GObject.Signal(name="started") + def started(self) -> None: + """Signal emitted when a manager is started""" + pass + + @GObject.Signal(name="done") + def done(self) -> None: + """Signal emitted when a manager is done""" pass diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index a87885d..d01e3f3 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -1,17 +1,17 @@ from requests import HTTPError from src.game import Game -from src.store.managers.manager import Manager +from src.store.managers.async_manager import AsyncManager from src.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper from src.store.managers.steam_api_manager import SteamAPIManager -class SGDBManager(Manager): +class SGDBManager(AsyncManager): """Manager in charge of downloading a game's cover from steamgriddb""" run_after = set((SteamAPIManager,)) - def run(self, game: Game) -> None: + def final_run(self, game: Game) -> None: try: sgdb = SGDBHelper() sgdb.conditionaly_update_cover(game) diff --git a/src/store/managers/steam_api_manager.py b/src/store/managers/steam_api_manager.py index 93e4d42..599580d 100644 --- a/src/store/managers/steam_api_manager.py +++ b/src/store/managers/steam_api_manager.py @@ -1,14 +1,14 @@ from requests import HTTPError, JSONDecodeError from src.game import Game -from src.store.managers.manager import Manager +from src.store.managers.async_manager import AsyncManager from src.utils.steam import SteamGameNotFoundError, SteamHelper, SteamNotAGameError -class SteamAPIManager(Manager): +class SteamAPIManager(AsyncManager): """Manager in charge of completing a game's data from the Steam API""" - def run(self, game: Game) -> None: + def final_run(self, game: Game) -> None: # Skip non-steam games if not game.source.startswith("steam_"): return diff --git a/src/store/pipeline.py b/src/store/pipeline.py index feea237..7453630 100644 --- a/src/store/pipeline.py +++ b/src/store/pipeline.py @@ -51,34 +51,29 @@ class Pipeline(GObject.Object): def advance(self): """Spawn tasks for managers that are able to run for a game""" - for manager in self.ready: - self.waiting.remove(manager) - self.running.add(manager) - data = (manager,) - task = Task.new(self, manager.cancellable, self.manager_task_callback, data) - task.set_task_data(data) - task.run_in_thread(self.manager_task_thread_func) + + # Separate blocking / async managers + managers = self.ready + blocking = set(filter(lambda manager: manager.blocking, managers)) + parallel = managers - parallel + + # Schedule parallel managers, then run the blocking ones + for manager in (*parallel, *blocking): + manager.run(self.game) @GObject.Signal(name="manager-started", arg_types=(object,)) def manager_started(self, manager: Manager) -> None: """Signal emitted when a manager is started""" pass - def manager_task_thread_func(self, _task, _source_object, data, cancellable): - """Thread function for manager tasks""" - manager, *_rest = data - self.emit("manager-started", manager) - manager.run(self.game) - @GObject.Signal(name="manager-done", arg_types=(object,)) def manager_done(self, manager: Manager) -> None: """Signal emitted when a manager is done""" pass - def manager_task_callback(self, _source_object, _result, data): - """Callback function for manager tasks""" - manager, *_rest = data - self.running.remove(manager) - self.done.add(manager) + def on_manager_started(self, manager: Manager) -> None: + self.emit("manager-started", manager) + + def on_manager_done(self, manager: Manager) -> None: self.emit("manager-done", manager) self.advance() From 8ddb110cbbe0c49073fb94a6f9551e2fb3c36e06 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 29 May 2023 00:23:25 +0200 Subject: [PATCH 073/173] Managers use callback functions instead of signals --- src/store/managers/async_manager.py | 15 ++++++++------- src/store/managers/manager.py | 20 ++++---------------- src/store/pipeline.py | 18 +++++++----------- 3 files changed, 19 insertions(+), 34 deletions(-) diff --git a/src/store/managers/async_manager.py b/src/store/managers/async_manager.py index 046bb6e..bd5c148 100644 --- a/src/store/managers/async_manager.py +++ b/src/store/managers/async_manager.py @@ -1,3 +1,5 @@ +from typing import Callable + from gi.repository import Gio from src.game import Game @@ -24,18 +26,17 @@ class AsyncManager(Manager): Already scheduled Tasks will no longer be cancellable.""" self.cancellable = Gio.Cancellable() - def run(self, game: Game) -> None: - data = (game,) - task = Task.new(self, self.cancellable, self._task_callback, data) - task.set_task_data(data) + def run(self, game: Game, callback: Callable) -> None: + task = Task.new(self, self.cancellable, self._task_callback, (callback,)) + task.set_task_data((game,)) task.run_in_thread(self._task_thread_func) def _task_thread_func(self, _task, _source_object, data, cancellable): """Task thread entry point""" game, *_rest = data - self.emit("started") self.final_run(game) - def _task_callback(self, _source_object, _result, _data): + def _task_callback(self, _source_object, _result, data): """Method run after the async task is done""" - self.emit("done") + _game, callback, *_rest = data + callback(self) diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 8d73999..cd3b37d 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -1,11 +1,10 @@ from abc import abstractmethod - -from gi.repository import GObject +from typing import Callable from src.game import Game -class Manager(GObject.Object): +class Manager: """Class in charge of handling a post creation action for games. * May connect to signals on the game to handle them. @@ -39,19 +38,8 @@ class Manager(GObject.Object): * May not raise exceptions, as they will be silently ignored """ - def run(self, game: Game) -> None: + def run(self, game: Game, callback: Callable[["Manager"]]) -> None: """Pass the game through the manager. In charge of calling the final_run method.""" - self.emit("started") self.final_run(game) - self.emit("done") - - @GObject.Signal(name="started") - def started(self) -> None: - """Signal emitted when a manager is started""" - pass - - @GObject.Signal(name="done") - def done(self) -> None: - """Signal emitted when a manager is done""" - pass + callback(self) diff --git a/src/store/pipeline.py b/src/store/pipeline.py index 7453630..fd0ad5a 100644 --- a/src/store/pipeline.py +++ b/src/store/pipeline.py @@ -55,25 +55,21 @@ class Pipeline(GObject.Object): # Separate blocking / async managers managers = self.ready blocking = set(filter(lambda manager: manager.blocking, managers)) - parallel = managers - parallel + parallel = managers - blocking # Schedule parallel managers, then run the blocking ones for manager in (*parallel, *blocking): - manager.run(self.game) + self.emit("manager-started", manager) + manager.run(self.game, self._manager_callback) + + def _manager_callback(self, manager: Manager) -> None: + """Method called by a manager when it's done""" + self.emit("manager-done", manager) @GObject.Signal(name="manager-started", arg_types=(object,)) def manager_started(self, manager: Manager) -> None: """Signal emitted when a manager is started""" - pass @GObject.Signal(name="manager-done", arg_types=(object,)) def manager_done(self, manager: Manager) -> None: """Signal emitted when a manager is done""" - pass - - def on_manager_started(self, manager: Manager) -> None: - self.emit("manager-started", manager) - - def on_manager_done(self, manager: Manager) -> None: - self.emit("manager-done", manager) - self.advance() From 0645808ac437ef7ffd5c7346b38fd119cebb7c5e Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 29 May 2023 01:38:36 +0200 Subject: [PATCH 074/173] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20GTK=20race=20con?= =?UTF-8?q?dition=20in=20pipelines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 14 ++------------ src/store/managers/async_manager.py | 8 ++++---- src/store/managers/display_manager.py | 6 +++--- src/store/managers/manager.py | 8 ++++++-- src/store/pipeline.py | 25 +++++++++++++------------ 5 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index a64fdfa..681628a 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -122,8 +122,7 @@ class Importer: pipeline: Pipeline = shared.store.add_game(game) if pipeline is not None: logging.info("Imported %s (%s)", game.name, game.game_id) - pipeline.connect("manager-done", self.manager_done_callback) - pipeline.connect("manager-started", self.manager_started_callback) + pipeline.connect("advanced", self.pipeline_advanced_callback) self.game_pipelines.add(pipeline) def update_progressbar(self): @@ -136,17 +135,8 @@ class Importer: logging.debug("Import done for source %s", source.id) self.n_source_tasks_done += 1 - def manager_started_callback(self, pipeline: Pipeline, manager: Manager): - """Callback called when a game manager has started""" - logging.debug( - "Manager %s for %s started", manager.__class__.__name__, pipeline.game.name - ) - - def manager_done_callback(self, pipeline: Pipeline, manager: Manager): + def pipeline_advanced_callback(self, pipeline: Pipeline): """Callback called when a pipeline for a game has advanced""" - logging.debug( - "Manager %s for %s done", manager.__class__.__name__, pipeline.game.name - ) if pipeline.is_done: self.n_pipelines_done += 1 self.update_progressbar() diff --git a/src/store/managers/async_manager.py b/src/store/managers/async_manager.py index bd5c148..c3dfb73 100644 --- a/src/store/managers/async_manager.py +++ b/src/store/managers/async_manager.py @@ -1,4 +1,4 @@ -from typing import Callable +from typing import Callable, Any from gi.repository import Gio @@ -26,8 +26,8 @@ class AsyncManager(Manager): Already scheduled Tasks will no longer be cancellable.""" self.cancellable = Gio.Cancellable() - def run(self, game: Game, callback: Callable) -> None: - task = Task.new(self, self.cancellable, self._task_callback, (callback,)) + def run(self, game: Game, callback: Callable[["Manager"], Any]) -> None: + task = Task.new(None, self.cancellable, self._task_callback, (callback,)) task.set_task_data((game,)) task.run_in_thread(self._task_thread_func) @@ -38,5 +38,5 @@ class AsyncManager(Manager): def _task_callback(self, _source_object, _result, data): """Method run after the async task is done""" - _game, callback, *_rest = data + callback, *_rest = data callback(self) diff --git a/src/store/managers/display_manager.py b/src/store/managers/display_manager.py index 81a0f0b..2e0ef39 100644 --- a/src/store/managers/display_manager.py +++ b/src/store/managers/display_manager.py @@ -1,16 +1,16 @@ from src import shared from src.game import Game -from src.store.managers.file_manager import FileManager +from src.store.managers.sgdb_manager import SGDBManager +from src.store.managers.steam_api_manager import SteamAPIManager from src.store.managers.manager import Manager class DisplayManager(Manager): """Manager in charge of adding a game to the UI""" - run_after = set((FileManager,)) + run_after = set((SteamAPIManager, SGDBManager)) def final_run(self, game: Game) -> None: # TODO decouple a game from its widget - # TODO make the display manager async shared.win.games[game.game_id] = game game.update() diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index cd3b37d..9c67205 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import Callable +from typing import Callable, Any from src.game import Game @@ -16,6 +16,10 @@ class Manager: errors: list[Exception] blocking: bool = True + @property + def name(self): + return type(self).__name__ + def __init__(self) -> None: super().__init__() self.errors = [] @@ -38,7 +42,7 @@ class Manager: * May not raise exceptions, as they will be silently ignored """ - def run(self, game: Game, callback: Callable[["Manager"]]) -> None: + def run(self, game: Game, callback: Callable[["Manager"], Any]) -> None: """Pass the game through the manager. In charge of calling the final_run method.""" self.final_run(game) diff --git a/src/store/pipeline.py b/src/store/pipeline.py index fd0ad5a..3fe4976 100644 --- a/src/store/pipeline.py +++ b/src/store/pipeline.py @@ -1,10 +1,10 @@ +import logging from typing import Iterable from gi.repository import GObject from src.game import Game from src.store.managers.manager import Manager -from src.utils.task import Task class Pipeline(GObject.Object): @@ -59,17 +59,18 @@ class Pipeline(GObject.Object): # Schedule parallel managers, then run the blocking ones for manager in (*parallel, *blocking): - self.emit("manager-started", manager) - manager.run(self.game, self._manager_callback) + self.waiting.remove(manager) + self.running.add(manager) + manager.run(self.game, self.manager_callback) - def _manager_callback(self, manager: Manager) -> None: + def manager_callback(self, manager: Manager) -> None: """Method called by a manager when it's done""" - self.emit("manager-done", manager) + logging.debug("%s done for %s", manager.name, self.game.game_id) + self.running.remove(manager) + self.done.add(manager) + self.emit("advanced") + self.advance() - @GObject.Signal(name="manager-started", arg_types=(object,)) - def manager_started(self, manager: Manager) -> None: - """Signal emitted when a manager is started""" - - @GObject.Signal(name="manager-done", arg_types=(object,)) - def manager_done(self, manager: Manager) -> None: - """Signal emitted when a manager is done""" + @GObject.Signal(name="advanced") + def advanced(self) -> None: + """Signal emitted when the pipeline has advanced""" From 0b188136a208d8dd3cbf6730b2ab84c018966800 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 31 May 2023 15:22:08 +0200 Subject: [PATCH 075/173] =?UTF-8?q?=F0=9F=9A=A7=20Initial=20work=20on=20re?= =?UTF-8?q?tryable=20managers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/source.py | 6 +--- src/store/managers/manager.py | 44 +++++++++++++++++++++---- src/store/managers/sgdb_manager.py | 7 ++-- src/store/managers/steam_api_manager.py | 14 ++++---- src/utils/steam.py | 3 +- src/utils/steamgriddb.py | 4 +-- 6 files changed, 52 insertions(+), 26 deletions(-) diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index a45667a..921a8c0 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -54,11 +54,7 @@ class Source(Iterable): @property def game_id_format(self) -> str: """The string format used to construct game IDs""" - format_ = self.name.lower() - if self.variant is not None: - format_ += f"_{self.variant.lower()}" - format_ += "_{game_id}" - return format_ + return self.name.lower() + "_{game_id}" @property @abstractmethod diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 9c67205..19b20c2 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -1,5 +1,6 @@ +import logging from abc import abstractmethod -from typing import Callable, Any +from typing import Any, Callable from src.game import Game @@ -10,11 +11,15 @@ class Manager: * May connect to signals on the game to handle them. * May cancel its running tasks on critical error, in that case a new cancellable must be generated for new tasks to run. + * May be retried on some specific error types """ run_after: set[type["Manager"]] = set() - errors: list[Exception] blocking: bool = True + retryable_on: set[type[Exception]] = set() + max_tries: int = 3 + + errors: list[Exception] @property def name(self): @@ -37,13 +42,38 @@ class Manager: @abstractmethod def final_run(self, game: Game) -> None: """ - Abstract method overriden by final child classes, called by the run method. + Manager specific logic triggered by the run method + * Implemented by final child classes + * Called by the run method, not used directly * May block its thread - * May not raise exceptions, as they will be silently ignored + * May raise retryable exceptions that will be be retried if possible + * May raise other exceptions that will be reported """ def run(self, game: Game, callback: Callable[["Manager"], Any]) -> None: - """Pass the game through the manager. - In charge of calling the final_run method.""" - self.final_run(game) + """ + Pass the game through the manager + * Public method called by a pipeline + * In charge of calling the final_run method and handling its errors + """ + + for remaining_tries in range(self.max_tries, -1, -1): + try: + self.final_run(game, self.max_tries) + except Exception as error: + if type(error) in self.retryable_on: + # Handle unretryable errors + logging.error("Unretryable error in %s", self.name, exc_info=error) + self.report_error(error) + break + elif remaining_tries == 0: + # Handle being out of retries + logging.error("Out of retries in %s", self.name, exc_info=error) + self.report_error(error) + break + else: + # Retry + logging.debug("Retrying %s (%s)", self.name, type(error).__name__) + continue + callback(self) diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index d01e3f3..d2c1384 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -2,14 +2,15 @@ from requests import HTTPError from src.game import Game from src.store.managers.async_manager import AsyncManager -from src.utils.steamgriddb import SGDBAuthError, SGDBError, SGDBHelper from src.store.managers.steam_api_manager import SteamAPIManager +from src.utils.steamgriddb import HTTPError, SGDBAuthError, SGDBError, SGDBHelper class SGDBManager(AsyncManager): """Manager in charge of downloading a game's cover from steamgriddb""" run_after = set((SteamAPIManager,)) + retryable_on = set((HTTPError,)) def final_run(self, game: Game) -> None: try: @@ -19,7 +20,3 @@ class SGDBManager(AsyncManager): # If invalid auth, cancel all SGDBManager tasks self.cancellable.cancel() self.report_error(error) - except (HTTPError, SGDBError) as error: - # On other error, just report it - self.report_error(error) - pass diff --git a/src/store/managers/steam_api_manager.py b/src/store/managers/steam_api_manager.py index 599580d..15ad35b 100644 --- a/src/store/managers/steam_api_manager.py +++ b/src/store/managers/steam_api_manager.py @@ -1,13 +1,18 @@ -from requests import HTTPError, JSONDecodeError - from src.game import Game from src.store.managers.async_manager import AsyncManager -from src.utils.steam import SteamGameNotFoundError, SteamHelper, SteamNotAGameError +from src.utils.steam import ( + HTTPError, + SteamGameNotFoundError, + SteamHelper, + SteamNotAGameError, +) class SteamAPIManager(AsyncManager): """Manager in charge of completing a game's data from the Steam API""" + retryable_on = set((HTTPError,)) + def final_run(self, game: Game) -> None: # Skip non-steam games if not game.source.startswith("steam_"): @@ -18,9 +23,6 @@ class SteamAPIManager(AsyncManager): steam = SteamHelper() try: online_data = steam.get_api_data(appid=appid) - except (HTTPError, JSONDecodeError) as error: - # On minor error, just report it - self.report_error(error) except (SteamNotAGameError, SteamGameNotFoundError): game.update_values({"blacklisted": True}) else: diff --git a/src/utils/steam.py b/src/utils/steam.py index f332456..ba5c649 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -75,7 +75,8 @@ class SteamHelper: logging.debug("Appid %s not found", appid) raise SteamGameNotFoundError() - if data["data"]["type"] != "game": + game_types = ("game", "demo") + if data["data"]["type"] not in game_types: logging.debug("Appid %s is not a game", appid) raise SteamNotAGameError() diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index ea3ecd9..3841e9b 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -96,7 +96,7 @@ class SGDBHelper: sgdb_id = self.get_game_id(game) except (HTTPError, SGDBError) as error: logging.warning( - "%s while getting SGDB ID for %s", error.__class__.__name__, game.name + "%s while getting SGDB ID for %s", type(error).__name__, game.name ) raise error @@ -120,7 +120,7 @@ class SGDBHelper: except (HTTPError, SGDBError) as error: logging.warning( "%s while getting image for %s kwargs=%s", - error.__class__.__name__, + type(error).__name__, game.name, str(uri_kwargs), ) From d204737339a416ceb7a44d056ce7cb2b980e1079 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 31 May 2023 16:54:51 +0200 Subject: [PATCH 076/173] =?UTF-8?q?=F0=9F=9A=A7=20More=20work=20on=20resil?= =?UTF-8?q?ient=20managers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main.py | 17 ++++++++++--- src/store/managers/async_manager.py | 7 +++--- src/store/managers/display_manager.py | 2 +- src/store/managers/file_manager.py | 2 +- src/store/managers/manager.py | 32 ++++++++++++------------- src/store/managers/sgdb_manager.py | 6 ++--- src/store/managers/steam_api_manager.py | 2 +- src/store/pipeline.py | 2 +- 8 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/main.py b/src/main.py index bb135dc..05f6204 100644 --- a/src/main.py +++ b/src/main.py @@ -241,9 +241,20 @@ class CartridgesApplication(Adw.Application): def main(version): # pylint: disable=unused-argument # Initiate logger - default_log_level = "DEBUG" if shared.PROFILE == "development" else "WARNING" - log_level = os.environ.get("LOGLEVEL", default_log_level).upper() - logging.basicConfig(level=log_level) + # (silence debug info from external libraries) + profile_base_log_level = "DEBUG" if shared.PROFILE == "development" else "WARNING" + profile_lib_log_level = "INFO" if shared.PROFILE == "development" else "WARNING" + base_log_level = os.environ.get("LOGLEVEL", profile_base_log_level).upper() + lib_log_level = os.environ.get("LIBLOGLEVEL", profile_lib_log_level).upper() + log_levels = { + __name__: base_log_level, + "PIL": lib_log_level, + "urllib3": lib_log_level, + } + logging.basicConfig() + for logger, level in log_levels.items(): + logging.getLogger(logger).setLevel(level) + # Start app app = CartridgesApplication() return app.run(sys.argv) diff --git a/src/store/managers/async_manager.py b/src/store/managers/async_manager.py index c3dfb73..8d54681 100644 --- a/src/store/managers/async_manager.py +++ b/src/store/managers/async_manager.py @@ -26,7 +26,8 @@ class AsyncManager(Manager): Already scheduled Tasks will no longer be cancellable.""" self.cancellable = Gio.Cancellable() - def run(self, game: Game, callback: Callable[["Manager"], Any]) -> None: + def process_game(self, game: Game, callback: Callable[["Manager"], Any]) -> None: + """Create a task to process the game in a separate thread""" task = Task.new(None, self.cancellable, self._task_callback, (callback,)) task.set_task_data((game,)) task.run_in_thread(self._task_thread_func) @@ -34,9 +35,9 @@ class AsyncManager(Manager): def _task_thread_func(self, _task, _source_object, data, cancellable): """Task thread entry point""" game, *_rest = data - self.final_run(game) + self.execute_resilient_manager_logic(game) def _task_callback(self, _source_object, _result, data): - """Method run after the async task is done""" + """Method run after the task is done""" callback, *_rest = data callback(self) diff --git a/src/store/managers/display_manager.py b/src/store/managers/display_manager.py index 2e0ef39..90a7bde 100644 --- a/src/store/managers/display_manager.py +++ b/src/store/managers/display_manager.py @@ -10,7 +10,7 @@ class DisplayManager(Manager): run_after = set((SteamAPIManager, SGDBManager)) - def final_run(self, game: Game) -> None: + def manager_logic(self, game: Game) -> None: # TODO decouple a game from its widget shared.win.games[game.game_id] = game game.update() diff --git a/src/store/managers/file_manager.py b/src/store/managers/file_manager.py index 12884ed..07c725f 100644 --- a/src/store/managers/file_manager.py +++ b/src/store/managers/file_manager.py @@ -8,5 +8,5 @@ class FileManager(AsyncManager): run_after = set((SteamAPIManager,)) - def final_run(self, game: Game) -> None: + def manager_logic(self, game: Game) -> None: game.save() diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 19b20c2..01535bc 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -40,40 +40,38 @@ class Manager: return errors @abstractmethod - def final_run(self, game: Game) -> None: + def manager_logic(self, game: Game) -> None: """ Manager specific logic triggered by the run method * Implemented by final child classes - * Called by the run method, not used directly * May block its thread - * May raise retryable exceptions that will be be retried if possible + * May raise retryable exceptions that will trigger a retry if possible * May raise other exceptions that will be reported """ - def run(self, game: Game, callback: Callable[["Manager"], Any]) -> None: - """ - Pass the game through the manager - * Public method called by a pipeline - * In charge of calling the final_run method and handling its errors - """ - + def execute_resilient_manager_logic(self, game: Game) -> None: + """Execute the manager logic and handle its errors by reporting them or retrying""" for remaining_tries in range(self.max_tries, -1, -1): try: - self.final_run(game, self.max_tries) + self.manager_logic(game) except Exception as error: + # Handle unretryable errors + log_args = (type(error).__name__, self.name, game.game_id) if type(error) in self.retryable_on: - # Handle unretryable errors - logging.error("Unretryable error in %s", self.name, exc_info=error) + logging.error("Unretryable %s in %s for %s", *log_args) self.report_error(error) break + # Handle being out of retries elif remaining_tries == 0: - # Handle being out of retries - logging.error("Out of retries in %s", self.name, exc_info=error) + logging.error("Too many retries due to %s in %s for %s", *log_args) self.report_error(error) break + # Retry else: - # Retry - logging.debug("Retrying %s (%s)", self.name, type(error).__name__) + logging.debug("Retry caused by %s in %s for %s", *log_args) continue + def process_game(self, game: Game, callback: Callable[["Manager"], Any]) -> None: + """Pass the game through the manager""" + self.execute_resilient_manager_logic(game, tries=0) callback(self) diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index d2c1384..27ca8de 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -12,11 +12,11 @@ class SGDBManager(AsyncManager): run_after = set((SteamAPIManager,)) retryable_on = set((HTTPError,)) - def final_run(self, game: Game) -> None: + def manager_logic(self, game: Game) -> None: try: sgdb = SGDBHelper() sgdb.conditionaly_update_cover(game) - except SGDBAuthError as error: + except SGDBAuthError: # If invalid auth, cancel all SGDBManager tasks self.cancellable.cancel() - self.report_error(error) + raise diff --git a/src/store/managers/steam_api_manager.py b/src/store/managers/steam_api_manager.py index 15ad35b..3b319c0 100644 --- a/src/store/managers/steam_api_manager.py +++ b/src/store/managers/steam_api_manager.py @@ -13,7 +13,7 @@ class SteamAPIManager(AsyncManager): retryable_on = set((HTTPError,)) - def final_run(self, game: Game) -> None: + def manager_logic(self, game: Game) -> None: # Skip non-steam games if not game.source.startswith("steam_"): return diff --git a/src/store/pipeline.py b/src/store/pipeline.py index 3fe4976..4372709 100644 --- a/src/store/pipeline.py +++ b/src/store/pipeline.py @@ -61,7 +61,7 @@ class Pipeline(GObject.Object): for manager in (*parallel, *blocking): self.waiting.remove(manager) self.running.add(manager) - manager.run(self.game, self.manager_callback) + manager.process_game(self.game, self.manager_callback) def manager_callback(self, manager: Manager) -> None: """Method called by a manager when it's done""" From ef63210a8ffd49adb96b366f851584f124173d24 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 31 May 2023 17:21:01 +0200 Subject: [PATCH 077/173] =?UTF-8?q?=F0=9F=8E=A8=20Better=20error=20handlin?= =?UTF-8?q?g=20in=20managers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main.py | 2 +- src/store/managers/manager.py | 14 ++++++++------ src/store/managers/sgdb_manager.py | 4 +--- src/utils/steam.py | 6 ++---- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/main.py b/src/main.py index 05f6204..fc90bf4 100644 --- a/src/main.py +++ b/src/main.py @@ -247,7 +247,7 @@ def main(version): # pylint: disable=unused-argument base_log_level = os.environ.get("LOGLEVEL", profile_base_log_level).upper() lib_log_level = os.environ.get("LIBLOGLEVEL", profile_lib_log_level).upper() log_levels = { - __name__: base_log_level, + None: base_log_level, "PIL": lib_log_level, "urllib3": lib_log_level, } diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 01535bc..c2c94f8 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -56,22 +56,24 @@ class Manager: self.manager_logic(game) except Exception as error: # Handle unretryable errors - log_args = (type(error).__name__, self.name, game.game_id) - if type(error) in self.retryable_on: - logging.error("Unretryable %s in %s for %s", *log_args) + log_args = (type(error).__name__, self.name, game.name, game.game_id) + if type(error) not in self.retryable_on: + logging.error("Unretryable %s in %s for %s (%s)", *log_args) self.report_error(error) break # Handle being out of retries elif remaining_tries == 0: - logging.error("Too many retries due to %s in %s for %s", *log_args) + logging.error( + "Too many retries due to %s in %s for %s (%s)", *log_args + ) self.report_error(error) break # Retry else: - logging.debug("Retry caused by %s in %s for %s", *log_args) + logging.debug("Retry caused by %s in %s for %s (%s)", *log_args) continue def process_game(self, game: Game, callback: Callable[["Manager"], Any]) -> None: """Pass the game through the manager""" - self.execute_resilient_manager_logic(game, tries=0) + self.execute_resilient_manager_logic(game) callback(self) diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index 27ca8de..2179ee9 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -1,9 +1,7 @@ -from requests import HTTPError - from src.game import Game from src.store.managers.async_manager import AsyncManager from src.store.managers.steam_api_manager import SteamAPIManager -from src.utils.steamgriddb import HTTPError, SGDBAuthError, SGDBError, SGDBHelper +from src.utils.steamgriddb import HTTPError, SGDBAuthError, SGDBHelper class SGDBManager(AsyncManager): diff --git a/src/utils/steam.py b/src/utils/steam.py index ba5c649..458dc77 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -66,11 +66,9 @@ class SteamHelper: ) as response: response.raise_for_status() data = response.json()[appid] - - except (HTTPError, JSONDecodeError) as error: - logging.warning("Error while querying Steam API for %s", appid) + except HTTPError as error: + logging.warning("Steam API HTTP error for %s", appid, exc_info=error) raise error - if not data["success"]: logging.debug("Appid %s not found", appid) raise SteamGameNotFoundError() From a213abe4da0e16c3e1ff8cca9b79517f2e03eb37 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 31 May 2023 18:18:58 +0200 Subject: [PATCH 078/173] =?UTF-8?q?=F0=9F=8E=A8=20SourceIterator=20is=20no?= =?UTF-8?q?t=20sized=20anymore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/lutris_source.py | 17 ----------------- src/importer/sources/source.py | 8 ++------ src/importer/sources/steam_source.py | 3 --- 3 files changed, 2 insertions(+), 26 deletions(-) diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 850c4d5..8e3dcf2 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -16,17 +16,6 @@ class LutrisSourceIterator(SourceIterator): db_connection = None db_cursor = None db_location = None - db_len_request = """ - SELECT count(*) - FROM 'games' - WHERE - name IS NOT NULL - AND slug IS NOT NULL - AND configPath IS NOT NULL - AND installed - AND (runner IS NOT "steam" OR :import_steam) - ; - """ db_games_request = """ SELECT id, name, slug, runner, hidden FROM 'games' @@ -46,16 +35,10 @@ class LutrisSourceIterator(SourceIterator): self.db_location = self.source.location / "pga.db" self.db_connection = connect(self.db_location) self.db_request_params = {"import_steam": self.import_steam} - self.__len__() # Init iterator length self.db_cursor = self.db_connection.execute( self.db_games_request, self.db_request_params ) - @lru_cache(maxsize=1) - def __len__(self): - cursor = self.db_connection.execute(self.db_len_request, self.db_request_params) - return cursor.fetchone()[0] - def __next__(self): """Produce games""" diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index 921a8c0..2506edb 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -1,11 +1,11 @@ from abc import abstractmethod -from collections.abc import Iterable, Iterator, Sized +from collections.abc import Iterable, Iterator from typing import Optional from src.game import Game -class SourceIterator(Iterator, Sized): +class SourceIterator(Iterator): """Data producer for a source of games""" source: "Source" = None @@ -17,10 +17,6 @@ class SourceIterator(Iterator, Sized): def __iter__(self) -> "SourceIterator": return self - @abstractmethod - def __len__(self) -> int: - """Get a rough estimate of the number of games produced by the source""" - @abstractmethod def __next__(self) -> Optional[Game]: """Get the next generated game from the source. diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index af79195..0ce0d54 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -51,9 +51,6 @@ class SteamSourceIterator(SourceIterator): self.manifests_iterator = iter(self.manifests) - def __len__(self): - return len(self.manifests) - def __next__(self): """Produce games""" From 344aa7057dfc2b6978a92ee00a19d55cbcddc3b7 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 31 May 2023 18:54:00 +0200 Subject: [PATCH 079/173] =?UTF-8?q?=F0=9F=8E=A8=20Consistency=20in=20sourc?= =?UTF-8?q?e=20typing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/lutris_source.py | 2 +- src/importer/sources/steam_source.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 8e3dcf2..663ba9a 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,5 +1,4 @@ from abc import abstractmethod -from functools import lru_cache from pathlib import Path from sqlite3 import connect from time import time @@ -12,6 +11,7 @@ from src.utils.save_cover import resize_cover, save_cover class LutrisSourceIterator(SourceIterator): + source: "LutrisSource" import_steam = False db_connection = None db_cursor = None diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 0ce0d54..1a13899 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -18,7 +18,6 @@ from src.utils.steam import SteamHelper, SteamInvalidManifestError class SteamSourceIterator(SourceIterator): source: "SteamSource" - manifests: set = None manifests_iterator: Iterator[Path] = None installed_state_mask: int = 4 From ed6610940415fa46b1049d635fc8425f7691c20f Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 31 May 2023 21:47:55 +0200 Subject: [PATCH 080/173] =?UTF-8?q?=F0=9F=9A=A7=20Ground=20work=20for=20he?= =?UTF-8?q?roic=20source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/heroic_source.py | 176 ++++++++++++++++++++++++++ src/main.py | 9 ++ 2 files changed, 185 insertions(+) create mode 100644 src/importer/sources/heroic_source.py diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py new file mode 100644 index 0000000..1dc1d8f --- /dev/null +++ b/src/importer/sources/heroic_source.py @@ -0,0 +1,176 @@ +import json +from abc import abstractmethod +from hashlib import sha256 +from json import JSONDecodeError +from pathlib import Path +from time import time +from typing import Generator, Optional, TypedDict + +from src import shared +from src.game import Game +from src.importer.sources.source import Source, SourceIterator +from src.utils.decorators import ( + replaced_by_env_path, + replaced_by_path, + replaced_by_schema_key, +) +from src.utils.save_cover import resize_cover, save_cover + + +class HeroicLibraryEntry(TypedDict): + app_name: str + installed: Optional[bool] + runner: str + title: str + developer: str + art_square: str + + +class HeroicSubSource(TypedDict): + service: str + path: tuple[str] + + +class HeroicSourceIterator(SourceIterator): + source: "HeroicSource" + generator: Generator = None + sub_sources: dict[str, HeroicSubSource] = { + "sideload": { + "service": "sideload", + "path": ("sideload_apps", "library.json"), + }, + "legendary": { + "service": "epic", + "path": ("store_cache", "legendary_library.json"), + }, + "gog": { + "service": "gog", + "path": ("store_cache", "gog_library.json"), + }, + } + + def game_from_library_entry(self, entry: HeroicLibraryEntry) -> Optional[Game]: + """Helper method used to build a Game from a Heroic library entry""" + + # Skip games that are not installed + if not entry["installed"]: + return None + + # Build game + app_name = entry["app_name"] + runner = entry["runner"] + service = self.sub_sources[runner]["service"] + values = { + "version": shared.SPEC_VERSION, + "hidden": False, + "source": self.source.id, + "added": int(time()), + "name": entry["title"], + "developer": entry["developer"], + "game_id": self.source.game_id_format.format( + service=service, game_id=app_name + ), + "executable": self.source.executable_format.format(app_name=app_name), + } + + # Save image from the heroic cache + # Filenames are derived from the URL that heroic used to get the file + uri: str = entry["art_square"] + if service == "epic": + uri += "?h=400&resize=1&w=300" + digest = sha256(uri.encode()).hexdigest() + image_path = self.source.location / "images-cache" / digest + if image_path.is_file(): + save_cover(values["game_id"], resize_cover(image_path)) + + return Game(values, allow_side_effects=False) + + def sub_sources_generator(self): + """Generator method producing games from all the Heroic sub-sources""" + for key, sub_source in self.sub_sources.items(): + # Skip disabled sub-sources + if not shared.schema.get_boolean("heroic-import-" + key): + continue + # Load games from JSON + try: + file = self.source.location.joinpath(*sub_source["path"]) + library = json.load(file.open())["library"] + except (JSONDecodeError, OSError, KeyError): + # Invalid library.json file, skip it + continue + for entry in library: + try: + game = self.game_from_library_entry(entry) + except KeyError: + # Skip invalid games + continue + yield game + + def __init__(self, source: "HeroicSource") -> None: + self.source = source + self.generator = self.sub_sources_generator() + + def __next__(self) -> Optional[Game]: + try: + game = next(self.generator) + except StopIteration: + raise + return game + + +class HeroicSource(Source): + """Generic heroic games launcher source""" + + name = "Heroic" + executable_format = "xdg-open heroic://launch/{app_name}" + + @property + @abstractmethod + def location(self) -> Path: + pass + + @property + def game_id_format(self) -> str: + """The string format used to construct game IDs""" + return self.name.lower() + "_{service}_{game_id}" + + @property + def is_installed(self): + # pylint: disable=pointless-statement + try: + self.location + except FileNotFoundError: + return False + return True + + def __iter__(self): + return HeroicSourceIterator(source=self) + + +class HeroicNativeSource(HeroicSource): + variant = "native" + + @replaced_by_schema_key("heroic-location") + @replaced_by_env_path("XDG_CONFIG_HOME", "heroic/") + @replaced_by_path("~/.config/heroic/") + def location(self) -> Path: + raise FileNotFoundError() + + +class HeroicFlatpakSource(HeroicSource): + variant = "flatpak" + + @replaced_by_schema_key("heroic-flatpak-location") + @replaced_by_path("~/.var/app/com.heroicgameslauncher.hgl/config/heroic/") + def location(self) -> Path: + raise FileNotFoundError() + + +class HeroicWindowsSource(HeroicSource): + variant = "windows" + executable_format = "start heroic://launch/{app_name}" + + @replaced_by_schema_key("heroic-windows-location") + @replaced_by_env_path("appdata", "heroic/") + def location(self) -> Path: + raise FileNotFoundError() diff --git a/src/main.py b/src/main.py index fc90bf4..8646622 100644 --- a/src/main.py +++ b/src/main.py @@ -34,6 +34,11 @@ from src import shared from src.details_window import DetailsWindow from src.game import Game from src.importer.importer import Importer +from src.importer.sources.heroic_source import ( + HeroicFlatpakSource, + HeroicNativeSource, + HeroicWindowsSource, +) from src.importer.sources.lutris_source import LutrisFlatpakSource, LutrisNativeSource from src.importer.sources.steam_source import ( SteamFlatpakSource, @@ -193,6 +198,10 @@ class CartridgesApplication(Adw.Application): importer.add_source(SteamNativeSource()) importer.add_source(SteamFlatpakSource()) importer.add_source(SteamWindowsSource()) + if shared.schema.get_boolean("heroic"): + importer.add_source(HeroicNativeSource()) + importer.add_source(HeroicFlatpakSource()) + importer.add_source(HeroicWindowsSource()) importer.run() def on_remove_game_action(self, *_args): From 97b770cbf29f78faf7dc31793e4c54a64bf0bd37 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 31 May 2023 22:43:30 +0200 Subject: [PATCH 081/173] =?UTF-8?q?=F0=9F=9A=A7=20Various=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Platform-dependent sources - Added heroic schema keys - Moved location and is_installed to Source --- data/hu.kramo.Cartridges.gschema.xml.in | 48 ++++++++++++++----------- src/importer/sources/heroic_source.py | 31 ++++++---------- src/importer/sources/lutris_source.py | 22 ++---------- src/importer/sources/source.py | 42 +++++++++++++++++++--- src/importer/sources/steam_source.py | 23 +++--------- 5 files changed, 81 insertions(+), 85 deletions(-) diff --git a/data/hu.kramo.Cartridges.gschema.xml.in b/data/hu.kramo.Cartridges.gschema.xml.in index a2f04de..0ae63b7 100644 --- a/data/hu.kramo.Cartridges.gschema.xml.in +++ b/data/hu.kramo.Cartridges.gschema.xml.in @@ -1,9 +1,9 @@ - - - false - + + + false + false @@ -44,21 +44,27 @@ true + "~/.config/heroic/" + + "~/.var/app/com.heroicgameslauncher.hgl/config/heroic/" - - true - - - true - - - true - + + "" + + + true + + + true + + + true + true - + "~/.var/app/com.usebottles.bottles/data/bottles/" @@ -79,7 +85,7 @@ false - + 1110 @@ -92,13 +98,13 @@ - - - - - + + + + + "a-z" - + \ No newline at end of file diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 1dc1d8f..9b44b7a 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -1,5 +1,5 @@ import json -from abc import abstractmethod +import logging from hashlib import sha256 from json import JSONDecodeError from pathlib import Path @@ -8,7 +8,7 @@ from typing import Generator, Optional, TypedDict from src import shared from src.game import Game -from src.importer.sources.source import Source, SourceIterator +from src.importer.sources.source import NTSource, PosixSource, Source, SourceIterator from src.utils.decorators import ( replaced_by_env_path, replaced_by_path, @@ -87,9 +87,9 @@ class HeroicSourceIterator(SourceIterator): def sub_sources_generator(self): """Generator method producing games from all the Heroic sub-sources""" - for key, sub_source in self.sub_sources.items(): + for _key, sub_source in self.sub_sources.items(): # Skip disabled sub-sources - if not shared.schema.get_boolean("heroic-import-" + key): + if not shared.schema.get_boolean("heroic-import-" + sub_source["service"]): continue # Load games from JSON try: @@ -124,32 +124,19 @@ class HeroicSource(Source): name = "Heroic" executable_format = "xdg-open heroic://launch/{app_name}" - @property - @abstractmethod - def location(self) -> Path: - pass - @property def game_id_format(self) -> str: """The string format used to construct game IDs""" return self.name.lower() + "_{service}_{game_id}" - @property - def is_installed(self): - # pylint: disable=pointless-statement - try: - self.location - except FileNotFoundError: - return False - return True - def __iter__(self): return HeroicSourceIterator(source=self) -class HeroicNativeSource(HeroicSource): +class HeroicNativeSource(HeroicSource, PosixSource): variant = "native" + @property @replaced_by_schema_key("heroic-location") @replaced_by_env_path("XDG_CONFIG_HOME", "heroic/") @replaced_by_path("~/.config/heroic/") @@ -157,19 +144,21 @@ class HeroicNativeSource(HeroicSource): raise FileNotFoundError() -class HeroicFlatpakSource(HeroicSource): +class HeroicFlatpakSource(HeroicSource, PosixSource): variant = "flatpak" + @property @replaced_by_schema_key("heroic-flatpak-location") @replaced_by_path("~/.var/app/com.heroicgameslauncher.hgl/config/heroic/") def location(self) -> Path: raise FileNotFoundError() -class HeroicWindowsSource(HeroicSource): +class HeroicWindowsSource(HeroicSource, NTSource): variant = "windows" executable_format = "start heroic://launch/{app_name}" + @property @replaced_by_schema_key("heroic-windows-location") @replaced_by_env_path("appdata", "heroic/") def location(self) -> Path: diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 663ba9a..befadc4 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,11 +1,9 @@ -from abc import abstractmethod -from pathlib import Path from sqlite3 import connect from time import time from src import shared from src.game import Game -from src.importer.sources.source import Source, SourceIterator +from src.importer.sources.source import PosixSource, Source, SourceIterator from src.utils.decorators import replaced_by_path, replaced_by_schema_key from src.utils.save_cover import resize_cover, save_cover @@ -78,29 +76,15 @@ class LutrisSource(Source): name = "Lutris" executable_format = "xdg-open lutris:rungameid/{game_id}" - @property - @abstractmethod - def location(self) -> Path: - pass - @property def game_id_format(self): return super().game_id_format + "_{game_internal_id}" - @property - def is_installed(self): - # pylint: disable=pointless-statement - try: - self.location - except FileNotFoundError: - return False - return True - def __iter__(self): return LutrisSourceIterator(source=self) -class LutrisNativeSource(LutrisSource): +class LutrisNativeSource(LutrisSource, PosixSource): variant = "native" @property @@ -110,7 +94,7 @@ class LutrisNativeSource(LutrisSource): raise FileNotFoundError() -class LutrisFlatpakSource(LutrisSource): +class LutrisFlatpakSource(LutrisSource, PosixSource): variant = "flatpak" @property diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index 2506edb..c65a845 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -1,5 +1,7 @@ +import os from abc import abstractmethod from collections.abc import Iterable, Iterator +from pathlib import Path from typing import Optional from src.game import Game @@ -30,6 +32,11 @@ class Source(Iterable): name: str variant: str + available_on: set[str] + + def __init__(self) -> None: + super().__init__() + self.available_on = set() @property def full_name(self) -> str: @@ -52,16 +59,41 @@ class Source(Iterable): """The string format used to construct game IDs""" return self.name.lower() + "_{game_id}" + @property + def is_installed(self): + # pylint: disable=pointless-statement + try: + self.location + except FileNotFoundError: + return False + return os.name in self.available_on + + @property + @abstractmethod + def location(self) -> Path: + """The source's location on disk""" + @property @abstractmethod def executable_format(self) -> str: """The executable format used to construct game executables""" - @property - @abstractmethod - def is_installed(self) -> bool: - """Whether the source is detected as installed""" - @abstractmethod def __iter__(self) -> SourceIterator: """Get the source's iterator, to use in for loops""" + + +class NTSource(Source): + """Mixin for sources available on Windows""" + + def __init__(self) -> None: + super().__init__() + self.available_on.add("nt") + + +class PosixSource(Source): + """Mixin for sources available on POXIX-compliant systems""" + + def __init__(self) -> None: + super().__init__() + self.available_on.add("posix") diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 1a13899..312551a 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -1,12 +1,11 @@ import re -from abc import abstractmethod from pathlib import Path from time import time from typing import Iterator from src import shared from src.game import Game -from src.importer.sources.source import Source, SourceIterator +from src.importer.sources.source import NTSource, PosixSource, Source, SourceIterator from src.utils.decorators import ( replaced_by_env_path, replaced_by_path, @@ -94,25 +93,11 @@ class SteamSource(Source): name = "Steam" executable_format = "xdg-open steam://rungameid/{game_id}" - @property - @abstractmethod - def location(self) -> Path: - pass - - @property - def is_installed(self): - # pylint: disable=pointless-statement - try: - self.location - except FileNotFoundError: - return False - return True - def __iter__(self): return SteamSourceIterator(source=self) -class SteamNativeSource(SteamSource): +class SteamNativeSource(SteamSource, PosixSource): variant = "native" @property @@ -124,7 +109,7 @@ class SteamNativeSource(SteamSource): raise FileNotFoundError() -class SteamFlatpakSource(SteamSource): +class SteamFlatpakSource(SteamSource, PosixSource): variant = "flatpak" @property @@ -134,7 +119,7 @@ class SteamFlatpakSource(SteamSource): raise FileNotFoundError() -class SteamWindowsSource(SteamSource): +class SteamWindowsSource(SteamSource, NTSource): variant = "windows" executable_format = "start steam://rungameid/{game_id}" From f0948c422fd8c101f8f06f2623fb9735d65db56f Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Thu, 1 Jun 2023 00:01:19 +0200 Subject: [PATCH 082/173] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20importer=20not?= =?UTF-8?q?=20finishing=20if=20no=20game=20found?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/importer.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 681628a..1cdf2cd 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -134,12 +134,23 @@ class Importer: source, *_rest = data logging.debug("Import done for source %s", source.id) self.n_source_tasks_done += 1 + self.progress_changed_callback() def pipeline_advanced_callback(self, pipeline: Pipeline): """Callback called when a pipeline for a game has advanced""" if pipeline.is_done: self.n_pipelines_done += 1 - self.update_progressbar() + self.progress_changed_callback() + + def progress_changed_callback(self): + """ + Callback called when the import process has progressed + + Triggered when: + * A source finishes + * A pipeline finishes + """ + self.update_progressbar() if self.finished: self.import_callback() From aa33e79963907f451d586b2124bff6236991fbfd Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Thu, 1 Jun 2023 00:34:46 +0200 Subject: [PATCH 083/173] =?UTF-8?q?=F0=9F=90=9B=20Blacklist=20on=20SteamAP?= =?UTF-8?q?I=20403?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/managers/steam_api_manager.py | 3 ++- src/utils/steam.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/store/managers/steam_api_manager.py b/src/store/managers/steam_api_manager.py index 3b319c0..8a1b2f9 100644 --- a/src/store/managers/steam_api_manager.py +++ b/src/store/managers/steam_api_manager.py @@ -2,6 +2,7 @@ from src.game import Game from src.store.managers.async_manager import AsyncManager from src.utils.steam import ( HTTPError, + SteamForbiddenError, SteamGameNotFoundError, SteamHelper, SteamNotAGameError, @@ -23,7 +24,7 @@ class SteamAPIManager(AsyncManager): steam = SteamHelper() try: online_data = steam.get_api_data(appid=appid) - except (SteamNotAGameError, SteamGameNotFoundError): + except (SteamNotAGameError, SteamGameNotFoundError, SteamForbiddenError): game.update_values({"blacklisted": True}) else: game.update_values(online_data) diff --git a/src/utils/steam.py b/src/utils/steam.py index 458dc77..9e576d7 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -22,6 +22,10 @@ class SteamInvalidManifestError(SteamError): pass +class SteamForbiddenError(SteamError): + pass + + class SteamManifestData(TypedDict): """Dict returned by SteamHelper.get_manifest_data""" @@ -64,6 +68,8 @@ class SteamHelper: with requests.get( f"{self.base_url}/appdetails?appids={appid}", timeout=5 ) as response: + if response.status_code == 403: + raise SteamForbiddenError() response.raise_for_status() data = response.json()[appid] except HTTPError as error: From f05d1e702b3502450c8702ed56e0e87b5a46c254 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Thu, 1 Jun 2023 00:40:28 +0200 Subject: [PATCH 084/173] =?UTF-8?q?=F0=9F=9A=A7=20Removed=20blacklist=20on?= =?UTF-8?q?=20403?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/game.py | 6 +++--- src/importer/sources/heroic_source.py | 1 - src/store/managers/steam_api_manager.py | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/game.py b/src/game.py index e269885..70dac82 100644 --- a/src/game.py +++ b/src/game.py @@ -52,12 +52,12 @@ class Game(Gtk.Box): executable = None game_id = None source = None - hidden = None + hidden = False last_played = 0 name = None developer = None - removed = None - blacklisted = None + removed = False + blacklisted = False game_cover = None version = 0 diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 9b44b7a..b1bfb17 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -62,7 +62,6 @@ class HeroicSourceIterator(SourceIterator): service = self.sub_sources[runner]["service"] values = { "version": shared.SPEC_VERSION, - "hidden": False, "source": self.source.id, "added": int(time()), "name": entry["title"], diff --git a/src/store/managers/steam_api_manager.py b/src/store/managers/steam_api_manager.py index 8a1b2f9..3b319c0 100644 --- a/src/store/managers/steam_api_manager.py +++ b/src/store/managers/steam_api_manager.py @@ -2,7 +2,6 @@ from src.game import Game from src.store.managers.async_manager import AsyncManager from src.utils.steam import ( HTTPError, - SteamForbiddenError, SteamGameNotFoundError, SteamHelper, SteamNotAGameError, @@ -24,7 +23,7 @@ class SteamAPIManager(AsyncManager): steam = SteamHelper() try: online_data = steam.get_api_data(appid=appid) - except (SteamNotAGameError, SteamGameNotFoundError, SteamForbiddenError): + except (SteamNotAGameError, SteamGameNotFoundError): game.update_values({"blacklisted": True}) else: game.update_values(online_data) From 2b2355e2c26fe202150a8e747fcc5e1682b56ed6 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Thu, 1 Jun 2023 14:10:59 +0200 Subject: [PATCH 085/173] =?UTF-8?q?=F0=9F=94=A5=20Removed=20SteamForbidden?= =?UTF-8?q?Error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/steam.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/utils/steam.py b/src/utils/steam.py index 9e576d7..458dc77 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -22,10 +22,6 @@ class SteamInvalidManifestError(SteamError): pass -class SteamForbiddenError(SteamError): - pass - - class SteamManifestData(TypedDict): """Dict returned by SteamHelper.get_manifest_data""" @@ -68,8 +64,6 @@ class SteamHelper: with requests.get( f"{self.base_url}/appdetails?appids={appid}", timeout=5 ) as response: - if response.status_code == 403: - raise SteamForbiddenError() response.raise_for_status() data = response.json()[appid] except HTTPError as error: From 2009003dea31bf3b64dfa16b07ef95275145bc8d Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Fri, 2 Jun 2023 10:28:43 +0200 Subject: [PATCH 086/173] =?UTF-8?q?=F0=9F=9A=A7=20Added=20todo=20before=20?= =?UTF-8?q?push?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/steam.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/steam.py b/src/utils/steam.py index 458dc77..4afa874 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -59,6 +59,7 @@ class SteamHelper: def get_api_data(self, appid) -> SteamAPIData: """Get online data for a game from its appid""" + # TODO throttle to not get access denied try: with requests.get( From 06b6ee45934441d227d6b486d55a91402263385d Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Fri, 2 Jun 2023 22:23:36 +0200 Subject: [PATCH 087/173] =?UTF-8?q?=F0=9F=9A=A7=20Unfinished=20rate=20limi?= =?UTF-8?q?ter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/rate_limiter.py | 90 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 src/utils/rate_limiter.py diff --git a/src/utils/rate_limiter.py b/src/utils/rate_limiter.py new file mode 100644 index 0000000..43e1150 --- /dev/null +++ b/src/utils/rate_limiter.py @@ -0,0 +1,90 @@ +from threading import Lock, Thread +from time import time_ns, sleep + + +class RateLimiter: + """ + Thread-safe and blocking rate limiter. + * There are at most X tokens available in the limiter + * Tokens can't be picked faster than every Y nanoseconds + * Acquire will block until those conditions are met + * The first to request a token will also be the first to acquire one + """ + + PICK_SPACING_NS: float + MAX_TOKENS: int + + _last_pick_time: int = 0 + _n_tokens: int = 0 + _queue: list[Lock] = None + _queue_lock: Lock = None + _tokens_lock: Lock = None + _last_pick_time_lock: Lock = None + + def __init__(self, pick_spacing_ns: float, max_tokens: int) -> None: + self.PICK_SPACING_NS = pick_spacing_ns + self.MAX_TOKENS = max_tokens + self._last_pick_time = 0 + self._last_pick_time_lock = Lock() + self._queue = [] + self._queue_lock = Lock() + self._n_tokens = max_tokens + self._tokens_lock = Lock() + + def update_queue(self) -> None: + """ + Move the queue forward if possible. + Non-blocking, logic runs in a daemon thread. + """ + thread = Thread(target=self.queue_update_thread_func, daemon=True) + thread.start() + + def queue_update_thread_func(self) -> None: + """Queue-updating thread's entry point""" + with self._queue_lock, self._tokens_lock: + # Unlock as many locks in the queue as there are tokens available + n_unlocked = min(len(self._queue), self._n_tokens) + for _ in range(n_unlocked): + lock = self._queue.pop(0) + lock.release() + # Consume the tokens used + self._n_tokens -= n_unlocked + + def add_to_queue(self) -> Lock: + """Create a lock, add it to the queue and return it""" + lock = Lock() + lock.acquire() + with self._queue_lock: + self._queue.append(lock) + return lock + + def acquire(self) -> None: + """ + Pick a token from the limiter. + Will block: + * Until your turn in queue + * Until the minimum pick spacing is satified + """ + + # Wait our turn in queue + # (no need for with since queue locks are unique, will be destroyed after that) + lock = self.add_to_queue() + self.update_queue() + lock.acquire() + + # TODO move to queue unlock (else order is not ensured) + # Satisfy the minimum pick spacing + now = time_ns() + with self._last_pick_time_lock: + elapsed = now - self._last_pick_time + ns_to_sleep = self.PICK_SPACING_NS - elapsed + self._last_pick_time = now + if ns_to_sleep > 0: + sleep(ns_to_sleep / 10**9) + self._last_pick_time += ns_to_sleep + + def release(self) -> None: + """Return a token to the bucket""" + with self._tokens_lock: + self._n_tokens += 1 + self.update_queue() From 10a635fc78a779541be3f6d477d8d1099aad0398 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 3 Jun 2023 14:18:07 +0200 Subject: [PATCH 088/173] =?UTF-8?q?=F0=9F=9A=A7=20Thread-safe=20manager=20?= =?UTF-8?q?error=20reporting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/managers/manager.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index c2c94f8..6df6d0f 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -1,6 +1,7 @@ import logging from abc import abstractmethod from typing import Any, Callable +from threading import Lock from src.game import Game @@ -20,6 +21,7 @@ class Manager: max_tries: int = 3 errors: list[Exception] + errors_lock: Lock = None @property def name(self): @@ -28,15 +30,18 @@ class Manager: def __init__(self) -> None: super().__init__() self.errors = [] + self.errors_lock = Lock() def report_error(self, error: Exception): """Report an error that happened in Manager.run""" - self.errors.append(error) + with self.errors_lock: + self.errors.append(error) def collect_errors(self) -> list[Exception]: """Get the errors produced by the manager and remove them from self.errors""" - errors = list(self.errors) - self.errors.clear() + with self.errors_lock: + errors = self.errors.copy() + self.errors.clear() return errors @abstractmethod From 6d6e830cc96bde5916956907bfbeac7016d2b3c9 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 3 Jun 2023 14:18:26 +0200 Subject: [PATCH 089/173] =?UTF-8?q?=F0=9F=9A=A7=20Intial=20work=20on=20a?= =?UTF-8?q?=20generic=20rate=20limiter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/rate_limiter.py | 124 +++++++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 50 deletions(-) diff --git a/src/utils/rate_limiter.py b/src/utils/rate_limiter.py index 43e1150..280846b 100644 --- a/src/utils/rate_limiter.py +++ b/src/utils/rate_limiter.py @@ -1,35 +1,51 @@ from threading import Lock, Thread from time import time_ns, sleep +from collections import deque +from contextlib import AbstractContextManager -class RateLimiter: +class RateLimiter(AbstractContextManager): """ Thread-safe and blocking rate limiter. - * There are at most X tokens available in the limiter - * Tokens can't be picked faster than every Y nanoseconds - * Acquire will block until those conditions are met - * The first to request a token will also be the first to acquire one + + There are at most X tokens available in the limiter, acquiring removes one + and releasing gives back one. + + Acquire will block until those conditions are met: + - There is a token available + - At least Y nanoseconds have passed since the last token was acquired + + The order in which tokens are requested is the order in which they will be given. + Works on a FIFO model. + + Can be used in a `with` statement like `threading.Lock` + [Using locks, conditions, and semaphores in the with statement](https://docs.python.org/3/library/threading.html#using-locks-conditions-and-semaphores-in-the-with-statement) """ - PICK_SPACING_NS: float + # Number of tokens available in the limiter + # = Max number of cuncurrent operations allowed MAX_TOKENS: int + available_tokens: int = 0 + tokens_lock: Lock = None - _last_pick_time: int = 0 - _n_tokens: int = 0 - _queue: list[Lock] = None - _queue_lock: Lock = None - _tokens_lock: Lock = None - _last_pick_time_lock: Lock = None + # Minimum time elapsed between two token being distributed + # = Rate limit + PICK_SPACING_NS: int + last_pick_time: int = 0 + last_pick_time_lock: Lock = None - def __init__(self, pick_spacing_ns: float, max_tokens: int) -> None: + # Queue containing locks unlocked when a token can be acquired + # Doesn't need a thread lock, deques have thread-safe append and pop on both ends + queue: deque[Lock] = None + + def __init__(self, pick_spacing_ns: int, max_tokens: int) -> None: self.PICK_SPACING_NS = pick_spacing_ns self.MAX_TOKENS = max_tokens - self._last_pick_time = 0 - self._last_pick_time_lock = Lock() - self._queue = [] - self._queue_lock = Lock() - self._n_tokens = max_tokens - self._tokens_lock = Lock() + self.last_pick_time = 0 + self.last_pick_time_lock = Lock() + self.queue = deque() + self.available_tokens = max_tokens + self.tokens_lock = Lock() def update_queue(self) -> None: """ @@ -41,50 +57,58 @@ class RateLimiter: def queue_update_thread_func(self) -> None: """Queue-updating thread's entry point""" - with self._queue_lock, self._tokens_lock: - # Unlock as many locks in the queue as there are tokens available - n_unlocked = min(len(self._queue), self._n_tokens) - for _ in range(n_unlocked): - lock = self._queue.pop(0) - lock.release() - # Consume the tokens used - self._n_tokens -= n_unlocked + + # Consume a token, if none is available do nothing + with self.tokens_lock: + if self.available_tokens == 0: + return + self.available_tokens -= 1 + + # Get the next lock in queue, if none is available do nothing + try: + lock = self.queue.pop() + except IndexError: + return + + # Satisfy the minimum pick spacing + with self.last_pick_time_lock: + elapsed = time_ns() - self.last_pick_time + if (ns_to_sleep := self.PICK_SPACING_NS - elapsed) > 0: + sleep(ns_to_sleep / 10**9) + self.last_pick_time = time_ns() + + # Finally unlock the acquire call linked to that lock + lock.release() def add_to_queue(self) -> Lock: """Create a lock, add it to the queue and return it""" lock = Lock() lock.acquire() - with self._queue_lock: - self._queue.append(lock) + self.queue.appendleft(lock) return lock def acquire(self) -> None: - """ - Pick a token from the limiter. - Will block: - * Until your turn in queue - * Until the minimum pick spacing is satified - """ + """Pick a token from the limiter""" # Wait our turn in queue - # (no need for with since queue locks are unique, will be destroyed after that) lock = self.add_to_queue() self.update_queue() - lock.acquire() - # TODO move to queue unlock (else order is not ensured) - # Satisfy the minimum pick spacing - now = time_ns() - with self._last_pick_time_lock: - elapsed = now - self._last_pick_time - ns_to_sleep = self.PICK_SPACING_NS - elapsed - self._last_pick_time = now - if ns_to_sleep > 0: - sleep(ns_to_sleep / 10**9) - self._last_pick_time += ns_to_sleep + # Block until lock is released (= its turn in queue) + # Single-use (this call to acquire), so no need to release it + lock.acquire() + del lock def release(self) -> None: - """Return a token to the bucket""" - with self._tokens_lock: - self._n_tokens += 1 + """Return a token to the limiter""" + with self.tokens_lock: + self.available_tokens += 1 self.update_queue() + + # --- Support for use in with statements + + def __enter__(self): + self.acquire() + + def __exit__(self): + self.release() From 58054f1c26f6c424cfad5ebbabe6dc185185dca9 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 3 Jun 2023 16:31:15 +0200 Subject: [PATCH 090/173] =?UTF-8?q?=F0=9F=90=9B=20Added=20rate=20limiter?= =?UTF-8?q?=20for=20Steam?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data/hu.kramo.Cartridges.gschema.xml.in | 6 + src/utils/rate_limiter.py | 162 ++++++++++++------------ src/utils/steam.py | 68 ++++++++-- 3 files changed, 141 insertions(+), 95 deletions(-) diff --git a/data/hu.kramo.Cartridges.gschema.xml.in b/data/hu.kramo.Cartridges.gschema.xml.in index 0ae63b7..326bdf9 100644 --- a/data/hu.kramo.Cartridges.gschema.xml.in +++ b/data/hu.kramo.Cartridges.gschema.xml.in @@ -106,5 +106,11 @@ "a-z" + + 200 + + + 0 + \ No newline at end of file diff --git a/src/utils/rate_limiter.py b/src/utils/rate_limiter.py index 280846b..b90f93d 100644 --- a/src/utils/rate_limiter.py +++ b/src/utils/rate_limiter.py @@ -1,114 +1,112 @@ -from threading import Lock, Thread -from time import time_ns, sleep +from typing import Optional +from threading import Lock, Thread, BoundedSemaphore +from time import sleep from collections import deque from contextlib import AbstractContextManager -class RateLimiter(AbstractContextManager): - """ - Thread-safe and blocking rate limiter. +class TokenBucketRateLimiter(AbstractContextManager): + """Rate limiter implementing the token bucket algorithm""" - There are at most X tokens available in the limiter, acquiring removes one - and releasing gives back one. - - Acquire will block until those conditions are met: - - There is a token available - - At least Y nanoseconds have passed since the last token was acquired - - The order in which tokens are requested is the order in which they will be given. - Works on a FIFO model. - - Can be used in a `with` statement like `threading.Lock` - [Using locks, conditions, and semaphores in the with statement](https://docs.python.org/3/library/threading.html#using-locks-conditions-and-semaphores-in-the-with-statement) - """ - - # Number of tokens available in the limiter - # = Max number of cuncurrent operations allowed + REFILL_SPACING_SECONDS: int MAX_TOKENS: int - available_tokens: int = 0 - tokens_lock: Lock = None - # Minimum time elapsed between two token being distributed - # = Rate limit - PICK_SPACING_NS: int - last_pick_time: int = 0 - last_pick_time_lock: Lock = None - - # Queue containing locks unlocked when a token can be acquired - # Doesn't need a thread lock, deques have thread-safe append and pop on both ends + bucket: BoundedSemaphore = None queue: deque[Lock] = None + queue_lock: Lock = None - def __init__(self, pick_spacing_ns: int, max_tokens: int) -> None: - self.PICK_SPACING_NS = pick_spacing_ns - self.MAX_TOKENS = max_tokens - self.last_pick_time = 0 - self.last_pick_time_lock = Lock() - self.queue = deque() - self.available_tokens = max_tokens - self.tokens_lock = Lock() + # Protect the number of tokens behind a lock + __n_tokens_lock: Lock = None + __n_tokens = 0 + + @property + def n_tokens(self): + with self.__n_tokens_lock: + return self.__n_tokens + + @n_tokens.setter + def n_tokens(self, value: int): + with self.__n_tokens_lock: + self.n_tokens = value + + def __init__( + self, + refill_spacing_seconds: Optional[int] = None, + max_tokens: Optional[int] = None, + initial_tokens: Optional[int] = None, + ) -> None: + # Initialize default values + self.queue_lock = Lock() + if max_tokens is not None: + self.MAX_TOKENS = max_tokens + if refill_spacing_seconds is not None: + self.REFILL_SPACING_SECONDS = refill_spacing_seconds + + # Initialize the bucket + self.bucket = BoundedSemaphore(self.MAX_TOKENS) + missing = 0 if initial_tokens is None else self.MAX_TOKENS - initial_tokens + missing = max(0, min(missing, max_tokens)) + for _ in range(missing): + self.bucket.acquire() + + # Initialize the counter + self.__n_tokens_lock = Lock() + self.n_tokens = self.MAX_TOKENS - missing + + # Spawn daemon thread that refills the bucket + refill_thread = Thread(target=self.refill_thread_func, daemon=True) + refill_thread.start() + + def refill(self): + """Method used by the refill thread""" + sleep(self.REFILL_SPACING_SECONDS) + try: + self.bucket.release() + except ValueError: + # Bucket was full + pass + else: + self.n_tokens += 1 + self.update_queue() + + def refill_thread_func(self): + """Entry point for the daemon thread that is refilling the bucket""" + while True: + self.refill() def update_queue(self) -> None: - """ - Move the queue forward if possible. - Non-blocking, logic runs in a daemon thread. - """ - thread = Thread(target=self.queue_update_thread_func, daemon=True) - thread.start() + """Update the queue, moving it forward if possible. Non-blocking.""" + update_thread = Thread(target=self.queue_update_thread_func, daemon=True) + update_thread.start() def queue_update_thread_func(self) -> None: """Queue-updating thread's entry point""" - - # Consume a token, if none is available do nothing - with self.tokens_lock: - if self.available_tokens == 0: + with self.queue_lock: + if len(self.queue) == 0: return - self.available_tokens -= 1 - - # Get the next lock in queue, if none is available do nothing - try: + self.bucket.acquire() + self.n_tokens -= 1 lock = self.queue.pop() - except IndexError: - return - - # Satisfy the minimum pick spacing - with self.last_pick_time_lock: - elapsed = time_ns() - self.last_pick_time - if (ns_to_sleep := self.PICK_SPACING_NS - elapsed) > 0: - sleep(ns_to_sleep / 10**9) - self.last_pick_time = time_ns() - - # Finally unlock the acquire call linked to that lock - lock.release() + lock.release() def add_to_queue(self) -> Lock: """Create a lock, add it to the queue and return it""" lock = Lock() lock.acquire() - self.queue.appendleft(lock) + with self.queue_lock: + self.queue.appendleft(lock) return lock - def acquire(self) -> None: - """Pick a token from the limiter""" - - # Wait our turn in queue + def acquire(self): + """Acquires a token from the bucket when it's your turn in queue""" lock = self.add_to_queue() self.update_queue() - - # Block until lock is released (= its turn in queue) - # Single-use (this call to acquire), so no need to release it lock.acquire() - del lock - - def release(self) -> None: - """Return a token to the limiter""" - with self.tokens_lock: - self.available_tokens += 1 - self.update_queue() # --- Support for use in with statements def __enter__(self): self.acquire() - def __exit__(self): - self.release() + def __exit__(self, *_args): + pass diff --git a/src/utils/steam.py b/src/utils/steam.py index 4afa874..cdef804 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -1,9 +1,14 @@ -import re import logging +import re +from time import time from typing import TypedDict +from math import floor, ceil import requests -from requests import HTTPError, JSONDecodeError +from requests import HTTPError + +from src import shared +from src.utils.rate_limiter import TokenBucketRateLimiter class SteamError(Exception): @@ -36,11 +41,40 @@ class SteamAPIData(TypedDict): developers: str +class SteamRateLimiter(TokenBucketRateLimiter): + """Rate limiter for the Steam web API""" + + # Steam web API limit + # 200 requests per 5 min seems to be the limit + # https://stackoverflow.com/questions/76047820/how-am-i-exceeding-steam-apis-rate-limit + # https://stackoverflow.com/questions/51795457/avoiding-error-429-too-many-requests-steam-web-api + REFILL_SPACING_SECONDS = 1.5 + MAX_TOKENS = 200 + + def __init__(self) -> None: + # Load initial tokens from schema + # (Remember API limits through restarts of Cartridges) + last_tokens = shared.state_schema.get_int("steam-api-tokens") + last_time = shared.state_schema.get_int("steam-api-tokens-timestamp") + produced = floor((time() - last_time) / self.REFILL_SPACING_SECONDS) + inital_tokens = last_tokens + produced + super().__init__(initial_tokens=inital_tokens) + + def refill(self): + """Refill the bucket and store its number of tokens in the schema""" + super().refill() + shared.state_schema.set_int("steam-api-tokens-timestamp", ceil(time())) + shared.state_schema.set_int("steam-api-tokens", self.n_tokens) + + class SteamHelper: """Helper around the Steam API""" base_url = "https://store.steampowered.com/api" + # Shared across instances + rate_limiter: SteamRateLimiter = SteamRateLimiter() + def get_manifest_data(self, manifest_path) -> SteamManifestData: """Get local data for a game from its manifest""" @@ -58,22 +92,30 @@ class SteamHelper: return SteamManifestData(**data) def get_api_data(self, appid) -> SteamAPIData: - """Get online data for a game from its appid""" - # TODO throttle to not get access denied + """ + Get online data for a game from its appid. - try: - with requests.get( - f"{self.base_url}/appdetails?appids={appid}", timeout=5 - ) as response: - response.raise_for_status() - data = response.json()[appid] - except HTTPError as error: - logging.warning("Steam API HTTP error for %s", appid, exc_info=error) - raise error + May block to satisfy the Steam web API limitations. + """ + + # Get data from the API (way block to satisfy its limits) + with self.rate_limiter: + try: + with requests.get( + f"{self.base_url}/appdetails?appids={appid}", timeout=5 + ) as response: + response.raise_for_status() + data = response.json()[appid] + except HTTPError as error: + logging.warning("Steam API HTTP error for %s", appid, exc_info=error) + raise error + + # Handle not found if not data["success"]: logging.debug("Appid %s not found", appid) raise SteamGameNotFoundError() + # Handle appid is not a game game_types = ("game", "demo") if data["data"]["type"] not in game_types: logging.debug("Appid %s is not a game", appid) From b1476a744dd4b024796ab67c00a79e26af27126e Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 3 Jun 2023 16:35:25 +0200 Subject: [PATCH 091/173] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20wrong=20variable?= =?UTF-8?q?=20in=20rate=20limiter=20init?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/rate_limiter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/rate_limiter.py b/src/utils/rate_limiter.py index b90f93d..ca118b4 100644 --- a/src/utils/rate_limiter.py +++ b/src/utils/rate_limiter.py @@ -45,7 +45,7 @@ class TokenBucketRateLimiter(AbstractContextManager): # Initialize the bucket self.bucket = BoundedSemaphore(self.MAX_TOKENS) missing = 0 if initial_tokens is None else self.MAX_TOKENS - initial_tokens - missing = max(0, min(missing, max_tokens)) + missing = max(0, min(missing, self.MAX_TOKENS)) for _ in range(missing): self.bucket.acquire() From dc47c850ceee67696b261cfac05a17a4908b79ce Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 3 Jun 2023 17:33:51 +0200 Subject: [PATCH 092/173] =?UTF-8?q?=F0=9F=90=9B=20Moved=20Steam=20rate=20l?= =?UTF-8?q?imiter=20init=20to=20runtime?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/steam.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/utils/steam.py b/src/utils/steam.py index cdef804..58d8ee1 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -71,9 +71,13 @@ class SteamHelper: """Helper around the Steam API""" base_url = "https://store.steampowered.com/api" + rate_limiter: SteamRateLimiter = None - # Shared across instances - rate_limiter: SteamRateLimiter = SteamRateLimiter() + def __init__(self) -> None: + # Instanciate the rate limiter on the class to share across instances + # Can't be done at class creation time, schema isn't available yet + if self.__class__.rate_limiter is None: + self.__class__.rate_limiter = SteamRateLimiter() def get_manifest_data(self, manifest_path) -> SteamManifestData: """Get local data for a game from its manifest""" From 216c3f5dae71d3688fab005d24230894d6d16f58 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 3 Jun 2023 17:50:00 +0200 Subject: [PATCH 093/173] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20deadlock=20in=20?= =?UTF-8?q?rate=20limiter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/rate_limiter.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/utils/rate_limiter.py b/src/utils/rate_limiter.py index ca118b4..a8a0e55 100644 --- a/src/utils/rate_limiter.py +++ b/src/utils/rate_limiter.py @@ -27,7 +27,7 @@ class TokenBucketRateLimiter(AbstractContextManager): @n_tokens.setter def n_tokens(self, value: int): with self.__n_tokens_lock: - self.n_tokens = value + self.__n_tokens = value def __init__( self, @@ -35,22 +35,24 @@ class TokenBucketRateLimiter(AbstractContextManager): max_tokens: Optional[int] = None, initial_tokens: Optional[int] = None, ) -> None: + """Initialize the limiter""" + # Initialize default values - self.queue_lock = Lock() if max_tokens is not None: self.MAX_TOKENS = max_tokens if refill_spacing_seconds is not None: self.REFILL_SPACING_SECONDS = refill_spacing_seconds - # Initialize the bucket + # Create locks + self.__n_tokens_lock = Lock() + self.queue_lock = Lock() + + # Initialize the number of tokens in the bucket self.bucket = BoundedSemaphore(self.MAX_TOKENS) missing = 0 if initial_tokens is None else self.MAX_TOKENS - initial_tokens missing = max(0, min(missing, self.MAX_TOKENS)) for _ in range(missing): self.bucket.acquire() - - # Initialize the counter - self.__n_tokens_lock = Lock() self.n_tokens = self.MAX_TOKENS - missing # Spawn daemon thread that refills the bucket From 6f77e0d30de58e2013a6a1a6161919662f7bc350 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 3 Jun 2023 18:49:47 +0200 Subject: [PATCH 094/173] =?UTF-8?q?=E2=9C=A8=20Added=20Steam=20API=20rate?= =?UTF-8?q?=20limiter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/managers/manager.py | 4 +++- src/utils/rate_limiter.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 6df6d0f..96a88fc 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -63,7 +63,9 @@ class Manager: # Handle unretryable errors log_args = (type(error).__name__, self.name, game.name, game.game_id) if type(error) not in self.retryable_on: - logging.error("Unretryable %s in %s for %s (%s)", *log_args) + logging.error( + "Unretryable %s in %s for %s (%s)", *log_args, exc_info=error + ) self.report_error(error) break # Handle being out of retries diff --git a/src/utils/rate_limiter.py b/src/utils/rate_limiter.py index a8a0e55..9332691 100644 --- a/src/utils/rate_limiter.py +++ b/src/utils/rate_limiter.py @@ -43,9 +43,10 @@ class TokenBucketRateLimiter(AbstractContextManager): if refill_spacing_seconds is not None: self.REFILL_SPACING_SECONDS = refill_spacing_seconds - # Create locks + # Create synchro data self.__n_tokens_lock = Lock() self.queue_lock = Lock() + self.queue = deque() # Initialize the number of tokens in the bucket self.bucket = BoundedSemaphore(self.MAX_TOKENS) From 729ca8244517e82c389785b72553ced475720df4 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 3 Jun 2023 20:55:03 +0200 Subject: [PATCH 095/173] =?UTF-8?q?=F0=9F=8E=A8=20Simplified=20sources?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data/hu.kramo.Cartridges.gschema.xml.in | 18 ------------ src/importer/sources/heroic_source.py | 26 +++++++---------- src/importer/sources/lutris_source.py | 19 ++++--------- src/importer/sources/source.py | 14 ++++----- src/importer/sources/steam_source.py | 38 +++++++++++++------------ 5 files changed, 43 insertions(+), 72 deletions(-) diff --git a/data/hu.kramo.Cartridges.gschema.xml.in b/data/hu.kramo.Cartridges.gschema.xml.in index 326bdf9..c56b82b 100644 --- a/data/hu.kramo.Cartridges.gschema.xml.in +++ b/data/hu.kramo.Cartridges.gschema.xml.in @@ -16,12 +16,6 @@ "~/.steam/" - - "~/.var/app/com.valvesoftware.Steam/data/Steam/" - - - "C:\Program Files (x86)\Steam" - true @@ -31,12 +25,6 @@ "~/.var/app/net.lutris.Lutris/cache/lutris" - - "~/.var/app/net.lutris.Lutris/data/lutris/" - - - "~/.var/app/net.lutris.Lutris/cache/lutris" - false @@ -46,12 +34,6 @@ "~/.config/heroic/" - - "~/.var/app/com.heroicgameslauncher.hgl/config/heroic/" - - - "" - true diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index b1bfb17..57d521a 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -8,7 +8,12 @@ from typing import Generator, Optional, TypedDict from src import shared from src.game import Game -from src.importer.sources.source import NTSource, PosixSource, Source, SourceIterator +from src.importer.sources.source import ( + LinuxSource, + Source, + SourceIterator, + WindowsSource, +) from src.utils.decorators import ( replaced_by_env_path, replaced_by_path, @@ -121,7 +126,6 @@ class HeroicSource(Source): """Generic heroic games launcher source""" name = "Heroic" - executable_format = "xdg-open heroic://launch/{app_name}" @property def game_id_format(self) -> str: @@ -132,28 +136,20 @@ class HeroicSource(Source): return HeroicSourceIterator(source=self) -class HeroicNativeSource(HeroicSource, PosixSource): - variant = "native" +class HeroicNativeSource(HeroicSource, LinuxSource): + variant = "linux" + executable_format = "xdg-open heroic://launch/{app_name}" @property @replaced_by_schema_key("heroic-location") + @replaced_by_path("~/.var/app/com.heroicgameslauncher.hgl/config/heroic/") @replaced_by_env_path("XDG_CONFIG_HOME", "heroic/") @replaced_by_path("~/.config/heroic/") def location(self) -> Path: raise FileNotFoundError() -class HeroicFlatpakSource(HeroicSource, PosixSource): - variant = "flatpak" - - @property - @replaced_by_schema_key("heroic-flatpak-location") - @replaced_by_path("~/.var/app/com.heroicgameslauncher.hgl/config/heroic/") - def location(self) -> Path: - raise FileNotFoundError() - - -class HeroicWindowsSource(HeroicSource, NTSource): +class HeroicWindowsSource(HeroicSource, WindowsSource): variant = "windows" executable_format = "start heroic://launch/{app_name}" diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index befadc4..bec4a8b 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -3,7 +3,7 @@ from time import time from src import shared from src.game import Game -from src.importer.sources.source import PosixSource, Source, SourceIterator +from src.importer.sources.source import LinuxSource, Source, SourceIterator from src.utils.decorators import replaced_by_path, replaced_by_schema_key from src.utils.save_cover import resize_cover, save_cover @@ -74,7 +74,6 @@ class LutrisSource(Source): """Generic lutris source""" name = "Lutris" - executable_format = "xdg-open lutris:rungameid/{game_id}" @property def game_id_format(self): @@ -84,21 +83,13 @@ class LutrisSource(Source): return LutrisSourceIterator(source=self) -class LutrisNativeSource(LutrisSource, PosixSource): - variant = "native" +class LutrisNativeSource(LutrisSource, LinuxSource): + variant = "linux" + executable_format = "xdg-open lutris:rungameid/{game_id}" @property @replaced_by_schema_key("lutris-location") + @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/") @replaced_by_path("~/.local/share/lutris/") def location(self): raise FileNotFoundError() - - -class LutrisFlatpakSource(LutrisSource, PosixSource): - variant = "flatpak" - - @property - @replaced_by_schema_key("lutris-flatpak-location") - @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/") - def location(self): - raise FileNotFoundError() diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index c65a845..44e2290 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -1,4 +1,4 @@ -import os +import sys from abc import abstractmethod from collections.abc import Iterable, Iterator from pathlib import Path @@ -66,7 +66,7 @@ class Source(Iterable): self.location except FileNotFoundError: return False - return os.name in self.available_on + return sys.platform in self.available_on @property @abstractmethod @@ -83,17 +83,17 @@ class Source(Iterable): """Get the source's iterator, to use in for loops""" -class NTSource(Source): +class WindowsSource(Source): """Mixin for sources available on Windows""" def __init__(self) -> None: super().__init__() - self.available_on.add("nt") + self.available_on.add("win32") -class PosixSource(Source): - """Mixin for sources available on POXIX-compliant systems""" +class LinuxSource(Source): + """Mixin for sources available on Linux""" def __init__(self) -> None: super().__init__() - self.available_on.add("posix") + self.available_on.add("linux") diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 312551a..1258bf9 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -5,7 +5,12 @@ from typing import Iterator from src import shared from src.game import Game -from src.importer.sources.source import NTSource, PosixSource, Source, SourceIterator +from src.importer.sources.source import ( + LinuxSource, + Source, + SourceIterator, + WindowsSource, +) from src.utils.decorators import ( replaced_by_env_path, replaced_by_path, @@ -20,10 +25,12 @@ class SteamSourceIterator(SourceIterator): manifests: set = None manifests_iterator: Iterator[Path] = None installed_state_mask: int = 4 + appid_cache: set = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.appid_cache = set() self.manifests = set() # Get dirs that contain steam app manifests @@ -64,8 +71,13 @@ class SteamSourceIterator(SourceIterator): if not int(local_data["stateflags"]) & self.installed_state_mask: return None - # Build game from local data + # Skip duplicate appids appid = local_data["appid"] + if appid in self.appid_cache: + return None + self.appid_cache.add(appid) + + # Build game from local data values = { "version": shared.SPEC_VERSION, "added": int(time()), @@ -91,17 +103,18 @@ class SteamSourceIterator(SourceIterator): class SteamSource(Source): name = "Steam" - executable_format = "xdg-open steam://rungameid/{game_id}" def __iter__(self): return SteamSourceIterator(source=self) -class SteamNativeSource(SteamSource, PosixSource): - variant = "native" +class SteamNativeSource(SteamSource, LinuxSource): + variant = "linux" + executable_format = "xdg-open steam://rungameid/{game_id}" @property @replaced_by_schema_key("steam-location") + @replaced_by_path("~/.var/app/com.valvesoftware.Steam/data/Steam/") @replaced_by_env_path("XDG_DATA_HOME", "Steam/") @replaced_by_path("~/.steam/") @replaced_by_path("~/.local/share/Steam/") @@ -109,23 +122,12 @@ class SteamNativeSource(SteamSource, PosixSource): raise FileNotFoundError() -class SteamFlatpakSource(SteamSource, PosixSource): - variant = "flatpak" - - @property - @replaced_by_schema_key("steam-flatpak-location") - @replaced_by_path("~/.var/app/com.valvesoftware.Steam/data/Steam/") - def location(self): - raise FileNotFoundError() - - -class SteamWindowsSource(SteamSource, NTSource): +class SteamWindowsSource(SteamSource, WindowsSource): variant = "windows" executable_format = "start steam://rungameid/{game_id}" @property - @replaced_by_schema_key("steam-windows-location") + @replaced_by_schema_key("steam-location") @replaced_by_env_path("programfiles(x86)", "Steam") - @replaced_by_path("C:\\Program Files (x86)\\Steam") def location(self): raise FileNotFoundError() From 7d8a7a894fd27caec0c027f4b9b707b68f67ddb0 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 3 Jun 2023 21:41:04 +0200 Subject: [PATCH 096/173] =?UTF-8?q?=F0=9F=90=9B=20fixed=20source=20imports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/heroic_source.py | 2 +- src/importer/sources/lutris_source.py | 2 +- src/importer/sources/steam_source.py | 2 +- src/main.py | 23 ++++++----------------- 4 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 57d521a..61400d6 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -136,7 +136,7 @@ class HeroicSource(Source): return HeroicSourceIterator(source=self) -class HeroicNativeSource(HeroicSource, LinuxSource): +class HeroicLinuxSource(HeroicSource, LinuxSource): variant = "linux" executable_format = "xdg-open heroic://launch/{app_name}" diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index bec4a8b..3d56db1 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -83,7 +83,7 @@ class LutrisSource(Source): return LutrisSourceIterator(source=self) -class LutrisNativeSource(LutrisSource, LinuxSource): +class LutrisLinuxSource(LutrisSource, LinuxSource): variant = "linux" executable_format = "xdg-open lutris:rungameid/{game_id}" diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 1258bf9..b77158c 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -108,7 +108,7 @@ class SteamSource(Source): return SteamSourceIterator(source=self) -class SteamNativeSource(SteamSource, LinuxSource): +class SteamLinuxSource(SteamSource, LinuxSource): variant = "linux" executable_format = "xdg-open steam://rungameid/{game_id}" diff --git a/src/main.py b/src/main.py index 8646622..01d32d9 100644 --- a/src/main.py +++ b/src/main.py @@ -34,17 +34,9 @@ from src import shared from src.details_window import DetailsWindow from src.game import Game from src.importer.importer import Importer -from src.importer.sources.heroic_source import ( - HeroicFlatpakSource, - HeroicNativeSource, - HeroicWindowsSource, -) -from src.importer.sources.lutris_source import LutrisFlatpakSource, LutrisNativeSource -from src.importer.sources.steam_source import ( - SteamFlatpakSource, - SteamNativeSource, - SteamWindowsSource, -) +from src.importer.sources.heroic_source import HeroicLinuxSource, HeroicWindowsSource +from src.importer.sources.lutris_source import LutrisLinuxSource +from src.importer.sources.steam_source import SteamLinuxSource, SteamWindowsSource from src.preferences import PreferencesWindow from src.store.managers.display_manager import DisplayManager from src.store.managers.file_manager import FileManager @@ -192,15 +184,12 @@ class CartridgesApplication(Adw.Application): def on_import_action(self, *_args): importer = Importer() if shared.schema.get_boolean("lutris"): - importer.add_source(LutrisNativeSource()) - importer.add_source(LutrisFlatpakSource()) + importer.add_source(LutrisLinuxSource()) if shared.schema.get_boolean("steam"): - importer.add_source(SteamNativeSource()) - importer.add_source(SteamFlatpakSource()) + importer.add_source(SteamLinuxSource()) importer.add_source(SteamWindowsSource()) if shared.schema.get_boolean("heroic"): - importer.add_source(HeroicNativeSource()) - importer.add_source(HeroicFlatpakSource()) + importer.add_source(HeroicLinuxSource()) importer.add_source(HeroicWindowsSource()) importer.run() From ebd22e27da01bd77e5e7ef9490a57a2f935ca590 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 4 Jun 2023 02:45:52 +0200 Subject: [PATCH 097/173] Various fixes - Improved pipeline performance - Improver importer progress - Steam API slow down to not get 429-ed (but still allow bursts on smaller steam libraries) --- src/importer/importer.py | 3 ++- src/store/pipeline.py | 13 ++++++++++++- src/utils/steam.py | 7 ++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 1cdf2cd..92e41f1 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -39,8 +39,9 @@ class Importer: @property def pipelines_progress(self): + progress = sum([pipeline.progress for pipeline in self.game_pipelines]) try: - progress = self.n_pipelines_done / len(self.game_pipelines) + progress = progress / len(self.game_pipelines) except ZeroDivisionError: progress = 1 return progress diff --git a/src/store/pipeline.py b/src/store/pipeline.py index 4372709..63fc7d3 100644 --- a/src/store/pipeline.py +++ b/src/store/pipeline.py @@ -30,7 +30,7 @@ class Pipeline(GObject.Object): @property def is_done(self) -> bool: - return len(self.not_done) == 0 + return len(self.waiting) == 0 and len(self.running) == 0 @property def blocked(self) -> set[Manager]: @@ -49,6 +49,17 @@ class Pipeline(GObject.Object): """Get the managers that can be run""" return self.waiting - self.blocked + @property + def progress(self) -> float: + """Get the pipeline progress. Should only be a rough idea.""" + n_done = len(self.done) + n_total = len(self.waiting) + len(self.running) + n_done + try: + progress = n_done / n_total + except ZeroDivisionError: + progress = 1 + return progress + def advance(self): """Spawn tasks for managers that are able to run for a game""" diff --git a/src/utils/steam.py b/src/utils/steam.py index 58d8ee1..5389382 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -48,8 +48,8 @@ class SteamRateLimiter(TokenBucketRateLimiter): # 200 requests per 5 min seems to be the limit # https://stackoverflow.com/questions/76047820/how-am-i-exceeding-steam-apis-rate-limit # https://stackoverflow.com/questions/51795457/avoiding-error-429-too-many-requests-steam-web-api - REFILL_SPACING_SECONDS = 1.5 - MAX_TOKENS = 200 + REFILL_SPACING_SECONDS = 5 * 60 / 100 + MAX_TOKENS = 100 def __init__(self) -> None: # Load initial tokens from schema @@ -98,8 +98,9 @@ class SteamHelper: def get_api_data(self, appid) -> SteamAPIData: """ Get online data for a game from its appid. - May block to satisfy the Steam web API limitations. + + See https://wiki.teamfortress.com/wiki/User:RJackson/StorefrontAPI#appdetails """ # Get data from the API (way block to satisfy its limits) From 7cf4d8199c732dd3bb0f9611a53463ccdac2d9f0 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 4 Jun 2023 17:03:59 +0200 Subject: [PATCH 098/173] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Improved=20rate=20?= =?UTF-8?q?limiting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Ensures that the target rate isn't overshot - Aware of the requests sliding window - Allows for a defined burst size - Remembers the request timestamps between app restarts --- data/hu.kramo.Cartridges.gschema.xml.in | 7 +- src/utils/rate_limiter.py | 125 +++++++++++++++++++----- src/utils/steam.py | 40 ++++---- 3 files changed, 127 insertions(+), 45 deletions(-) diff --git a/data/hu.kramo.Cartridges.gschema.xml.in b/data/hu.kramo.Cartridges.gschema.xml.in index c56b82b..9de7f7b 100644 --- a/data/hu.kramo.Cartridges.gschema.xml.in +++ b/data/hu.kramo.Cartridges.gschema.xml.in @@ -88,11 +88,8 @@ "a-z" - - 200 - - - 0 + + "[]" \ No newline at end of file diff --git a/src/utils/rate_limiter.py b/src/utils/rate_limiter.py index 9332691..1a537a8 100644 --- a/src/utils/rate_limiter.py +++ b/src/utils/rate_limiter.py @@ -1,16 +1,74 @@ -from typing import Optional +from typing import Optional, Sized from threading import Lock, Thread, BoundedSemaphore -from time import sleep +from time import sleep, time from collections import deque from contextlib import AbstractContextManager -class TokenBucketRateLimiter(AbstractContextManager): +class PickHistory(Sized): + """Utility class used for rate limiters, counting how many picks + happened in a given period""" + + PERIOD: int + + timestamps: list[int] = None + timestamps_lock: Lock = None + + def __init__(self, period: int) -> None: + self.PERIOD = period + self.timestamps = [] + self.timestamps_lock = Lock() + + def remove_old_entries(self): + """Remove history entries older than the period""" + now = time() + cutoff = now - self.PERIOD + with self.timestamps_lock: + self.timestamps = [entry for entry in self.timestamps if entry > cutoff] + + def add(self, *new_timestamps: Optional[int]): + """Add timestamps to the history. + If none given, will add the current timestamp""" + if len(new_timestamps) == 0: + new_timestamps = (time(),) + with self.timestamps_lock: + self.timestamps.extend(new_timestamps) + + def __len__(self) -> int: + """How many entries were logged in the period""" + self.remove_old_entries() + with self.timestamps_lock: + return len(self.timestamps) + + @property + def start(self) -> int: + """Get the time at which the history started""" + self.remove_old_entries() + with self.timestamps_lock: + try: + entry = self.timestamps[0] + except KeyError: + entry = time() + return entry + + def copy_timestamps(self) -> str: + """Get a copy of the timestamps history""" + self.remove_old_entries() + with self.timestamps_lock: + return self.timestamps.copy() + + +class RateLimiter(AbstractContextManager): """Rate limiter implementing the token bucket algorithm""" - REFILL_SPACING_SECONDS: int - MAX_TOKENS: int + # Period in which we have a max amount of tokens + REFILL_PERIOD_SECONDS: int + # Number of tokens allowed in this period + REFILL_PERIOD_TOKENS: int + # Max number of tokens that can be consumed instantly + BURST_TOKENS: int + pick_history: PickHistory = None bucket: BoundedSemaphore = None queue: deque[Lock] = None queue_lock: Lock = None @@ -31,38 +89,59 @@ class TokenBucketRateLimiter(AbstractContextManager): def __init__( self, - refill_spacing_seconds: Optional[int] = None, - max_tokens: Optional[int] = None, - initial_tokens: Optional[int] = None, + refill_period_seconds: Optional[int] = None, + refill_period_tokens: Optional[int] = None, + burst_tokens: Optional[int] = None, ) -> None: """Initialize the limiter""" # Initialize default values - if max_tokens is not None: - self.MAX_TOKENS = max_tokens - if refill_spacing_seconds is not None: - self.REFILL_SPACING_SECONDS = refill_spacing_seconds + if refill_period_seconds is not None: + self.REFILL_PERIOD_SECONDS = refill_period_seconds + if refill_period_tokens is not None: + self.REFILL_PERIOD_TOKENS = refill_period_tokens + if burst_tokens is not None: + self.BURST_TOKENS = burst_tokens + if self.pick_history is None: + self.pick_history = PickHistory(self.REFILL_PERIOD_SECONDS) - # Create synchro data + # Create synchronization data self.__n_tokens_lock = Lock() self.queue_lock = Lock() self.queue = deque() - # Initialize the number of tokens in the bucket - self.bucket = BoundedSemaphore(self.MAX_TOKENS) - missing = 0 if initial_tokens is None else self.MAX_TOKENS - initial_tokens - missing = max(0, min(missing, self.MAX_TOKENS)) - for _ in range(missing): - self.bucket.acquire() - self.n_tokens = self.MAX_TOKENS - missing + # Initialize the token bucket + self.bucket = BoundedSemaphore(self.BURST_TOKENS) + self.n_tokens = self.BURST_TOKENS # Spawn daemon thread that refills the bucket refill_thread = Thread(target=self.refill_thread_func, daemon=True) refill_thread.start() + @property + def refill_spacing(self) -> float: + """ + Get the current refill spacing. + + Ensures that even with a burst in the period, the limit will not be exceeded. + """ + + # Compute ideal spacing + tokens_left = self.REFILL_PERIOD_TOKENS - len(self.pick_history) + seconds_left = self.pick_history.start + self.REFILL_PERIOD_SECONDS - time() + try: + spacing_seconds = seconds_left / tokens_left + except ZeroDivisionError: + # There were no remaining tokens, gotta wait until end of the period + spacing_seconds = seconds_left + + # Prevent spacing dropping down lower than the natural spacing + natural_spacing = self.REFILL_PERIOD_SECONDS / self.REFILL_PERIOD_TOKENS + return max(natural_spacing, spacing_seconds) + def refill(self): - """Method used by the refill thread""" - sleep(self.REFILL_SPACING_SECONDS) + """Add a token back in the bucket""" + sleep(self.refill_spacing) try: self.bucket.release() except ValueError: @@ -70,7 +149,6 @@ class TokenBucketRateLimiter(AbstractContextManager): pass else: self.n_tokens += 1 - self.update_queue() def refill_thread_func(self): """Entry point for the daemon thread that is refilling the bucket""" @@ -105,6 +183,7 @@ class TokenBucketRateLimiter(AbstractContextManager): lock = self.add_to_queue() self.update_queue() lock.acquire() + self.pick_history.add() # --- Support for use in with statements diff --git a/src/utils/steam.py b/src/utils/steam.py index 5389382..8008f77 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -1,14 +1,13 @@ +import json import logging import re -from time import time from typing import TypedDict -from math import floor, ceil import requests from requests import HTTPError from src import shared -from src.utils.rate_limiter import TokenBucketRateLimiter +from src.utils.rate_limiter import PickHistory, RateLimiter class SteamError(Exception): @@ -41,30 +40,37 @@ class SteamAPIData(TypedDict): developers: str -class SteamRateLimiter(TokenBucketRateLimiter): +class SteamRateLimiter(RateLimiter): """Rate limiter for the Steam web API""" # Steam web API limit # 200 requests per 5 min seems to be the limit # https://stackoverflow.com/questions/76047820/how-am-i-exceeding-steam-apis-rate-limit # https://stackoverflow.com/questions/51795457/avoiding-error-429-too-many-requests-steam-web-api - REFILL_SPACING_SECONDS = 5 * 60 / 100 - MAX_TOKENS = 100 + REFILL_PERIOD_SECONDS = 5 * 60 + REFILL_PERIOD_TOKENS = 200 + BURST_TOKENS = 100 def __init__(self) -> None: - # Load initial tokens from schema + # Load pick history from schema # (Remember API limits through restarts of Cartridges) - last_tokens = shared.state_schema.get_int("steam-api-tokens") - last_time = shared.state_schema.get_int("steam-api-tokens-timestamp") - produced = floor((time() - last_time) / self.REFILL_SPACING_SECONDS) - inital_tokens = last_tokens + produced - super().__init__(initial_tokens=inital_tokens) + timestamps_str = shared.state_schema.get_string("steam-limiter-tokens-history") + self.pick_history = PickHistory(self.REFILL_PERIOD_SECONDS) + self.pick_history.add(*json.loads(timestamps_str)) + self.pick_history.remove_old_entries() + super().__init__() - def refill(self): - """Refill the bucket and store its number of tokens in the schema""" - super().refill() - shared.state_schema.set_int("steam-api-tokens-timestamp", ceil(time())) - shared.state_schema.set_int("steam-api-tokens", self.n_tokens) + @property + def refill_spacing(self) -> float: + spacing = super().refill_spacing + logging.debug("Next Steam API request token in %f seconds", spacing) + return spacing + + def acquire(self): + """Get a token from the bucket and store the pick history in the schema""" + super().acquire() + timestamps_str = json.dumps(self.pick_history.copy_timestamps()) + shared.state_schema.set_string("steam-limiter-tokens-history", timestamps_str) class SteamHelper: From 1e4004329c9740d4aff4a8cb02eea67bc9167afb Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 4 Jun 2023 22:51:54 +0200 Subject: [PATCH 099/173] =?UTF-8?q?=E2=9C=A8=20New=20Heroic=20source=20-?= =?UTF-8?q?=20Fixed=20wrong=20`installed`=20key,=20shoud=20be=20`is=5Finst?= =?UTF-8?q?alled`=20-=20Log=20warnings=20on=20invalid=20games=20found=20in?= =?UTF-8?q?=20library?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/heroic_source.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 61400d6..3ce0973 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -58,7 +58,7 @@ class HeroicSourceIterator(SourceIterator): """Helper method used to build a Game from a Heroic library entry""" # Skip games that are not installed - if not entry["installed"]: + if not entry["is_installed"]: return None # Build game @@ -96,17 +96,19 @@ class HeroicSourceIterator(SourceIterator): if not shared.schema.get_boolean("heroic-import-" + sub_source["service"]): continue # Load games from JSON + file = self.source.location.joinpath(*sub_source["path"]) try: - file = self.source.location.joinpath(*sub_source["path"]) library = json.load(file.open())["library"] except (JSONDecodeError, OSError, KeyError): # Invalid library.json file, skip it + logging.warning("Couldn't open Heroic file: %s", str(file)) continue for entry in library: try: game = self.game_from_library_entry(entry) except KeyError: # Skip invalid games + logging.warning("Invalid Heroic game skipped in %s", str(file)) continue yield game @@ -154,7 +156,7 @@ class HeroicWindowsSource(HeroicSource, WindowsSource): executable_format = "start heroic://launch/{app_name}" @property - @replaced_by_schema_key("heroic-windows-location") + @replaced_by_schema_key("heroic-location") @replaced_by_env_path("appdata", "heroic/") def location(self) -> Path: raise FileNotFoundError() From ff0ba00733876456f6ac912d03997492b505215a Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 5 Jun 2023 00:17:41 +0200 Subject: [PATCH 100/173] =?UTF-8?q?=F0=9F=8E=A8=20=20Added=20delay=20befor?= =?UTF-8?q?e=20manager=20retry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/managers/manager.py | 56 ++++++++++++++----------- src/store/managers/steam_api_manager.py | 4 +- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 96a88fc..9675aeb 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -1,7 +1,8 @@ import logging from abc import abstractmethod -from typing import Any, Callable from threading import Lock +from time import sleep +from typing import Any, Callable from src.game import Game @@ -17,7 +18,10 @@ class Manager: run_after: set[type["Manager"]] = set() blocking: bool = True + retryable_on: set[type[Exception]] = set() + continue_on: set[type[Exception]] = set() + retry_delay: int = 3 max_tries: int = 3 errors: list[Exception] @@ -54,31 +58,35 @@ class Manager: * May raise other exceptions that will be reported """ - def execute_resilient_manager_logic(self, game: Game) -> None: + def execute_resilient_manager_logic(self, game: Game, try_index: int = 0) -> None: """Execute the manager logic and handle its errors by reporting them or retrying""" - for remaining_tries in range(self.max_tries, -1, -1): - try: - self.manager_logic(game) - except Exception as error: - # Handle unretryable errors - log_args = (type(error).__name__, self.name, game.name, game.game_id) - if type(error) not in self.retryable_on: - logging.error( - "Unretryable %s in %s for %s (%s)", *log_args, exc_info=error - ) - self.report_error(error) - break - # Handle being out of retries - elif remaining_tries == 0: - logging.error( - "Too many retries due to %s in %s for %s (%s)", *log_args - ) - self.report_error(error) - break - # Retry + try: + self.manager_logic(game) + except Exception as error: + if error in self.continue_on: + # Handle skippable errors (skip silently) + return + elif error in self.retryable_on: + if try_index < self.max_tries: + # Handle retryable errors + logging_format = "Retrying %s in %s for %s" + sleep(self.retry_delay) + self.execute_resilient_manager_logic(game, try_index + 1) else: - logging.debug("Retry caused by %s in %s for %s (%s)", *log_args) - continue + # Handle being out of retries + logging_format = "Out of retries dues to %s in %s for %s" + self.report_error(error) + else: + # Handle unretryable errors + logging_format = "Unretryable %s in %s for %s" + self.report_error(error) + # Finally log errors + logging.error( + logging_format, + type(error).__name__, + self.name, + f"{game.name} ({game.game_id})", + ) def process_game(self, game: Game, callback: Callable[["Manager"], Any]) -> None: """Pass the game through the manager""" diff --git a/src/store/managers/steam_api_manager.py b/src/store/managers/steam_api_manager.py index 3b319c0..5874735 100644 --- a/src/store/managers/steam_api_manager.py +++ b/src/store/managers/steam_api_manager.py @@ -1,3 +1,5 @@ +from urllib3.exceptions import SSLError + from src.game import Game from src.store.managers.async_manager import AsyncManager from src.utils.steam import ( @@ -11,7 +13,7 @@ from src.utils.steam import ( class SteamAPIManager(AsyncManager): """Manager in charge of completing a game's data from the Steam API""" - retryable_on = set((HTTPError,)) + retryable_on = set((HTTPError, SSLError)) def manager_logic(self, game: Game) -> None: # Skip non-steam games From 67a5a364f708ef6a161ea94f098f4d8548c4158e Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 5 Jun 2023 00:19:22 +0200 Subject: [PATCH 101/173] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20pick=20history?= =?UTF-8?q?=20start=20error=20when=20empty?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils/rate_limiter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/rate_limiter.py b/src/utils/rate_limiter.py index 1a537a8..2a80fc8 100644 --- a/src/utils/rate_limiter.py +++ b/src/utils/rate_limiter.py @@ -47,7 +47,7 @@ class PickHistory(Sized): with self.timestamps_lock: try: entry = self.timestamps[0] - except KeyError: + except IndexError: entry = time() return entry From cf9d4059b3749cc5012c37765565da5afb93c437 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 5 Jun 2023 00:34:47 +0200 Subject: [PATCH 102/173] =?UTF-8?q?=F0=9F=93=9D=20Consistent=20typing=20in?= =?UTF-8?q?=20exsiting=20sources?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/heroic_source.py | 4 ++-- src/importer/sources/lutris_source.py | 7 +++---- src/importer/sources/steam_source.py | 8 +++----- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 3ce0973..73d6d22 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -112,8 +112,8 @@ class HeroicSourceIterator(SourceIterator): continue yield game - def __init__(self, source: "HeroicSource") -> None: - self.source = source + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) self.generator = self.sub_sources_generator() def __next__(self) -> Optional[Game]: diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 3d56db1..6ec7cf2 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,5 +1,6 @@ from sqlite3 import connect from time import time +from typing import Optional from src import shared from src.game import Game @@ -27,7 +28,7 @@ class LutrisSourceIterator(SourceIterator): """ db_request_params = None - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.import_steam = shared.schema.get_boolean("lutris-import-steam") self.db_location = self.source.location / "pga.db" @@ -37,9 +38,7 @@ class LutrisSourceIterator(SourceIterator): self.db_games_request, self.db_request_params ) - def __next__(self): - """Produce games""" - + def __next__(self) -> Optional[Game]: row = None try: row = self.db_cursor.__next__() diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index b77158c..be062cf 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -1,7 +1,7 @@ import re from pathlib import Path from time import time -from typing import Iterator +from typing import Iterator, Optional from src import shared from src.game import Game @@ -27,7 +27,7 @@ class SteamSourceIterator(SourceIterator): installed_state_mask: int = 4 appid_cache: set = None - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.appid_cache = set() @@ -56,9 +56,7 @@ class SteamSourceIterator(SourceIterator): self.manifests_iterator = iter(self.manifests) - def __next__(self): - """Produce games""" - + def __next__(self) -> Optional[Game]: # Get metadata from manifest manifest_path = next(self.manifests_iterator) steam = SteamHelper() From 43d4a50bf7c87a6ef39629addaa8990be6a63cc7 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 5 Jun 2023 01:57:25 +0200 Subject: [PATCH 103/173] Added logging info to game launch --- src/game.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/game.py b/src/game.py index 70dac82..20504f9 100644 --- a/src/game.py +++ b/src/game.py @@ -18,6 +18,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import json +import logging import os import shlex import subprocess @@ -185,6 +186,7 @@ class Game(Gtk.Box): else string # Others ) + logging.info("Starting %s: %s", self.name, str(args)) subprocess.Popen( args, cwd=Path.home(), From e91aeddd3bde1253324809019e05b8fe6fdbc104 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 5 Jun 2023 01:57:38 +0200 Subject: [PATCH 104/173] =?UTF-8?q?=E2=9C=A8=20Added=20Bottles=20source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/bottles_source.py | 85 ++++++++++++++++++++++++++ src/main.py | 3 + 2 files changed, 88 insertions(+) create mode 100644 src/importer/sources/bottles_source.py diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py new file mode 100644 index 0000000..34d93cd --- /dev/null +++ b/src/importer/sources/bottles_source.py @@ -0,0 +1,85 @@ +from pathlib import Path +from time import time +from typing import Generator, Optional + +import yaml + +from src import shared +from src.game import Game +from src.importer.sources.source import LinuxSource, Source, SourceIterator +from src.utils.decorators import ( + replaced_by_env_path, + replaced_by_path, + replaced_by_schema_key, +) +from src.utils.save_cover import resize_cover, save_cover + + +class BottlesSourceIterator(SourceIterator): + source: "BottlesSource" + generator: Generator = None + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.generator = self.generator_builder() + + def generator_builder(self) -> Optional[Game]: + """Generator method producing games""" + + data = (self.source.location / "library.yml").read_text("utf-8") + library: dict = yaml.safe_load(data) + + for entry in library.values(): + # Build game + values = { + "version": shared.SPEC_VERSION, + "source": self.source.id, + "added": int(time()), + "name": entry["name"], + "game_id": self.source.game_id_format.format(game_id=entry["id"]), + "executable": self.source.executable_format.format( + bottle_name=entry["bottle"]["name"], game_name=entry["name"] + ), + } + game = Game(values, allow_side_effects=False) + + # Save official cover + bottle_path = entry["bottle"]["path"] + image_name = entry["thumbnail"].split(":")[1] + image_path = ( + self.source.location / "bottles" / bottle_path / "grids" / image_name + ) + if image_path.is_file(): + save_cover(values["game_id"], resize_cover(image_path)) + + # Produce game + yield game + + def __next__(self) -> Optional[Game]: + try: + game = next(self.generator) + except StopIteration: + raise + return game + + +class BottlesSource(Source): + """Generic Bottles source""" + + name = "Bottles" + + def __iter__(self) -> SourceIterator: + return BottlesSourceIterator(self) + + +class BottlesLinuxSource(BottlesSource, LinuxSource): + variant = "linux" + executable_format = 'xdg-open bottles:run/"{bottle_name}"/"{game_name}"' + + @property + @replaced_by_schema_key("bottles-location") + @replaced_by_path("~/.var/app/com.usebottles.bottles/data/bottles/") + @replaced_by_env_path("XDG_DATA_HOME", "bottles/") + @replaced_by_path("~/.local/share/bottles/") + def location(self) -> Path: + raise FileNotFoundError() diff --git a/src/main.py b/src/main.py index 01d32d9..d379d10 100644 --- a/src/main.py +++ b/src/main.py @@ -34,6 +34,7 @@ from src import shared from src.details_window import DetailsWindow from src.game import Game from src.importer.importer import Importer +from src.importer.sources.bottles_source import BottlesLinuxSource from src.importer.sources.heroic_source import HeroicLinuxSource, HeroicWindowsSource from src.importer.sources.lutris_source import LutrisLinuxSource from src.importer.sources.steam_source import SteamLinuxSource, SteamWindowsSource @@ -191,6 +192,8 @@ class CartridgesApplication(Adw.Application): if shared.schema.get_boolean("heroic"): importer.add_source(HeroicLinuxSource()) importer.add_source(HeroicWindowsSource()) + if shared.schema.get_boolean("bottles"): + importer.add_source(BottlesLinuxSource()) importer.run() def on_remove_game_action(self, *_args): From 1dcfe38253e71e322f0057f2f7ca136c36833cb2 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 5 Jun 2023 12:40:41 +0200 Subject: [PATCH 105/173] =?UTF-8?q?=F0=9F=8E=A8=20Simplified=20SourceItera?= =?UTF-8?q?tor-s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Using generator functions - Common generator init and next in base class - Explicited that error handling should happen in generator --- src/importer/sources/bottles_source.py | 12 --- src/importer/sources/heroic_source.py | 18 +---- src/importer/sources/lutris_source.py | 92 ++++++++++------------ src/importer/sources/source.py | 20 +++-- src/importer/sources/steam_source.py | 105 ++++++++++++------------- 5 files changed, 109 insertions(+), 138 deletions(-) diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index 34d93cd..8e84869 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -17,11 +17,6 @@ from src.utils.save_cover import resize_cover, save_cover class BottlesSourceIterator(SourceIterator): source: "BottlesSource" - generator: Generator = None - - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.generator = self.generator_builder() def generator_builder(self) -> Optional[Game]: """Generator method producing games""" @@ -55,13 +50,6 @@ class BottlesSourceIterator(SourceIterator): # Produce game yield game - def __next__(self) -> Optional[Game]: - try: - game = next(self.generator) - except StopIteration: - raise - return game - class BottlesSource(Source): """Generic Bottles source""" diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 73d6d22..1e2259d 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -38,7 +38,7 @@ class HeroicSubSource(TypedDict): class HeroicSourceIterator(SourceIterator): source: "HeroicSource" - generator: Generator = None + sub_sources: dict[str, HeroicSubSource] = { "sideload": { "service": "sideload", @@ -89,9 +89,10 @@ class HeroicSourceIterator(SourceIterator): return Game(values, allow_side_effects=False) - def sub_sources_generator(self): + def generator_builder(self): """Generator method producing games from all the Heroic sub-sources""" - for _key, sub_source in self.sub_sources.items(): + + for sub_source in self.sub_sources.values(): # Skip disabled sub-sources if not shared.schema.get_boolean("heroic-import-" + sub_source["service"]): continue @@ -112,17 +113,6 @@ class HeroicSourceIterator(SourceIterator): continue yield game - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.generator = self.sub_sources_generator() - - def __next__(self) -> Optional[Game]: - try: - game = next(self.generator) - except StopIteration: - raise - return game - class HeroicSource(Source): """Generic heroic games launcher source""" diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 6ec7cf2..7bb662b 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,6 +1,6 @@ from sqlite3 import connect from time import time -from typing import Optional +from typing import Optional, Generator from src import shared from src.game import Game @@ -11,62 +11,50 @@ from src.utils.save_cover import resize_cover, save_cover class LutrisSourceIterator(SourceIterator): source: "LutrisSource" - import_steam = False - db_connection = None - db_cursor = None - db_location = None - db_games_request = """ - SELECT id, name, slug, runner, hidden - FROM 'games' - WHERE - name IS NOT NULL - AND slug IS NOT NULL - AND configPath IS NOT NULL - AND installed - AND (runner IS NOT "steam" OR :import_steam) - ; - """ - db_request_params = None - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.import_steam = shared.schema.get_boolean("lutris-import-steam") - self.db_location = self.source.location / "pga.db" - self.db_connection = connect(self.db_location) - self.db_request_params = {"import_steam": self.import_steam} - self.db_cursor = self.db_connection.execute( - self.db_games_request, self.db_request_params - ) + def generator_builder(self) -> Optional[Game]: + """Generator method producing games""" - def __next__(self) -> Optional[Game]: - row = None - try: - row = self.db_cursor.__next__() - except StopIteration as error: - self.db_connection.close() - raise error + # Query the database + request = """ + SELECT id, name, slug, runner, hidden + FROM 'games' + WHERE + name IS NOT NULL + AND slug IS NOT NULL + AND configPath IS NOT NULL + AND installed + AND (runner IS NOT "steam" OR :import_steam) + ; + """ + params = {"import_steam": shared.schema.get_boolean("lutris-import-steam")} + connection = connect(self.source.location / "pga.db") + cursor = connection.execute(request, params) - # Create game - values = { - "version": shared.SPEC_VERSION, - "added": int(time()), - "hidden": row[4], - "name": row[1], - "source": f"{self.source.id}_{row[3]}", - "game_id": self.source.game_id_format.format( - game_id=row[2], game_internal_id=row[0] - ), - "executable": self.source.executable_format.format(game_id=row[2]), - "developer": None, # TODO get developer metadata on Lutris - } - game = Game(values, allow_side_effects=False) + # Create games from the DB results + for row in cursor: + # Create game + values = { + "version": shared.SPEC_VERSION, + "added": int(time()), + "hidden": row[4], + "name": row[1], + "source": f"{self.source.id}_{row[3]}", + "game_id": self.source.game_id_format.format( + game_id=row[2], game_internal_id=row[0] + ), + "executable": self.source.executable_format.format(game_id=row[2]), + "developer": None, # TODO get developer metadata on Lutris + } + game = Game(values, allow_side_effects=False) - # Save official image - image_path = self.source.location / "covers" / "coverart" / f"{row[2]}.jpg" - if image_path.exists(): - save_cover(values["game_id"], resize_cover(image_path)) + # Save official image + image_path = self.source.location / "covers" / "coverart" / f"{row[2]}.jpg" + if image_path.exists(): + save_cover(values["game_id"], resize_cover(image_path)) - return game + # Produce game + yield game class LutrisSource(Source): diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index 44e2290..f0da552 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -2,7 +2,7 @@ import sys from abc import abstractmethod from collections.abc import Iterable, Iterator from pathlib import Path -from typing import Optional +from typing import Optional, Generator from src.game import Game @@ -11,20 +11,28 @@ class SourceIterator(Iterator): """Data producer for a source of games""" source: "Source" = None + generator: Generator = None def __init__(self, source: "Source") -> None: super().__init__() self.source = source + self.generator = self.generator_builder() def __iter__(self) -> "SourceIterator": return self - @abstractmethod def __next__(self) -> Optional[Game]: - """Get the next generated game from the source. - Raises StopIteration when exhausted. - May raise any other exception signifying an error on this specific game. - May return None when a game has been skipped without an error.""" + return next(self.generator) + + @abstractmethod + def generator_builder(self) -> Generator[Optional[Game], None, None]: + """ + Method that returns a generator that produces games + * Should be implemented as a generator method + * May yield `None` when an iteration hasn't produced a game + * In charge of handling per-game errors + * Returns when exhausted + """ class Source(Iterable): diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index be062cf..5e59252 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -1,7 +1,7 @@ import re from pathlib import Path from time import time -from typing import Iterator, Optional +from typing import Iterable, Optional, Generator from src import shared from src.game import Game @@ -22,81 +22,78 @@ from src.utils.steam import SteamHelper, SteamInvalidManifestError class SteamSourceIterator(SourceIterator): source: "SteamSource" - manifests: set = None - manifests_iterator: Iterator[Path] = None - installed_state_mask: int = 4 - appid_cache: set = None - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - - self.appid_cache = set() - self.manifests = set() - - # Get dirs that contain steam app manifests + def get_manifest_dirs(self) -> Iterable[Path]: + """Get dirs that contain steam app manifests""" libraryfolders_path = self.source.location / "steamapps" / "libraryfolders.vdf" with open(libraryfolders_path, "r") as file: contents = file.read() - steamapps_dirs = [ + return [ Path(path) / "steamapps" for path in re.findall('"path"\s+"(.*)"\n', contents, re.IGNORECASE) ] - # Get app manifests - for steamapps_dir in steamapps_dirs: + def get_manifests(self) -> Iterable[Path]: + """Get app manifests""" + manifests = set() + for steamapps_dir in self.get_manifest_dirs(): if not steamapps_dir.is_dir(): continue - self.manifests.update( + manifests.update( [ manifest for manifest in steamapps_dir.glob("appmanifest_*.acf") if manifest.is_file() ] ) + return manifests - self.manifests_iterator = iter(self.manifests) + def generator_builder(self) -> Optional[Game]: + """Generator method producing games""" + appid_cache = set() + manifests = self.get_manifests() + for manifest in manifests: + # Get metadata from manifest + steam = SteamHelper() + try: + local_data = steam.get_manifest_data(manifest) + except (OSError, SteamInvalidManifestError): + continue - def __next__(self) -> Optional[Game]: - # Get metadata from manifest - manifest_path = next(self.manifests_iterator) - steam = SteamHelper() - try: - local_data = steam.get_manifest_data(manifest_path) - except (OSError, SteamInvalidManifestError): - return None + # Skip non installed games + INSTALLED_MASK: int = 4 + if not int(local_data["stateflags"]) & INSTALLED_MASK: + continue - # Skip non installed games - if not int(local_data["stateflags"]) & self.installed_state_mask: - return None + # Skip duplicate appids + appid = local_data["appid"] + if appid in appid_cache: + continue + appid_cache.add(appid) - # Skip duplicate appids - appid = local_data["appid"] - if appid in self.appid_cache: - return None - self.appid_cache.add(appid) + # Build game from local data + values = { + "version": shared.SPEC_VERSION, + "added": int(time()), + "name": local_data["name"], + "source": self.source.id, + "game_id": self.source.game_id_format.format(game_id=appid), + "executable": self.source.executable_format.format(game_id=appid), + } + game = Game(values, allow_side_effects=False) - # Build game from local data - values = { - "version": shared.SPEC_VERSION, - "added": int(time()), - "name": local_data["name"], - "source": self.source.id, - "game_id": self.source.game_id_format.format(game_id=appid), - "executable": self.source.executable_format.format(game_id=appid), - } - game = Game(values, allow_side_effects=False) + # Add official cover image + image_path = ( + self.source.location + / "appcache" + / "librarycache" + / f"{appid}_library_600x900.jpg" + ) + if image_path.is_file(): + save_cover(game.game_id, resize_cover(image_path)) - # Add official cover image - cover_path = ( - self.source.location - / "appcache" - / "librarycache" - / f"{appid}_library_600x900.jpg" - ) - if cover_path.is_file(): - save_cover(game.game_id, resize_cover(cover_path)) - - return game + # Produce game + yield game class SteamSource(Source): From 1e3e6484e40f8c56723269297fdd78e197730678 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 5 Jun 2023 13:11:05 +0200 Subject: [PATCH 106/173] =?UTF-8?q?=F0=9F=8E=A8=20Simplified=20source=20lo?= =?UTF-8?q?cation=20user=20override?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/bottles_source.py | 11 +++----- src/importer/sources/heroic_source.py | 13 ++++------ src/importer/sources/lutris_source.py | 7 +++--- src/importer/sources/source.py | 29 ++++++++++++++++++++- src/importer/sources/steam_source.py | 13 ++++------ src/utils/decorators.py | 35 -------------------------- 6 files changed, 46 insertions(+), 62 deletions(-) diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index 8e84869..b2c0a2d 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -1,17 +1,13 @@ from pathlib import Path from time import time -from typing import Generator, Optional +from typing import Optional import yaml from src import shared from src.game import Game from src.importer.sources.source import LinuxSource, Source, SourceIterator -from src.utils.decorators import ( - replaced_by_env_path, - replaced_by_path, - replaced_by_schema_key, -) +from src.utils.decorators import replaced_by_env_path, replaced_by_path from src.utils.save_cover import resize_cover, save_cover @@ -55,6 +51,7 @@ class BottlesSource(Source): """Generic Bottles source""" name = "Bottles" + location_key = "bottles-location" def __iter__(self) -> SourceIterator: return BottlesSourceIterator(self) @@ -65,7 +62,7 @@ class BottlesLinuxSource(BottlesSource, LinuxSource): executable_format = 'xdg-open bottles:run/"{bottle_name}"/"{game_name}"' @property - @replaced_by_schema_key("bottles-location") + @Source.replaced_by_schema_key() @replaced_by_path("~/.var/app/com.usebottles.bottles/data/bottles/") @replaced_by_env_path("XDG_DATA_HOME", "bottles/") @replaced_by_path("~/.local/share/bottles/") diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 1e2259d..998bf65 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -4,7 +4,7 @@ from hashlib import sha256 from json import JSONDecodeError from pathlib import Path from time import time -from typing import Generator, Optional, TypedDict +from typing import Optional, TypedDict from src import shared from src.game import Game @@ -14,11 +14,7 @@ from src.importer.sources.source import ( SourceIterator, WindowsSource, ) -from src.utils.decorators import ( - replaced_by_env_path, - replaced_by_path, - replaced_by_schema_key, -) +from src.utils.decorators import replaced_by_env_path, replaced_by_path from src.utils.save_cover import resize_cover, save_cover @@ -118,6 +114,7 @@ class HeroicSource(Source): """Generic heroic games launcher source""" name = "Heroic" + location_key = "heroic-location" @property def game_id_format(self) -> str: @@ -133,7 +130,7 @@ class HeroicLinuxSource(HeroicSource, LinuxSource): executable_format = "xdg-open heroic://launch/{app_name}" @property - @replaced_by_schema_key("heroic-location") + @Source.replaced_by_schema_key() @replaced_by_path("~/.var/app/com.heroicgameslauncher.hgl/config/heroic/") @replaced_by_env_path("XDG_CONFIG_HOME", "heroic/") @replaced_by_path("~/.config/heroic/") @@ -146,7 +143,7 @@ class HeroicWindowsSource(HeroicSource, WindowsSource): executable_format = "start heroic://launch/{app_name}" @property - @replaced_by_schema_key("heroic-location") + @Source.replaced_by_schema_key() @replaced_by_env_path("appdata", "heroic/") def location(self) -> Path: raise FileNotFoundError() diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 7bb662b..1633445 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,11 +1,11 @@ from sqlite3 import connect from time import time -from typing import Optional, Generator +from typing import Optional from src import shared from src.game import Game from src.importer.sources.source import LinuxSource, Source, SourceIterator -from src.utils.decorators import replaced_by_path, replaced_by_schema_key +from src.utils.decorators import replaced_by_path from src.utils.save_cover import resize_cover, save_cover @@ -61,6 +61,7 @@ class LutrisSource(Source): """Generic lutris source""" name = "Lutris" + location_key = "lutris-location" @property def game_id_format(self): @@ -75,7 +76,7 @@ class LutrisLinuxSource(LutrisSource, LinuxSource): executable_format = "xdg-open lutris:rungameid/{game_id}" @property - @replaced_by_schema_key("lutris-location") + @Source.replaced_by_schema_key() @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/") @replaced_by_path("~/.local/share/lutris/") def location(self): diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index f0da552..cd9857d 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -1,10 +1,13 @@ import sys from abc import abstractmethod from collections.abc import Iterable, Iterator +from functools import wraps from pathlib import Path -from typing import Optional, Generator +from typing import Generator, Optional +from src import shared from src.game import Game +from src.utils.decorators import replaced_by_path class SourceIterator(Iterator): @@ -40,11 +43,13 @@ class Source(Iterable): name: str variant: str + location_key: str available_on: set[str] def __init__(self) -> None: super().__init__() self.available_on = set() + self.update_location_schema_key() @property def full_name(self) -> str: @@ -76,6 +81,28 @@ class Source(Iterable): return False return sys.platform in self.available_on + def update_location_schema_key(self): + """Update the schema value for this source's location if possible""" + try: + location = self.location + except FileNotFoundError: + return + shared.schema.set_string(self.location_key, location) + + @classmethod + def replaced_by_schema_key(cls): # Decorator builder + """Replace the returned path with schema's path if valid""" + + def decorator(original_function): # Built decorator (closure) + @wraps(original_function) + def wrapper(*args, **kwargs): # func's override + override = shared.schema.get_string(cls.location_key) + return replaced_by_path(override)(original_function)(*args, **kwargs) + + return wrapper + + return decorator + @property @abstractmethod def location(self) -> Path: diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 5e59252..cff8c87 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -1,7 +1,7 @@ import re from pathlib import Path from time import time -from typing import Iterable, Optional, Generator +from typing import Iterable, Optional from src import shared from src.game import Game @@ -11,11 +11,7 @@ from src.importer.sources.source import ( SourceIterator, WindowsSource, ) -from src.utils.decorators import ( - replaced_by_env_path, - replaced_by_path, - replaced_by_schema_key, -) +from src.utils.decorators import replaced_by_env_path, replaced_by_path from src.utils.save_cover import resize_cover, save_cover from src.utils.steam import SteamHelper, SteamInvalidManifestError @@ -98,6 +94,7 @@ class SteamSourceIterator(SourceIterator): class SteamSource(Source): name = "Steam" + location_key = "steam-location" def __iter__(self): return SteamSourceIterator(source=self) @@ -108,7 +105,7 @@ class SteamLinuxSource(SteamSource, LinuxSource): executable_format = "xdg-open steam://rungameid/{game_id}" @property - @replaced_by_schema_key("steam-location") + @Source.replaced_by_schema_key() @replaced_by_path("~/.var/app/com.valvesoftware.Steam/data/Steam/") @replaced_by_env_path("XDG_DATA_HOME", "Steam/") @replaced_by_path("~/.steam/") @@ -122,7 +119,7 @@ class SteamWindowsSource(SteamSource, WindowsSource): executable_format = "start steam://rungameid/{game_id}" @property - @replaced_by_schema_key("steam-location") + @Source.replaced_by_schema_key() @replaced_by_env_path("programfiles(x86)", "Steam") def location(self): raise FileNotFoundError() diff --git a/src/utils/decorators.py b/src/utils/decorators.py index cce9a09..f836945 100644 --- a/src/utils/decorators.py +++ b/src/utils/decorators.py @@ -1,24 +1,7 @@ -""" -A decorator takes a callable A and returns a callable B that will override the name of A. -A decorator with arguments returns a closure decorator having access to the arguments. - -Example usage for the location decorators: - -class MyClass(): - @cached_property - @replaced_by_schema_key(key="source-location") - @replaced_by_path(override="/somewhere/that/doesnt/exist") - @replaced_by_path(override="/somewhere/that/exists") - def location(self): - return None -""" - from pathlib import Path from os import PathLike, environ from functools import wraps -from src import shared - def replaced_by_path(override: PathLike): # Decorator builder """Replace the method's returned path with the override @@ -37,24 +20,6 @@ def replaced_by_path(override: PathLike): # Decorator builder return decorator -def replaced_by_schema_key(key: str): # Decorator builder - """Replace the method's returned path with the path pointed by the key - if it exists on disk""" - - def decorator(original_function): # Built decorator (closure) - @wraps(original_function) - def wrapper(*args, **kwargs): # func's override - try: - override = shared.schema.get_string(key) - except Exception: # pylint: disable=broad-exception-caught - return original_function(*args, **kwargs) - return replaced_by_path(override)(original_function)(*args, **kwargs) - - return wrapper - - return decorator - - def replaced_by_env_path(env_var_name: str, suffix: PathLike | None = None): """Replace the method's returned path with a path whose root is the env variable""" From 725bab5c9306b0150dc372f67a5e2f90df7eb572 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 5 Jun 2023 13:21:19 +0200 Subject: [PATCH 107/173] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20new=20location?= =?UTF-8?q?=20override=20syntax?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/bottles_source.py | 2 +- src/importer/sources/heroic_source.py | 4 ++-- src/importer/sources/lutris_source.py | 2 +- src/importer/sources/source.py | 2 +- src/importer/sources/steam_source.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index b2c0a2d..4abc3f3 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -62,7 +62,7 @@ class BottlesLinuxSource(BottlesSource, LinuxSource): executable_format = 'xdg-open bottles:run/"{bottle_name}"/"{game_name}"' @property - @Source.replaced_by_schema_key() + @BottlesSource.replaced_by_schema_key() @replaced_by_path("~/.var/app/com.usebottles.bottles/data/bottles/") @replaced_by_env_path("XDG_DATA_HOME", "bottles/") @replaced_by_path("~/.local/share/bottles/") diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 998bf65..3515125 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -130,7 +130,7 @@ class HeroicLinuxSource(HeroicSource, LinuxSource): executable_format = "xdg-open heroic://launch/{app_name}" @property - @Source.replaced_by_schema_key() + @HeroicSource.replaced_by_schema_key() @replaced_by_path("~/.var/app/com.heroicgameslauncher.hgl/config/heroic/") @replaced_by_env_path("XDG_CONFIG_HOME", "heroic/") @replaced_by_path("~/.config/heroic/") @@ -143,7 +143,7 @@ class HeroicWindowsSource(HeroicSource, WindowsSource): executable_format = "start heroic://launch/{app_name}" @property - @Source.replaced_by_schema_key() + @HeroicSource.replaced_by_schema_key() @replaced_by_env_path("appdata", "heroic/") def location(self) -> Path: raise FileNotFoundError() diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 1633445..d9b8c35 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -76,7 +76,7 @@ class LutrisLinuxSource(LutrisSource, LinuxSource): executable_format = "xdg-open lutris:rungameid/{game_id}" @property - @Source.replaced_by_schema_key() + @LutrisSource.replaced_by_schema_key() @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/") @replaced_by_path("~/.local/share/lutris/") def location(self): diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index cd9857d..904368b 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -49,7 +49,7 @@ class Source(Iterable): def __init__(self) -> None: super().__init__() self.available_on = set() - self.update_location_schema_key() + @property def full_name(self) -> str: diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index cff8c87..fbde1be 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -105,7 +105,7 @@ class SteamLinuxSource(SteamSource, LinuxSource): executable_format = "xdg-open steam://rungameid/{game_id}" @property - @Source.replaced_by_schema_key() + @SteamSource.replaced_by_schema_key() @replaced_by_path("~/.var/app/com.valvesoftware.Steam/data/Steam/") @replaced_by_env_path("XDG_DATA_HOME", "Steam/") @replaced_by_path("~/.steam/") @@ -119,7 +119,7 @@ class SteamWindowsSource(SteamSource, WindowsSource): executable_format = "start steam://rungameid/{game_id}" @property - @Source.replaced_by_schema_key() + @SteamSource.replaced_by_schema_key() @replaced_by_env_path("programfiles(x86)", "Steam") def location(self): raise FileNotFoundError() From b50a0a1a045a78442ba061a0620b0a58004a1f40 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 5 Jun 2023 13:29:54 +0200 Subject: [PATCH 108/173] =?UTF-8?q?=F0=9F=93=9D=20Updated=20SourceIterator?= =?UTF-8?q?=20type=20hints?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/bottles_source.py | 4 ++-- src/importer/sources/heroic_source.py | 4 ++-- src/importer/sources/lutris_source.py | 4 ++-- src/importer/sources/steam_source.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index 4abc3f3..9fa1849 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -1,6 +1,6 @@ from pathlib import Path from time import time -from typing import Optional +from typing import Optional, Generator import yaml @@ -14,7 +14,7 @@ from src.utils.save_cover import resize_cover, save_cover class BottlesSourceIterator(SourceIterator): source: "BottlesSource" - def generator_builder(self) -> Optional[Game]: + def generator_builder(self) -> Generator[Optional[Game], None, None]: """Generator method producing games""" data = (self.source.location / "library.yml").read_text("utf-8") diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 3515125..c6ef498 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -4,7 +4,7 @@ from hashlib import sha256 from json import JSONDecodeError from pathlib import Path from time import time -from typing import Optional, TypedDict +from typing import Optional, TypedDict, Generator from src import shared from src.game import Game @@ -85,7 +85,7 @@ class HeroicSourceIterator(SourceIterator): return Game(values, allow_side_effects=False) - def generator_builder(self): + def generator_builder(self) -> Generator[Optional[Game], None, None]: """Generator method producing games from all the Heroic sub-sources""" for sub_source in self.sub_sources.values(): diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index d9b8c35..63e3344 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,6 +1,6 @@ from sqlite3 import connect from time import time -from typing import Optional +from typing import Optional, Generator from src import shared from src.game import Game @@ -12,7 +12,7 @@ from src.utils.save_cover import resize_cover, save_cover class LutrisSourceIterator(SourceIterator): source: "LutrisSource" - def generator_builder(self) -> Optional[Game]: + def generator_builder(self) -> Generator[Optional[Game], None, None]: """Generator method producing games""" # Query the database diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index fbde1be..e93b9f3 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -1,7 +1,7 @@ import re from pathlib import Path from time import time -from typing import Iterable, Optional +from typing import Iterable, Optional, Generator from src import shared from src.game import Game @@ -44,7 +44,7 @@ class SteamSourceIterator(SourceIterator): ) return manifests - def generator_builder(self) -> Optional[Game]: + def generator_builder(self) -> Generator[Optional[Game], None, None]: """Generator method producing games""" appid_cache = set() manifests = self.get_manifests() From 7eef050a64080889b1b84ce1023e972d0c5ee852 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 7 Jun 2023 12:12:12 +0200 Subject: [PATCH 109/173] =?UTF-8?q?=F0=9F=9A=A7=20WIP=20Itch=20source=20(o?= =?UTF-8?q?nly=20game=20discovery)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/itch_source.py | 86 +++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 src/importer/sources/itch_source.py diff --git a/src/importer/sources/itch_source.py b/src/importer/sources/itch_source.py new file mode 100644 index 0000000..1b90ffc --- /dev/null +++ b/src/importer/sources/itch_source.py @@ -0,0 +1,86 @@ +from sqlite3 import connect +from pathlib import Path +from time import time +from typing import Optional, Generator + +from src import shared +from src.utils.decorators import replaced_by_env_path, replaced_by_path +from src.game import Game +from src.importer.sources.source import ( + Source, + SourceIterator, + LinuxSource, + WindowsSource, +) + + +class ItchSourceIterator(SourceIterator): + source: "ItchSource" + + def generator_builder(self) -> Generator[Optional[Game], None, None]: + """Generator method producing games""" + + # Query the database + db_request = """ + SELECT + games.id, + games.title, + games.cover_url, + games.still_cover_url, + caves.id + FROM + 'caves' + INNER JOIN + 'games' + ON + caves.game_id = games.id + ; + """ + connection = connect(self.source.location / "db" / "butler.db") + cursor = connection.execute(db_request) + + # Create games from the db results + for row in cursor: + values = { + "version": shared.SPEC_VERSION, + "added": int(time()), + "source": self.source.id, + "game_id": self.source.game_id_format.format(game_id=row[0]), + "executable": self.source.executable_format.format(cave_id=row[4]), + } + yield Game(values, allow_side_effects=False) + + # TODO pass image URIs to the pipeline somehow + # - Add a reserved field to the Game object + # - Reconstruct those from the pipeline (we already have them) + # - Pass game and additional data to the pipeline separately (requires deep changes) + + +class ItchSource(Source): + name = "Itch" + location_key = "itch-location" + + def __iter__(self) -> SourceIterator: + return ItchSourceIterator(self) + + +class ItchLinuxSource(ItchSource, LinuxSource): + variant = "linux" + executable_format = "xdg-open itch://caves/{cave_id}/launch" + + @ItchSource.replaced_by_schema_key() + @replaced_by_path("~/.var/app/io.itch.itch/config/itch/") + @replaced_by_env_path("XDG_DATA_HOME", "itch/") + @replaced_by_path("~/.config/itch") + def location(self) -> Path: + raise FileNotFoundError() + + +class ItchWindowsSource(ItchSource, WindowsSource): + variant = "windows" + executable_format = "start itch://caves/{cave_id}/launch" + + @ItchSource.replaced_by_schema_key() + @replaced_by_env_path("appdata", "itch/") + def location(self) -> Path: + raise FileNotFoundError() From 98f02da36c5452f50c0988b11842582b84c5642b Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 7 Jun 2023 14:01:06 +0200 Subject: [PATCH 110/173] =?UTF-8?q?=F0=9F=8E=A8=20SourceIterator=20can=20y?= =?UTF-8?q?ield=20addtitional=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SourceIterator-s can yield a game and a tuple of additional data. This data will be passed to the Store, Pipeline and Managers. --- src/importer/importer.py | 28 +++++++++++++++++++------ src/importer/sources/itch_source.py | 9 +++----- src/importer/sources/source.py | 10 +++++---- src/main.py | 2 +- src/store/managers/async_manager.py | 10 +++++---- src/store/managers/display_manager.py | 2 +- src/store/managers/file_manager.py | 2 +- src/store/managers/manager.py | 20 +++++++++++------- src/store/managers/sgdb_manager.py | 6 ++++-- src/store/managers/steam_api_manager.py | 2 +- src/store/pipeline.py | 8 +++++-- src/store/store.py | 6 ++++-- 12 files changed, 68 insertions(+), 37 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 92e41f1..bd463e1 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -3,9 +3,9 @@ import logging from gi.repository import Adw, Gio, Gtk from src import shared +from src.game import Game from src.utils.task import Task from src.store.pipeline import Pipeline -from src.store.managers.manager import Manager from src.importer.sources.source import Source @@ -92,6 +92,7 @@ class Importer: def source_task_thread_func(self, _task, _obj, data, _cancellable): """Source import task code""" + source: Source source, *_rest = data # Early exit if not installed @@ -107,20 +108,35 @@ class Importer: while True: # Handle exceptions raised when iterating try: - game = next(iterator) + iteration_result = next(iterator) except StopIteration: break except Exception as exception: # pylint: disable=broad-exception-caught logging.exception( - msg=f"Exception in source {source.id}", - exc_info=exception, + "Exception in source %s", source.id, exc_info=exception ) continue - if game is None: + + # Handle the result depending on its type + if isinstance(iteration_result, Game): + game = iteration_result + additional_data = tuple() + elif isinstance(iteration_result, tuple): + game, additional_data = iteration_result + elif iteration_result is None: + continue + else: + # Warn source implementers that an invalid type was produced + # Should not happen on production code + logging.warn( + "%s produced an invalid iteration return type %s", + source.id, + type(iteration_result), + ) continue # Register game - pipeline: Pipeline = shared.store.add_game(game) + pipeline: Pipeline = shared.store.add_game(game, additional_data) if pipeline is not None: logging.info("Imported %s (%s)", game.name, game.game_id) pipeline.connect("advanced", self.pipeline_advanced_callback) diff --git a/src/importer/sources/itch_source.py b/src/importer/sources/itch_source.py index 1b90ffc..eddbb2e 100644 --- a/src/importer/sources/itch_source.py +++ b/src/importer/sources/itch_source.py @@ -48,12 +48,9 @@ class ItchSourceIterator(SourceIterator): "game_id": self.source.game_id_format.format(game_id=row[0]), "executable": self.source.executable_format.format(cave_id=row[4]), } - yield Game(values, allow_side_effects=False) - - # TODO pass image URIs to the pipeline somehow - # - Add a reserved field to the Game object - # - Reconstruct those from the pipeline (we already have them) - # - Pass game and additional data to the pipeline separately (requires deep changes) + additional_data = (row[3], row[2]) + game = Game(values, allow_side_effects=False) + yield (game, additional_data) class ItchSource(Source): diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index 904368b..8fa4283 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -3,12 +3,15 @@ from abc import abstractmethod from collections.abc import Iterable, Iterator from functools import wraps from pathlib import Path -from typing import Generator, Optional +from typing import Generator, Any from src import shared from src.game import Game from src.utils.decorators import replaced_by_path +# Type of the data returned by iterating on a Source +SourceIterationResult = None | Game | tuple[Game, tuple[Any]] + class SourceIterator(Iterator): """Data producer for a source of games""" @@ -24,11 +27,11 @@ class SourceIterator(Iterator): def __iter__(self) -> "SourceIterator": return self - def __next__(self) -> Optional[Game]: + def __next__(self) -> SourceIterationResult: return next(self.generator) @abstractmethod - def generator_builder(self) -> Generator[Optional[Game], None, None]: + def generator_builder(self) -> Generator[SourceIterationResult, None, None]: """ Method that returns a generator that produces games * Should be implemented as a generator method @@ -49,7 +52,6 @@ class Source(Iterable): def __init__(self) -> None: super().__init__() self.available_on = set() - @property def full_name(self) -> str: diff --git a/src/main.py b/src/main.py index d379d10..1c05625 100644 --- a/src/main.py +++ b/src/main.py @@ -133,7 +133,7 @@ class CartridgesApplication(Adw.Application): for game_file in shared.games_dir.iterdir(): data = json.load(game_file.open()) game = Game(data, allow_side_effects=False) - shared.store.add_game(game) + shared.store.add_game(game, tuple()) def on_about_action(self, *_args): about = Adw.AboutWindow( diff --git a/src/store/managers/async_manager.py b/src/store/managers/async_manager.py index 8d54681..a69f161 100644 --- a/src/store/managers/async_manager.py +++ b/src/store/managers/async_manager.py @@ -26,16 +26,18 @@ class AsyncManager(Manager): Already scheduled Tasks will no longer be cancellable.""" self.cancellable = Gio.Cancellable() - def process_game(self, game: Game, callback: Callable[["Manager"], Any]) -> None: + def process_game( + self, game: Game, additional_data: tuple, callback: Callable[["Manager"], Any] + ) -> None: """Create a task to process the game in a separate thread""" task = Task.new(None, self.cancellable, self._task_callback, (callback,)) - task.set_task_data((game,)) + task.set_task_data((game, additional_data)) task.run_in_thread(self._task_thread_func) def _task_thread_func(self, _task, _source_object, data, cancellable): """Task thread entry point""" - game, *_rest = data - self.execute_resilient_manager_logic(game) + game, additional_data, *_rest = data + self.execute_resilient_manager_logic(game, additional_data) def _task_callback(self, _source_object, _result, data): """Method run after the task is done""" diff --git a/src/store/managers/display_manager.py b/src/store/managers/display_manager.py index 90a7bde..cc0d82d 100644 --- a/src/store/managers/display_manager.py +++ b/src/store/managers/display_manager.py @@ -10,7 +10,7 @@ class DisplayManager(Manager): run_after = set((SteamAPIManager, SGDBManager)) - def manager_logic(self, game: Game) -> None: + def manager_logic(self, game: Game, _additional_data: tuple) -> None: # TODO decouple a game from its widget shared.win.games[game.game_id] = game game.update() diff --git a/src/store/managers/file_manager.py b/src/store/managers/file_manager.py index 07c725f..69901ee 100644 --- a/src/store/managers/file_manager.py +++ b/src/store/managers/file_manager.py @@ -8,5 +8,5 @@ class FileManager(AsyncManager): run_after = set((SteamAPIManager,)) - def manager_logic(self, game: Game) -> None: + def manager_logic(self, game: Game, _additional_data: tuple) -> None: game.save() diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 9675aeb..5b1b7d6 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -37,7 +37,7 @@ class Manager: self.errors_lock = Lock() def report_error(self, error: Exception): - """Report an error that happened in Manager.run""" + """Report an error that happened in Manager.process_game""" with self.errors_lock: self.errors.append(error) @@ -49,7 +49,7 @@ class Manager: return errors @abstractmethod - def manager_logic(self, game: Game) -> None: + def manager_logic(self, game: Game, additional_data: tuple) -> None: """ Manager specific logic triggered by the run method * Implemented by final child classes @@ -58,10 +58,12 @@ class Manager: * May raise other exceptions that will be reported """ - def execute_resilient_manager_logic(self, game: Game, try_index: int = 0) -> None: + def execute_resilient_manager_logic( + self, game: Game, additional_data: tuple, try_index: int = 0 + ) -> None: """Execute the manager logic and handle its errors by reporting them or retrying""" try: - self.manager_logic(game) + self.manager_logic(game, additional_data) except Exception as error: if error in self.continue_on: # Handle skippable errors (skip silently) @@ -71,7 +73,9 @@ class Manager: # Handle retryable errors logging_format = "Retrying %s in %s for %s" sleep(self.retry_delay) - self.execute_resilient_manager_logic(game, try_index + 1) + self.execute_resilient_manager_logic( + game, additional_data, try_index + 1 + ) else: # Handle being out of retries logging_format = "Out of retries dues to %s in %s for %s" @@ -88,7 +92,9 @@ class Manager: f"{game.name} ({game.game_id})", ) - def process_game(self, game: Game, callback: Callable[["Manager"], Any]) -> None: + def process_game( + self, game: Game, additional_data: tuple, callback: Callable[["Manager"], Any] + ) -> None: """Pass the game through the manager""" - self.execute_resilient_manager_logic(game) + self.execute_resilient_manager_logic(game, additional_data) callback(self) diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index 2179ee9..5b902df 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -1,3 +1,5 @@ +from urllib3.exceptions import SSLError + from src.game import Game from src.store.managers.async_manager import AsyncManager from src.store.managers.steam_api_manager import SteamAPIManager @@ -8,9 +10,9 @@ class SGDBManager(AsyncManager): """Manager in charge of downloading a game's cover from steamgriddb""" run_after = set((SteamAPIManager,)) - retryable_on = set((HTTPError,)) + retryable_on = set((HTTPError, SSLError)) - def manager_logic(self, game: Game) -> None: + def manager_logic(self, game: Game, _additional_data: tuple) -> None: try: sgdb = SGDBHelper() sgdb.conditionaly_update_cover(game) diff --git a/src/store/managers/steam_api_manager.py b/src/store/managers/steam_api_manager.py index 5874735..500a358 100644 --- a/src/store/managers/steam_api_manager.py +++ b/src/store/managers/steam_api_manager.py @@ -15,7 +15,7 @@ class SteamAPIManager(AsyncManager): retryable_on = set((HTTPError, SSLError)) - def manager_logic(self, game: Game) -> None: + def manager_logic(self, game: Game, _additional_data: tuple) -> None: # Skip non-steam games if not game.source.startswith("steam_"): return diff --git a/src/store/pipeline.py b/src/store/pipeline.py index 63fc7d3..a13253c 100644 --- a/src/store/pipeline.py +++ b/src/store/pipeline.py @@ -11,14 +11,18 @@ class Pipeline(GObject.Object): """Class representing a set of managers for a game""" game: Game + additional_data: tuple waiting: set[Manager] running: set[Manager] done: set[Manager] - def __init__(self, game: Game, managers: Iterable[Manager]) -> None: + def __init__( + self, game: Game, additional_data: tuple, managers: Iterable[Manager] + ) -> None: super().__init__() self.game = game + self.additional_data = additional_data self.waiting = set(managers) self.running = set() self.done = set() @@ -72,7 +76,7 @@ class Pipeline(GObject.Object): for manager in (*parallel, *blocking): self.waiting.remove(manager) self.running.add(manager) - manager.process_game(self.game, self.manager_callback) + manager.process_game(self.game, self.additional_data, self.manager_callback) def manager_callback(self, manager: Manager) -> None: """Method called by a manager when it's done""" diff --git a/src/store/store.py b/src/store/store.py index 4c260c9..5ecd79a 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -20,7 +20,9 @@ class Store: """Add a manager class that will run when games are added""" self.managers.add(manager) - def add_game(self, game: Game, replace=False) -> Pipeline | None: + def add_game( + self, game: Game, additional_data: tuple, replace=False + ) -> Pipeline | None: """Add a game to the app if not already there :param replace bool: Replace the game if it already exists @@ -49,7 +51,7 @@ class Store: return None # Run the pipeline for the game - pipeline = Pipeline(game, self.managers) + pipeline = Pipeline(game, additional_data, self.managers) self.games[game.game_id] = game self.pipelines[game.game_id] = pipeline pipeline.advance() From 5dc6ec899ab60b982f71f34f8c03d077b43f4e67 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 7 Jun 2023 15:00:42 +0200 Subject: [PATCH 111/173] =?UTF-8?q?=F0=9F=8E=A8=20Various=20changes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Changed source additional data to dict - Moved local cover saving into a manager - Added stub for itch cover manager --- src/importer/importer.py | 2 +- src/importer/sources/bottles_source.py | 18 ++++++++++-------- src/importer/sources/heroic_source.py | 22 ++++++++++++---------- src/importer/sources/itch_source.py | 14 +++++++------- src/importer/sources/lutris_source.py | 18 ++++++++++-------- src/importer/sources/source.py | 2 +- src/importer/sources/steam_source.py | 11 +++++------ src/main.py | 4 ++++ src/store/managers/async_manager.py | 2 +- src/store/managers/display_manager.py | 2 +- src/store/managers/file_manager.py | 2 +- src/store/managers/itch_cover_manager.py | 19 +++++++++++++++++++ src/store/managers/local_cover_manager.py | 23 +++++++++++++++++++++++ src/store/managers/manager.py | 6 +++--- src/store/managers/sgdb_manager.py | 6 ++++-- src/store/managers/steam_api_manager.py | 3 +-- src/store/pipeline.py | 4 ++-- src/store/store.py | 2 +- 18 files changed, 106 insertions(+), 54 deletions(-) create mode 100644 src/store/managers/itch_cover_manager.py create mode 100644 src/store/managers/local_cover_manager.py diff --git a/src/importer/importer.py b/src/importer/importer.py index bd463e1..fdbee81 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -120,7 +120,7 @@ class Importer: # Handle the result depending on its type if isinstance(iteration_result, Game): game = iteration_result - additional_data = tuple() + additional_data = {} elif isinstance(iteration_result, tuple): game, additional_data = iteration_result elif iteration_result is None: diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index 9fa1849..da70df6 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -1,20 +1,23 @@ from pathlib import Path from time import time -from typing import Optional, Generator import yaml from src import shared from src.game import Game -from src.importer.sources.source import LinuxSource, Source, SourceIterator +from src.importer.sources.source import ( + LinuxSource, + Source, + SourceIterationResult, + SourceIterator, +) from src.utils.decorators import replaced_by_env_path, replaced_by_path -from src.utils.save_cover import resize_cover, save_cover class BottlesSourceIterator(SourceIterator): source: "BottlesSource" - def generator_builder(self) -> Generator[Optional[Game], None, None]: + def generator_builder(self) -> SourceIterationResult: """Generator method producing games""" data = (self.source.location / "library.yml").read_text("utf-8") @@ -34,17 +37,16 @@ class BottlesSourceIterator(SourceIterator): } game = Game(values, allow_side_effects=False) - # Save official cover + # Get official cover path bottle_path = entry["bottle"]["path"] image_name = entry["thumbnail"].split(":")[1] image_path = ( self.source.location / "bottles" / bottle_path / "grids" / image_name ) - if image_path.is_file(): - save_cover(values["game_id"], resize_cover(image_path)) + additional_data = {"local_image_path": image_path} # Produce game - yield game + yield (game, additional_data) class BottlesSource(Source): diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index c6ef498..4d74ae7 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -4,18 +4,18 @@ from hashlib import sha256 from json import JSONDecodeError from pathlib import Path from time import time -from typing import Optional, TypedDict, Generator +from typing import Optional, TypedDict from src import shared from src.game import Game from src.importer.sources.source import ( LinuxSource, Source, + SourceIterationResult, SourceIterator, WindowsSource, ) from src.utils.decorators import replaced_by_env_path, replaced_by_path -from src.utils.save_cover import resize_cover, save_cover class HeroicLibraryEntry(TypedDict): @@ -50,7 +50,9 @@ class HeroicSourceIterator(SourceIterator): }, } - def game_from_library_entry(self, entry: HeroicLibraryEntry) -> Optional[Game]: + def game_from_library_entry( + self, entry: HeroicLibraryEntry + ) -> SourceIterationResult: """Helper method used to build a Game from a Heroic library entry""" # Skip games that are not installed @@ -72,20 +74,20 @@ class HeroicSourceIterator(SourceIterator): ), "executable": self.source.executable_format.format(app_name=app_name), } + game = Game(values, allow_side_effects=False) - # Save image from the heroic cache + # Get the image path from the heroic cache # Filenames are derived from the URL that heroic used to get the file uri: str = entry["art_square"] if service == "epic": uri += "?h=400&resize=1&w=300" digest = sha256(uri.encode()).hexdigest() image_path = self.source.location / "images-cache" / digest - if image_path.is_file(): - save_cover(values["game_id"], resize_cover(image_path)) + additional_data = {"local_image_path": image_path} - return Game(values, allow_side_effects=False) + return (game, additional_data) - def generator_builder(self) -> Generator[Optional[Game], None, None]: + def generator_builder(self) -> SourceIterationResult: """Generator method producing games from all the Heroic sub-sources""" for sub_source in self.sub_sources.values(): @@ -102,12 +104,12 @@ class HeroicSourceIterator(SourceIterator): continue for entry in library: try: - game = self.game_from_library_entry(entry) + result = self.game_from_library_entry(entry) except KeyError: # Skip invalid games logging.warning("Invalid Heroic game skipped in %s", str(file)) continue - yield game + yield result class HeroicSource(Source): diff --git a/src/importer/sources/itch_source.py b/src/importer/sources/itch_source.py index eddbb2e..4ffda7b 100644 --- a/src/importer/sources/itch_source.py +++ b/src/importer/sources/itch_source.py @@ -1,23 +1,23 @@ -from sqlite3 import connect from pathlib import Path +from sqlite3 import connect from time import time -from typing import Optional, Generator from src import shared -from src.utils.decorators import replaced_by_env_path, replaced_by_path from src.game import Game from src.importer.sources.source import ( - Source, - SourceIterator, LinuxSource, + Source, + SourceIterationResult, + SourceIterator, WindowsSource, ) +from src.utils.decorators import replaced_by_env_path, replaced_by_path class ItchSourceIterator(SourceIterator): source: "ItchSource" - def generator_builder(self) -> Generator[Optional[Game], None, None]: + def generator_builder(self) -> SourceIterationResult: """Generator method producing games""" # Query the database @@ -48,7 +48,7 @@ class ItchSourceIterator(SourceIterator): "game_id": self.source.game_id_format.format(game_id=row[0]), "executable": self.source.executable_format.format(cave_id=row[4]), } - additional_data = (row[3], row[2]) + additional_data = {"itch_cover_url": row[2], "itch_still_cover_url": row[3]} game = Game(values, allow_side_effects=False) yield (game, additional_data) diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 63e3344..641b75f 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,18 +1,21 @@ from sqlite3 import connect from time import time -from typing import Optional, Generator from src import shared from src.game import Game -from src.importer.sources.source import LinuxSource, Source, SourceIterator +from src.importer.sources.source import ( + LinuxSource, + Source, + SourceIterationResult, + SourceIterator, +) from src.utils.decorators import replaced_by_path -from src.utils.save_cover import resize_cover, save_cover class LutrisSourceIterator(SourceIterator): source: "LutrisSource" - def generator_builder(self) -> Generator[Optional[Game], None, None]: + def generator_builder(self) -> SourceIterationResult: """Generator method producing games""" # Query the database @@ -48,13 +51,12 @@ class LutrisSourceIterator(SourceIterator): } game = Game(values, allow_side_effects=False) - # Save official image + # Get official image path image_path = self.source.location / "covers" / "coverart" / f"{row[2]}.jpg" - if image_path.exists(): - save_cover(values["game_id"], resize_cover(image_path)) + additional_data = {"local_image_path": image_path} # Produce game - yield game + yield (game, additional_data) class LutrisSource(Source): diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index 8fa4283..1df02b1 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -3,7 +3,7 @@ from abc import abstractmethod from collections.abc import Iterable, Iterator from functools import wraps from pathlib import Path -from typing import Generator, Any +from typing import Generator, Any, TypedDict from src import shared from src.game import Game diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index e93b9f3..9c6488c 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -1,18 +1,18 @@ import re from pathlib import Path from time import time -from typing import Iterable, Optional, Generator +from typing import Iterable from src import shared from src.game import Game from src.importer.sources.source import ( LinuxSource, Source, + SourceIterationResult, SourceIterator, WindowsSource, ) from src.utils.decorators import replaced_by_env_path, replaced_by_path -from src.utils.save_cover import resize_cover, save_cover from src.utils.steam import SteamHelper, SteamInvalidManifestError @@ -44,7 +44,7 @@ class SteamSourceIterator(SourceIterator): ) return manifests - def generator_builder(self) -> Generator[Optional[Game], None, None]: + def generator_builder(self) -> SourceIterationResult: """Generator method producing games""" appid_cache = set() manifests = self.get_manifests() @@ -85,11 +85,10 @@ class SteamSourceIterator(SourceIterator): / "librarycache" / f"{appid}_library_600x900.jpg" ) - if image_path.is_file(): - save_cover(game.game_id, resize_cover(image_path)) + additional_data = {"local_image_path": image_path} # Produce game - yield game + yield (game, additional_data) class SteamSource(Source): diff --git a/src/main.py b/src/main.py index 1c05625..1ffa240 100644 --- a/src/main.py +++ b/src/main.py @@ -43,6 +43,8 @@ from src.store.managers.display_manager import DisplayManager from src.store.managers.file_manager import FileManager from src.store.managers.sgdb_manager import SGDBManager from src.store.managers.steam_api_manager import SteamAPIManager +from src.store.managers.local_cover_manager import LocalCoverManager +from src.store.managers.itch_cover_manager import ItchCoverManager from src.store.store import Store from src.window import CartridgesWindow @@ -85,7 +87,9 @@ class CartridgesApplication(Adw.Application): self.load_games_from_disk() # Add rest of the managers for game imports + shared.store.add_manager(LocalCoverManager()) shared.store.add_manager(SteamAPIManager()) + shared.store.add_manager(ItchCoverManager()) shared.store.add_manager(SGDBManager()) shared.store.add_manager(FileManager()) diff --git a/src/store/managers/async_manager.py b/src/store/managers/async_manager.py index a69f161..636b2bd 100644 --- a/src/store/managers/async_manager.py +++ b/src/store/managers/async_manager.py @@ -27,7 +27,7 @@ class AsyncManager(Manager): self.cancellable = Gio.Cancellable() def process_game( - self, game: Game, additional_data: tuple, callback: Callable[["Manager"], Any] + self, game: Game, additional_data: dict, callback: Callable[["Manager"], Any] ) -> None: """Create a task to process the game in a separate thread""" task = Task.new(None, self.cancellable, self._task_callback, (callback,)) diff --git a/src/store/managers/display_manager.py b/src/store/managers/display_manager.py index cc0d82d..e79746e 100644 --- a/src/store/managers/display_manager.py +++ b/src/store/managers/display_manager.py @@ -10,7 +10,7 @@ class DisplayManager(Manager): run_after = set((SteamAPIManager, SGDBManager)) - def manager_logic(self, game: Game, _additional_data: tuple) -> None: + def manager_logic(self, game: Game, _additional_data: dict) -> None: # TODO decouple a game from its widget shared.win.games[game.game_id] = game game.update() diff --git a/src/store/managers/file_manager.py b/src/store/managers/file_manager.py index 69901ee..66cebc4 100644 --- a/src/store/managers/file_manager.py +++ b/src/store/managers/file_manager.py @@ -8,5 +8,5 @@ class FileManager(AsyncManager): run_after = set((SteamAPIManager,)) - def manager_logic(self, game: Game, _additional_data: tuple) -> None: + def manager_logic(self, game: Game, _additional_data: dict) -> None: game.save() diff --git a/src/store/managers/itch_cover_manager.py b/src/store/managers/itch_cover_manager.py new file mode 100644 index 0000000..39b7928 --- /dev/null +++ b/src/store/managers/itch_cover_manager.py @@ -0,0 +1,19 @@ +from urllib3.exceptions import SSLError + +import requests +from requests import HTTPError + +from src.game import Game +from src.store.managers.async_manager import AsyncManager +from src.store.managers.local_cover_manager import LocalCoverManager + + +class ItchCoverManager(AsyncManager): + """Manager in charge of downloading the game's cover from itch.io""" + + run_after = set((LocalCoverManager,)) + retryable_on = set((HTTPError, SSLError)) + + def manager_logic(self, game: Game, additional_data: dict) -> None: + # TODO move itch cover logic here + pass diff --git a/src/store/managers/local_cover_manager.py b/src/store/managers/local_cover_manager.py new file mode 100644 index 0000000..a191f36 --- /dev/null +++ b/src/store/managers/local_cover_manager.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from src.game import Game +from src.store.managers.manager import Manager +from src.store.managers.steam_api_manager import SteamAPIManager +from src.utils.save_cover import save_cover, resize_cover + + +class LocalCoverManager(Manager): + """Manager in charge of adding the local cover image of the game""" + + run_after = set((SteamAPIManager,)) + + def manager_logic(self, game: Game, additional_data: dict) -> None: + # Ensure that the cover path is in the additional data + try: + image_path: Path = additional_data["local_image_path"] + except KeyError: + return + if not image_path.is_file(): + return + # Save the image + save_cover(game.game_id, resize_cover(image_path)) diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 5b1b7d6..732b868 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -49,7 +49,7 @@ class Manager: return errors @abstractmethod - def manager_logic(self, game: Game, additional_data: tuple) -> None: + def manager_logic(self, game: Game, additional_data: dict) -> None: """ Manager specific logic triggered by the run method * Implemented by final child classes @@ -59,7 +59,7 @@ class Manager: """ def execute_resilient_manager_logic( - self, game: Game, additional_data: tuple, try_index: int = 0 + self, game: Game, additional_data: dict, try_index: int = 0 ) -> None: """Execute the manager logic and handle its errors by reporting them or retrying""" try: @@ -93,7 +93,7 @@ class Manager: ) def process_game( - self, game: Game, additional_data: tuple, callback: Callable[["Manager"], Any] + self, game: Game, additional_data: dict, callback: Callable[["Manager"], Any] ) -> None: """Pass the game through the manager""" self.execute_resilient_manager_logic(game, additional_data) diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index 5b902df..08cfa88 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -2,6 +2,8 @@ from urllib3.exceptions import SSLError from src.game import Game from src.store.managers.async_manager import AsyncManager +from src.store.managers.itch_cover_manager import ItchCoverManager +from src.store.managers.local_cover_manager import LocalCoverManager from src.store.managers.steam_api_manager import SteamAPIManager from src.utils.steamgriddb import HTTPError, SGDBAuthError, SGDBHelper @@ -9,10 +11,10 @@ from src.utils.steamgriddb import HTTPError, SGDBAuthError, SGDBHelper class SGDBManager(AsyncManager): """Manager in charge of downloading a game's cover from steamgriddb""" - run_after = set((SteamAPIManager,)) + run_after = set((SteamAPIManager, LocalCoverManager, ItchCoverManager)) retryable_on = set((HTTPError, SSLError)) - def manager_logic(self, game: Game, _additional_data: tuple) -> None: + def manager_logic(self, game: Game, _additional_data: dict) -> None: try: sgdb = SGDBHelper() sgdb.conditionaly_update_cover(game) diff --git a/src/store/managers/steam_api_manager.py b/src/store/managers/steam_api_manager.py index 500a358..4f7d0c3 100644 --- a/src/store/managers/steam_api_manager.py +++ b/src/store/managers/steam_api_manager.py @@ -15,11 +15,10 @@ class SteamAPIManager(AsyncManager): retryable_on = set((HTTPError, SSLError)) - def manager_logic(self, game: Game, _additional_data: tuple) -> None: + def manager_logic(self, game: Game, _additional_data: dict) -> None: # Skip non-steam games if not game.source.startswith("steam_"): return - # Get online metadata appid = str(game.game_id).split("_")[-1] steam = SteamHelper() diff --git a/src/store/pipeline.py b/src/store/pipeline.py index a13253c..af1b64b 100644 --- a/src/store/pipeline.py +++ b/src/store/pipeline.py @@ -11,14 +11,14 @@ class Pipeline(GObject.Object): """Class representing a set of managers for a game""" game: Game - additional_data: tuple + additional_data: dict waiting: set[Manager] running: set[Manager] done: set[Manager] def __init__( - self, game: Game, additional_data: tuple, managers: Iterable[Manager] + self, game: Game, additional_data: dict, managers: Iterable[Manager] ) -> None: super().__init__() self.game = game diff --git a/src/store/store.py b/src/store/store.py index 5ecd79a..14ad917 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -21,7 +21,7 @@ class Store: self.managers.add(manager) def add_game( - self, game: Game, additional_data: tuple, replace=False + self, game: Game, additional_data: dict, replace=False ) -> Pipeline | None: """Add a game to the app if not already there From 9ebd7cf7eeef5aedfb056eba1c37a1e2b0f7cac9 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 7 Jun 2023 15:33:00 +0200 Subject: [PATCH 112/173] =?UTF-8?q?=E2=9C=A8=20Added=20Itch=20source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added call stack to unretryable errors in managers - Added existing itch cover downloading code - Fixed importer not closing if no source enabled TODO - Tidying the itch cover downloading code - If possible, make save_cover and resize_cover work in AsyncManager-s --- src/importer/importer.py | 7 ++-- src/importer/sources/itch_source.py | 3 ++ src/main.py | 8 +++- src/store/managers/itch_cover_manager.py | 48 ++++++++++++++++++++++-- src/store/managers/manager.py | 22 ++++++----- 5 files changed, 69 insertions(+), 19 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index fdbee81..d824b1a 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -61,10 +61,6 @@ class Importer: self.create_dialog() - # Single SGDB cancellable shared by all its tasks - # (If SGDB auth is bad, cancel all SGDB tasks) - self.sgdb_cancellable = Gio.Cancellable() - for source in self.sources: logging.debug("Importing games from source %s", source.id) task = Task.new(None, None, self.source_callback, (source,)) @@ -72,6 +68,8 @@ class Importer: task.set_task_data((source,)) task.run_in_thread(self.source_task_thread_func) + self.progress_changed_callback() + def create_dialog(self): """Create the import dialog""" self.progressbar = Gtk.ProgressBar(margin_start=12, margin_end=12) @@ -164,6 +162,7 @@ class Importer: Callback called when the import process has progressed Triggered when: + * All sources have been started * A source finishes * A pipeline finishes """ diff --git a/src/importer/sources/itch_source.py b/src/importer/sources/itch_source.py index 4ffda7b..5682f03 100644 --- a/src/importer/sources/itch_source.py +++ b/src/importer/sources/itch_source.py @@ -45,6 +45,7 @@ class ItchSourceIterator(SourceIterator): "version": shared.SPEC_VERSION, "added": int(time()), "source": self.source.id, + "name": row[1], "game_id": self.source.game_id_format.format(game_id=row[0]), "executable": self.source.executable_format.format(cave_id=row[4]), } @@ -65,6 +66,7 @@ class ItchLinuxSource(ItchSource, LinuxSource): variant = "linux" executable_format = "xdg-open itch://caves/{cave_id}/launch" + @property @ItchSource.replaced_by_schema_key() @replaced_by_path("~/.var/app/io.itch.itch/config/itch/") @replaced_by_env_path("XDG_DATA_HOME", "itch/") @@ -77,6 +79,7 @@ class ItchWindowsSource(ItchSource, WindowsSource): variant = "windows" executable_format = "start itch://caves/{cave_id}/launch" + @property @ItchSource.replaced_by_schema_key() @replaced_by_env_path("appdata", "itch/") def location(self) -> Path: diff --git a/src/main.py b/src/main.py index 1ffa240..834e204 100644 --- a/src/main.py +++ b/src/main.py @@ -36,15 +36,16 @@ from src.game import Game from src.importer.importer import Importer from src.importer.sources.bottles_source import BottlesLinuxSource from src.importer.sources.heroic_source import HeroicLinuxSource, HeroicWindowsSource +from src.importer.sources.itch_source import ItchLinuxSource, ItchWindowsSource from src.importer.sources.lutris_source import LutrisLinuxSource from src.importer.sources.steam_source import SteamLinuxSource, SteamWindowsSource from src.preferences import PreferencesWindow from src.store.managers.display_manager import DisplayManager from src.store.managers.file_manager import FileManager +from src.store.managers.itch_cover_manager import ItchCoverManager +from src.store.managers.local_cover_manager import LocalCoverManager from src.store.managers.sgdb_manager import SGDBManager from src.store.managers.steam_api_manager import SteamAPIManager -from src.store.managers.local_cover_manager import LocalCoverManager -from src.store.managers.itch_cover_manager import ItchCoverManager from src.store.store import Store from src.window import CartridgesWindow @@ -198,6 +199,9 @@ class CartridgesApplication(Adw.Application): importer.add_source(HeroicWindowsSource()) if shared.schema.get_boolean("bottles"): importer.add_source(BottlesLinuxSource()) + if shared.schema.get_boolean("itch"): + importer.add_source(ItchLinuxSource()) + importer.add_source(ItchWindowsSource()) importer.run() def on_remove_game_action(self, *_args): diff --git a/src/store/managers/itch_cover_manager.py b/src/store/managers/itch_cover_manager.py index 39b7928..cc8b783 100644 --- a/src/store/managers/itch_cover_manager.py +++ b/src/store/managers/itch_cover_manager.py @@ -1,11 +1,15 @@ -from urllib3.exceptions import SSLError +from pathlib import Path import requests +from gi.repository import GdkPixbuf, Gio from requests import HTTPError +from urllib3.exceptions import SSLError +from src import shared from src.game import Game from src.store.managers.async_manager import AsyncManager from src.store.managers.local_cover_manager import LocalCoverManager +from src.utils.save_cover import resize_cover, save_cover class ItchCoverManager(AsyncManager): @@ -15,5 +19,43 @@ class ItchCoverManager(AsyncManager): retryable_on = set((HTTPError, SSLError)) def manager_logic(self, game: Game, additional_data: dict) -> None: - # TODO move itch cover logic here - pass + # Get the first matching cover url + base_cover_url: str = additional_data.get("itch_cover_url", None) + still_cover_url: str = additional_data.get("itch_still_cover_url", None) + cover_url = still_cover_url or base_cover_url + if not cover_url: + return + + # Download cover + tmp_file = Gio.File.new_tmp()[0] + with requests.get(cover_url, timeout=5) as cover: + cover.raise_for_status() + Path(tmp_file.get_path()).write_bytes(cover.content) + + # TODO comment the following blocks of code + game_cover = GdkPixbuf.Pixbuf.new_from_stream_at_scale( + tmp_file.read(), 2, 2, False + ).scale_simple(*shared.image_size, GdkPixbuf.InterpType.BILINEAR) + + itch_pixbuf = GdkPixbuf.Pixbuf.new_from_stream(tmp_file.read()) + itch_pixbuf = itch_pixbuf.scale_simple( + shared.image_size[0], + itch_pixbuf.get_height() * (shared.image_size[0] / itch_pixbuf.get_width()), + GdkPixbuf.InterpType.BILINEAR, + ) + itch_pixbuf.composite( + game_cover, + 0, + (shared.image_size[1] - itch_pixbuf.get_height()) / 2, + itch_pixbuf.get_width(), + itch_pixbuf.get_height(), + 0, + (shared.image_size[1] - itch_pixbuf.get_height()) / 2, + 1.0, + 1.0, + GdkPixbuf.InterpType.BILINEAR, + 255, + ) + + # Resize and save the cover + save_cover(game.game_id, resize_cover(pixbuf=game_cover)) diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 732b868..6d82f61 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -65,32 +65,34 @@ class Manager: try: self.manager_logic(game, additional_data) except Exception as error: + logging_args = ( + type(error).__name__, + self.name, + f"{game.name} ({game.game_id})", + ) if error in self.continue_on: # Handle skippable errors (skip silently) return elif error in self.retryable_on: if try_index < self.max_tries: # Handle retryable errors - logging_format = "Retrying %s in %s for %s" + logging.error("Retrying %s in %s for %s", *logging_args) sleep(self.retry_delay) self.execute_resilient_manager_logic( game, additional_data, try_index + 1 ) else: # Handle being out of retries - logging_format = "Out of retries dues to %s in %s for %s" + logging.error( + "Out of retries dues to %s in %s for %s", *logging_args + ) self.report_error(error) else: # Handle unretryable errors - logging_format = "Unretryable %s in %s for %s" + logging.error( + "Unretryable %s in %s for %s", *logging_args, exc_info=error + ) self.report_error(error) - # Finally log errors - logging.error( - logging_format, - type(error).__name__, - self.name, - f"{game.name} ({game.game_id})", - ) def process_game( self, game: Game, additional_data: dict, callback: Callable[["Manager"], Any] From b895c8ebe2daf23f2325f8d4ec9f01766daffb9d Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 7 Jun 2023 19:31:15 +0200 Subject: [PATCH 113/173] =?UTF-8?q?=F0=9F=8E=A8=20Made=20itch=20cover=20ma?= =?UTF-8?q?nager=20sync?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/managers/itch_cover_manager.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/store/managers/itch_cover_manager.py b/src/store/managers/itch_cover_manager.py index cc8b783..79506ea 100644 --- a/src/store/managers/itch_cover_manager.py +++ b/src/store/managers/itch_cover_manager.py @@ -7,12 +7,12 @@ from urllib3.exceptions import SSLError from src import shared from src.game import Game -from src.store.managers.async_manager import AsyncManager +from src.store.managers.manager import Manager from src.store.managers.local_cover_manager import LocalCoverManager from src.utils.save_cover import resize_cover, save_cover -class ItchCoverManager(AsyncManager): +class ItchCoverManager(Manager): """Manager in charge of downloading the game's cover from itch.io""" run_after = set((LocalCoverManager,)) @@ -32,17 +32,20 @@ class ItchCoverManager(AsyncManager): cover.raise_for_status() Path(tmp_file.get_path()).write_bytes(cover.content) - # TODO comment the following blocks of code + # Create background blur game_cover = GdkPixbuf.Pixbuf.new_from_stream_at_scale( tmp_file.read(), 2, 2, False ).scale_simple(*shared.image_size, GdkPixbuf.InterpType.BILINEAR) + # Resize square image itch_pixbuf = GdkPixbuf.Pixbuf.new_from_stream(tmp_file.read()) itch_pixbuf = itch_pixbuf.scale_simple( shared.image_size[0], itch_pixbuf.get_height() * (shared.image_size[0] / itch_pixbuf.get_width()), GdkPixbuf.InterpType.BILINEAR, ) + + # Composite itch_pixbuf.composite( game_cover, 0, From 51922ad4c6f870c63e8b5af57ad7bcf37b5933d7 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Thu, 8 Jun 2023 10:50:09 +0200 Subject: [PATCH 114/173] =?UTF-8?q?=F0=9F=9A=A7=20Base=20Legendary=20sourc?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/legendary_source.py | 93 ++++++++++++++++++++++ src/store/managers/online_cover_manager.py | 31 ++++++++ 2 files changed, 124 insertions(+) create mode 100644 src/importer/sources/legendary_source.py create mode 100644 src/store/managers/online_cover_manager.py diff --git a/src/importer/sources/legendary_source.py b/src/importer/sources/legendary_source.py new file mode 100644 index 0000000..73ad76f --- /dev/null +++ b/src/importer/sources/legendary_source.py @@ -0,0 +1,93 @@ +import logging +from pathlib import Path +from typing import Generator +import json +from json import JSONDecodeError +from time import time + +from src import shared +from src.game import Game +from src.importer.sources.source import ( + LinuxSource, + Source, + SourceIterationResult, + SourceIterator, +) +from src.utils.decorators import replaced_by_env_path, replaced_by_path + + +class LegendarySourceIterator(SourceIterator): + source: "LegendarySource" + + def game_from_library_entry(self, entry: dict) -> SourceIterationResult: + # Skip non-games + if entry["is_dlc"]: + return None + + # Build game + app_name = entry["app_name"] + values = { + "version": shared.SPEC_VERSION, + "added": int(time()), + "source": self.source.id, + "name": entry["title"], + "game_id": self.source.game_id_format.format(game_id=app_name), + "executable": self.source.game_id_format.format(app_name=app_name), + } + data = {} + + # Get additional metadata from file (optional) + metadata_file = self.source.location / "metadata" / f"{app_name}.json" + try: + metadata = json.load(metadata_file.open()) + values["developer"] = metadata["metadata"]["developer"] + for image_entry in metadata["metadata"]["keyImages"]: + if image_entry["type"] == "DieselGameBoxTall": + data["online_cover_url"] = image_entry["url"] + break + except (JSONDecodeError, OSError, KeyError): + pass + + game = Game(values, allow_side_effects=False) + return (game, data) + + def generator_builder(self) -> Generator[SourceIterationResult, None, None]: + # Open library + file = self.source.location / "installed.json" + try: + library = json.load(file.open()) + except (JSONDecodeError, OSError): + logging.warning("Couldn't open Legendary file: %s", str(file)) + return + # Generate games from library + for entry in library: + try: + result = self.game_from_library_entry(entry) + except KeyError: + # Skip invalid games + logging.warning("Invalid Legendary game skipped in %s", str(file)) + continue + yield result + + +class LegendarySource(Source): + name = "Legendary" + location_key = "legendary-location" + + def __iter__(self) -> SourceIterator: + return LegendarySourceIterator(self) + + +# TODO add Legendary windows variant + + +class LegendaryLinuxSource(LegendarySource, LinuxSource): + variant = "linux" + executable_format = "legendary launch {app_name}" + + @property + @LegendarySource.replaced_by_schema_key() + @replaced_by_env_path("XDG_CONFIG_HOME", "legendary/") + @replaced_by_path("~/.config/legendary/") + def location(self) -> Path: + raise FileNotFoundError() diff --git a/src/store/managers/online_cover_manager.py b/src/store/managers/online_cover_manager.py new file mode 100644 index 0000000..40753a2 --- /dev/null +++ b/src/store/managers/online_cover_manager.py @@ -0,0 +1,31 @@ +from pathlib import Path + +import requests +from gi.repository import Gio +from requests import HTTPError +from urllib3.exceptions import SSLError + +from src.game import Game +from src.store.managers.manager import Manager +from src.store.managers.local_cover_manager import LocalCoverManager +from src.utils.save_cover import resize_cover, save_cover + + +class OnlineCoverManager(Manager): + """Manager that downloads game covers from URLs""" + + run_after = set((LocalCoverManager,)) + retryable_on = set((HTTPError, SSLError)) + + def manager_logic(self, game: Game, additional_data: dict) -> None: + # Ensure that we have a cover to download + cover_url = additional_data.get("online_cover_url", None) + if not cover_url: + return + # Download cover + tmp_file = Gio.File.new_tmp()[0] + with requests.get(cover_url, timeout=5) as cover: + cover.raise_for_status() + Path(tmp_file.get_path()).write_bytes(cover.content) + # Resize and save + save_cover(game.game_id, resize_cover(tmp_file.get_path())) From 070d875ff89e758f4c16b7f362309dbeed6e23ac Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Fri, 9 Jun 2023 17:06:33 +0200 Subject: [PATCH 115/173] =?UTF-8?q?=F0=9F=8E=A8=20Improved=20Legendary=20s?= =?UTF-8?q?ource?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fixed wrong library iteration - Fixed executable format - Added toggle in the preferences - Added legendary to on_import_action --- data/gtk/preferences.blp | 14 ++++++++++++++ data/hu.kramo.Cartridges.gschema.xml.in | 6 ++++++ src/importer/sources/legendary_source.py | 12 +++++++----- src/main.py | 3 +++ src/preferences.py | 7 +++++++ 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/data/gtk/preferences.blp b/data/gtk/preferences.blp index 5026889..f7a71b9 100644 --- a/data/gtk/preferences.blp +++ b/data/gtk/preferences.blp @@ -186,6 +186,20 @@ template $PreferencesWindow : Adw.PreferencesWindow { } } } + + Adw.ExpanderRow legendary_expander_row { + title: _("Legendary"); + show-enable-switch: true; + + Adw.ActionRow legendary_action_row { + title: _("Legendary Install Location"); + + Button legendary_file_chooser_button { + icon-name: "folder-symbolic"; + valign: center; + } + } + } } } diff --git a/data/hu.kramo.Cartridges.gschema.xml.in b/data/hu.kramo.Cartridges.gschema.xml.in index 9de7f7b..4738807 100644 --- a/data/hu.kramo.Cartridges.gschema.xml.in +++ b/data/hu.kramo.Cartridges.gschema.xml.in @@ -55,6 +55,12 @@ "~/.var/app/io.itch.itch/config/itch/" + + true + + + "~/.config/legendary/" + "" diff --git a/src/importer/sources/legendary_source.py b/src/importer/sources/legendary_source.py index 73ad76f..bbc5b5e 100644 --- a/src/importer/sources/legendary_source.py +++ b/src/importer/sources/legendary_source.py @@ -32,7 +32,7 @@ class LegendarySourceIterator(SourceIterator): "source": self.source.id, "name": entry["title"], "game_id": self.source.game_id_format.format(game_id=app_name), - "executable": self.source.game_id_format.format(app_name=app_name), + "executable": self.source.executable_format.format(app_name=app_name), } data = {} @@ -55,17 +55,19 @@ class LegendarySourceIterator(SourceIterator): # Open library file = self.source.location / "installed.json" try: - library = json.load(file.open()) + library: dict = json.load(file.open()) except (JSONDecodeError, OSError): logging.warning("Couldn't open Legendary file: %s", str(file)) return # Generate games from library - for entry in library: + for entry in library.values(): try: result = self.game_from_library_entry(entry) - except KeyError: + except KeyError as error: # Skip invalid games - logging.warning("Invalid Legendary game skipped in %s", str(file)) + logging.warning( + "Invalid Legendary game skipped in %s", str(file), exc_info=error + ) continue yield result diff --git a/src/main.py b/src/main.py index 834e204..52dd8e5 100644 --- a/src/main.py +++ b/src/main.py @@ -37,6 +37,7 @@ from src.importer.importer import Importer from src.importer.sources.bottles_source import BottlesLinuxSource from src.importer.sources.heroic_source import HeroicLinuxSource, HeroicWindowsSource from src.importer.sources.itch_source import ItchLinuxSource, ItchWindowsSource +from src.importer.sources.legendary_source import LegendaryLinuxSource from src.importer.sources.lutris_source import LutrisLinuxSource from src.importer.sources.steam_source import SteamLinuxSource, SteamWindowsSource from src.preferences import PreferencesWindow @@ -202,6 +203,8 @@ class CartridgesApplication(Adw.Application): if shared.schema.get_boolean("itch"): importer.add_source(ItchLinuxSource()) importer.add_source(ItchWindowsSource()) + if shared.schema.get_boolean("legendary"): + importer.add_source(LegendaryLinuxSource()) importer.run() def on_remove_game_action(self, *_args): diff --git a/src/preferences.py b/src/preferences.py index e0a5aff..784fab9 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -76,6 +76,10 @@ class PreferencesWindow(Adw.PreferencesWindow): itch_action_row = Gtk.Template.Child() itch_file_chooser_button = Gtk.Template.Child() + legendary_expander_row = Gtk.Template.Child() + legendary_action_row = Gtk.Template.Child() + legendary_file_chooser_button = Gtk.Template.Child() + sgdb_key_group = Gtk.Template.Child() sgdb_key_entry_row = Gtk.Template.Child() sgdb_switch = Gtk.Template.Child() @@ -156,6 +160,9 @@ class PreferencesWindow(Adw.PreferencesWindow): # itch self.create_preferences(self, "itch", "itch", True) + # Legendary + self.create_preferences(self, "legendary", "Legendary", True) + # SteamGridDB def sgdb_key_changed(*_args): shared.schema.set_string("sgdb-key", self.sgdb_key_entry_row.get_text()) From 842f9fe5226db465e1c1d9685f5b792a40587496 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 10 Jun 2023 02:59:41 +0200 Subject: [PATCH 116/173] =?UTF-8?q?=F0=9F=8E=A8=20Various=20code=20style?= =?UTF-8?q?=20/=20behaviour=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Merged platform sources when possible - Added URLExecutableSource class - Moved replaced_by_schema_key to utils/decorators - Better retryable exception handling in some managers - Split SteamHelper into SteamFileHelper and SteamAPIHelper - Delegated SteamRateLimiter creation to SteamAPIManager init - Using additional_data for appid in SteamAPIManager - Added Windows support for Legendary - Stylistic changed suggested by pylint --- src/importer/sources/bottles_source.py | 25 ++++---- src/importer/sources/heroic_source.py | 36 ++++------- src/importer/sources/itch_source.py | 36 ++++------- src/importer/sources/legendary_source.py | 33 ++++------- src/importer/sources/lutris_source.py | 22 +++---- src/importer/sources/source.py | 69 ++++++++++------------ src/importer/sources/steam_source.py | 50 ++++++---------- src/main.py | 34 ++++++----- src/store/managers/async_manager.py | 2 +- src/store/managers/itch_cover_manager.py | 7 +-- src/store/managers/manager.py | 2 +- src/store/managers/online_cover_manager.py | 5 +- src/store/managers/sgdb_manager.py | 8 ++- src/store/managers/steam_api_manager.py | 23 +++++--- src/utils/decorators.py | 17 ++++++ src/utils/steam.py | 35 +++++------ src/utils/steamgriddb.py | 2 +- 17 files changed, 182 insertions(+), 224 deletions(-) diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index da70df6..d77bee3 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -6,12 +6,15 @@ import yaml from src import shared from src.game import Game from src.importer.sources.source import ( - LinuxSource, - Source, SourceIterationResult, SourceIterator, + URLExecutableSource, +) +from src.utils.decorators import ( + replaced_by_env_path, + replaced_by_path, + replaced_by_schema_key, ) -from src.utils.decorators import replaced_by_env_path, replaced_by_path class BottlesSourceIterator(SourceIterator): @@ -49,22 +52,16 @@ class BottlesSourceIterator(SourceIterator): yield (game, additional_data) -class BottlesSource(Source): +class BottlesSource(URLExecutableSource): """Generic Bottles source""" name = "Bottles" - location_key = "bottles-location" - - def __iter__(self) -> SourceIterator: - return BottlesSourceIterator(self) - - -class BottlesLinuxSource(BottlesSource, LinuxSource): - variant = "linux" - executable_format = 'xdg-open bottles:run/"{bottle_name}"/"{game_name}"' + iterator_class = BottlesSourceIterator + url_format = 'bottles:run/"{bottle_name}"/"{game_name}"' + available_on = set(("linux",)) @property - @BottlesSource.replaced_by_schema_key() + @replaced_by_schema_key @replaced_by_path("~/.var/app/com.usebottles.bottles/data/bottles/") @replaced_by_env_path("XDG_DATA_HOME", "bottles/") @replaced_by_path("~/.local/share/bottles/") diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 4d74ae7..01faf24 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -9,13 +9,15 @@ from typing import Optional, TypedDict from src import shared from src.game import Game from src.importer.sources.source import ( - LinuxSource, - Source, + URLExecutableSource, SourceIterationResult, SourceIterator, - WindowsSource, ) -from src.utils.decorators import replaced_by_env_path, replaced_by_path +from src.utils.decorators import ( + replaced_by_env_path, + replaced_by_path, + replaced_by_schema_key, +) class HeroicLibraryEntry(TypedDict): @@ -112,40 +114,24 @@ class HeroicSourceIterator(SourceIterator): yield result -class HeroicSource(Source): +class HeroicSource(URLExecutableSource): """Generic heroic games launcher source""" name = "Heroic" - location_key = "heroic-location" + iterator_class = HeroicSourceIterator + url_format = "heroic://launch/{app_name}" + available_on = set(("linux", "win32")) @property def game_id_format(self) -> str: """The string format used to construct game IDs""" return self.name.lower() + "_{service}_{game_id}" - def __iter__(self): - return HeroicSourceIterator(source=self) - - -class HeroicLinuxSource(HeroicSource, LinuxSource): - variant = "linux" - executable_format = "xdg-open heroic://launch/{app_name}" - @property - @HeroicSource.replaced_by_schema_key() + @replaced_by_schema_key @replaced_by_path("~/.var/app/com.heroicgameslauncher.hgl/config/heroic/") @replaced_by_env_path("XDG_CONFIG_HOME", "heroic/") @replaced_by_path("~/.config/heroic/") - def location(self) -> Path: - raise FileNotFoundError() - - -class HeroicWindowsSource(HeroicSource, WindowsSource): - variant = "windows" - executable_format = "start heroic://launch/{app_name}" - - @property - @HeroicSource.replaced_by_schema_key() @replaced_by_env_path("appdata", "heroic/") def location(self) -> Path: raise FileNotFoundError() diff --git a/src/importer/sources/itch_source.py b/src/importer/sources/itch_source.py index 5682f03..29ef507 100644 --- a/src/importer/sources/itch_source.py +++ b/src/importer/sources/itch_source.py @@ -5,13 +5,15 @@ from time import time from src import shared from src.game import Game from src.importer.sources.source import ( - LinuxSource, - Source, SourceIterationResult, SourceIterator, - WindowsSource, + URLExecutableSource, +) +from src.utils.decorators import ( + replaced_by_env_path, + replaced_by_path, + replaced_by_schema_key, ) -from src.utils.decorators import replaced_by_env_path, replaced_by_path class ItchSourceIterator(SourceIterator): @@ -54,33 +56,17 @@ class ItchSourceIterator(SourceIterator): yield (game, additional_data) -class ItchSource(Source): +class ItchSource(URLExecutableSource): name = "Itch" - location_key = "itch-location" - - def __iter__(self) -> SourceIterator: - return ItchSourceIterator(self) - - -class ItchLinuxSource(ItchSource, LinuxSource): - variant = "linux" - executable_format = "xdg-open itch://caves/{cave_id}/launch" + iterator_class = ItchSourceIterator + url_format = "itch://caves/{cave_id}/launch" + available_on = set(("linux", "win32")) @property - @ItchSource.replaced_by_schema_key() + @replaced_by_schema_key @replaced_by_path("~/.var/app/io.itch.itch/config/itch/") @replaced_by_env_path("XDG_DATA_HOME", "itch/") @replaced_by_path("~/.config/itch") - def location(self) -> Path: - raise FileNotFoundError() - - -class ItchWindowsSource(ItchSource, WindowsSource): - variant = "windows" - executable_format = "start itch://caves/{cave_id}/launch" - - @property - @ItchSource.replaced_by_schema_key() @replaced_by_env_path("appdata", "itch/") def location(self) -> Path: raise FileNotFoundError() diff --git a/src/importer/sources/legendary_source.py b/src/importer/sources/legendary_source.py index bbc5b5e..91f8b0a 100644 --- a/src/importer/sources/legendary_source.py +++ b/src/importer/sources/legendary_source.py @@ -1,19 +1,18 @@ -import logging -from pathlib import Path -from typing import Generator import json +import logging from json import JSONDecodeError +from pathlib import Path from time import time +from typing import Generator from src import shared from src.game import Game -from src.importer.sources.source import ( - LinuxSource, - Source, - SourceIterationResult, - SourceIterator, +from src.importer.sources.source import Source, SourceIterationResult, SourceIterator +from src.utils.decorators import ( + replaced_by_env_path, + replaced_by_path, + replaced_by_schema_key, ) -from src.utils.decorators import replaced_by_env_path, replaced_by_path class LegendarySourceIterator(SourceIterator): @@ -74,22 +73,14 @@ class LegendarySourceIterator(SourceIterator): class LegendarySource(Source): name = "Legendary" - location_key = "legendary-location" - - def __iter__(self) -> SourceIterator: - return LegendarySourceIterator(self) - - -# TODO add Legendary windows variant - - -class LegendaryLinuxSource(LegendarySource, LinuxSource): - variant = "linux" executable_format = "legendary launch {app_name}" + iterator_class = LegendarySourceIterator + available_on = set(("linux", "win32")) @property - @LegendarySource.replaced_by_schema_key() + @replaced_by_schema_key @replaced_by_env_path("XDG_CONFIG_HOME", "legendary/") @replaced_by_path("~/.config/legendary/") + @replaced_by_path("~\\.config\\legendary\\") def location(self) -> Path: raise FileNotFoundError() diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 641b75f..7a2e51d 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -4,12 +4,11 @@ from time import time from src import shared from src.game import Game from src.importer.sources.source import ( - LinuxSource, - Source, SourceIterationResult, SourceIterator, + URLExecutableSource, ) -from src.utils.decorators import replaced_by_path +from src.utils.decorators import replaced_by_path, replaced_by_schema_key class LutrisSourceIterator(SourceIterator): @@ -47,7 +46,6 @@ class LutrisSourceIterator(SourceIterator): game_id=row[2], game_internal_id=row[0] ), "executable": self.source.executable_format.format(game_id=row[2]), - "developer": None, # TODO get developer metadata on Lutris } game = Game(values, allow_side_effects=False) @@ -59,26 +57,20 @@ class LutrisSourceIterator(SourceIterator): yield (game, additional_data) -class LutrisSource(Source): +class LutrisSource(URLExecutableSource): """Generic lutris source""" name = "Lutris" - location_key = "lutris-location" + iterator_class = LutrisSourceIterator + url_format = "lutris:rungameid/{game_id}" + available_on = set(("linux",)) @property def game_id_format(self): return super().game_id_format + "_{game_internal_id}" - def __iter__(self): - return LutrisSourceIterator(source=self) - - -class LutrisLinuxSource(LutrisSource, LinuxSource): - variant = "linux" - executable_format = "xdg-open lutris:rungameid/{game_id}" - @property - @LutrisSource.replaced_by_schema_key() + @replaced_by_schema_key @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/") @replaced_by_path("~/.local/share/lutris/") def location(self): diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index 1df02b1..16963dc 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -1,13 +1,11 @@ import sys from abc import abstractmethod from collections.abc import Iterable, Iterator -from functools import wraps from pathlib import Path -from typing import Generator, Any, TypedDict +from typing import Generator, Any from src import shared from src.game import Game -from src.utils.decorators import replaced_by_path # Type of the data returned by iterating on a Source SourceIterationResult = None | Game | tuple[Game, tuple[Any]] @@ -45,13 +43,9 @@ class Source(Iterable): """Source of games. E.g an installed app with a config file that lists game directories""" name: str - variant: str - location_key: str - available_on: set[str] - - def __init__(self) -> None: - super().__init__() - self.available_on = set() + iterator_class: type[SourceIterator] + variant: str = None + available_on: set[str] = set() @property def full_name(self) -> str: @@ -83,6 +77,14 @@ class Source(Iterable): return False return sys.platform in self.available_on + @property + def location_key(self) -> str: + """ + The schema key pointing to the user-set location for the source. + May be overriden by inherinting classes. + """ + return f"{self.name.lower()}-location" + def update_location_schema_key(self): """Update the schema value for this source's location if possible""" try: @@ -91,19 +93,9 @@ class Source(Iterable): return shared.schema.set_string(self.location_key, location) - @classmethod - def replaced_by_schema_key(cls): # Decorator builder - """Replace the returned path with schema's path if valid""" - - def decorator(original_function): # Built decorator (closure) - @wraps(original_function) - def wrapper(*args, **kwargs): # func's override - override = shared.schema.get_string(cls.location_key) - return replaced_by_path(override)(original_function)(*args, **kwargs) - - return wrapper - - return decorator + def __iter__(self) -> SourceIterator: + """Get an iterator for the source""" + return self.iterator_class(self) @property @abstractmethod @@ -115,22 +107,21 @@ class Source(Iterable): def executable_format(self) -> str: """The executable format used to construct game executables""" - @abstractmethod - def __iter__(self) -> SourceIterator: - """Get the source's iterator, to use in for loops""" +# pylint: disable=abstract-method +class URLExecutableSource(Source): + """Source class that use custom URLs to start games""" -class WindowsSource(Source): - """Mixin for sources available on Windows""" + url_format: str - def __init__(self) -> None: - super().__init__() - self.available_on.add("win32") - - -class LinuxSource(Source): - """Mixin for sources available on Linux""" - - def __init__(self) -> None: - super().__init__() - self.available_on.add("linux") + @property + def executable_format(self) -> str: + match sys.platform: + case "win32": + return "start " + self.url_format + case "linux": + return "xdg-open " + self.url_format + case other: + raise NotImplementedError( + f"No URL handler command available for {other}" + ) diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 9c6488c..7e724ab 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -6,14 +6,16 @@ from typing import Iterable from src import shared from src.game import Game from src.importer.sources.source import ( - LinuxSource, - Source, SourceIterationResult, SourceIterator, - WindowsSource, + URLExecutableSource, ) -from src.utils.decorators import replaced_by_env_path, replaced_by_path -from src.utils.steam import SteamHelper, SteamInvalidManifestError +from src.utils.decorators import ( + replaced_by_env_path, + replaced_by_path, + replaced_by_schema_key, +) +from src.utils.steam import SteamFileHelper, SteamInvalidManifestError class SteamSourceIterator(SourceIterator): @@ -22,11 +24,11 @@ class SteamSourceIterator(SourceIterator): def get_manifest_dirs(self) -> Iterable[Path]: """Get dirs that contain steam app manifests""" libraryfolders_path = self.source.location / "steamapps" / "libraryfolders.vdf" - with open(libraryfolders_path, "r") as file: + with open(libraryfolders_path, "r", encoding="utf-8") as file: contents = file.read() return [ Path(path) / "steamapps" - for path in re.findall('"path"\s+"(.*)"\n', contents, re.IGNORECASE) + for path in re.findall('"path"\\s+"(.*)"\n', contents, re.IGNORECASE) ] def get_manifests(self) -> Iterable[Path]: @@ -50,15 +52,15 @@ class SteamSourceIterator(SourceIterator): manifests = self.get_manifests() for manifest in manifests: # Get metadata from manifest - steam = SteamHelper() + steam = SteamFileHelper() try: local_data = steam.get_manifest_data(manifest) except (OSError, SteamInvalidManifestError): continue # Skip non installed games - INSTALLED_MASK: int = 4 - if not int(local_data["stateflags"]) & INSTALLED_MASK: + installed_mask = 4 + if not int(local_data["stateflags"]) & installed_mask: continue # Skip duplicate appids @@ -85,40 +87,24 @@ class SteamSourceIterator(SourceIterator): / "librarycache" / f"{appid}_library_600x900.jpg" ) - additional_data = {"local_image_path": image_path} + additional_data = {"local_image_path": image_path, "steam_appid": appid} # Produce game yield (game, additional_data) -class SteamSource(Source): +class SteamSource(URLExecutableSource): name = "Steam" - location_key = "steam-location" - - def __iter__(self): - return SteamSourceIterator(source=self) - - -class SteamLinuxSource(SteamSource, LinuxSource): - variant = "linux" - executable_format = "xdg-open steam://rungameid/{game_id}" + iterator_class = SteamSourceIterator + url_format = "steam://rungameid/{game_id}" + available_on = set(("linux", "win32")) @property - @SteamSource.replaced_by_schema_key() + @replaced_by_schema_key @replaced_by_path("~/.var/app/com.valvesoftware.Steam/data/Steam/") @replaced_by_env_path("XDG_DATA_HOME", "Steam/") @replaced_by_path("~/.steam/") @replaced_by_path("~/.local/share/Steam/") - def location(self): - raise FileNotFoundError() - - -class SteamWindowsSource(SteamSource, WindowsSource): - variant = "windows" - executable_format = "start steam://rungameid/{game_id}" - - @property - @SteamSource.replaced_by_schema_key() @replaced_by_env_path("programfiles(x86)", "Steam") def location(self): raise FileNotFoundError() diff --git a/src/main.py b/src/main.py index 52dd8e5..5cd8dc3 100644 --- a/src/main.py +++ b/src/main.py @@ -34,12 +34,12 @@ from src import shared from src.details_window import DetailsWindow from src.game import Game from src.importer.importer import Importer -from src.importer.sources.bottles_source import BottlesLinuxSource -from src.importer.sources.heroic_source import HeroicLinuxSource, HeroicWindowsSource -from src.importer.sources.itch_source import ItchLinuxSource, ItchWindowsSource -from src.importer.sources.legendary_source import LegendaryLinuxSource -from src.importer.sources.lutris_source import LutrisLinuxSource -from src.importer.sources.steam_source import SteamLinuxSource, SteamWindowsSource +from src.importer.sources.bottles_source import BottlesSource +from src.importer.sources.heroic_source import HeroicSource +from src.importer.sources.itch_source import ItchSource +from src.importer.sources.legendary_source import LegendarySource +from src.importer.sources.lutris_source import LutrisSource +from src.importer.sources.steam_source import SteamSource from src.preferences import PreferencesWindow from src.store.managers.display_manager import DisplayManager from src.store.managers.file_manager import FileManager @@ -190,21 +190,25 @@ class CartridgesApplication(Adw.Application): def on_import_action(self, *_args): importer = Importer() + if shared.schema.get_boolean("lutris"): - importer.add_source(LutrisLinuxSource()) + importer.add_source(LutrisSource()) + if shared.schema.get_boolean("steam"): - importer.add_source(SteamLinuxSource()) - importer.add_source(SteamWindowsSource()) + importer.add_source(SteamSource()) + if shared.schema.get_boolean("heroic"): - importer.add_source(HeroicLinuxSource()) - importer.add_source(HeroicWindowsSource()) + importer.add_source(HeroicSource()) + if shared.schema.get_boolean("bottles"): - importer.add_source(BottlesLinuxSource()) + importer.add_source(BottlesSource()) + if shared.schema.get_boolean("itch"): - importer.add_source(ItchLinuxSource()) - importer.add_source(ItchWindowsSource()) + importer.add_source(ItchSource()) + if shared.schema.get_boolean("legendary"): - importer.add_source(LegendaryLinuxSource()) + importer.add_source(LegendarySource()) + importer.run() def on_remove_game_action(self, *_args): diff --git a/src/store/managers/async_manager.py b/src/store/managers/async_manager.py index 636b2bd..373d51f 100644 --- a/src/store/managers/async_manager.py +++ b/src/store/managers/async_manager.py @@ -34,7 +34,7 @@ class AsyncManager(Manager): task.set_task_data((game, additional_data)) task.run_in_thread(self._task_thread_func) - def _task_thread_func(self, _task, _source_object, data, cancellable): + def _task_thread_func(self, _task, _source_object, data, _cancellable): """Task thread entry point""" game, additional_data, *_rest = data self.execute_resilient_manager_logic(game, additional_data) diff --git a/src/store/managers/itch_cover_manager.py b/src/store/managers/itch_cover_manager.py index 79506ea..3abc474 100644 --- a/src/store/managers/itch_cover_manager.py +++ b/src/store/managers/itch_cover_manager.py @@ -2,13 +2,12 @@ from pathlib import Path import requests from gi.repository import GdkPixbuf, Gio -from requests import HTTPError -from urllib3.exceptions import SSLError +from requests.exceptions import HTTPError, SSLError from src import shared from src.game import Game -from src.store.managers.manager import Manager from src.store.managers.local_cover_manager import LocalCoverManager +from src.store.managers.manager import Manager from src.utils.save_cover import resize_cover, save_cover @@ -45,7 +44,7 @@ class ItchCoverManager(Manager): GdkPixbuf.InterpType.BILINEAR, ) - # Composite + # Composite itch_pixbuf.composite( game_cover, 0, diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 6d82f61..e5581b7 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -64,7 +64,7 @@ class Manager: """Execute the manager logic and handle its errors by reporting them or retrying""" try: self.manager_logic(game, additional_data) - except Exception as error: + except Exception as error: # pylint: disable=broad-exception-caught logging_args = ( type(error).__name__, self.name, diff --git a/src/store/managers/online_cover_manager.py b/src/store/managers/online_cover_manager.py index 40753a2..b4e5bf0 100644 --- a/src/store/managers/online_cover_manager.py +++ b/src/store/managers/online_cover_manager.py @@ -2,12 +2,11 @@ from pathlib import Path import requests from gi.repository import Gio -from requests import HTTPError -from urllib3.exceptions import SSLError +from requests.exceptions import HTTPError, SSLError from src.game import Game -from src.store.managers.manager import Manager from src.store.managers.local_cover_manager import LocalCoverManager +from src.store.managers.manager import Manager from src.utils.save_cover import resize_cover, save_cover diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index 08cfa88..c1831fc 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -1,18 +1,20 @@ -from urllib3.exceptions import SSLError +from json import JSONDecodeError + +from requests.exceptions import HTTPError, SSLError from src.game import Game from src.store.managers.async_manager import AsyncManager from src.store.managers.itch_cover_manager import ItchCoverManager from src.store.managers.local_cover_manager import LocalCoverManager from src.store.managers.steam_api_manager import SteamAPIManager -from src.utils.steamgriddb import HTTPError, SGDBAuthError, SGDBHelper +from src.utils.steamgriddb import SGDBAuthError, SGDBHelper class SGDBManager(AsyncManager): """Manager in charge of downloading a game's cover from steamgriddb""" run_after = set((SteamAPIManager, LocalCoverManager, ItchCoverManager)) - retryable_on = set((HTTPError, SSLError)) + retryable_on = set((HTTPError, SSLError, ConnectionError, JSONDecodeError)) def manager_logic(self, game: Game, _additional_data: dict) -> None: try: diff --git a/src/store/managers/steam_api_manager.py b/src/store/managers/steam_api_manager.py index 4f7d0c3..faad9d7 100644 --- a/src/store/managers/steam_api_manager.py +++ b/src/store/managers/steam_api_manager.py @@ -1,12 +1,12 @@ -from urllib3.exceptions import SSLError +from requests.exceptions import HTTPError, SSLError from src.game import Game from src.store.managers.async_manager import AsyncManager from src.utils.steam import ( - HTTPError, SteamGameNotFoundError, - SteamHelper, + SteamAPIHelper, SteamNotAGameError, + SteamRateLimiter, ) @@ -15,15 +15,22 @@ class SteamAPIManager(AsyncManager): retryable_on = set((HTTPError, SSLError)) - def manager_logic(self, game: Game, _additional_data: dict) -> None: + steam_api_helper: SteamAPIHelper = None + steam_rate_limiter: SteamRateLimiter = None + + def __init__(self) -> None: + super().__init__() + self.steam_rate_limiter = SteamRateLimiter() + self.steam_api_helper = SteamAPIHelper(self.steam_rate_limiter) + + def manager_logic(self, game: Game, additional_data: dict) -> None: # Skip non-steam games - if not game.source.startswith("steam_"): + appid = additional_data.get("steam_appid", None) + if appid is None: return # Get online metadata - appid = str(game.game_id).split("_")[-1] - steam = SteamHelper() try: - online_data = steam.get_api_data(appid=appid) + online_data = self.steam_api_helper.get_api_data(appid=appid) except (SteamNotAGameError, SteamGameNotFoundError): game.update_values({"blacklisted": True}) else: diff --git a/src/utils/decorators.py b/src/utils/decorators.py index f836945..09763fa 100644 --- a/src/utils/decorators.py +++ b/src/utils/decorators.py @@ -2,6 +2,8 @@ from pathlib import Path from os import PathLike, environ from functools import wraps +from src import shared + def replaced_by_path(override: PathLike): # Decorator builder """Replace the method's returned path with the override @@ -36,3 +38,18 @@ def replaced_by_env_path(env_var_name: str, suffix: PathLike | None = None): return wrapper return decorator + + +def replaced_by_schema_key(original_method): # Built decorator (closure) + """ + Replace the original method's value by the path pointed at in the schema + by the class' location key (if that override exists) + """ + + @wraps(original_method) + def wrapper(*args, **kwargs): # func's override + source = args[0] + override = shared.schema.get_string(source.location_key) + return replaced_by_path(override)(original_method)(*args, **kwargs) + + return wrapper diff --git a/src/utils/steam.py b/src/utils/steam.py index 8008f77..517f9bf 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -4,7 +4,7 @@ import re from typing import TypedDict import requests -from requests import HTTPError +from requests.exceptions import HTTPError from src import shared from src.utils.rate_limiter import PickHistory, RateLimiter @@ -27,7 +27,7 @@ class SteamInvalidManifestError(SteamError): class SteamManifestData(TypedDict): - """Dict returned by SteamHelper.get_manifest_data""" + """Dict returned by SteamFileHelper.get_manifest_data""" name: str appid: str @@ -35,7 +35,7 @@ class SteamManifestData(TypedDict): class SteamAPIData(TypedDict): - """Dict returned by SteamHelper.get_api_data""" + """Dict returned by SteamAPIHelper.get_api_data""" developers: str @@ -73,34 +73,35 @@ class SteamRateLimiter(RateLimiter): shared.state_schema.set_string("steam-limiter-tokens-history", timestamps_str) -class SteamHelper: - """Helper around the Steam API""" - - base_url = "https://store.steampowered.com/api" - rate_limiter: SteamRateLimiter = None - - def __init__(self) -> None: - # Instanciate the rate limiter on the class to share across instances - # Can't be done at class creation time, schema isn't available yet - if self.__class__.rate_limiter is None: - self.__class__.rate_limiter = SteamRateLimiter() +class SteamFileHelper: + """Helper for steam file formats""" def get_manifest_data(self, manifest_path) -> SteamManifestData: """Get local data for a game from its manifest""" - with open(manifest_path) as file: + with open(manifest_path, "r", encoding="utf-8") as file: contents = file.read() data = {} - for key in SteamManifestData.__required_keys__: - regex = f'"{key}"\s+"(.*)"\n' + for key in SteamManifestData.__required_keys__: # pylint: disable=no-member + regex = f'"{key}"\\s+"(.*)"\n' if (match := re.search(regex, contents, re.IGNORECASE)) is None: raise SteamInvalidManifestError() data[key] = match.group(1) return SteamManifestData(**data) + +class SteamAPIHelper: + """Helper around the Steam API""" + + base_url = "https://store.steampowered.com/api" + rate_limiter: RateLimiter + + def __init__(self, rate_limiter: RateLimiter) -> None: + self.rate_limiter = rate_limiter + def get_api_data(self, appid) -> SteamAPIData: """ Get online data for a game from its appid. diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index 3841e9b..5351898 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -3,7 +3,7 @@ from pathlib import Path import requests from gi.repository import Gio -from requests import HTTPError +from requests.exceptions import HTTPError from src import shared from src.utils.create_dialog import create_dialog From eeb0f3e501af18a0264fe8ab3592991a9b0be165 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 10 Jun 2023 03:06:53 +0200 Subject: [PATCH 117/173] =?UTF-8?q?=F0=9F=91=B7=20Added=20.pylintrc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .pylintrc | 637 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 637 insertions(+) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..2dbc403 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,637 @@ +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=importers + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.11 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + too-few-public-methods, + missing-function-docstring, + missing-class-docstring, + missing-module-docstring, + relative-beyond-top-level, + import-error + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work.. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=Child + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins=_ + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io From c9a96f5eec2f02019b19a51b328298dedaf9af7a Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 10 Jun 2023 03:30:09 +0200 Subject: [PATCH 118/173] =?UTF-8?q?=F0=9F=8E=A8=20Fixed=20some=20linter=20?= =?UTF-8?q?warnings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applied suggested pylint fixes and suppressed unhelpful pylint messages --- src/details_window.py | 1 + src/game.py | 2 + src/importer/importer.py | 12 +-- src/store/managers/itch_cover_manager.py | 3 + src/store/managers/manager.py | 2 +- src/utils/importer.py | 132 ----------------------- src/utils/rate_limiter.py | 40 +++---- src/utils/steam.py | 8 +- 8 files changed, 38 insertions(+), 162 deletions(-) delete mode 100644 src/utils/importer.py diff --git a/src/details_window.py b/src/details_window.py index d896bc2..767110e 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -110,6 +110,7 @@ class DetailsWindow(Adw.Window): file_path = _("/path/to/{}").format(file_name) command = "xdg-open" + # pylint: disable=line-too-long exec_info_text = _( 'To launch the executable "{}", use the command:\n\n"{}"\n\nTo open the file "{}" with the default application, use:\n\n{} "{}"\n\nIf the path contains spaces, make sure to wrap it in double quotes!' ).format(exe_name, exe_path, file_name, command, file_path) diff --git a/src/game.py b/src/game.py index 20504f9..1b4aeef 100644 --- a/src/game.py +++ b/src/game.py @@ -31,6 +31,7 @@ from src import shared from src.game_cover import GameCover +# pylint: disable=too-many-instance-attributes @Gtk.Template(resource_path=shared.PREFIX + "/gtk/game.ui") class Game(Gtk.Box): __gtype_name__ = "Game" @@ -187,6 +188,7 @@ class Game(Gtk.Box): ) logging.info("Starting %s: %s", self.name, str(args)) + # pylint: disable=consider-using-with subprocess.Popen( args, cwd=Path.home(), diff --git a/src/importer/importer.py b/src/importer/importer.py index d824b1a..eae8565 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -1,6 +1,6 @@ import logging -from gi.repository import Adw, Gio, Gtk +from gi.repository import Adw, Gtk from src import shared from src.game import Game @@ -31,15 +31,13 @@ class Importer: @property def n_games_added(self): return sum( - [ - 1 if not (pipeline.game.blacklisted or pipeline.game.removed) else 0 - for pipeline in self.game_pipelines - ] + 1 if not (pipeline.game.blacklisted or pipeline.game.removed) else 0 + for pipeline in self.game_pipelines ) @property def pipelines_progress(self): - progress = sum([pipeline.progress for pipeline in self.game_pipelines]) + progress = sum(pipeline.progress for pipeline in self.game_pipelines) try: progress = progress / len(self.game_pipelines) except ZeroDivisionError: @@ -126,7 +124,7 @@ class Importer: else: # Warn source implementers that an invalid type was produced # Should not happen on production code - logging.warn( + logging.warning( "%s produced an invalid iteration return type %s", source.id, type(iteration_result), diff --git a/src/store/managers/itch_cover_manager.py b/src/store/managers/itch_cover_manager.py index 3abc474..abf61b2 100644 --- a/src/store/managers/itch_cover_manager.py +++ b/src/store/managers/itch_cover_manager.py @@ -11,6 +11,9 @@ from src.store.managers.manager import Manager from src.utils.save_cover import resize_cover, save_cover +# TODO Remove by generalizing OnlineCoverManager + + class ItchCoverManager(Manager): """Manager in charge of downloading the game's cover from itch.io""" diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index e5581b7..e73eb5a 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -73,7 +73,7 @@ class Manager: if error in self.continue_on: # Handle skippable errors (skip silently) return - elif error in self.retryable_on: + if error in self.retryable_on: if try_index < self.max_tries: # Handle retryable errors logging.error("Retrying %s in %s for %s", *logging_args) diff --git a/src/utils/importer.py b/src/utils/importer.py deleted file mode 100644 index 2d651ac..0000000 --- a/src/utils/importer.py +++ /dev/null @@ -1,132 +0,0 @@ -# importer.py -# -# Copyright 2022-2023 kramo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from gi.repository import Adw, GLib, Gtk - -from src import shared -from .create_dialog import create_dialog -from .game import Game -from .save_cover import resize_cover, save_cover -from .steamgriddb import SGDBSave - - -class Importer: - def __init__(self): - self.win = shared.win - self.total_queue = 0 - self.queue = 0 - self.games_no = 0 - self.blocker = False - self.games = set() - self.sgdb_exception = None - - self.progressbar = Gtk.ProgressBar(margin_start=12, margin_end=12) - self.import_statuspage = Adw.StatusPage( - title=_("Importing Games…"), - child=self.progressbar, - ) - - self.import_dialog = Adw.Window( - content=self.import_statuspage, - modal=True, - default_width=350, - default_height=-1, - transient_for=self.win, - deletable=False, - ) - - self.import_dialog.present() - - def save_game(self, values=None, cover_path=None): - if values: - game = Game(values) - - if save_cover: - save_cover(game.game_id, resize_cover(cover_path)) - - self.games.add(game) - - self.games_no += 1 - if game.blacklisted: - self.games_no -= 1 - - self.queue -= 1 - self.update_progressbar() - - if self.queue == 0 and not self.blocker: - if self.games: - self.total_queue = len(self.games) - self.queue = len(self.games) - self.import_statuspage.set_title(_("Importing Covers…")) - self.update_progressbar() - SGDBSave(self.games, self) - else: - self.done() - - def done(self): - self.update_progressbar() - if self.queue == 0: - self.import_dialog.close() - - toast = Adw.Toast() - toast.set_priority(Adw.ToastPriority.HIGH) - - if self.games_no == 0: - toast.set_title(_("No new games found")) - toast.set_button_label(_("Preferences")) - toast.connect( - "button-clicked", self.response, "open_preferences", "import" - ) - - elif self.games_no == 1: - toast.set_title(_("1 game imported")) - - elif self.games_no > 1: - games_no = self.games_no - toast.set_title( - # The variable is the number of games - _("{} games imported").format(games_no) - ) - - self.win.toast_overlay.add_toast(toast) - # Add timeout to make it the last thing to happen - GLib.timeout_add(0, self.warning, None, None) - - def response(self, _widget, response, page_name=None, expander_row=None): - if response == "open_preferences": - self.win.get_application().on_preferences_action( - None, page_name=page_name, expander_row=expander_row - ) - - def warning(self, *_args): - if self.sgdb_exception: - create_dialog( - self.win, - _("Couldn't Connect to SteamGridDB"), - self.sgdb_exception, - "open_preferences", - _("Preferences"), - ).connect("response", self.response, "sgdb") - self.sgdb_exception = None - - def update_progressbar(self): - try: - self.progressbar.set_fraction(1 - (self.queue / self.total_queue)) - except ZeroDivisionError: - self.progressbar.set_fraction(1) diff --git a/src/utils/rate_limiter.py b/src/utils/rate_limiter.py index 2a80fc8..d0f1e29 100644 --- a/src/utils/rate_limiter.py +++ b/src/utils/rate_limiter.py @@ -9,20 +9,20 @@ class PickHistory(Sized): """Utility class used for rate limiters, counting how many picks happened in a given period""" - PERIOD: int + period: int timestamps: list[int] = None timestamps_lock: Lock = None def __init__(self, period: int) -> None: - self.PERIOD = period + self.period = period self.timestamps = [] self.timestamps_lock = Lock() def remove_old_entries(self): """Remove history entries older than the period""" now = time() - cutoff = now - self.PERIOD + cutoff = now - self.period with self.timestamps_lock: self.timestamps = [entry for entry in self.timestamps if entry > cutoff] @@ -58,15 +58,16 @@ class PickHistory(Sized): return self.timestamps.copy() +# pylint: disable=too-many-instance-attributes class RateLimiter(AbstractContextManager): """Rate limiter implementing the token bucket algorithm""" # Period in which we have a max amount of tokens - REFILL_PERIOD_SECONDS: int + refill_period_seconds: int # Number of tokens allowed in this period - REFILL_PERIOD_TOKENS: int + refill_period_tokens: int # Max number of tokens that can be consumed instantly - BURST_TOKENS: int + burst_tokens: int pick_history: PickHistory = None bucket: BoundedSemaphore = None @@ -97,13 +98,13 @@ class RateLimiter(AbstractContextManager): # Initialize default values if refill_period_seconds is not None: - self.REFILL_PERIOD_SECONDS = refill_period_seconds + self.refill_period_seconds = refill_period_seconds if refill_period_tokens is not None: - self.REFILL_PERIOD_TOKENS = refill_period_tokens + self.refill_period_tokens = refill_period_tokens if burst_tokens is not None: - self.BURST_TOKENS = burst_tokens + self.burst_tokens = burst_tokens if self.pick_history is None: - self.pick_history = PickHistory(self.REFILL_PERIOD_SECONDS) + self.pick_history = PickHistory(self.refill_period_seconds) # Create synchronization data self.__n_tokens_lock = Lock() @@ -111,8 +112,8 @@ class RateLimiter(AbstractContextManager): self.queue = deque() # Initialize the token bucket - self.bucket = BoundedSemaphore(self.BURST_TOKENS) - self.n_tokens = self.BURST_TOKENS + self.bucket = BoundedSemaphore(self.burst_tokens) + self.n_tokens = self.burst_tokens # Spawn daemon thread that refills the bucket refill_thread = Thread(target=self.refill_thread_func, daemon=True) @@ -127,8 +128,8 @@ class RateLimiter(AbstractContextManager): """ # Compute ideal spacing - tokens_left = self.REFILL_PERIOD_TOKENS - len(self.pick_history) - seconds_left = self.pick_history.start + self.REFILL_PERIOD_SECONDS - time() + tokens_left = self.refill_period_tokens - len(self.pick_history) + seconds_left = self.pick_history.start + self.refill_period_seconds - time() try: spacing_seconds = seconds_left / tokens_left except ZeroDivisionError: @@ -136,7 +137,7 @@ class RateLimiter(AbstractContextManager): spacing_seconds = seconds_left # Prevent spacing dropping down lower than the natural spacing - natural_spacing = self.REFILL_PERIOD_SECONDS / self.REFILL_PERIOD_TOKENS + natural_spacing = self.refill_period_seconds / self.refill_period_tokens return max(natural_spacing, spacing_seconds) def refill(self): @@ -165,7 +166,8 @@ class RateLimiter(AbstractContextManager): with self.queue_lock: if len(self.queue) == 0: return - self.bucket.acquire() + # Not using with because we don't want to release to the bucket + self.bucket.acquire() # pylint: disable=consider-using-with self.n_tokens -= 1 lock = self.queue.pop() lock.release() @@ -173,7 +175,8 @@ class RateLimiter(AbstractContextManager): def add_to_queue(self) -> Lock: """Create a lock, add it to the queue and return it""" lock = Lock() - lock.acquire() + # We want the lock locked until its turn in queue + lock.acquire() # pylint: disable=consider-using-with with self.queue_lock: self.queue.appendleft(lock) return lock @@ -182,7 +185,8 @@ class RateLimiter(AbstractContextManager): """Acquires a token from the bucket when it's your turn in queue""" lock = self.add_to_queue() self.update_queue() - lock.acquire() + # Wait until our turn in queue + lock.acquire() # pylint: disable=consider-using-with self.pick_history.add() # --- Support for use in with statements diff --git a/src/utils/steam.py b/src/utils/steam.py index 517f9bf..c0a0677 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -47,15 +47,15 @@ class SteamRateLimiter(RateLimiter): # 200 requests per 5 min seems to be the limit # https://stackoverflow.com/questions/76047820/how-am-i-exceeding-steam-apis-rate-limit # https://stackoverflow.com/questions/51795457/avoiding-error-429-too-many-requests-steam-web-api - REFILL_PERIOD_SECONDS = 5 * 60 - REFILL_PERIOD_TOKENS = 200 - BURST_TOKENS = 100 + refill_period_seconds = 5 * 60 + refill_period_tokens = 200 + burst_tokens = 100 def __init__(self) -> None: # Load pick history from schema # (Remember API limits through restarts of Cartridges) timestamps_str = shared.state_schema.get_string("steam-limiter-tokens-history") - self.pick_history = PickHistory(self.REFILL_PERIOD_SECONDS) + self.pick_history = PickHistory(self.refill_period_seconds) self.pick_history.add(*json.loads(timestamps_str)) self.pick_history.remove_old_entries() super().__init__() From e7fd01f50993ee878c52902419be182fa5210226 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 10 Jun 2023 12:03:16 +0200 Subject: [PATCH 119/173] =?UTF-8?q?=F0=9F=8E=A8=20Made=20manager=20attribu?= =?UTF-8?q?tes=20more=20flexible?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changed run_after, retryable_on and continue_on to be type Container. We don't need them to be sets. The performance gain of sets over small tuples is nonexistant for in checks and the syntax is more verbose. --- src/store/managers/display_manager.py | 2 +- src/store/managers/file_manager.py | 2 +- src/store/managers/itch_cover_manager.py | 4 ++-- src/store/managers/local_cover_manager.py | 2 +- src/store/managers/manager.py | 8 ++++---- src/store/managers/online_cover_manager.py | 4 ++-- src/store/managers/sgdb_manager.py | 4 ++-- src/store/managers/steam_api_manager.py | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/store/managers/display_manager.py b/src/store/managers/display_manager.py index e79746e..2d1377a 100644 --- a/src/store/managers/display_manager.py +++ b/src/store/managers/display_manager.py @@ -8,7 +8,7 @@ from src.store.managers.manager import Manager class DisplayManager(Manager): """Manager in charge of adding a game to the UI""" - run_after = set((SteamAPIManager, SGDBManager)) + run_after = (SteamAPIManager, SGDBManager) def manager_logic(self, game: Game, _additional_data: dict) -> None: # TODO decouple a game from its widget diff --git a/src/store/managers/file_manager.py b/src/store/managers/file_manager.py index 66cebc4..7a72e5d 100644 --- a/src/store/managers/file_manager.py +++ b/src/store/managers/file_manager.py @@ -6,7 +6,7 @@ from src.store.managers.steam_api_manager import SteamAPIManager class FileManager(AsyncManager): """Manager in charge of saving a game to a file""" - run_after = set((SteamAPIManager,)) + run_after = (SteamAPIManager,) def manager_logic(self, game: Game, _additional_data: dict) -> None: game.save() diff --git a/src/store/managers/itch_cover_manager.py b/src/store/managers/itch_cover_manager.py index abf61b2..03bfcd1 100644 --- a/src/store/managers/itch_cover_manager.py +++ b/src/store/managers/itch_cover_manager.py @@ -17,8 +17,8 @@ from src.utils.save_cover import resize_cover, save_cover class ItchCoverManager(Manager): """Manager in charge of downloading the game's cover from itch.io""" - run_after = set((LocalCoverManager,)) - retryable_on = set((HTTPError, SSLError)) + run_after = (LocalCoverManager,) + retryable_on = (HTTPError, SSLError) def manager_logic(self, game: Game, additional_data: dict) -> None: # Get the first matching cover url diff --git a/src/store/managers/local_cover_manager.py b/src/store/managers/local_cover_manager.py index a191f36..eba3abc 100644 --- a/src/store/managers/local_cover_manager.py +++ b/src/store/managers/local_cover_manager.py @@ -9,7 +9,7 @@ from src.utils.save_cover import save_cover, resize_cover class LocalCoverManager(Manager): """Manager in charge of adding the local cover image of the game""" - run_after = set((SteamAPIManager,)) + run_after = (SteamAPIManager,) def manager_logic(self, game: Game, additional_data: dict) -> None: # Ensure that the cover path is in the additional data diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index e73eb5a..927d8ca 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -2,7 +2,7 @@ import logging from abc import abstractmethod from threading import Lock from time import sleep -from typing import Any, Callable +from typing import Any, Callable, Container from src.game import Game @@ -16,11 +16,11 @@ class Manager: * May be retried on some specific error types """ - run_after: set[type["Manager"]] = set() + run_after: Container[type["Manager"]] = tuple() blocking: bool = True - retryable_on: set[type[Exception]] = set() - continue_on: set[type[Exception]] = set() + retryable_on: Container[type[Exception]] = tuple() + continue_on: Container[type[Exception]] = tuple() retry_delay: int = 3 max_tries: int = 3 diff --git a/src/store/managers/online_cover_manager.py b/src/store/managers/online_cover_manager.py index b4e5bf0..e9b8411 100644 --- a/src/store/managers/online_cover_manager.py +++ b/src/store/managers/online_cover_manager.py @@ -13,8 +13,8 @@ from src.utils.save_cover import resize_cover, save_cover class OnlineCoverManager(Manager): """Manager that downloads game covers from URLs""" - run_after = set((LocalCoverManager,)) - retryable_on = set((HTTPError, SSLError)) + run_after = (LocalCoverManager,) + retryable_on = (HTTPError, SSLError) def manager_logic(self, game: Game, additional_data: dict) -> None: # Ensure that we have a cover to download diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index c1831fc..5844204 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -13,8 +13,8 @@ from src.utils.steamgriddb import SGDBAuthError, SGDBHelper class SGDBManager(AsyncManager): """Manager in charge of downloading a game's cover from steamgriddb""" - run_after = set((SteamAPIManager, LocalCoverManager, ItchCoverManager)) - retryable_on = set((HTTPError, SSLError, ConnectionError, JSONDecodeError)) + run_after = (SteamAPIManager, LocalCoverManager, ItchCoverManager) + retryable_on = (HTTPError, SSLError, ConnectionError, JSONDecodeError) def manager_logic(self, game: Game, _additional_data: dict) -> None: try: diff --git a/src/store/managers/steam_api_manager.py b/src/store/managers/steam_api_manager.py index faad9d7..efe82e2 100644 --- a/src/store/managers/steam_api_manager.py +++ b/src/store/managers/steam_api_manager.py @@ -13,7 +13,7 @@ from src.utils.steam import ( class SteamAPIManager(AsyncManager): """Manager in charge of completing a game's data from the Steam API""" - retryable_on = set((HTTPError, SSLError)) + retryable_on = (HTTPError, SSLError) steam_api_helper: SteamAPIHelper = None steam_rate_limiter: SteamRateLimiter = None From 8eb2a270f651f4d2fa6b66460884967fad140165 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 10 Jun 2023 14:51:52 +0200 Subject: [PATCH 120/173] =?UTF-8?q?=F0=9F=8E=A8=20Improved=20manager=20err?= =?UTF-8?q?or=20handling=20structure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/managers/manager.py | 56 +++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 927d8ca..9444ea9 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -58,42 +58,54 @@ class Manager: * May raise other exceptions that will be reported """ - def execute_resilient_manager_logic( - self, game: Game, additional_data: dict, try_index: int = 0 - ) -> None: - """Execute the manager logic and handle its errors by reporting them or retrying""" - try: - self.manager_logic(game, additional_data) - except Exception as error: # pylint: disable=broad-exception-caught - logging_args = ( + def execute_resilient_manager_logic(self, game: Game, additional_data: dict): + """Handle errors (retry, ignore or raise) that occur the manager logic""" + + # Keep track of the number of tries + tries = 1 + + def handle_error(error: Exception): + nonlocal tries + + log_args = ( type(error).__name__, self.name, f"{game.name} ({game.game_id})", ) + + out_of_retries_format = "Out of retries dues to %s in %s for %s" + retrying_format = "Retrying %s in %s for %s" + unretryable_format = "Unretryable %s in %s for %s" + if error in self.continue_on: # Handle skippable errors (skip silently) return + if error in self.retryable_on: - if try_index < self.max_tries: - # Handle retryable errors - logging.error("Retrying %s in %s for %s", *logging_args) - sleep(self.retry_delay) - self.execute_resilient_manager_logic( - game, additional_data, try_index + 1 - ) - else: + if tries > self.max_tries: # Handle being out of retries - logging.error( - "Out of retries dues to %s in %s for %s", *logging_args - ) + logging.error(out_of_retries_format, *log_args) self.report_error(error) + else: + # Handle retryable errors + logging.error(retrying_format, *log_args) + sleep(self.retry_delay) + tries += 1 + try_manager_logic() + else: # Handle unretryable errors - logging.error( - "Unretryable %s in %s for %s", *logging_args, exc_info=error - ) + logging.error(unretryable_format, *log_args, exc_info=error) self.report_error(error) + def try_manager_logic(): + try: + self.manager_logic(game, additional_data) + except Exception as error: # pylint: disable=broad-exception-caught + handle_error(error) + + try_manager_logic() + def process_game( self, game: Game, additional_data: dict, callback: Callable[["Manager"], Any] ) -> None: From 3a0911e7425b658c522456cb500e00ef1a2be5ef Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 10 Jun 2023 15:31:54 +0200 Subject: [PATCH 121/173] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20UI=20not=20updat?= =?UTF-8?q?ing=20on=20some=20game=20actions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/details_window.py | 18 +++++++++++++++--- src/game.py | 3 +++ src/preferences.py | 2 ++ src/window.py | 1 + 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/details_window.py b/src/details_window.py index 767110e..0aedb44 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -17,20 +17,21 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +import logging import os import shlex from time import time from gi.repository import Adw, Gio, GLib, Gtk from PIL import Image +from requests.exceptions import HTTPError, SSLError -# TODO use SGDBHelper from src import shared from src.game import Game from src.game_cover import GameCover from src.utils.create_dialog import create_dialog from src.utils.save_cover import resize_cover, save_cover -from src.utils.steamgriddb import SGDBSave +from src.utils.steamgriddb import SGDBError, SGDBHelper @Gtk.Template(resource_path=shared.PREFIX + "/gtk/details_window.ui") @@ -202,9 +203,20 @@ class DetailsWindow(Adw.Window): ) self.game.save() + self.game.update() + # Try to get a cover if none is present + # TODO inform the user + # TODO wrap in a task and mark loading if not self.game_cover.get_pixbuf(): - SGDBSave({self.game}) + print("test 1212") + sgdb = SGDBHelper() + try: + sgdb.conditionaly_update_cover(self.game) + except SGDBError as error: + logging.error("Could not update cover", exc_info=error) + except (HTTPError, SSLError, ConnectionError): + logging.warning("Could not connect to SteamGridDB") self.game_cover.pictures.remove(self.cover) diff --git a/src/game.py b/src/game.py index 1b4aeef..77f0fec 100644 --- a/src/game.py +++ b/src/game.py @@ -174,6 +174,7 @@ class Game(Gtk.Box): def launch(self): self.last_played = int(time()) self.save() + self.update() string = ( self.executable @@ -206,6 +207,7 @@ class Game(Gtk.Box): def toggle_hidden(self, toast=True): self.hidden = not self.hidden self.save() + self.update() if self.win.stack.get_visible_child() == self.win.details_view: self.win.on_go_back_action() @@ -221,6 +223,7 @@ class Game(Gtk.Box): # Add "removed=True" to the game properties so it can be deleted on next init self.removed = True self.save() + self.update() if self.win.stack.get_visible_child() == self.win.details_view: self.win.on_go_back_action() diff --git a/src/preferences.py b/src/preferences.py index 784fab9..9010aee 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -245,6 +245,7 @@ class PreferencesWindow(Adw.PreferencesWindow): for game in self.removed_games: game.removed = False game.save() + game.update() self.removed_games = set() self.toast.dismiss() @@ -256,6 +257,7 @@ class PreferencesWindow(Adw.PreferencesWindow): game.removed = True game.save() + game.update() if self.win.stack.get_visible_child() == self.win.details_view: self.win.on_go_back_action() diff --git a/src/window.py b/src/window.py index d2b8ddf..0135a24 100644 --- a/src/window.py +++ b/src/window.py @@ -339,6 +339,7 @@ class CartridgesWindow(Adw.ApplicationWindow): elif undo == "remove": game.removed = False game.save() + game.update() self.toasts[(game, undo)].dismiss() self.toasts.pop((game, undo)) From dcd4357e57edad1716ee64da64387511a5ca2421 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 10 Jun 2023 16:22:09 +0200 Subject: [PATCH 122/173] =?UTF-8?q?=F0=9F=8E=A8=20No=20longer=20using=20SG?= =?UTF-8?q?DBSave?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Details window uses a Pipeline with SGDBTask - Store saves managers in a type: instance dict - Removed SGDBSave --- src/details_window.py | 52 +++++++++++----- src/store/store.py | 10 +-- src/utils/steamgriddb.py | 131 --------------------------------------- 3 files changed, 43 insertions(+), 150 deletions(-) diff --git a/src/details_window.py b/src/details_window.py index 0aedb44..89e7772 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -17,21 +17,21 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -import logging import os import shlex from time import time from gi.repository import Adw, Gio, GLib, Gtk from PIL import Image -from requests.exceptions import HTTPError, SSLError from src import shared from src.game import Game from src.game_cover import GameCover +from src.store.managers.sgdb_manager import SGDBManager +from src.store.pipeline import Pipeline from src.utils.create_dialog import create_dialog from src.utils.save_cover import resize_cover, save_cover -from src.utils.steamgriddb import SGDBError, SGDBHelper +from src.utils.steamgriddb import SGDBAuthError @Gtk.Template(resource_path=shared.PREFIX + "/gtk/details_window.ui") @@ -205,24 +205,48 @@ class DetailsWindow(Adw.Window): self.game.save() self.game.update() - # Try to get a cover if none is present - # TODO inform the user - # TODO wrap in a task and mark loading + # Get a cover from SGDB if none is present if not self.game_cover.get_pixbuf(): - print("test 1212") - sgdb = SGDBHelper() - try: - sgdb.conditionaly_update_cover(self.game) - except SGDBError as error: - logging.error("Could not update cover", exc_info=error) - except (HTTPError, SSLError, ConnectionError): - logging.warning("Could not connect to SteamGridDB") + self.game.set_loading(1) + sgdb_manager: SGDBManager = shared.store.managers[SGDBManager] + sgdb_manager.reset_cancellable() + pipeline = Pipeline(self.game, {}, (sgdb_manager,)) + pipeline.connect("advanced", self.update_cover_callback) + pipeline.advance() self.game_cover.pictures.remove(self.cover) self.close() self.win.show_details_view(self.game) + def update_cover_callback(self, pipeline: Pipeline): + # Check that managers are done + if not pipeline.is_done: + return + + # Set the game as not loading + self.game.set_loading(-1) + self.game.update() + + # Handle errors that occured + errors = [] + for manager in pipeline.done: + errors.extend(manager.collect_errors()) + for error in errors: + # On auth error, inform the user + if isinstance(error, SGDBAuthError): + create_dialog( + shared.win, + _("Couldn't Connect to SteamGridDB"), + str(error), + "open_preferences", + _("Preferences"), + ).connect("response", self.update_cover_error_response) + + def update_cover_error_response(self, _widget, response): + if response == "open_preferences": + shared.win.get_application().on_preferences_action(page_name="sgdb") + def focus_executable(self, *_args): self.set_focus(self.executable) diff --git a/src/store/store.py b/src/store/store.py index 14ad917..744491e 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -7,18 +7,18 @@ from src.store.pipeline import Pipeline class Store: """Class in charge of handling games being added to the app.""" - managers: set[Manager] + managers: dict[type[Manager], Manager] pipelines: dict[str, Pipeline] games: dict[str, Game] def __init__(self) -> None: - self.managers = set() + self.managers = {} self.games = {} self.pipelines = {} def add_manager(self, manager: Manager): - """Add a manager class that will run when games are added""" - self.managers.add(manager) + """Add a manager that will run when games are added""" + self.managers[type(manager)] = manager def add_game( self, game: Game, additional_data: dict, replace=False @@ -51,7 +51,7 @@ class Store: return None # Run the pipeline for the game - pipeline = Pipeline(game, additional_data, self.managers) + pipeline = Pipeline(game, additional_data, self.managers.values()) self.games[game.game_id] = game self.pipelines[game.game_id] = pipeline pipeline.advance() diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index 5351898..6398f75 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -6,7 +6,6 @@ from gi.repository import Gio from requests.exceptions import HTTPError from src import shared -from src.utils.create_dialog import create_dialog from src.utils.save_cover import resize_cover, save_cover @@ -136,133 +135,3 @@ class SGDBHelper: sgdb_id, ) raise SGDBNoImageFoundError() - - -# Current steps to save image for N games -# Create a task for every game -# Call update_cover -# If using sgdb and (prefer or no image) and not blacklisted -# Search for game -# Get image from sgdb (animated if preferred and found, or still) -# Exit task and enter task_done -# If error, create popup - - -class SGDBSave: - def __init__(self, games, importer=None): - self.win = shared.win - self.importer = importer - self.exception = None - - # Wrap the function in another one as Gio.Task.run_in_thread does not allow for passing args - def create_func(game): - def wrapper(task, *_args): - self.update_cover( - task, - game, - ) - - return wrapper - - for game in games: - Gio.Task.new(None, None, self.task_done).run_in_thread(create_func(game)) - - def update_cover(self, task, game): - game.set_loading(1) - - if ( - not ( - shared.schema.get_boolean("sgdb") - and ( - (shared.schema.get_boolean("sgdb-prefer")) - or not ( - (shared.covers_dir / f"{game.game_id}.gif").is_file() - or (shared.covers_dir / f"{game.game_id}.tiff").is_file() - ) - ) - ) - or game.blacklisted - ): - task.return_value(game) - return - - url = "https://www.steamgriddb.com/api/v2/" - headers = {"Authorization": f'Bearer {shared.schema.get_string("sgdb-key")}'} - - try: - search_result = requests.get( - f"{url}search/autocomplete/{game.name}", - headers=headers, - timeout=5, - ) - if search_result.status_code != 200: - self.exception = str( - search_result.json()["errors"][0] - if "errors" in tuple(search_result.json()) - else search_result.status_code - ) - search_result.raise_for_status() - except requests.exceptions.RequestException: - task.return_value(game) - return - - response = None - - try: - if shared.schema.get_boolean("sgdb-animated"): - try: - grid = requests.get( - f'{url}grids/game/{search_result.json()["data"][0]["id"]}?dimensions=600x900&types=animated', - headers=headers, - timeout=5, - ) - response = requests.get(grid.json()["data"][0]["url"], timeout=5) - except IndexError: - pass - if not response: - grid = requests.get( - f'{url}grids/game/{search_result.json()["data"][0]["id"]}?dimensions=600x900', - headers=headers, - timeout=5, - ) - response = requests.get(grid.json()["data"][0]["url"], timeout=5) - except (requests.exceptions.RequestException, IndexError): - task.return_value(game) - return - - tmp_file = Gio.File.new_tmp()[0] - Path(tmp_file.get_path()).write_bytes(response.content) - - save_cover( - game.game_id, - resize_cover(tmp_file.get_path()), - ) - - task.return_value(game) - - def task_done(self, _task, result): - if self.importer: - self.importer.queue -= 1 - self.importer.done() - self.importer.sgdb_exception = self.exception - - if self.exception and not self.importer: - create_dialog( - self.win, - _("Couldn't Connect to SteamGridDB"), - self.exception, - "open_preferences", - _("Preferences"), - ).connect("response", self.response) - - game = result.propagate_value()[1] - game.set_loading(-1) - - if self.importer: - game.save() - else: - game.update() - - def response(self, _widget, response): - if response == "open_preferences": - self.win.get_application().on_preferences_action(page_name="sgdb") From 0db636b375dc8ee1e91ae1ba05ba7c860084842f Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Sat, 10 Jun 2023 19:13:25 +0200 Subject: [PATCH 123/173] Port bottles fix from main --- src/importer/sources/bottles_source.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index d77bee3..b664651 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -41,11 +41,20 @@ class BottlesSourceIterator(SourceIterator): game = Game(values, allow_side_effects=False) # Get official cover path + try: + # This will not work if both Cartridges and Bottles are installed via Flatpak + # as Cartridges can't access directories picked via Bottles' file picker portal + bottles_location = Path( + yaml.safe_load( + (self.source.location / "data.yml").read_text("utf-8") + )["custom_bottles_path"] + ) + except (FileNotFoundError, KeyError): + bottles_location = self.source.location / "bottles" + bottle_path = entry["bottle"]["path"] image_name = entry["thumbnail"].split(":")[1] - image_path = ( - self.source.location / "bottles" / bottle_path / "grids" / image_name - ) + image_path = bottles_location / bottle_path / "grids" / image_name additional_data = {"local_image_path": image_path} # Produce game From 5a7ada1c0ef3b3cb941380f14f143b0df5b19440 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Sat, 10 Jun 2023 19:29:28 +0200 Subject: [PATCH 124/173] Trim .pylintrc --- .pylintrc | 608 +----------------------------------------------------- 1 file changed, 1 insertion(+), 607 deletions(-) diff --git a/.pylintrc b/.pylintrc index 2dbc403..b4518b4 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,426 +1,10 @@ [MAIN] -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Clear in-memory caches upon conclusion of linting. Useful if running pylint -# in a server-like mode. -clear-cache-post-run=no - -# Load and enable all available extensions. Use --list-extensions to see a list -# all available extensions. -#enable-all-extensions= - -# In error mode, messages with a category besides ERROR or FATAL are -# suppressed, and no reports are done by default. Error mode is compatible with -# disabling specific errors. -#errors-only= - -# Always return a 0 (non-error) status code, even if lint errors are found. -# This is primarily useful in continuous integration scripts. -#exit-zero= - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. -extension-pkg-allow-list= - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. (This is an alternative name to extension-pkg-allow-list -# for backward compatibility.) -extension-pkg-whitelist= - -# Return non-zero exit code if any of these messages/categories are detected, -# even if score is above --fail-under value. Syntax same as enable. Messages -# specified are enabled, while categories only check already-enabled messages. -fail-on= - -# Specify a score threshold under which the program will exit with error. -fail-under=10 - -# Interpret the stdin as a python script, whose filename needs to be passed as -# the module_or_package argument. -#from-stdin= - -# Files or directories to be skipped. They should be base names, not paths. ignore=importers -# Add files or directories matching the regular expressions patterns to the -# ignore-list. The regex matches against paths and can be in Posix or Windows -# format. Because '\\' represents the directory delimiter on Windows systems, -# it can't be used as an escape character. -ignore-paths= - -# Files or directories matching the regular expression patterns are skipped. -# The regex matches against base names, not paths. The default value ignores -# Emacs file locks -ignore-patterns=^\.# - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis). It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use, and will cap the count on Windows to -# avoid hangs. -jobs=1 - -# Control the amount of potential inferred values when inferring a single -# object. This can help the performance when dealing with large functions or -# complex, nested conditions. -limit-inference-results=100 - -# List of plugins (as comma separated values of python module names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Minimum Python version to use for version dependent checks. Will default to -# the version used to run pylint. -py-version=3.11 - -# Discover python modules and packages in the file system subtree. -recursive=no - -# Add paths to the list of the source roots. Supports globbing patterns. The -# source root is an absolute path or a path relative to the current working -# directory used to determine a package namespace for modules located under the -# source root. -source-roots= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - -# In verbose mode, extra non-checker-related info will be displayed. -#verbose= - - -[BASIC] - -# Naming style matching correct argument names. -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style. If left empty, argument names will be checked with the set -# naming style. -#argument-rgx= - -# Naming style matching correct attribute names. -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style. If left empty, attribute names will be checked with the set naming -# style. -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma. -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - -# Bad variable names regexes, separated by a comma. If names match any regex, -# they will always be refused -bad-names-rgxs= - -# Naming style matching correct class attribute names. -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. If left empty, class attribute names will be checked -# with the set naming style. -#class-attribute-rgx= - -# Naming style matching correct class constant names. -class-const-naming-style=UPPER_CASE - -# Regular expression matching correct class constant names. Overrides class- -# const-naming-style. If left empty, class constant names will be checked with -# the set naming style. -#class-const-rgx= - -# Naming style matching correct class names. -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming- -# style. If left empty, class names will be checked with the set naming style. -#class-rgx= - -# Naming style matching correct constant names. -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style. If left empty, constant names will be checked with the set naming -# style. -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names. -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style. If left empty, function names will be checked with the set -# naming style. -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma. -good-names=i, - j, - k, - ex, - Run, - _ - -# Good variable names regexes, separated by a comma. If names match any regex, -# they will always be accepted -good-names-rgxs= - -# Include a hint for the correct naming format with invalid-name. -include-naming-hint=no - -# Naming style matching correct inline iteration names. -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. If left empty, inline iteration names will be checked -# with the set naming style. -#inlinevar-rgx= - -# Naming style matching correct method names. -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style. If left empty, method names will be checked with the set naming style. -#method-rgx= - -# Naming style matching correct module names. -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style. If left empty, module names will be checked with the set naming style. -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -# These decorators are taken in consideration only for invalid-name. -property-classes=abc.abstractproperty - -# Regular expression matching correct type alias names. If left empty, type -# alias names will be checked with the set naming style. -#typealias-rgx= - -# Regular expression matching correct type variable names. If left empty, type -# variable names will be checked with the set naming style. -#typevar-rgx= - -# Naming style matching correct variable names. -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style. If left empty, variable names will be checked with the set -# naming style. -#variable-rgx= - - -[CLASSES] - -# Warn about protected attribute access inside special methods -check-protected-access-in-special-methods=no - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp, - asyncSetUp, - __post_init__ - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[DESIGN] - -# List of regular expressions of class ancestor names to ignore when counting -# public methods (see R0903) -exclude-too-few-public-methods= - -# List of qualified class names to ignore when counting class parents (see -# R0901) -ignored-parents= - -# Maximum number of arguments for function / method. -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in an if statement (see R0916). -max-bool-expr=5 - -# Maximum number of branch for function / method body. -max-branches=12 - -# Maximum number of locals for function / method body. -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body. -max-returns=6 - -# Maximum number of statements in function / method body. -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when caught. -overgeneral-exceptions=builtins.BaseException,builtins.Exception - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module. -max-module-lines=1000 - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[IMPORTS] - -# List of modules that can be imported at any level, not just the top level -# one. -allow-any-import-level= - -# Allow explicit reexports by alias from a package __init__. -allow-reexport-from-package=no - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Deprecated modules which should not be used, separated by a comma. -deprecated-modules= - -# Output a graph (.gv or any supported image format) of external dependencies -# to the given file (report RP0402 must not be disabled). -ext-import-graph= - -# Output a graph (.gv or any supported image format) of all (i.e. internal and -# external) dependencies to the given file (report RP0402 must not be -# disabled). -import-graph= - -# Output a graph (.gv or any supported image format) of internal dependencies -# to the given file (report RP0402 must not be disabled). -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - -# Couples of modules and preferred modules, separated by a comma. -preferred-modules= - - -[LOGGING] - -# The type of string formatting that logging methods do. `old` means using % -# formatting, `new` is for `{}` formatting. -logging-format-style=old - -# Logging modules to check that the string format arguments are in logging -# function parameter format. -logging-modules=logging - [MESSAGES CONTROL] -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, -# UNDEFINED. -confidence=HIGH, - CONTROL_FLOW, - INFERENCE, - INFERENCE_FAILURE, - UNDEFINED - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then re-enable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". disable=raw-checker-failed, bad-inline-option, locally-disabled, @@ -436,202 +20,12 @@ disable=raw-checker-failed, relative-beyond-top-level, import-error -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[METHOD_ARGS] - -# List of qualified names (i.e., library.method) which require a timeout -# parameter e.g. 'requests.api.get,requests.api.post' -timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - -# Regular expression of note tags to take in consideration. -notes-rgx= - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=sys.exit,argparse.parse_error - - -[REPORTS] - -# Python expression which should return a score less than or equal to 10. You -# have access to the variables 'fatal', 'error', 'warning', 'refactor', -# 'convention', and 'info' which contain the number of messages in each -# category, as well as 'statement' which is the total number of statements -# analyzed. This score is used by the global evaluation report (RP0004). -evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details. -msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. -# mypackage.mymodule.MyReporterClass. -#output-format= - -# Tells whether to display a full report or only the messages. -reports=no - -# Activate the evaluation score. -score=yes - - -[SIMILARITIES] - -# Comments are removed from the similarity computation -ignore-comments=yes - -# Docstrings are removed from the similarity computation -ignore-docstrings=yes - -# Imports are removed from the similarity computation -ignore-imports=yes - -# Signatures are removed from the similarity computation -ignore-signatures=yes - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. No available dictionaries : You need to install -# both the python package and the system dependency for enchant to work.. -spelling-dict= - -# List of comma separated words that should be considered directives if they -# appear at the beginning of a comment and should not be checked. -spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains the private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to the private dictionary (see the -# --spelling-private-dict-file option) instead of raising a message. -spelling-store-unknown-words=no - - -[STRING] - -# This flag controls whether inconsistent-quotes generates a warning when the -# character used as a quote delimiter is used inconsistently within a module. -check-quote-consistency=no - -# This flag controls whether the implicit-str-concat should generate a warning -# on implicit string concatenation in sequences defined over several lines. -check-str-concat-over-line-jumps=no - [TYPECHECK] -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether to warn about missing members when the owner of the attribute -# is inferred to be None. -ignore-none=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of symbolic message names to ignore for Mixin members. -ignored-checks-for-mixins=no-member, - not-async-context-manager, - not-context-manager, - attribute-defined-outside-init - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. ignored-classes=Child -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - -# Regex pattern to define which classes are considered mixins. -mixin-class-rgx=.*[Mm]ixin - -# List of decorators that change the signature of a decorated function. -signature-mutators= - [VARIABLES] -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid defining new builtins when possible. -additional-builtins=_ - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of names allowed to shadow builtins -allowed-redefined-builtins= - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expected to -# not be used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io +additional-builtins=_ \ No newline at end of file From de3ef5314855636b56d33ce805ead2b0476c938d Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Sat, 10 Jun 2023 19:34:00 +0200 Subject: [PATCH 125/173] Make pylint happy --- src/details_window.py | 2 +- src/game.py | 2 +- src/game_cover.py | 2 +- src/importer/importer.py | 2 +- src/importer/sources/bottles_source.py | 2 +- src/importer/sources/heroic_source.py | 2 +- src/importer/sources/itch_source.py | 2 +- src/importer/sources/legendary_source.py | 2 +- src/importer/sources/lutris_source.py | 2 +- src/importer/sources/source.py | 2 +- src/importer/sources/steam_source.py | 2 +- src/importers/bottles_importer.py | 2 +- src/importers/heroic_importer.py | 2 +- src/importers/itch_importer.py | 2 +- src/importers/lutris_importer.py | 2 +- src/importers/steam_importer.py | 2 +- src/main.py | 2 +- src/preferences.py | 2 +- src/store/managers/display_manager.py | 2 +- src/store/managers/itch_cover_manager.py | 2 +- src/store/store.py | 2 +- src/utils/decorators.py | 2 +- src/utils/save_cover.py | 2 +- src/utils/steam.py | 2 +- src/utils/steamgriddb.py | 2 +- src/window.py | 2 +- 26 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/details_window.py b/src/details_window.py index 89e7772..86c3a83 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -24,7 +24,7 @@ from time import time from gi.repository import Adw, Gio, GLib, Gtk from PIL import Image -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.game_cover import GameCover from src.store.managers.sgdb_manager import SGDBManager diff --git a/src/game.py b/src/game.py index 77f0fec..090dedb 100644 --- a/src/game.py +++ b/src/game.py @@ -27,7 +27,7 @@ from time import time from gi.repository import Adw, Gtk -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.game_cover import GameCover diff --git a/src/game_cover.py b/src/game_cover.py index cca0eba..793bd8c 100644 --- a/src/game_cover.py +++ b/src/game_cover.py @@ -20,7 +20,7 @@ from gi.repository import GdkPixbuf, Gio, GLib from PIL import Image, ImageFilter, ImageStat -from src import shared +from src import shared # pylint: disable=no-name-in-module class GameCover: diff --git a/src/importer/importer.py b/src/importer/importer.py index eae8565..ba0b2e4 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -2,7 +2,7 @@ import logging from gi.repository import Adw, Gtk -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.utils.task import Task from src.store.pipeline import Pipeline diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index b664651..c98d431 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -3,7 +3,7 @@ from time import time import yaml -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.importer.sources.source import ( SourceIterationResult, diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 01faf24..76c34d6 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -6,7 +6,7 @@ from pathlib import Path from time import time from typing import Optional, TypedDict -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.importer.sources.source import ( URLExecutableSource, diff --git a/src/importer/sources/itch_source.py b/src/importer/sources/itch_source.py index 29ef507..7d282dc 100644 --- a/src/importer/sources/itch_source.py +++ b/src/importer/sources/itch_source.py @@ -2,7 +2,7 @@ from pathlib import Path from sqlite3 import connect from time import time -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.importer.sources.source import ( SourceIterationResult, diff --git a/src/importer/sources/legendary_source.py b/src/importer/sources/legendary_source.py index 91f8b0a..150f518 100644 --- a/src/importer/sources/legendary_source.py +++ b/src/importer/sources/legendary_source.py @@ -5,7 +5,7 @@ from pathlib import Path from time import time from typing import Generator -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.importer.sources.source import Source, SourceIterationResult, SourceIterator from src.utils.decorators import ( diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 7a2e51d..ddf6778 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,7 +1,7 @@ from sqlite3 import connect from time import time -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.importer.sources.source import ( SourceIterationResult, diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index 16963dc..73b3874 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -4,7 +4,7 @@ from collections.abc import Iterable, Iterator from pathlib import Path from typing import Generator, Any -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.game import Game # Type of the data returned by iterating on a Source diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 7e724ab..68aa861 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -3,7 +3,7 @@ from pathlib import Path from time import time from typing import Iterable -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.importer.sources.source import ( SourceIterationResult, diff --git a/src/importers/bottles_importer.py b/src/importers/bottles_importer.py index dc431af..cba2f73 100644 --- a/src/importers/bottles_importer.py +++ b/src/importers/bottles_importer.py @@ -22,7 +22,7 @@ from time import time import yaml -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.utils.check_install import check_install diff --git a/src/importers/heroic_importer.py b/src/importers/heroic_importer.py index 549b4c1..619e647 100644 --- a/src/importers/heroic_importer.py +++ b/src/importers/heroic_importer.py @@ -23,7 +23,7 @@ from hashlib import sha256 from pathlib import Path from time import time -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.utils.check_install import check_install diff --git a/src/importers/itch_importer.py b/src/importers/itch_importer.py index ba8ad6b..6a19c91 100644 --- a/src/importers/itch_importer.py +++ b/src/importers/itch_importer.py @@ -26,7 +26,7 @@ from time import time import requests from gi.repository import GdkPixbuf, Gio -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.utils.check_install import check_install from src.utils.save_cover import resize_cover diff --git a/src/importers/lutris_importer.py b/src/importers/lutris_importer.py index a698caf..9fe43ab 100644 --- a/src/importers/lutris_importer.py +++ b/src/importers/lutris_importer.py @@ -22,7 +22,7 @@ from shutil import copyfile from sqlite3 import connect from time import time -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.utils.check_install import check_install diff --git a/src/importers/steam_importer.py b/src/importers/steam_importer.py index f0e12dd..09c504f 100644 --- a/src/importers/steam_importer.py +++ b/src/importers/steam_importer.py @@ -25,7 +25,7 @@ from time import time import requests from gi.repository import Gio -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.utils.check_install import check_install diff --git a/src/main.py b/src/main.py index 5cd8dc3..6fdea73 100644 --- a/src/main.py +++ b/src/main.py @@ -30,7 +30,7 @@ gi.require_version("Adw", "1") # pylint: disable=wrong-import-position from gi.repository import Adw, Gio, GLib, Gtk -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.details_window import DetailsWindow from src.game import Game from src.importer.importer import Importer diff --git a/src/preferences.py b/src/preferences.py index 9010aee..311ab98 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -24,7 +24,7 @@ from pathlib import Path from gi.repository import Adw, Gio, GLib, Gtk # pylint: disable=unused-import -from src import shared +from src import shared # pylint: disable=no-name-in-module # TODO use the new sources from src.importers.bottles_importer import bottles_installed diff --git a/src/store/managers/display_manager.py b/src/store/managers/display_manager.py index 2d1377a..13418d4 100644 --- a/src/store/managers/display_manager.py +++ b/src/store/managers/display_manager.py @@ -1,4 +1,4 @@ -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.store.managers.sgdb_manager import SGDBManager from src.store.managers.steam_api_manager import SteamAPIManager diff --git a/src/store/managers/itch_cover_manager.py b/src/store/managers/itch_cover_manager.py index 03bfcd1..3d96b0e 100644 --- a/src/store/managers/itch_cover_manager.py +++ b/src/store/managers/itch_cover_manager.py @@ -4,7 +4,7 @@ import requests from gi.repository import GdkPixbuf, Gio from requests.exceptions import HTTPError, SSLError -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.store.managers.local_cover_manager import LocalCoverManager from src.store.managers.manager import Manager diff --git a/src/store/store.py b/src/store/store.py index 744491e..c33ded2 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -1,4 +1,4 @@ -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.store.managers.manager import Manager from src.store.pipeline import Pipeline diff --git a/src/utils/decorators.py b/src/utils/decorators.py index 09763fa..4664808 100644 --- a/src/utils/decorators.py +++ b/src/utils/decorators.py @@ -2,7 +2,7 @@ from pathlib import Path from os import PathLike, environ from functools import wraps -from src import shared +from src import shared # pylint: disable=no-name-in-module def replaced_by_path(override: PathLike): # Decorator builder diff --git a/src/utils/save_cover.py b/src/utils/save_cover.py index a3e6c67..217cf7c 100644 --- a/src/utils/save_cover.py +++ b/src/utils/save_cover.py @@ -24,7 +24,7 @@ from shutil import copyfile from gi.repository import Gio from PIL import Image, ImageSequence -from src import shared +from src import shared # pylint: disable=no-name-in-module def resize_cover(cover_path=None, pixbuf=None): diff --git a/src/utils/steam.py b/src/utils/steam.py index c0a0677..703b315 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -6,7 +6,7 @@ from typing import TypedDict import requests from requests.exceptions import HTTPError -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.utils.rate_limiter import PickHistory, RateLimiter diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index 6398f75..db31c6c 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -5,7 +5,7 @@ import requests from gi.repository import Gio from requests.exceptions import HTTPError -from src import shared +from src import shared # pylint: disable=no-name-in-module from src.utils.save_cover import resize_cover, save_cover diff --git a/src/window.py b/src/window.py index 0135a24..59205c8 100644 --- a/src/window.py +++ b/src/window.py @@ -21,7 +21,7 @@ from datetime import datetime from gi.repository import Adw, GLib, Gtk -from src import shared +from src import shared # pylint: disable=no-name-in-module @Gtk.Template(resource_path=shared.PREFIX + "/gtk/window.ui") From d340e007e359144905fd085dbc3cec585f1c8a7c Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Sat, 10 Jun 2023 20:54:49 +0200 Subject: [PATCH 126/173] Remove replaced_by_env_path decorator --- src/importer/sources/bottles_source.py | 4 +--- src/importer/sources/heroic_source.py | 6 ++---- src/importer/sources/itch_source.py | 6 ++---- src/importer/sources/legendary_source.py | 10 ++-------- src/importer/sources/steam_source.py | 6 ++---- src/shared.py.in | 3 +++ src/utils/decorators.py | 18 ------------------ 7 files changed, 12 insertions(+), 41 deletions(-) diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index c98d431..5ce598d 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -11,7 +11,6 @@ from src.importer.sources.source import ( URLExecutableSource, ) from src.utils.decorators import ( - replaced_by_env_path, replaced_by_path, replaced_by_schema_key, ) @@ -72,7 +71,6 @@ class BottlesSource(URLExecutableSource): @property @replaced_by_schema_key @replaced_by_path("~/.var/app/com.usebottles.bottles/data/bottles/") - @replaced_by_env_path("XDG_DATA_HOME", "bottles/") - @replaced_by_path("~/.local/share/bottles/") + @replaced_by_path(shared.data_dir / "bottles") def location(self) -> Path: raise FileNotFoundError() diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 76c34d6..adcf34d 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -14,7 +14,6 @@ from src.importer.sources.source import ( SourceIterator, ) from src.utils.decorators import ( - replaced_by_env_path, replaced_by_path, replaced_by_schema_key, ) @@ -130,8 +129,7 @@ class HeroicSource(URLExecutableSource): @property @replaced_by_schema_key @replaced_by_path("~/.var/app/com.heroicgameslauncher.hgl/config/heroic/") - @replaced_by_env_path("XDG_CONFIG_HOME", "heroic/") - @replaced_by_path("~/.config/heroic/") - @replaced_by_env_path("appdata", "heroic/") + @replaced_by_path(shared.config_dir / "heroic") + @replaced_by_path(shared.appdata_dir / "heroic") def location(self) -> Path: raise FileNotFoundError() diff --git a/src/importer/sources/itch_source.py b/src/importer/sources/itch_source.py index 7d282dc..6d8a157 100644 --- a/src/importer/sources/itch_source.py +++ b/src/importer/sources/itch_source.py @@ -10,7 +10,6 @@ from src.importer.sources.source import ( URLExecutableSource, ) from src.utils.decorators import ( - replaced_by_env_path, replaced_by_path, replaced_by_schema_key, ) @@ -65,8 +64,7 @@ class ItchSource(URLExecutableSource): @property @replaced_by_schema_key @replaced_by_path("~/.var/app/io.itch.itch/config/itch/") - @replaced_by_env_path("XDG_DATA_HOME", "itch/") - @replaced_by_path("~/.config/itch") - @replaced_by_env_path("appdata", "itch/") + @replaced_by_path(shared.config_dir / "itch") + @replaced_by_path(shared.appdata_dir / "itch") def location(self) -> Path: raise FileNotFoundError() diff --git a/src/importer/sources/legendary_source.py b/src/importer/sources/legendary_source.py index 150f518..87f1f05 100644 --- a/src/importer/sources/legendary_source.py +++ b/src/importer/sources/legendary_source.py @@ -8,11 +8,7 @@ from typing import Generator from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.importer.sources.source import Source, SourceIterationResult, SourceIterator -from src.utils.decorators import ( - replaced_by_env_path, - replaced_by_path, - replaced_by_schema_key, -) +from src.utils.decorators import replaced_by_path, replaced_by_schema_key class LegendarySourceIterator(SourceIterator): @@ -79,8 +75,6 @@ class LegendarySource(Source): @property @replaced_by_schema_key - @replaced_by_env_path("XDG_CONFIG_HOME", "legendary/") - @replaced_by_path("~/.config/legendary/") - @replaced_by_path("~\\.config\\legendary\\") + @replaced_by_path(shared.config_dir / "legendary") def location(self) -> Path: raise FileNotFoundError() diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 68aa861..c275dc7 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -11,7 +11,6 @@ from src.importer.sources.source import ( URLExecutableSource, ) from src.utils.decorators import ( - replaced_by_env_path, replaced_by_path, replaced_by_schema_key, ) @@ -102,9 +101,8 @@ class SteamSource(URLExecutableSource): @property @replaced_by_schema_key @replaced_by_path("~/.var/app/com.valvesoftware.Steam/data/Steam/") - @replaced_by_env_path("XDG_DATA_HOME", "Steam/") + @replaced_by_path(shared.data_dir / "Steam") @replaced_by_path("~/.steam/") - @replaced_by_path("~/.local/share/Steam/") - @replaced_by_env_path("programfiles(x86)", "Steam") + @replaced_by_path(shared.programfiles32_dir / "Steam") def location(self): raise FileNotFoundError() diff --git a/src/shared.py.in b/src/shared.py.in index 6f08a29..d7d60a7 100644 --- a/src/shared.py.in +++ b/src/shared.py.in @@ -50,6 +50,9 @@ cache_dir = ( games_dir = data_dir / "cartridges" / "games" covers_dir = data_dir / "cartridges" / "covers" +appdata_dir = Path(os.getenv("appdata") or "C:\\Users\\Default\\AppData\\Roaming") +programfiles32_dir = Path(os.getenv("programfiles(x86)") or "C:\\Program Files (x86)") + scale_factor = max( monitor.get_scale_factor() for monitor in Gdk.Display.get_default().get_monitors() ) diff --git a/src/utils/decorators.py b/src/utils/decorators.py index 4664808..be572ec 100644 --- a/src/utils/decorators.py +++ b/src/utils/decorators.py @@ -22,24 +22,6 @@ def replaced_by_path(override: PathLike): # Decorator builder return decorator -def replaced_by_env_path(env_var_name: str, suffix: PathLike | None = None): - """Replace the method's returned path with a path whose root is the env variable""" - - def decorator(original_function): # Built decorator (closure) - @wraps(original_function) - def wrapper(*args, **kwargs): # func's override - try: - env_var = environ[env_var_name] - except KeyError: - return original_function(*args, **kwargs) - override = Path(env_var) / suffix - return replaced_by_path(override)(original_function)(*args, **kwargs) - - return wrapper - - return decorator - - def replaced_by_schema_key(original_method): # Built decorator (closure) """ Replace the original method's value by the path pointed at in the schema From 6a099b2bdd9bd147c681cf72544a0c8ef6a209ac Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sun, 11 Jun 2023 21:11:53 +0200 Subject: [PATCH 127/173] =?UTF-8?q?=E2=9C=A8=20New=20logging=20system?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/logging/color_log_formatter.py | 25 +++++++++++++ src/logging/setup.py | 58 ++++++++++++++++++++++++++++++ src/main.py | 18 ++-------- src/meson.build | 1 + 4 files changed, 87 insertions(+), 15 deletions(-) create mode 100644 src/logging/color_log_formatter.py create mode 100644 src/logging/setup.py diff --git a/src/logging/color_log_formatter.py b/src/logging/color_log_formatter.py new file mode 100644 index 0000000..54a873b --- /dev/null +++ b/src/logging/color_log_formatter.py @@ -0,0 +1,25 @@ +from logging import Formatter, LogRecord + + +class ColorLogFormatter(Formatter): + """Formatter that outputs logs in a colored format""" + + RESET = "\033[0m" + DIM = "\033[2m" + BOLD = "\033[1m" + RED = "\033[31m" + YELLOW = "\033[33m" + + def format(self, record: LogRecord): + super_format = super().format(record) + match record.levelname: + case "CRITICAL": + return self.BOLD + self.RED + super_format + self.RESET + case "ERROR": + return self.RED + super_format + self.RESET + case "WARNING": + return self.YELLOW + super_format + self.RESET + case "DEBUG": + return self.DIM + super_format + self.RESET + case _other: + return super_format diff --git a/src/logging/setup.py b/src/logging/setup.py new file mode 100644 index 0000000..83dbc9b --- /dev/null +++ b/src/logging/setup.py @@ -0,0 +1,58 @@ +import logging.config as logging_dot_config +import os +from datetime import datetime + +from src import shared + + +def setup_logging(): + """Intitate the app's logging""" + + # Prepare log file + log_dir = shared.data_dir / "cartridges" / "logs" + log_dir.mkdir(exist_ok=True) + log_file = log_dir / f'{datetime.now().isoformat(timespec="seconds")}.log' + + # Define log levels + profile_main_log_level = "DEBUG" if shared.PROFILE == "development" else "WARNING" + profile_lib_log_level = "INFO" if shared.PROFILE == "development" else "WARNING" + main_log_level = os.environ.get("LOGLEVEL", profile_main_log_level).upper() + lib_log_level = os.environ.get("LIBLOGLEVEL", profile_lib_log_level).upper() + + # Load config + config = { + "version": 1, + "formatters": { + "console_formatter": { + "class": "src.logging.color_log_formatter.ColorLogFormatter", + "format": "%(name)s %(levelname)s - %(message)s", + }, + "file_formatter": { + "format": "%(asctime)s | %(name)s | %(levelname)s | %(message)s" + }, + }, + "handlers": { + "main_console_handler": { + "class": "logging.StreamHandler", + "formatter": "console_formatter", + "level": main_log_level, + }, + "lib_console_handler": { + "class": "logging.StreamHandler", + "formatter": "console_formatter", + "level": lib_log_level, + }, + "file_handler": { + "class": "logging.FileHandler", + "level": "DEBUG", + "filename": str(log_file), + "formatter": "file_formatter", + }, + }, + "loggers": { + "PIL": {"handlers": ["lib_console_handler", "file_handler"]}, + "urllib3": {"handlers": ["lib_console_handler", "file_handler"]}, + "root": {"handlers": ["main_console_handler", "file_handler"]}, + }, + } + logging_dot_config.dictConfig(config) diff --git a/src/main.py b/src/main.py index 6fdea73..f4f413f 100644 --- a/src/main.py +++ b/src/main.py @@ -19,7 +19,6 @@ import json import logging -import os import sys import gi @@ -40,6 +39,7 @@ from src.importer.sources.itch_source import ItchSource from src.importer.sources.legendary_source import LegendarySource from src.importer.sources.lutris_source import LutrisSource from src.importer.sources.steam_source import SteamSource +from src.logging.setup import setup_logging from src.preferences import PreferencesWindow from src.store.managers.display_manager import DisplayManager from src.store.managers.file_manager import FileManager @@ -257,20 +257,8 @@ class CartridgesApplication(Adw.Application): def main(version): # pylint: disable=unused-argument # Initiate logger - # (silence debug info from external libraries) - profile_base_log_level = "DEBUG" if shared.PROFILE == "development" else "WARNING" - profile_lib_log_level = "INFO" if shared.PROFILE == "development" else "WARNING" - base_log_level = os.environ.get("LOGLEVEL", profile_base_log_level).upper() - lib_log_level = os.environ.get("LIBLOGLEVEL", profile_lib_log_level).upper() - log_levels = { - None: base_log_level, - "PIL": lib_log_level, - "urllib3": lib_log_level, - } - logging.basicConfig() - for logger, level in log_levels.items(): - logging.getLogger(logger).setLevel(level) - + logging.basicConfig(level="DEBUG") + setup_logging() # Start app app = CartridgesApplication() return app.run(sys.argv) diff --git a/src/meson.build b/src/meson.build index c5ef74f..6631a62 100644 --- a/src/meson.build +++ b/src/meson.build @@ -12,6 +12,7 @@ install_subdir('importer', install_dir: moduledir) install_subdir('importers', install_dir: moduledir) install_subdir('utils', install_dir: moduledir) install_subdir('store', install_dir: moduledir) +install_subdir('logging', install_dir: moduledir) install_data( [ 'main.py', From 2798097623ff848742912b501091b3469b693e3c Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 12 Jun 2023 03:27:43 +0200 Subject: [PATCH 128/173] =?UTF-8?q?=F0=9F=8E=A8=20Improved=20logging=20cod?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Silenced unwanted library messages - Logging to file with a max size of 8MB When the file size is passed, a backup of the file is created, and the file gets truncated. There can only be one current file and one backup file. --- src/logging/setup.py | 62 +++++++++++++++++++++++++++++--------------- src/main.py | 7 +---- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/src/logging/setup.py b/src/logging/setup.py index 83dbc9b..c8184ff 100644 --- a/src/logging/setup.py +++ b/src/logging/setup.py @@ -1,6 +1,6 @@ +import logging import logging.config as logging_dot_config import os -from datetime import datetime from src import shared @@ -8,51 +8,71 @@ from src import shared def setup_logging(): """Intitate the app's logging""" - # Prepare log file + # Prepare the log file log_dir = shared.data_dir / "cartridges" / "logs" log_dir.mkdir(exist_ok=True) - log_file = log_dir / f'{datetime.now().isoformat(timespec="seconds")}.log' + log_file_path = log_dir / "cartridges.log" + log_file_max_size_bytes = 8 * 10**6 # 8 MB # Define log levels - profile_main_log_level = "DEBUG" if shared.PROFILE == "development" else "WARNING" + profile_app_log_level = "DEBUG" if shared.PROFILE == "development" else "INFO" profile_lib_log_level = "INFO" if shared.PROFILE == "development" else "WARNING" - main_log_level = os.environ.get("LOGLEVEL", profile_main_log_level).upper() + app_log_level = os.environ.get("LOGLEVEL", profile_app_log_level).upper() lib_log_level = os.environ.get("LIBLOGLEVEL", profile_lib_log_level).upper() - # Load config config = { "version": 1, "formatters": { - "console_formatter": { - "class": "src.logging.color_log_formatter.ColorLogFormatter", - "format": "%(name)s %(levelname)s - %(message)s", - }, "file_formatter": { "format": "%(asctime)s | %(name)s | %(levelname)s | %(message)s" }, + "console_formatter": { + "format": "%(name)s %(levelname)s - %(message)s", + "class": "src.logging.color_log_formatter.ColorLogFormatter", + }, }, "handlers": { - "main_console_handler": { + "file_handler": { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "file_formatter", + "level": "DEBUG", + "filename": log_file_path, + "maxBytes": log_file_max_size_bytes, + "backupCount": 1, + }, + "app_console_handler": { "class": "logging.StreamHandler", "formatter": "console_formatter", - "level": main_log_level, + "level": app_log_level, }, "lib_console_handler": { "class": "logging.StreamHandler", "formatter": "console_formatter", "level": lib_log_level, }, - "file_handler": { - "class": "logging.FileHandler", - "level": "DEBUG", - "filename": str(log_file), - "formatter": "file_formatter", - }, }, "loggers": { - "PIL": {"handlers": ["lib_console_handler", "file_handler"]}, - "urllib3": {"handlers": ["lib_console_handler", "file_handler"]}, - "root": {"handlers": ["main_console_handler", "file_handler"]}, + "PIL": { + "handlers": ["lib_console_handler", "file_handler"], + "propagate": False, + "level": "NOTSET", + }, + "urllib3": { + "handlers": ["lib_console_handler", "file_handler"], + "propagate": False, + "level": "NOTSET", + }, + }, + "root": { + "level": "NOTSET", + "handlers": ["app_console_handler", "file_handler"], }, } logging_dot_config.dictConfig(config) + + # Inform of the logging behaviour + logging.info("Logging profile: %s", shared.PROFILE) + logging.info("Console logging level for application: %s", app_log_level) + logging.info("Console logging level for libraries: %s", lib_log_level) + logging.info("Use env vars LOGLEVEL, LIBLOGLEVEL to override") + logging.info("All message levels are written to the log file") diff --git a/src/main.py b/src/main.py index f4f413f..8ac30dd 100644 --- a/src/main.py +++ b/src/main.py @@ -16,9 +16,7 @@ # along with this program. If not, see . # # SPDX-License-Identifier: GPL-3.0-or-later - import json -import logging import sys import gi @@ -255,10 +253,7 @@ class CartridgesApplication(Adw.Application): scope.add_action(simple_action) -def main(version): # pylint: disable=unused-argument - # Initiate logger - logging.basicConfig(level="DEBUG") +def main(_version): setup_logging() - # Start app app = CartridgesApplication() return app.run(sys.argv) From 68273d9217deeb00bfae42f9a2091eeb0b9048c3 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 12 Jun 2023 23:11:09 +0200 Subject: [PATCH 129/173] =?UTF-8?q?=F0=9F=8E=A8=20Improved=20logging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - One unlimited log file per session - Up to 3 session logs kept at any time - Log compressed via lzma --- src/logging/session_file_handler.py | 70 +++++++++++++++++++++++++++++ src/logging/setup.py | 21 ++++----- 2 files changed, 78 insertions(+), 13 deletions(-) create mode 100644 src/logging/session_file_handler.py diff --git a/src/logging/session_file_handler.py b/src/logging/session_file_handler.py new file mode 100644 index 0000000..5562735 --- /dev/null +++ b/src/logging/session_file_handler.py @@ -0,0 +1,70 @@ +import lzma +from io import StringIO +from logging import StreamHandler +from lzma import FORMAT_XZ, PRESET_DEFAULT +from os import PathLike +from pathlib import Path + + +class SessionFileHandler(StreamHandler): + """ + A logging handler that writes to a new file on every app restart. + The files are compressed and older sessions logs are kept up to a small limit. + """ + + backup_count: int + filename: Path + log_file: StringIO = None + + def create_dir(self) -> None: + """Create the log dir if needed""" + self.filename.parent.mkdir(exist_ok=True) + + def rotate_file(self, file: Path): + """Rotate a file's number suffix and remove it if it's too old""" + + # Skip non interesting dir entries + if not (file.is_file() and file.name.startswith(self.filename.name)): + return + + # Compute the new number suffix + suffixes = file.suffixes + has_number = len(suffixes) != len(self.filename.suffixes) + current_number = 0 if not has_number else int(suffixes[-1][1:]) + new_number = current_number + 1 + + # Rename with new number suffix + if has_number: + suffixes.pop() + suffixes.append(f".{new_number}") + stem = file.name.split(".", maxsplit=1)[0] + new_name = stem + "".join(suffixes) + print(f"Log file renamed: {file.name} -> {new_name}") + file = file.rename(file.with_name(new_name)) + + # Remove older files + if new_number > self.backup_count: + print(f"Log file deleted: {file.name}") + file.unlink() + return + + def rotate(self) -> None: + """Rotate the numbered suffix on the log files and remove old ones""" + files = list(self.filename.parent.iterdir()) + files.sort(key=lambda file: file.name, reverse=True) + for file in files: + self.rotate_file(file) + + def __init__(self, filename: PathLike, backup_count: int = 2) -> None: + self.filename = Path(filename) + self.backup_count = backup_count + self.create_dir() + self.rotate() + self.log_file = lzma.open( + self.filename, "at", format=FORMAT_XZ, preset=PRESET_DEFAULT + ) + super().__init__(self.log_file) + + def close(self) -> None: + self.log_file.close() + super().close() diff --git a/src/logging/setup.py b/src/logging/setup.py index c8184ff..696d574 100644 --- a/src/logging/setup.py +++ b/src/logging/setup.py @@ -8,18 +8,14 @@ from src import shared def setup_logging(): """Intitate the app's logging""" - # Prepare the log file - log_dir = shared.data_dir / "cartridges" / "logs" - log_dir.mkdir(exist_ok=True) - log_file_path = log_dir / "cartridges.log" - log_file_max_size_bytes = 8 * 10**6 # 8 MB - - # Define log levels - profile_app_log_level = "DEBUG" if shared.PROFILE == "development" else "INFO" - profile_lib_log_level = "INFO" if shared.PROFILE == "development" else "WARNING" + is_dev = shared.PROFILE == "development" + profile_app_log_level = "DEBUG" if is_dev else "INFO" + profile_lib_log_level = "INFO" if is_dev else "WARNING" app_log_level = os.environ.get("LOGLEVEL", profile_app_log_level).upper() lib_log_level = os.environ.get("LIBLOGLEVEL", profile_lib_log_level).upper() + log_filename = shared.data_dir / "cartridges" / "logs" / "cartridges.log.xz" + config = { "version": 1, "formatters": { @@ -33,12 +29,11 @@ def setup_logging(): }, "handlers": { "file_handler": { - "class": "logging.handlers.RotatingFileHandler", + "class": "src.logging.session_file_handler.SessionFileHandler", "formatter": "file_formatter", "level": "DEBUG", - "filename": log_file_path, - "maxBytes": log_file_max_size_bytes, - "backupCount": 1, + "filename": log_filename, + "backup_count": 2, }, "app_console_handler": { "class": "logging.StreamHandler", From 59c2d6864278a02c1b5fefe251d5b9ce85125045 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Tue, 13 Jun 2023 00:25:24 +0200 Subject: [PATCH 130/173] Removed unnecessary prints --- src/logging/session_file_handler.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/logging/session_file_handler.py b/src/logging/session_file_handler.py index 5562735..1f765df 100644 --- a/src/logging/session_file_handler.py +++ b/src/logging/session_file_handler.py @@ -39,12 +39,10 @@ class SessionFileHandler(StreamHandler): suffixes.append(f".{new_number}") stem = file.name.split(".", maxsplit=1)[0] new_name = stem + "".join(suffixes) - print(f"Log file renamed: {file.name} -> {new_name}") file = file.rename(file.with_name(new_name)) # Remove older files if new_number > self.backup_count: - print(f"Log file deleted: {file.name}") file.unlink() return From 054089431f177d217d9e504b6f28d661cf747c46 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Tue, 13 Jun 2023 09:05:29 +0200 Subject: [PATCH 131/173] =?UTF-8?q?=F0=9F=8E=A8=20Made=20log=20file=20rota?= =?UTF-8?q?tion=20more=20robust?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/logging/session_file_handler.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/logging/session_file_handler.py b/src/logging/session_file_handler.py index 1f765df..9005c27 100644 --- a/src/logging/session_file_handler.py +++ b/src/logging/session_file_handler.py @@ -46,10 +46,21 @@ class SessionFileHandler(StreamHandler): file.unlink() return + def file_sort_key(self, file: Path) -> int: + """Key function used to sort files""" + if not file.name.startswith(self.filename.name): + # First all files that aren't logs + return -1 + if file.name == self.filename.name: + # Then the latest log file + return 0 + # Then in order the other log files + return int(file.suffixes[-1][1:]) + def rotate(self) -> None: """Rotate the numbered suffix on the log files and remove old ones""" files = list(self.filename.parent.iterdir()) - files.sort(key=lambda file: file.name, reverse=True) + files.sort(key=self.file_sort_key, reverse=True) for file in files: self.rotate_file(file) From dbb96a166bea374cdc7bcd77d21df465ee9b61cc Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Tue, 13 Jun 2023 09:48:18 +0200 Subject: [PATCH 132/173] =?UTF-8?q?=E2=9C=A8=20Added=20debug=20info=20to?= =?UTF-8?q?=20the=20beginning=20of=20log=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/logging/setup.py | 31 +++++++++++++++++++++++++------ src/main.py | 4 +++- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/logging/setup.py b/src/logging/setup.py index 696d574..da5e549 100644 --- a/src/logging/setup.py +++ b/src/logging/setup.py @@ -1,6 +1,8 @@ import logging import logging.config as logging_dot_config import os +import sys +import subprocess from src import shared @@ -65,9 +67,26 @@ def setup_logging(): } logging_dot_config.dictConfig(config) - # Inform of the logging behaviour - logging.info("Logging profile: %s", shared.PROFILE) - logging.info("Console logging level for application: %s", app_log_level) - logging.info("Console logging level for libraries: %s", lib_log_level) - logging.info("Use env vars LOGLEVEL, LIBLOGLEVEL to override") - logging.info("All message levels are written to the log file") + +def log_system_info(): + """Log system debug information""" + + logging.debug("Starting %s v%s (%s)", shared.APP_ID, shared.VERSION, shared.PROFILE) + logging.debug("System: %s", sys.platform) + logging.debug("Python version: %s", sys.version) + if os.getenv("FLATPAK_ID"): + process = subprocess.run( + ("flatpak-spawn", "--host", "flatpak", "--version"), + capture_output=True, + encoding="utf-8", + check=False, + ) + logging.debug("Flatpak version: %s", process.stdout.rstrip()) + if os.name == "posix": + uname = os.uname() + logging.debug("Uname info:") + logging.debug("\tsysname: %s", uname.sysname) + logging.debug("\trelease: %s", uname.release) + logging.debug("\tversion: %s", uname.version) + logging.debug("\tmachine: %s", uname.machine) + logging.debug("-" * 80) diff --git a/src/main.py b/src/main.py index 8ac30dd..588d8c1 100644 --- a/src/main.py +++ b/src/main.py @@ -37,7 +37,7 @@ from src.importer.sources.itch_source import ItchSource from src.importer.sources.legendary_source import LegendarySource from src.importer.sources.lutris_source import LutrisSource from src.importer.sources.steam_source import SteamSource -from src.logging.setup import setup_logging +from src.logging.setup import setup_logging, log_system_info from src.preferences import PreferencesWindow from src.store.managers.display_manager import DisplayManager from src.store.managers.file_manager import FileManager @@ -254,6 +254,8 @@ class CartridgesApplication(Adw.Application): def main(_version): + """App entry point""" setup_logging() + log_system_info() app = CartridgesApplication() return app.run(sys.argv) From 6dd8e3965f8ba88fc09450cce1005caf0f2a8216 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Tue, 13 Jun 2023 10:32:07 +0200 Subject: [PATCH 133/173] =?UTF-8?q?=F0=9F=90=9B=20Ported=20sqlite=20fix=20?= =?UTF-8?q?from=20main?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/itch_source.py | 13 ++++++++----- src/importer/sources/lutris_source.py | 8 +++++++- src/utils/sqlite.py | 17 +++++++++++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 src/utils/sqlite.py diff --git a/src/importer/sources/itch_source.py b/src/importer/sources/itch_source.py index 6d8a157..e93ad17 100644 --- a/src/importer/sources/itch_source.py +++ b/src/importer/sources/itch_source.py @@ -1,4 +1,5 @@ from pathlib import Path +from shutil import rmtree from sqlite3 import connect from time import time @@ -9,10 +10,8 @@ from src.importer.sources.source import ( SourceIterator, URLExecutableSource, ) -from src.utils.decorators import ( - replaced_by_path, - replaced_by_schema_key, -) +from src.utils.decorators import replaced_by_path, replaced_by_schema_key +from src.utils.sqlite import copy_db class ItchSourceIterator(SourceIterator): @@ -37,7 +36,8 @@ class ItchSourceIterator(SourceIterator): caves.game_id = games.id ; """ - connection = connect(self.source.location / "db" / "butler.db") + db_path = copy_db(self.source.location / "db" / "butler.db") + connection = connect(db_path) cursor = connection.execute(db_request) # Create games from the db results @@ -54,6 +54,9 @@ class ItchSourceIterator(SourceIterator): game = Game(values, allow_side_effects=False) yield (game, additional_data) + # Cleanup + rmtree(str(db_path.parent)) + class ItchSource(URLExecutableSource): name = "Itch" diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index ddf6778..0da9b87 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,3 +1,4 @@ +from shutil import rmtree from sqlite3 import connect from time import time @@ -9,6 +10,7 @@ from src.importer.sources.source import ( URLExecutableSource, ) from src.utils.decorators import replaced_by_path, replaced_by_schema_key +from src.utils.sqlite import copy_db class LutrisSourceIterator(SourceIterator): @@ -30,7 +32,8 @@ class LutrisSourceIterator(SourceIterator): ; """ params = {"import_steam": shared.schema.get_boolean("lutris-import-steam")} - connection = connect(self.source.location / "pga.db") + db_path = copy_db(self.source.location / "pga.db") + connection = connect(db_path) cursor = connection.execute(request, params) # Create games from the DB results @@ -56,6 +59,9 @@ class LutrisSourceIterator(SourceIterator): # Produce game yield (game, additional_data) + # Cleanup + rmtree(str(db_path.parent)) + class LutrisSource(URLExecutableSource): """Generic lutris source""" diff --git a/src/utils/sqlite.py b/src/utils/sqlite.py new file mode 100644 index 0000000..9661c2a --- /dev/null +++ b/src/utils/sqlite.py @@ -0,0 +1,17 @@ +from glob import escape +from pathlib import Path +from shutil import copyfile + +from gi.repository import GLib + + +def copy_db(original_path: Path) -> Path: + """ + Copy a sqlite database to a cache dir and return its new path. + The caller in in charge of deleting the returned path's parent dir. + """ + tmp = Path(GLib.Dir.make_tmp()) + for file in original_path.parent.glob(f"{escape(original_path.name)}*"): + copy = tmp / file.name + copyfile(str(file), str(copy)) + return tmp / original_path.name From 695cc88d76f8a69eef9a1a564ba7afd04ba9e079 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 14 Jun 2023 00:05:38 +0200 Subject: [PATCH 134/173] =?UTF-8?q?=F0=9F=8E=A8=20Made=20OnlineCoverManage?= =?UTF-8?q?r=20more=20general?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Does compositing of image with a blurred background - Stretches the original image if it's not too much - Handles images that are too wide and images that are too tall - Removed ItchCoverManager --- src/importer/sources/itch_source.py | 2 +- src/main.py | 6 +- src/store/managers/itch_cover_manager.py | 66 ---------------- src/store/managers/online_cover_manager.py | 89 ++++++++++++++++++++-- src/store/managers/sgdb_manager.py | 4 +- 5 files changed, 89 insertions(+), 78 deletions(-) delete mode 100644 src/store/managers/itch_cover_manager.py diff --git a/src/importer/sources/itch_source.py b/src/importer/sources/itch_source.py index e93ad17..43e546c 100644 --- a/src/importer/sources/itch_source.py +++ b/src/importer/sources/itch_source.py @@ -50,7 +50,7 @@ class ItchSourceIterator(SourceIterator): "game_id": self.source.game_id_format.format(game_id=row[0]), "executable": self.source.executable_format.format(cave_id=row[4]), } - additional_data = {"itch_cover_url": row[2], "itch_still_cover_url": row[3]} + additional_data = {"online_cover_url": row[3] or row[2]} game = Game(values, allow_side_effects=False) yield (game, additional_data) diff --git a/src/main.py b/src/main.py index 588d8c1..b59176c 100644 --- a/src/main.py +++ b/src/main.py @@ -37,12 +37,12 @@ from src.importer.sources.itch_source import ItchSource from src.importer.sources.legendary_source import LegendarySource from src.importer.sources.lutris_source import LutrisSource from src.importer.sources.steam_source import SteamSource -from src.logging.setup import setup_logging, log_system_info +from src.logging.setup import log_system_info, setup_logging from src.preferences import PreferencesWindow from src.store.managers.display_manager import DisplayManager from src.store.managers.file_manager import FileManager -from src.store.managers.itch_cover_manager import ItchCoverManager from src.store.managers.local_cover_manager import LocalCoverManager +from src.store.managers.online_cover_manager import OnlineCoverManager from src.store.managers.sgdb_manager import SGDBManager from src.store.managers.steam_api_manager import SteamAPIManager from src.store.store import Store @@ -89,7 +89,7 @@ class CartridgesApplication(Adw.Application): # Add rest of the managers for game imports shared.store.add_manager(LocalCoverManager()) shared.store.add_manager(SteamAPIManager()) - shared.store.add_manager(ItchCoverManager()) + shared.store.add_manager(OnlineCoverManager()) shared.store.add_manager(SGDBManager()) shared.store.add_manager(FileManager()) diff --git a/src/store/managers/itch_cover_manager.py b/src/store/managers/itch_cover_manager.py deleted file mode 100644 index 3d96b0e..0000000 --- a/src/store/managers/itch_cover_manager.py +++ /dev/null @@ -1,66 +0,0 @@ -from pathlib import Path - -import requests -from gi.repository import GdkPixbuf, Gio -from requests.exceptions import HTTPError, SSLError - -from src import shared # pylint: disable=no-name-in-module -from src.game import Game -from src.store.managers.local_cover_manager import LocalCoverManager -from src.store.managers.manager import Manager -from src.utils.save_cover import resize_cover, save_cover - - -# TODO Remove by generalizing OnlineCoverManager - - -class ItchCoverManager(Manager): - """Manager in charge of downloading the game's cover from itch.io""" - - run_after = (LocalCoverManager,) - retryable_on = (HTTPError, SSLError) - - def manager_logic(self, game: Game, additional_data: dict) -> None: - # Get the first matching cover url - base_cover_url: str = additional_data.get("itch_cover_url", None) - still_cover_url: str = additional_data.get("itch_still_cover_url", None) - cover_url = still_cover_url or base_cover_url - if not cover_url: - return - - # Download cover - tmp_file = Gio.File.new_tmp()[0] - with requests.get(cover_url, timeout=5) as cover: - cover.raise_for_status() - Path(tmp_file.get_path()).write_bytes(cover.content) - - # Create background blur - game_cover = GdkPixbuf.Pixbuf.new_from_stream_at_scale( - tmp_file.read(), 2, 2, False - ).scale_simple(*shared.image_size, GdkPixbuf.InterpType.BILINEAR) - - # Resize square image - itch_pixbuf = GdkPixbuf.Pixbuf.new_from_stream(tmp_file.read()) - itch_pixbuf = itch_pixbuf.scale_simple( - shared.image_size[0], - itch_pixbuf.get_height() * (shared.image_size[0] / itch_pixbuf.get_width()), - GdkPixbuf.InterpType.BILINEAR, - ) - - # Composite - itch_pixbuf.composite( - game_cover, - 0, - (shared.image_size[1] - itch_pixbuf.get_height()) / 2, - itch_pixbuf.get_width(), - itch_pixbuf.get_height(), - 0, - (shared.image_size[1] - itch_pixbuf.get_height()) / 2, - 1.0, - 1.0, - GdkPixbuf.InterpType.BILINEAR, - 255, - ) - - # Resize and save the cover - save_cover(game.game_id, resize_cover(pixbuf=game_cover)) diff --git a/src/store/managers/online_cover_manager.py b/src/store/managers/online_cover_manager.py index e9b8411..736e6c6 100644 --- a/src/store/managers/online_cover_manager.py +++ b/src/store/managers/online_cover_manager.py @@ -1,9 +1,12 @@ +import logging from pathlib import Path import requests -from gi.repository import Gio +from gi.repository import Gio, GdkPixbuf from requests.exceptions import HTTPError, SSLError +from PIL import Image +from src import shared from src.game import Game from src.store.managers.local_cover_manager import LocalCoverManager from src.store.managers.manager import Manager @@ -16,15 +19,89 @@ class OnlineCoverManager(Manager): run_after = (LocalCoverManager,) retryable_on = (HTTPError, SSLError) + def save_composited_cover( + self, + game: Game, + image_file: Gio.File, + original_width: int, + original_height: int, + target_width: int, + target_height: int, + ) -> None: + """Save the image composited with a background blur to fit the cover size""" + + logging.debug( + "Compositing image for %s (%s) %dx%d -> %dx%d", + game.name, + game.game_id, + original_width, + original_height, + target_width, + target_height, + ) + + # Load game image + image = GdkPixbuf.Pixbuf.new_from_stream(image_file.read()) + + # Create background blur of the size of the cover + cover = image.scale_simple(2, 2, GdkPixbuf.InterpType.BILINEAR).scale_simple( + target_width, target_height, GdkPixbuf.InterpType.BILINEAR + ) + + # Center the image above the blurred background + scale = min(target_width / original_width, target_height / original_height) + left_padding = (target_width - original_width * scale) / 2 + top_padding = (target_height - original_height * scale) / 2 + image.composite( + cover, + # Top left of overwritten area on the destination + left_padding, + top_padding, + # Size of the overwritten area on the destination + original_width * scale, + original_height * scale, + # Offset + left_padding, + top_padding, + # Scale to apply to the resized image + scale, + scale, + # Compositing stuff + GdkPixbuf.InterpType.BILINEAR, + 255, + ) + + # Resize and save the cover + save_cover(game.game_id, resize_cover(pixbuf=cover)) + def manager_logic(self, game: Game, additional_data: dict) -> None: # Ensure that we have a cover to download - cover_url = additional_data.get("online_cover_url", None) + cover_url = additional_data.get("online_cover_url") if not cover_url: return + # Download cover - tmp_file = Gio.File.new_tmp()[0] + image_file = Gio.File.new_tmp()[0] + image_path = Path(image_file.get_path()) with requests.get(cover_url, timeout=5) as cover: cover.raise_for_status() - Path(tmp_file.get_path()).write_bytes(cover.content) - # Resize and save - save_cover(game.game_id, resize_cover(tmp_file.get_path())) + image_path.write_bytes(cover.content) + + # Get image size + cover_width, cover_height = shared.image_size + with Image.open(image_path) as pil_image: + width, height = pil_image.size + + # Composite the image if its aspect ratio differs too much + # (allow the side that is smaller to be stretched by a small percentage) + max_diff_proportion = 0.12 + scale = min(cover_width / width, cover_height / height) + width_diff = (cover_width - (width * scale)) / cover_width + height_diff = (cover_height - (height * scale)) / cover_height + diff = width_diff + height_diff + if diff < max_diff_proportion: + save_cover(game.game_id, resize_cover(image_path)) + else: + self.save_composited_cover( + game, image_file, width, height, cover_width, cover_height + ) diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index 5844204..a3a5811 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -4,7 +4,7 @@ from requests.exceptions import HTTPError, SSLError from src.game import Game from src.store.managers.async_manager import AsyncManager -from src.store.managers.itch_cover_manager import ItchCoverManager +from src.store.managers.online_cover_manager import OnlineCoverManager from src.store.managers.local_cover_manager import LocalCoverManager from src.store.managers.steam_api_manager import SteamAPIManager from src.utils.steamgriddb import SGDBAuthError, SGDBHelper @@ -13,7 +13,7 @@ from src.utils.steamgriddb import SGDBAuthError, SGDBHelper class SGDBManager(AsyncManager): """Manager in charge of downloading a game's cover from steamgriddb""" - run_after = (SteamAPIManager, LocalCoverManager, ItchCoverManager) + run_after = (SteamAPIManager, LocalCoverManager, OnlineCoverManager) retryable_on = (HTTPError, SSLError, ConnectionError, JSONDecodeError) def manager_logic(self, game: Game, _additional_data: dict) -> None: From 3bc0df3881be00bb1ad488a94ad2d6b560a7bb24 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Wed, 14 Jun 2023 17:23:54 +0200 Subject: [PATCH 135/173] =?UTF-8?q?=F0=9F=8E=A8=20Change=20image=20composi?= =?UTF-8?q?tion=20logic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/managers/online_cover_manager.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/store/managers/online_cover_manager.py b/src/store/managers/online_cover_manager.py index 736e6c6..fa82d06 100644 --- a/src/store/managers/online_cover_manager.py +++ b/src/store/managers/online_cover_manager.py @@ -92,14 +92,14 @@ class OnlineCoverManager(Manager): with Image.open(image_path) as pil_image: width, height = pil_image.size - # Composite the image if its aspect ratio differs too much - # (allow the side that is smaller to be stretched by a small percentage) - max_diff_proportion = 0.12 - scale = min(cover_width / width, cover_height / height) - width_diff = (cover_width - (width * scale)) / cover_width - height_diff = (cover_height - (height * scale)) / cover_height - diff = width_diff + height_diff - if diff < max_diff_proportion: + # Composite if the image is shorter and the stretch amount is too high + aspect_ratio = width / height + target_aspect_ratio = cover_width / cover_height + is_taller = aspect_ratio < target_aspect_ratio + resized_height = height / width * cover_width + stretch = 1 - (resized_height / cover_height) + max_stretch = 0.12 + if is_taller or stretch <= max_stretch: save_cover(game.game_id, resize_cover(image_path)) else: self.save_composited_cover( From e6afed66787ec674934291877483f1dcedeb59a2 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Thu, 15 Jun 2023 15:22:08 +0200 Subject: [PATCH 136/173] Cleanups --- src/logging/session_file_handler.py | 2 +- src/logging/setup.py | 8 ++++---- src/store/managers/manager.py | 1 + src/store/pipeline.py | 10 +++++----- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/logging/session_file_handler.py b/src/logging/session_file_handler.py index 9005c27..03bf174 100644 --- a/src/logging/session_file_handler.py +++ b/src/logging/session_file_handler.py @@ -18,7 +18,7 @@ class SessionFileHandler(StreamHandler): def create_dir(self) -> None: """Create the log dir if needed""" - self.filename.parent.mkdir(exist_ok=True) + self.filename.parent.mkdir(exist_ok=True, parents=True) def rotate_file(self, file: Path): """Rotate a file's number suffix and remove it if it's too old""" diff --git a/src/logging/setup.py b/src/logging/setup.py index da5e549..73df738 100644 --- a/src/logging/setup.py +++ b/src/logging/setup.py @@ -1,10 +1,10 @@ import logging import logging.config as logging_dot_config import os -import sys import subprocess +import sys -from src import shared +from src import shared # pylint: disable=no-name-in-module def setup_logging(): @@ -16,7 +16,7 @@ def setup_logging(): app_log_level = os.environ.get("LOGLEVEL", profile_app_log_level).upper() lib_log_level = os.environ.get("LIBLOGLEVEL", profile_lib_log_level).upper() - log_filename = shared.data_dir / "cartridges" / "logs" / "cartridges.log.xz" + log_filename = shared.cache_dir / "cartridges" / "logs" / "cartridges.log.xz" config = { "version": 1, @@ -35,7 +35,7 @@ def setup_logging(): "formatter": "file_formatter", "level": "DEBUG", "filename": log_filename, - "backup_count": 2, + "backup_count": 3, }, "app_console_handler": { "class": "logging.StreamHandler", diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 9444ea9..b3fa098 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -110,5 +110,6 @@ class Manager: self, game: Game, additional_data: dict, callback: Callable[["Manager"], Any] ) -> None: """Pass the game through the manager""" + # TODO: connect to signals here self.execute_resilient_manager_logic(game, additional_data) callback(self) diff --git a/src/store/pipeline.py b/src/store/pipeline.py index af1b64b..4799fd2 100644 --- a/src/store/pipeline.py +++ b/src/store/pipeline.py @@ -40,12 +40,12 @@ class Pipeline(GObject.Object): def blocked(self) -> set[Manager]: """Get the managers that cannot run because their dependencies aren't done""" blocked = set() - for manager_a in self.waiting: - for manager_b in self.not_done: - if manager_a == manager_b: + for waiting in self.waiting: + for not_done in self.not_done: + if waiting == not_done: continue - if type(manager_b) in manager_a.run_after: - blocked.add(manager_a) + if type(not_done) in waiting.run_after: + blocked.add(waiting) return blocked @property From 39bc64c136fdce9881eac0f605ea1ae0a0a44f23 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Thu, 15 Jun 2023 17:37:54 +0200 Subject: [PATCH 137/173] Use signals for updating and saving games --- src/details_window.py | 1 + src/game.py | 79 +++++---------------------- src/main.py | 8 ++- src/store/managers/display_manager.py | 45 ++++++++++++++- src/store/managers/file_manager.py | 32 ++++++++++- src/store/managers/manager.py | 2 +- src/store/store.py | 18 +++++- 7 files changed, 107 insertions(+), 78 deletions(-) diff --git a/src/details_window.py b/src/details_window.py index 86c3a83..25d6069 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -205,6 +205,7 @@ class DetailsWindow(Adw.Window): self.game.save() self.game.update() + # TODO: this is fucked up # Get a cover from SGDB if none is present if not self.game_cover.get_pixbuf(): self.game.set_loading(1) diff --git a/src/game.py b/src/game.py index 090dedb..e7be76c 100644 --- a/src/game.py +++ b/src/game.py @@ -17,7 +17,6 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -import json import logging import os import shlex @@ -25,10 +24,9 @@ import subprocess from pathlib import Path from time import time -from gi.repository import Adw, Gtk +from gi.repository import Adw, GObject, Gtk from src import shared # pylint: disable=no-name-in-module -from src.game_cover import GameCover # pylint: disable=too-many-instance-attributes @@ -86,74 +84,15 @@ class Game(Gtk.Box): shared.schema.connect("changed", self.schema_changed) - def update(self): - if self.get_parent(): - self.get_parent().get_parent().remove(self) - if self.get_parent(): - self.get_parent().set_child() - - self.menu_button.set_menu_model( - self.hidden_game_options if self.hidden else self.game_options - ) - - self.title.set_label(self.name) - - self.menu_button.get_popover().connect( - "notify::visible", self.toggle_play, None - ) - self.menu_button.get_popover().connect( - "notify::visible", self.win.set_active_game, self - ) - - if self.game_id in self.win.game_covers: - self.game_cover = self.win.game_covers[self.game_id] - self.game_cover.add_picture(self.cover) - else: - self.game_cover = GameCover({self.cover}, self.get_cover_path()) - self.win.game_covers[self.game_id] = self.game_cover - - if ( - self.win.stack.get_visible_child() == self.win.details_view - and self.win.active_game == self - ): - self.win.show_details_view(self) - - if not self.removed and not self.blacklisted: - if self.hidden: - self.win.hidden_library.append(self) - else: - self.win.library.append(self) - self.get_parent().set_focusable(False) - - self.win.set_library_child() - def update_values(self, data): for key, value in data.items(): setattr(self, key, value) + def update(self): + self.emit("update-ready", {}) + def save(self): - shared.games_dir.mkdir(parents=True, exist_ok=True) - - attrs = ( - "added", - "executable", - "game_id", - "source", - "hidden", - "last_played", - "name", - "developer", - "removed", - "blacklisted", - "version", - ) - - json.dump( - {attr: getattr(self, attr) for attr in attrs if attr}, - (shared.games_dir / f"{self.game_id}.json").open("w"), - indent=4, - sort_keys=True, - ) + self.emit("save-ready", {}) def create_toast(self, title, action=None): toast = Adw.Toast.new(title.format(self.name)) @@ -270,3 +209,11 @@ class Game(Gtk.Box): def schema_changed(self, _settings, key): if key == "cover-launches-game": self.set_play_icon() + + @GObject.Signal(name="update-ready", arg_types=[object]) + def update_ready(self, _additional_data) -> None: + """Signal emitted when the game needs updating""" + + @GObject.Signal(name="save-ready", arg_types=[object]) + def save_ready(self, _additional_data) -> None: + """Signal emitted when the game needs saving""" diff --git a/src/main.py b/src/main.py index b59176c..c099ab1 100644 --- a/src/main.py +++ b/src/main.py @@ -82,6 +82,7 @@ class CartridgesApplication(Adw.Application): # Create the games store ready to load games from disk if not shared.store: shared.store = Store() + shared.store.add_manager(FileManager(), False) shared.store.add_manager(DisplayManager()) self.load_games_from_disk() @@ -91,7 +92,8 @@ class CartridgesApplication(Adw.Application): shared.store.add_manager(SteamAPIManager()) shared.store.add_manager(OnlineCoverManager()) shared.store.add_manager(SGDBManager()) - shared.store.add_manager(FileManager()) + + shared.store.manager_to_pipeline(FileManager) # Create actions self.create_actions( @@ -137,7 +139,7 @@ class CartridgesApplication(Adw.Application): for game_file in shared.games_dir.iterdir(): data = json.load(game_file.open()) game = Game(data, allow_side_effects=False) - shared.store.add_game(game, tuple()) + shared.store.add_game(game, {"skip_save": True}) def on_about_action(self, *_args): about = Adw.AboutWindow( @@ -148,9 +150,9 @@ class CartridgesApplication(Adw.Application): version=shared.VERSION, developers=[ "kramo https://kramo.hu", + "Geoffrey Coulaud https://geoffrey-coulaud.fr", "Arcitec https://github.com/Arcitec", "Domenico https://github.com/Domefemia", - "Geoffrey Coulaud https://geoffrey-coulaud.fr", "Paweł Lidwin https://github.com/imLinguin", "Rafael Mardojai CM https://mardojai.com", ], diff --git a/src/store/managers/display_manager.py b/src/store/managers/display_manager.py index 13418d4..847bdfb 100644 --- a/src/store/managers/display_manager.py +++ b/src/store/managers/display_manager.py @@ -1,16 +1,55 @@ from src import shared # pylint: disable=no-name-in-module from src.game import Game +from src.game_cover import GameCover +from src.store.managers.manager import Manager from src.store.managers.sgdb_manager import SGDBManager from src.store.managers.steam_api_manager import SteamAPIManager -from src.store.managers.manager import Manager class DisplayManager(Manager): """Manager in charge of adding a game to the UI""" run_after = (SteamAPIManager, SGDBManager) + signals = {"update-ready"} def manager_logic(self, game: Game, _additional_data: dict) -> None: - # TODO decouple a game from its widget shared.win.games[game.game_id] = game - game.update() + if game.get_parent(): + game.get_parent().get_parent().remove(game) + if game.get_parent(): + game.get_parent().set_child() + + game.menu_button.set_menu_model( + game.hidden_game_options if game.hidden else game.game_options + ) + + game.title.set_label(game.name) + + game.menu_button.get_popover().connect( + "notify::visible", game.toggle_play, None + ) + game.menu_button.get_popover().connect( + "notify::visible", game.win.set_active_game, game + ) + + if game.game_id in game.win.game_covers: + game.game_cover = game.win.game_covers[game.game_id] + game.game_cover.add_picture(game.cover) + else: + game.game_cover = GameCover({game.cover}, game.get_cover_path()) + game.win.game_covers[game.game_id] = game.game_cover + + if ( + game.win.stack.get_visible_child() == game.win.details_view + and game.win.active_game == game + ): + game.win.show_details_view(game) + + if not game.removed and not game.blacklisted: + if game.hidden: + game.win.hidden_library.append(game) + else: + game.win.library.append(game) + game.get_parent().set_focusable(False) + + game.win.set_library_child() diff --git a/src/store/managers/file_manager.py b/src/store/managers/file_manager.py index 7a72e5d..13d492f 100644 --- a/src/store/managers/file_manager.py +++ b/src/store/managers/file_manager.py @@ -1,3 +1,6 @@ +import json + +from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.store.managers.async_manager import AsyncManager from src.store.managers.steam_api_manager import SteamAPIManager @@ -7,6 +10,31 @@ class FileManager(AsyncManager): """Manager in charge of saving a game to a file""" run_after = (SteamAPIManager,) + signals = {"save-ready"} - def manager_logic(self, game: Game, _additional_data: dict) -> None: - game.save() + def manager_logic(self, game: Game, additional_data: dict) -> None: + if additional_data.get("skip_save"): # Skip saving when loading games from disk + return + + shared.games_dir.mkdir(parents=True, exist_ok=True) + + attrs = ( + "added", + "executable", + "game_id", + "source", + "hidden", + "last_played", + "name", + "developer", + "removed", + "blacklisted", + "version", + ) + + json.dump( + {attr: getattr(game, attr) for attr in attrs if attr}, + (shared.games_dir / f"{game.game_id}.json").open("w"), + indent=4, + sort_keys=True, + ) diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index b3fa098..1124d55 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -21,6 +21,7 @@ class Manager: retryable_on: Container[type[Exception]] = tuple() continue_on: Container[type[Exception]] = tuple() + signals: Container[type[str]] = set() retry_delay: int = 3 max_tries: int = 3 @@ -110,6 +111,5 @@ class Manager: self, game: Game, additional_data: dict, callback: Callable[["Manager"], Any] ) -> None: """Pass the game through the manager""" - # TODO: connect to signals here self.execute_resilient_manager_logic(game, additional_data) callback(self) diff --git a/src/store/store.py b/src/store/store.py index c33ded2..c65b7c8 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -16,9 +16,12 @@ class Store: self.games = {} self.pipelines = {} - def add_manager(self, manager: Manager): + def add_manager(self, manager: Manager, in_pipeline=True): """Add a manager that will run when games are added""" - self.managers[type(manager)] = manager + self.managers[type(manager)] = [manager, in_pipeline] + + def manager_to_pipeline(self, manager_type: type[Manager]): + self.managers[manager_type][1] = True def add_game( self, game: Game, additional_data: dict, replace=False @@ -50,8 +53,17 @@ class Store: path.unlink(missing_ok=True) return None + # Connect signals + for manager, _in_pipeline in self.managers.values(): + for signal in manager.signals: + game.connect(signal, manager.execute_resilient_manager_logic) + # Run the pipeline for the game - pipeline = Pipeline(game, additional_data, self.managers.values()) + pipeline = Pipeline( + game, + additional_data, + (manager[0] for manager in self.managers.values() if manager[1]), + ) self.games[game.game_id] = game self.pipelines[game.game_id] = pipeline pipeline.advance() From 90667b0f31330e16d733409f764f1451f63b2898 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Thu, 15 Jun 2023 17:53:48 +0200 Subject: [PATCH 138/173] Fix details_window logic --- src/details_window.py | 2 +- src/store/store.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/details_window.py b/src/details_window.py index 25d6069..33feff3 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -209,7 +209,7 @@ class DetailsWindow(Adw.Window): # Get a cover from SGDB if none is present if not self.game_cover.get_pixbuf(): self.game.set_loading(1) - sgdb_manager: SGDBManager = shared.store.managers[SGDBManager] + sgdb_manager: SGDBManager = shared.store.managers[SGDBManager][0] sgdb_manager.reset_cancellable() pipeline = Pipeline(self.game, {}, (sgdb_manager,)) pipeline.connect("advanced", self.update_cover_callback) diff --git a/src/store/store.py b/src/store/store.py index c65b7c8..fed0d60 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -45,6 +45,7 @@ class Store: # Cleanup removed games if game.removed: + # TODO: come back to this later for path in ( shared.games_dir / f"{game.game_id}.json", shared.covers_dir / f"{game.game_id}.tiff", From d060acb90a3055055a0bda38c02651ba739fb021 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Fri, 16 Jun 2023 13:16:48 +0200 Subject: [PATCH 139/173] Escape game titles in toasts --- src/game.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/game.py b/src/game.py index e7be76c..734e195 100644 --- a/src/game.py +++ b/src/game.py @@ -24,7 +24,7 @@ import subprocess from pathlib import Path from time import time -from gi.repository import Adw, GObject, Gtk +from gi.repository import Adw, GLib, GObject, Gtk from src import shared # pylint: disable=no-name-in-module @@ -154,7 +154,9 @@ class Game(Gtk.Box): if toast: self.create_toast( # The variable is the title of the game - (_("{} hidden") if self.hidden else _("{} unhidden")).format(self.name), + (_("{} hidden") if self.hidden else _("{} unhidden")).format( + GLib.markup_escape_text(self.name) + ), "hide", ) @@ -168,7 +170,9 @@ class Game(Gtk.Box): self.win.on_go_back_action() # The variable is the title of the game - self.create_toast(_("{} removed").format(self.name), "remove") + self.create_toast( + _("{} removed").format(GLib.markup_escape_text(self.name)), "remove" + ) def set_loading(self, state): self.loading += state From e694341a3123f58aae4ded8a81b0c9ad0c42b71f Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Fri, 16 Jun 2023 15:21:39 +0200 Subject: [PATCH 140/173] =?UTF-8?q?=F0=9F=90=9B=20Fix=20game=20import=20no?= =?UTF-8?q?t=20refreshing=20remove=20covers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/store/store.py | 52 +++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/src/store/store.py b/src/store/store.py index fed0d60..0d4a159 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -1,3 +1,5 @@ +import logging + from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.store.managers.manager import Manager @@ -23,35 +25,43 @@ class Store: def manager_to_pipeline(self, manager_type: type[Manager]): self.managers[manager_type][1] = True - def add_game( - self, game: Game, additional_data: dict, replace=False - ) -> Pipeline | None: - """Add a game to the app if not already there + def cleanup_game(self, game: Game) -> None: + """Remove a game's files""" + for path in ( + shared.games_dir / f"{game.game_id}.json", + shared.covers_dir / f"{game.game_id}.tiff", + shared.covers_dir / f"{game.game_id}.gif", + ): + path.unlink(missing_ok=True) - :param replace bool: Replace the game if it already exists - """ + def add_game(self, game: Game, additional_data: dict) -> Pipeline | None: + """Add a game to the app""" # Ignore games from a newer spec version if game.version > shared.SPEC_VERSION: return None - # Ignore games that are already there - if ( - game.game_id in self.games - and not self.games[game.game_id].removed - and not replace - ): + # Scanned game is already removed, just clean it up + if game.removed: + self.cleanup_game(game) return None - # Cleanup removed games - if game.removed: - # TODO: come back to this later - for path in ( - shared.games_dir / f"{game.game_id}.json", - shared.covers_dir / f"{game.game_id}.tiff", - shared.covers_dir / f"{game.game_id}.gif", - ): - path.unlink(missing_ok=True) + # Handle game duplicates + stored_game = self.games.get(game.game_id) + if not stored_game: + # New game, do as normal + logging.debug("New store game %s (%s)", game.name, game.game_id) + elif stored_game.removed: + # Will replace a removed game, cleanup its remains + logging.debug( + "New store game %s (%s) (replacing a removed one)", + game.name, + game.game_id, + ) + self.cleanup_game(stored_game) + else: + # Duplicate game, ignore it + logging.debug("Duplicate store game %s (%s)", game.name, game.game_id) return None # Connect signals From beba0ff1e20e5d4ae3107ace6d77d80fb8170caa Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Fri, 16 Jun 2023 15:38:05 +0200 Subject: [PATCH 141/173] =?UTF-8?q?=F0=9F=8E=A8=20Improved=20internal=20ma?= =?UTF-8?q?nager=20storage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Stored pipeline managers in a set - Renamed store method to enable_manager_in_pipeline - Simplified a bit the ugly code™ in details_window --- src/details_window.py | 19 +++++-------------- src/main.py | 3 +-- src/store/store.py | 24 +++++++++++++----------- 3 files changed, 19 insertions(+), 27 deletions(-) diff --git a/src/details_window.py b/src/details_window.py index 33feff3..8ec15ab 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -205,35 +205,26 @@ class DetailsWindow(Adw.Window): self.game.save() self.game.update() - # TODO: this is fucked up + # TODO: this is fucked up (less than before) # Get a cover from SGDB if none is present if not self.game_cover.get_pixbuf(): self.game.set_loading(1) - sgdb_manager: SGDBManager = shared.store.managers[SGDBManager][0] + sgdb_manager: SGDBManager = shared.store.managers[SGDBManager] sgdb_manager.reset_cancellable() - pipeline = Pipeline(self.game, {}, (sgdb_manager,)) - pipeline.connect("advanced", self.update_cover_callback) - pipeline.advance() + sgdb_manager.process_game(self.game, {}, self.update_cover_callback) self.game_cover.pictures.remove(self.cover) self.close() self.win.show_details_view(self.game) - def update_cover_callback(self, pipeline: Pipeline): - # Check that managers are done - if not pipeline.is_done: - return - + def update_cover_callback(self, manager: SGDBManager): # Set the game as not loading self.game.set_loading(-1) self.game.update() # Handle errors that occured - errors = [] - for manager in pipeline.done: - errors.extend(manager.collect_errors()) - for error in errors: + for error in manager.collect_errors(): # On auth error, inform the user if isinstance(error, SGDBAuthError): create_dialog( diff --git a/src/main.py b/src/main.py index c099ab1..d7c2167 100644 --- a/src/main.py +++ b/src/main.py @@ -92,8 +92,7 @@ class CartridgesApplication(Adw.Application): shared.store.add_manager(SteamAPIManager()) shared.store.add_manager(OnlineCoverManager()) shared.store.add_manager(SGDBManager()) - - shared.store.manager_to_pipeline(FileManager) + shared.store.enable_manager_in_pipelines(FileManager) # Create actions self.create_actions( diff --git a/src/store/store.py b/src/store/store.py index 0d4a159..2b8cbe5 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -10,20 +10,26 @@ class Store: """Class in charge of handling games being added to the app.""" managers: dict[type[Manager], Manager] + pipeline_managers: set[Manager] pipelines: dict[str, Pipeline] games: dict[str, Game] def __init__(self) -> None: self.managers = {} - self.games = {} + self.pipeline_managers = set() self.pipelines = {} + self.games = {} def add_manager(self, manager: Manager, in_pipeline=True): - """Add a manager that will run when games are added""" - self.managers[type(manager)] = [manager, in_pipeline] + """Add a manager to the store""" + manager_type = type(manager) + self.managers[manager_type] = manager + if in_pipeline: + self.enable_manager_in_pipelines(manager_type) - def manager_to_pipeline(self, manager_type: type[Manager]): - self.managers[manager_type][1] = True + def enable_manager_in_pipelines(self, manager_type: type[Manager]): + """Make a manager run in new pipelines""" + self.pipeline_managers.add(self.managers[manager_type]) def cleanup_game(self, game: Game) -> None: """Remove a game's files""" @@ -65,16 +71,12 @@ class Store: return None # Connect signals - for manager, _in_pipeline in self.managers.values(): + for manager in self.managers.values(): for signal in manager.signals: game.connect(signal, manager.execute_resilient_manager_logic) # Run the pipeline for the game - pipeline = Pipeline( - game, - additional_data, - (manager[0] for manager in self.managers.values() if manager[1]), - ) + pipeline = Pipeline(game, additional_data, self.pipeline_managers) self.games[game.game_id] = game self.pipelines[game.game_id] = pipeline pipeline.advance() From 366b68cf8f81f4d7febf578d93e4865692d8f046 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Fri, 16 Jun 2023 16:22:12 +0200 Subject: [PATCH 142/173] Add after import error dialog --- src/importer/importer.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index ba0b2e4..4d0bfee 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -4,9 +4,10 @@ from gi.repository import Adw, Gtk from src import shared # pylint: disable=no-name-in-module from src.game import Game -from src.utils.task import Task -from src.store.pipeline import Pipeline from src.importer.sources.source import Source +from src.store.pipeline import Pipeline +from src.utils.create_dialog import create_dialog +from src.utils.task import Task # pylint: disable=too-many-instance-attributes @@ -176,6 +177,25 @@ class Importer: self.create_summary_toast() # TODO create a summary of errors/warnings/tips popup (eg. SGDB, Steam libraries) # Get the error data from shared.store.managers) + self.create_error_dialog() + + def create_error_dialog(self): + """Dialog containing all errors raised by importers""" + string = _("The following errors occured during import:") + errors = "" + + for manager in shared.store.managers.values(): + for error in manager.collect_errors(): + errors += "\n\n" + str(error) + + if errors: + create_dialog( + shared.win, + "Warning", + string + errors, + "open_preferences", + _("Preferences"), + ).connect("response", self.dialog_response_callback) def create_summary_toast(self): """N games imported toast""" From f0dda997c316b2b5c9f5bfb041b78515ece3895a Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Fri, 16 Jun 2023 20:46:50 +0200 Subject: [PATCH 143/173] Only kill import toast after the user can click it --- src/importer/importer.py | 27 ++++++++++++++++++--------- src/main.py | 2 ++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 4d0bfee..adfc23b 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -1,6 +1,6 @@ import logging -from gi.repository import Adw, Gtk +from gi.repository import Adw, Gtk, GLib from src import shared # pylint: disable=no-name-in-module from src.game import Game @@ -17,6 +17,7 @@ class Importer: progressbar = None import_statuspage = None import_dialog = None + summary_toast = None sources: set[Source] = None @@ -173,10 +174,7 @@ class Importer: """Callback called when importing has finished""" logging.info("Import done") self.import_dialog.close() - # TODO replace by summary if necessary - self.create_summary_toast() - # TODO create a summary of errors/warnings/tips popup (eg. SGDB, Steam libraries) - # Get the error data from shared.store.managers) + self.summary_toast = self.create_summary_toast() self.create_error_dialog() def create_error_dialog(self): @@ -193,15 +191,16 @@ class Importer: shared.win, "Warning", string + errors, - "open_preferences", + "open_preferences_import", _("Preferences"), ).connect("response", self.dialog_response_callback) + else: + self.timeout_toast() def create_summary_toast(self): """N games imported toast""" - toast = Adw.Toast() - toast.set_priority(Adw.ToastPriority.HIGH) + toast = Adw.Toast(timeout=0, priority=Adw.ToastPriority.HIGH) if self.n_games_added == 0: toast.set_title(_("No new games found")) @@ -221,13 +220,23 @@ class Importer: toast.set_title(_("{} games imported").format(self.n_games_added)) shared.win.toast_overlay.add_toast(toast) + return toast def open_preferences(self, page=None, expander_row=None): - shared.win.get_application().on_preferences_action( + return shared.win.get_application().on_preferences_action( page_name=page, expander_row=expander_row ) + def timeout_toast(self, *_args): + """Manually timeout the toast after the user has dismissed all warnings""" + GLib.timeout_add_seconds(5, self.summary_toast.dismiss) + def dialog_response_callback(self, _widget, response, *args): """Handle after-import dialogs callback""" if response == "open_preferences": self.open_preferences(*args) + + elif response == "open_preferences_import": + self.open_preferences(*args).connect("close-request", self.timeout_toast) + else: + self.timeout_toast() diff --git a/src/main.py b/src/main.py index d7c2167..0b365cf 100644 --- a/src/main.py +++ b/src/main.py @@ -175,6 +175,8 @@ class CartridgesApplication(Adw.Application): getattr(win, expander_row).set_expanded(True) win.present() + return win + def on_launch_game_action(self, *_args): self.win.active_game.launch() From b2b17803747ded290745b711e5936655d1c25bea Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Sat, 17 Jun 2023 10:07:23 +0200 Subject: [PATCH 144/173] Remove auto-import feature from settings --- src/preferences.py | 50 ---------------------------------------------- 1 file changed, 50 deletions(-) diff --git a/src/preferences.py b/src/preferences.py index 311ab98..7d8ad2d 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -89,11 +89,6 @@ class PreferencesWindow(Adw.PreferencesWindow): removed_games = set() - # Whether to import after closing the window - import_changed = False - # Widgets and their properties to check whether to import after closing the window - import_changed_widgets = {} - def __init__(self, **kwargs): super().__init__(**kwargs) self.win = shared.win @@ -133,7 +128,6 @@ class PreferencesWindow(Adw.PreferencesWindow): self.choose_folder(widget, set_cache_dir) if lutris_cache_exists(path): - self.import_changed = True self.set_subtitle(self, "lutris-cache") else: @@ -203,25 +197,11 @@ class PreferencesWindow(Adw.PreferencesWindow): ) ) - # Connect the switches that change the behavior of importing to set_import_changed - self.connect_import_switches( - ( - "lutris-import-steam", - "heroic-import-epic", - "heroic-import-gog", - "heroic-import-sideload", - ) - ) - # Windows if os.name == "nt": self.sources_group.remove(self.lutris_expander_row) self.sources_group.remove(self.bottles_expander_row) - # When the user interacts with a widget that changes the behavior of importing, - # Cartridges should automatically import upon closing the preferences window - self.connect("close-request", self.check_import) - def get_switch(self, setting): return getattr(self, f'{setting.replace("-", "_")}_switch') @@ -234,10 +214,6 @@ class PreferencesWindow(Adw.PreferencesWindow): Gio.SettingsBindFlags.DEFAULT, ) - def connect_import_switches(self, settings): - for setting in settings: - self.get_switch(setting).connect("notify::active", self.set_import_changed) - def choose_folder(self, _widget, function): self.file_chooser.select_folder(self.win, None, function, None) @@ -288,7 +264,6 @@ class PreferencesWindow(Adw.PreferencesWindow): win.choose_folder(widget, set_dir) if globals()[f"{source_id}_installed"](path): - self.import_changed = True self.set_subtitle(win, source_id) else: @@ -315,28 +290,3 @@ class PreferencesWindow(Adw.PreferencesWindow): getattr(win, f"{source_id}_file_chooser_button").connect( "clicked", win.choose_folder, set_dir ) - - getattr(win, f"{source_id}_expander_row").connect( - "notify::enable-expansion", self.set_import_changed - ) - - def set_import_changed(self, widget, param): - if widget not in self.import_changed_widgets: - self.import_changed = True - self.import_changed_widgets[widget] = ( - param.name, - not widget.get_property(param.name), - ) - - def check_import(self, *_args): - # This checks whether any of the switches that did actually change their state - # would have an effect on the outcome of the import action - # and if they would, it initiates it. - - if self.import_changed and any( - (value := widget.get_property(prop[0])) and value != prop[1] - for widget, prop in self.import_changed_widgets.items() - ): - # The timeout is a hack to circumvent a GTK bug that I'm too lazy to report: - # The window would stay darkened because of the import dialog for some reason - GLib.timeout_add(1, self.win.get_application().on_import_action) From 8eb203cb06a6abab9f85c7fe8a7ecf746ee30223 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Sat, 17 Jun 2023 15:35:43 +0200 Subject: [PATCH 145/173] Convert executables to strings at init time --- docs/game_id.json.md | 4 ++-- src/details_window.py | 6 +----- src/game.py | 13 +++++-------- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/docs/game_id.json.md b/docs/game_id.json.md index 4b8b44c..c7e879b 100644 --- a/docs/game_id.json.md +++ b/docs/game_id.json.md @@ -35,9 +35,9 @@ Stored as a Unix time stamp. The executable to run when launching a game. -If the source has a URL handler, using that is preferred. In that case, the value should be `["xdg-open", "url://example/url"]` for Linux and `["start", "url://example/url"]` for Windows. +If the source has a URL handler, using that is preferred. In that case, the value should be `"xdg-open url://example/url"` for Linux and `"start url://example/url"` for Windows. -Stored as either a string or an argument vector to be passed to the shell through [subprocess.Popen](https://docs.python.org/3/library/subprocess.html#popen-constructor). +Stored as either a string (preferred) or an argument vector to be passed to the shell through [subprocess.Popen](https://docs.python.org/3/library/subprocess.html#popen-constructor). ### game_id diff --git a/src/details_window.py b/src/details_window.py index 8ec15ab..bde6132 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -69,11 +69,7 @@ class DetailsWindow(Adw.Window): self.name.set_text(self.game.name) if self.game.developer: self.developer.set_text(self.game.developer) - self.executable.set_text( - self.game.executable - if isinstance(self.game.executable, str) - else shlex.join(self.game.executable) - ) + self.executable.set_text(self.game.executable) self.apply_button.set_label(_("Apply")) self.game_cover.new_cover(self.game.get_cover_path()) diff --git a/src/game.py b/src/game.py index 734e195..365400a 100644 --- a/src/game.py +++ b/src/game.py @@ -86,6 +86,9 @@ class Game(Gtk.Box): def update_values(self, data): for key, value in data.items(): + # Convert executables to strings + if key == "executable" and isinstance(value, list): + value = shlex.join(value) setattr(self, key, value) def update(self): @@ -115,16 +118,10 @@ class Game(Gtk.Box): self.save() self.update() - string = ( - self.executable - if isinstance(self.executable, str) - else shlex.join(self.executable) - ) - args = ( - "flatpak-spawn --host /bin/sh -c " + shlex.quote(string) # Flatpak + "flatpak-spawn --host /bin/sh -c " + shlex.quote(self.executable) # Flatpak if os.getenv("FLATPAK_ID") == shared.APP_ID - else string # Others + else self.executable # Others ) logging.info("Starting %s: %s", self.name, str(args)) From eb915862161802182fb3d8e26d79c506315f1a39 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Sat, 17 Jun 2023 16:11:02 +0200 Subject: [PATCH 146/173] Clean up relative dates --- src/utils/relative_date.py | 44 ++++++++++++++++++++++++++++++++++++++ src/window.py | 22 ++++--------------- 2 files changed, 48 insertions(+), 18 deletions(-) create mode 100644 src/utils/relative_date.py diff --git a/src/utils/relative_date.py b/src/utils/relative_date.py new file mode 100644 index 0000000..6ecde67 --- /dev/null +++ b/src/utils/relative_date.py @@ -0,0 +1,44 @@ +# relative_date.py +# +# Copyright 2022-2023 kramo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + +from datetime import datetime + +from gi.repository import GLib + + +def relative_date(timestamp): # pylint: disable=too-many-return-statements + days_no = ((today := datetime.today()) - datetime.fromtimestamp(timestamp)).days + + if days_no == 0: + return _("Today") + if days_no == 1: + return _("Yesterday") + if days_no <= (day_of_week := today.weekday()): + return GLib.DateTime.new_from_unix_utc(timestamp).format("%A") + if days_no <= day_of_week + 7: + return _("Last Week") + if days_no <= (day_of_month := today.day): + return _("This Month") + if days_no <= day_of_month + 30: + return _("Last Month") + if days_no < (day_of_year := today.timetuple().tm_yday): + return GLib.DateTime.new_from_unix_utc(timestamp).format("%B") + if days_no <= day_of_year + 365: + return _("Last Year") + return GLib.DateTime.new_from_unix_utc(timestamp).format("%Y") diff --git a/src/window.py b/src/window.py index 59205c8..d82b07a 100644 --- a/src/window.py +++ b/src/window.py @@ -17,11 +17,10 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -from datetime import datetime - -from gi.repository import Adw, GLib, Gtk +from gi.repository import Adw, Gtk from src import shared # pylint: disable=no-name-in-module +from src.utils.relative_date import relative_date @Gtk.Template(resource_path=shared.PREFIX + "/gtk/window.ui") @@ -158,19 +157,6 @@ class CartridgesWindow(Adw.ApplicationWindow): def set_active_game(self, _widget, _pspec, game): self.active_game = game - def get_time(self, timestamp): - days_no = (datetime.today() - datetime.fromtimestamp(timestamp)).days - - if days_no == 0: - return _("Today") - if days_no == 1: - return _("Yesterday") - if days_no < 8: - return GLib.DateTime.new_from_unix_utc(timestamp).format("%A") - if days_no < 335: - return GLib.DateTime.new_from_unix_utc(timestamp).format("%B") - return GLib.DateTime.new_from_unix_utc(timestamp).format("%Y") - def show_details_view(self, game): self.active_game = game @@ -200,13 +186,13 @@ class CartridgesWindow(Adw.ApplicationWindow): self.details_view_title.set_label(game.name) self.details_view_header_bar_title.set_title(game.name) - date = self.get_time(game.added) + date = relative_date(game.added) self.details_view_added.set_label( # The variable is the date when the game was added _("Added: {}").format(date) ) last_played_date = ( - self.get_time(game.last_played) if game.last_played else _("Never") + relative_date(game.last_played) if game.last_played else _("Never") ) self.details_view_last_played.set_label( # The variable is the date when the game was last played From cff2a4ae6c932623b741b76aafb084658156bd3f Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 17 Jun 2023 16:15:53 +0200 Subject: [PATCH 147/173] Fix adding games manually --- src/details_window.py | 16 ++++++++++------ src/store/store.py | 6 +++++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/details_window.py b/src/details_window.py index bde6132..e4d7797 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -150,20 +150,23 @@ class DetailsWindow(Adw.Window): return # Increment the number after the game id (eg. imported_1, imported_2) - numbers = [0] - - for current_game in self.win.games: - if "imported_" in current_game: - numbers.append(int(current_game.replace("imported_", ""))) + game_id: str + for game_id in shared.store.games: + prefix = "imported_" + if not game_id.startswith(prefix): + continue + numbers.append(int(game_id.replace(prefix, "", count=1))) + game_number = max(numbers) + 1 self.game = Game( { - "game_id": f"imported_{str(max(numbers) + 1)}", + "game_id": f"imported_{game_number}", "hidden": False, "source": "imported", "added": int(time()), }, + allow_side_effects=False, ) else: @@ -198,6 +201,7 @@ class DetailsWindow(Adw.Window): self.game_cover.path, ) + shared.store.add_game(self.game, {}, run_pipeline=False) self.game.save() self.game.update() diff --git a/src/store/store.py b/src/store/store.py index 2b8cbe5..86e623b 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -40,7 +40,9 @@ class Store: ): path.unlink(missing_ok=True) - def add_game(self, game: Game, additional_data: dict) -> Pipeline | None: + def add_game( + self, game: Game, additional_data: dict, run_pipeline=True + ) -> Pipeline | None: """Add a game to the app""" # Ignore games from a newer spec version @@ -76,6 +78,8 @@ class Store: game.connect(signal, manager.execute_resilient_manager_logic) # Run the pipeline for the game + if not run_pipeline: + return None pipeline = Pipeline(game, additional_data, self.pipeline_managers) self.games[game.game_id] = game self.pipelines[game.game_id] = pipeline From 9d7a6d8ea44e5ae1f9b52fda943832136f19f858 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Sat, 17 Jun 2023 16:27:10 +0200 Subject: [PATCH 148/173] Add copyright headers --- src/importer/importer.py | 20 ++++++++++++++++++++ src/importer/sources/bottles_source.py | 20 ++++++++++++++++++++ src/importer/sources/heroic_source.py | 20 ++++++++++++++++++++ src/importer/sources/itch_source.py | 20 ++++++++++++++++++++ src/importer/sources/legendary_source.py | 19 +++++++++++++++++++ src/importer/sources/lutris_source.py | 20 ++++++++++++++++++++ src/importer/sources/source.py | 19 +++++++++++++++++++ src/importer/sources/steam_source.py | 20 ++++++++++++++++++++ src/logging/color_log_formatter.py | 19 +++++++++++++++++++ src/logging/session_file_handler.py | 19 +++++++++++++++++++ src/logging/setup.py | 19 +++++++++++++++++++ src/store/managers/async_manager.py | 19 +++++++++++++++++++ src/store/managers/display_manager.py | 19 +++++++++++++++++++ src/store/managers/file_manager.py | 19 +++++++++++++++++++ src/store/managers/local_cover_manager.py | 19 +++++++++++++++++++ src/store/managers/manager.py | 19 +++++++++++++++++++ src/store/managers/online_cover_manager.py | 19 +++++++++++++++++++ src/store/managers/sgdb_manager.py | 19 +++++++++++++++++++ src/store/managers/steam_api_manager.py | 19 +++++++++++++++++++ src/store/pipeline.py | 19 +++++++++++++++++++ src/store/store.py | 19 +++++++++++++++++++ src/utils/decorators.py | 21 ++++++++++++++++++++- src/utils/rate_limiter.py | 19 +++++++++++++++++++ src/utils/sqlite.py | 20 ++++++++++++++++++++ src/utils/steam.py | 20 ++++++++++++++++++++ src/utils/steamgriddb.py | 20 ++++++++++++++++++++ src/utils/task.py | 19 +++++++++++++++++++ 27 files changed, 523 insertions(+), 1 deletion(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index adfc23b..fa15ff6 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -1,3 +1,23 @@ +# importer.py +# +# Copyright 2022-2023 kramo +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import logging from gi.repository import Adw, Gtk, GLib diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index 5ce598d..410ed3e 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -1,3 +1,23 @@ +# bottles_source.py +# +# Copyright 2022-2023 kramo +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + from pathlib import Path from time import time diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index adcf34d..cfbf09c 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -1,3 +1,23 @@ +# heroic_source.py +# +# Copyright 2022-2023 kramo +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import json import logging from hashlib import sha256 diff --git a/src/importer/sources/itch_source.py b/src/importer/sources/itch_source.py index 43e546c..098ca87 100644 --- a/src/importer/sources/itch_source.py +++ b/src/importer/sources/itch_source.py @@ -1,3 +1,23 @@ +# itch_source.py +# +# Copyright 2022-2023 kramo +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + from pathlib import Path from shutil import rmtree from sqlite3 import connect diff --git a/src/importer/sources/legendary_source.py b/src/importer/sources/legendary_source.py index 87f1f05..08c3446 100644 --- a/src/importer/sources/legendary_source.py +++ b/src/importer/sources/legendary_source.py @@ -1,3 +1,22 @@ +# legendary_source.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import json import logging from json import JSONDecodeError diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 0da9b87..4d191d5 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -1,3 +1,23 @@ +# lutris_source.py +# +# Copyright 2022-2023 kramo +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + from shutil import rmtree from sqlite3 import connect from time import time diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index 73b3874..c0c6d6d 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -1,3 +1,22 @@ +# source.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import sys from abc import abstractmethod from collections.abc import Iterable, Iterator diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index c275dc7..97d9aac 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -1,3 +1,23 @@ +# steam_source.py +# +# Copyright 2022-2023 kramo +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import re from pathlib import Path from time import time diff --git a/src/logging/color_log_formatter.py b/src/logging/color_log_formatter.py index 54a873b..53fe261 100644 --- a/src/logging/color_log_formatter.py +++ b/src/logging/color_log_formatter.py @@ -1,3 +1,22 @@ +# color_log_formatter.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + from logging import Formatter, LogRecord diff --git a/src/logging/session_file_handler.py b/src/logging/session_file_handler.py index 03bf174..156be71 100644 --- a/src/logging/session_file_handler.py +++ b/src/logging/session_file_handler.py @@ -1,3 +1,22 @@ +# session_file_handler.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import lzma from io import StringIO from logging import StreamHandler diff --git a/src/logging/setup.py b/src/logging/setup.py index 73df738..122681c 100644 --- a/src/logging/setup.py +++ b/src/logging/setup.py @@ -1,3 +1,22 @@ +# setup.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import logging import logging.config as logging_dot_config import os diff --git a/src/store/managers/async_manager.py b/src/store/managers/async_manager.py index 373d51f..153ce82 100644 --- a/src/store/managers/async_manager.py +++ b/src/store/managers/async_manager.py @@ -1,3 +1,22 @@ +# async_manager.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + from typing import Callable, Any from gi.repository import Gio diff --git a/src/store/managers/display_manager.py b/src/store/managers/display_manager.py index 847bdfb..f6f1749 100644 --- a/src/store/managers/display_manager.py +++ b/src/store/managers/display_manager.py @@ -1,3 +1,22 @@ +# display_manager.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + from src import shared # pylint: disable=no-name-in-module from src.game import Game from src.game_cover import GameCover diff --git a/src/store/managers/file_manager.py b/src/store/managers/file_manager.py index 13d492f..80d448a 100644 --- a/src/store/managers/file_manager.py +++ b/src/store/managers/file_manager.py @@ -1,3 +1,22 @@ +# file_manager.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import json from src import shared # pylint: disable=no-name-in-module diff --git a/src/store/managers/local_cover_manager.py b/src/store/managers/local_cover_manager.py index eba3abc..9aa8703 100644 --- a/src/store/managers/local_cover_manager.py +++ b/src/store/managers/local_cover_manager.py @@ -1,3 +1,22 @@ +# local_cover_manager.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + from pathlib import Path from src.game import Game diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index 1124d55..c6b8ea9 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -1,3 +1,22 @@ +# manager.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import logging from abc import abstractmethod from threading import Lock diff --git a/src/store/managers/online_cover_manager.py b/src/store/managers/online_cover_manager.py index fa82d06..7398169 100644 --- a/src/store/managers/online_cover_manager.py +++ b/src/store/managers/online_cover_manager.py @@ -1,3 +1,22 @@ +# online_cover_manager.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import logging from pathlib import Path diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index a3a5811..ac59592 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -1,3 +1,22 @@ +# sgdb_manager.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + from json import JSONDecodeError from requests.exceptions import HTTPError, SSLError diff --git a/src/store/managers/steam_api_manager.py b/src/store/managers/steam_api_manager.py index efe82e2..75fa416 100644 --- a/src/store/managers/steam_api_manager.py +++ b/src/store/managers/steam_api_manager.py @@ -1,3 +1,22 @@ +# steam_api_manager.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + from requests.exceptions import HTTPError, SSLError from src.game import Game diff --git a/src/store/pipeline.py b/src/store/pipeline.py index 4799fd2..f3f2883 100644 --- a/src/store/pipeline.py +++ b/src/store/pipeline.py @@ -1,3 +1,22 @@ +# pipeline.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import logging from typing import Iterable diff --git a/src/store/store.py b/src/store/store.py index 86e623b..8639625 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -1,3 +1,22 @@ +# store.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import logging from src import shared # pylint: disable=no-name-in-module diff --git a/src/utils/decorators.py b/src/utils/decorators.py index be572ec..af37523 100644 --- a/src/utils/decorators.py +++ b/src/utils/decorators.py @@ -1,5 +1,24 @@ +# decorators.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + from pathlib import Path -from os import PathLike, environ +from os import PathLike from functools import wraps from src import shared # pylint: disable=no-name-in-module diff --git a/src/utils/rate_limiter.py b/src/utils/rate_limiter.py index d0f1e29..09b17a9 100644 --- a/src/utils/rate_limiter.py +++ b/src/utils/rate_limiter.py @@ -1,3 +1,22 @@ +# rate_limiter.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + from typing import Optional, Sized from threading import Lock, Thread, BoundedSemaphore from time import sleep, time diff --git a/src/utils/sqlite.py b/src/utils/sqlite.py index 9661c2a..c605dff 100644 --- a/src/utils/sqlite.py +++ b/src/utils/sqlite.py @@ -1,3 +1,23 @@ +# sqlite.py +# +# Copyright 2022-2023 kramo +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + from glob import escape from pathlib import Path from shutil import copyfile diff --git a/src/utils/steam.py b/src/utils/steam.py index 703b315..76305d3 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -1,3 +1,23 @@ +# steam.py +# +# Copyright 2022-2023 kramo +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import json import logging import re diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index db31c6c..674d019 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -1,3 +1,23 @@ +# steamgriddb.py +# +# Copyright 2022-2023 kramo +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import logging from pathlib import Path diff --git a/src/utils/task.py b/src/utils/task.py index c6b7076..190d7c4 100644 --- a/src/utils/task.py +++ b/src/utils/task.py @@ -1,3 +1,22 @@ +# task.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + from functools import wraps from gi.repository import Gio From 009e6f36419241a6216c6f3e62dce35885085b8d Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Sat, 17 Jun 2023 16:31:14 +0200 Subject: [PATCH 149/173] "Intellisense betrayed me" --- src/details_window.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/details_window.py b/src/details_window.py index e4d7797..d203873 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -156,7 +156,7 @@ class DetailsWindow(Adw.Window): prefix = "imported_" if not game_id.startswith(prefix): continue - numbers.append(int(game_id.replace(prefix, "", count=1))) + numbers.append(int(game_id.replace(prefix, ""))) game_number = max(numbers) + 1 self.game = Game( From 8dcbe56e76c3ffa6c915db94ddb6544c17df3066 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Sat, 17 Jun 2023 16:33:27 +0200 Subject: [PATCH 150/173] "Sigh..." --- src/details_window.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/details_window.py b/src/details_window.py index d203873..bf593b7 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -156,7 +156,7 @@ class DetailsWindow(Adw.Window): prefix = "imported_" if not game_id.startswith(prefix): continue - numbers.append(int(game_id.replace(prefix, ""))) + numbers.append(int(game_id.replace(prefix, "", 1))) game_number = max(numbers) + 1 self.game = Game( From 32adc68d829798694c6b9685356a43272a6d4abf Mon Sep 17 00:00:00 2001 From: kramo Date: Sun, 18 Jun 2023 12:58:24 +0200 Subject: [PATCH 151/173] Correct directory check --- src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.py b/src/main.py index 0b365cf..a160f02 100644 --- a/src/main.py +++ b/src/main.py @@ -134,7 +134,7 @@ class CartridgesApplication(Adw.Application): self.win.present() def load_games_from_disk(self): - if shared.games_dir.exists(): + if shared.games_dir.is_dir(): for game_file in shared.games_dir.iterdir(): data = json.load(game_file.open()) game = Game(data, allow_side_effects=False) From 2e97edcdb550ed8d85bdba84a56cd75f1f0f6ef8 Mon Sep 17 00:00:00 2001 From: kramo Date: Sun, 18 Jun 2023 13:15:03 +0200 Subject: [PATCH 152/173] Please pylint once and for all --- .pylintrc | 3 ++- src/details_window.py | 4 +--- src/game.py | 2 +- src/game_cover.py | 2 +- src/importer/importer.py | 2 +- src/importer/sources/bottles_source.py | 2 +- src/importer/sources/heroic_source.py | 2 +- src/importer/sources/itch_source.py | 2 +- src/importer/sources/legendary_source.py | 2 +- src/importer/sources/lutris_source.py | 2 +- src/importer/sources/source.py | 2 +- src/importer/sources/steam_source.py | 2 +- src/importers/bottles_importer.py | 2 +- src/importers/heroic_importer.py | 2 +- src/importers/itch_importer.py | 2 +- src/importers/lutris_importer.py | 2 +- src/importers/steam_importer.py | 2 +- src/logging/setup.py | 2 +- src/main.py | 2 +- src/preferences.py | 2 +- src/store/managers/display_manager.py | 2 +- src/store/managers/file_manager.py | 2 +- src/store/store.py | 2 +- src/utils/decorators.py | 2 +- src/utils/save_cover.py | 2 +- src/utils/steam.py | 2 +- src/utils/steamgriddb.py | 2 +- src/window.py | 2 +- 28 files changed, 29 insertions(+), 30 deletions(-) diff --git a/.pylintrc b/.pylintrc index b4518b4..e2f5b68 100644 --- a/.pylintrc +++ b/.pylintrc @@ -18,7 +18,8 @@ disable=raw-checker-failed, missing-class-docstring, missing-module-docstring, relative-beyond-top-level, - import-error + import-error, + no-name-in-module [TYPECHECK] diff --git a/src/details_window.py b/src/details_window.py index bf593b7..c75c302 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -18,17 +18,15 @@ # SPDX-License-Identifier: GPL-3.0-or-later import os -import shlex from time import time from gi.repository import Adw, Gio, GLib, Gtk from PIL import Image -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.game import Game from src.game_cover import GameCover from src.store.managers.sgdb_manager import SGDBManager -from src.store.pipeline import Pipeline from src.utils.create_dialog import create_dialog from src.utils.save_cover import resize_cover, save_cover from src.utils.steamgriddb import SGDBAuthError diff --git a/src/game.py b/src/game.py index 365400a..ce76216 100644 --- a/src/game.py +++ b/src/game.py @@ -26,7 +26,7 @@ from time import time from gi.repository import Adw, GLib, GObject, Gtk -from src import shared # pylint: disable=no-name-in-module +from src import shared # pylint: disable=too-many-instance-attributes diff --git a/src/game_cover.py b/src/game_cover.py index 793bd8c..cca0eba 100644 --- a/src/game_cover.py +++ b/src/game_cover.py @@ -20,7 +20,7 @@ from gi.repository import GdkPixbuf, Gio, GLib from PIL import Image, ImageFilter, ImageStat -from src import shared # pylint: disable=no-name-in-module +from src import shared class GameCover: diff --git a/src/importer/importer.py b/src/importer/importer.py index fa15ff6..e08af72 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -22,7 +22,7 @@ import logging from gi.repository import Adw, Gtk, GLib -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.game import Game from src.importer.sources.source import Source from src.store.pipeline import Pipeline diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index 410ed3e..a08c92d 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -23,7 +23,7 @@ from time import time import yaml -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.game import Game from src.importer.sources.source import ( SourceIterationResult, diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index cfbf09c..595afcf 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -26,7 +26,7 @@ from pathlib import Path from time import time from typing import Optional, TypedDict -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.game import Game from src.importer.sources.source import ( URLExecutableSource, diff --git a/src/importer/sources/itch_source.py b/src/importer/sources/itch_source.py index 098ca87..ddcb2d0 100644 --- a/src/importer/sources/itch_source.py +++ b/src/importer/sources/itch_source.py @@ -23,7 +23,7 @@ from shutil import rmtree from sqlite3 import connect from time import time -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.game import Game from src.importer.sources.source import ( SourceIterationResult, diff --git a/src/importer/sources/legendary_source.py b/src/importer/sources/legendary_source.py index 08c3446..64bd606 100644 --- a/src/importer/sources/legendary_source.py +++ b/src/importer/sources/legendary_source.py @@ -24,7 +24,7 @@ from pathlib import Path from time import time from typing import Generator -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.game import Game from src.importer.sources.source import Source, SourceIterationResult, SourceIterator from src.utils.decorators import replaced_by_path, replaced_by_schema_key diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 4d191d5..2830bd8 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -22,7 +22,7 @@ from shutil import rmtree from sqlite3 import connect from time import time -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.game import Game from src.importer.sources.source import ( SourceIterationResult, diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index c0c6d6d..376e5d9 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -23,7 +23,7 @@ from collections.abc import Iterable, Iterator from pathlib import Path from typing import Generator, Any -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.game import Game # Type of the data returned by iterating on a Source diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 97d9aac..f416d06 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -23,7 +23,7 @@ from pathlib import Path from time import time from typing import Iterable -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.game import Game from src.importer.sources.source import ( SourceIterationResult, diff --git a/src/importers/bottles_importer.py b/src/importers/bottles_importer.py index cba2f73..dc431af 100644 --- a/src/importers/bottles_importer.py +++ b/src/importers/bottles_importer.py @@ -22,7 +22,7 @@ from time import time import yaml -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.utils.check_install import check_install diff --git a/src/importers/heroic_importer.py b/src/importers/heroic_importer.py index 619e647..549b4c1 100644 --- a/src/importers/heroic_importer.py +++ b/src/importers/heroic_importer.py @@ -23,7 +23,7 @@ from hashlib import sha256 from pathlib import Path from time import time -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.utils.check_install import check_install diff --git a/src/importers/itch_importer.py b/src/importers/itch_importer.py index 6a19c91..ba8ad6b 100644 --- a/src/importers/itch_importer.py +++ b/src/importers/itch_importer.py @@ -26,7 +26,7 @@ from time import time import requests from gi.repository import GdkPixbuf, Gio -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.utils.check_install import check_install from src.utils.save_cover import resize_cover diff --git a/src/importers/lutris_importer.py b/src/importers/lutris_importer.py index 9fe43ab..a698caf 100644 --- a/src/importers/lutris_importer.py +++ b/src/importers/lutris_importer.py @@ -22,7 +22,7 @@ from shutil import copyfile from sqlite3 import connect from time import time -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.utils.check_install import check_install diff --git a/src/importers/steam_importer.py b/src/importers/steam_importer.py index 09c504f..f0e12dd 100644 --- a/src/importers/steam_importer.py +++ b/src/importers/steam_importer.py @@ -25,7 +25,7 @@ from time import time import requests from gi.repository import Gio -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.utils.check_install import check_install diff --git a/src/logging/setup.py b/src/logging/setup.py index 122681c..ab682f8 100644 --- a/src/logging/setup.py +++ b/src/logging/setup.py @@ -23,7 +23,7 @@ import os import subprocess import sys -from src import shared # pylint: disable=no-name-in-module +from src import shared def setup_logging(): diff --git a/src/main.py b/src/main.py index a160f02..29252e8 100644 --- a/src/main.py +++ b/src/main.py @@ -27,7 +27,7 @@ gi.require_version("Adw", "1") # pylint: disable=wrong-import-position from gi.repository import Adw, Gio, GLib, Gtk -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.details_window import DetailsWindow from src.game import Game from src.importer.importer import Importer diff --git a/src/preferences.py b/src/preferences.py index 7d8ad2d..3fb9ea8 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -24,7 +24,7 @@ from pathlib import Path from gi.repository import Adw, Gio, GLib, Gtk # pylint: disable=unused-import -from src import shared # pylint: disable=no-name-in-module +from src import shared # TODO use the new sources from src.importers.bottles_importer import bottles_installed diff --git a/src/store/managers/display_manager.py b/src/store/managers/display_manager.py index f6f1749..c8acf0d 100644 --- a/src/store/managers/display_manager.py +++ b/src/store/managers/display_manager.py @@ -17,7 +17,7 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.game import Game from src.game_cover import GameCover from src.store.managers.manager import Manager diff --git a/src/store/managers/file_manager.py b/src/store/managers/file_manager.py index 80d448a..4caa3b4 100644 --- a/src/store/managers/file_manager.py +++ b/src/store/managers/file_manager.py @@ -19,7 +19,7 @@ import json -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.game import Game from src.store.managers.async_manager import AsyncManager from src.store.managers.steam_api_manager import SteamAPIManager diff --git a/src/store/store.py b/src/store/store.py index 8639625..d56a746 100644 --- a/src/store/store.py +++ b/src/store/store.py @@ -19,7 +19,7 @@ import logging -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.game import Game from src.store.managers.manager import Manager from src.store.pipeline import Pipeline diff --git a/src/utils/decorators.py b/src/utils/decorators.py index af37523..fcb3cff 100644 --- a/src/utils/decorators.py +++ b/src/utils/decorators.py @@ -21,7 +21,7 @@ from pathlib import Path from os import PathLike from functools import wraps -from src import shared # pylint: disable=no-name-in-module +from src import shared def replaced_by_path(override: PathLike): # Decorator builder diff --git a/src/utils/save_cover.py b/src/utils/save_cover.py index 217cf7c..a3e6c67 100644 --- a/src/utils/save_cover.py +++ b/src/utils/save_cover.py @@ -24,7 +24,7 @@ from shutil import copyfile from gi.repository import Gio from PIL import Image, ImageSequence -from src import shared # pylint: disable=no-name-in-module +from src import shared def resize_cover(cover_path=None, pixbuf=None): diff --git a/src/utils/steam.py b/src/utils/steam.py index 76305d3..a15548e 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -26,7 +26,7 @@ from typing import TypedDict import requests from requests.exceptions import HTTPError -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.utils.rate_limiter import PickHistory, RateLimiter diff --git a/src/utils/steamgriddb.py b/src/utils/steamgriddb.py index 674d019..9cd9c3c 100644 --- a/src/utils/steamgriddb.py +++ b/src/utils/steamgriddb.py @@ -25,7 +25,7 @@ import requests from gi.repository import Gio from requests.exceptions import HTTPError -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.utils.save_cover import resize_cover, save_cover diff --git a/src/window.py b/src/window.py index d82b07a..5e34973 100644 --- a/src/window.py +++ b/src/window.py @@ -19,7 +19,7 @@ from gi.repository import Adw, Gtk -from src import shared # pylint: disable=no-name-in-module +from src import shared from src.utils.relative_date import relative_date From 2aea2fb377fb9dec7b86aa06f7e8dd164b467269 Mon Sep 17 00:00:00 2001 From: kramo Date: Sun, 18 Jun 2023 13:46:17 +0200 Subject: [PATCH 153/173] Move placeholders to Gdk.Texture --- data/cartridges.gresource.xml.in | 1 + data/library_placeholder_small.svg | 1 + src/game_cover.py | 25 +++++++++++++------------ src/window.py | 2 +- 4 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 data/library_placeholder_small.svg diff --git a/data/cartridges.gresource.xml.in b/data/cartridges.gresource.xml.in index 203edea..90d5c3b 100644 --- a/data/cartridges.gresource.xml.in +++ b/data/cartridges.gresource.xml.in @@ -9,5 +9,6 @@ gtk/style.css gtk/style-dark.css library_placeholder.svg + library_placeholder_small.svg diff --git a/data/library_placeholder_small.svg b/data/library_placeholder_small.svg new file mode 100644 index 0000000..541aec1 --- /dev/null +++ b/data/library_placeholder_small.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/game_cover.py b/src/game_cover.py index cca0eba..5a4621c 100644 --- a/src/game_cover.py +++ b/src/game_cover.py @@ -17,7 +17,7 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -from gi.repository import GdkPixbuf, Gio, GLib +from gi.repository import Gdk, GdkPixbuf, Gio, GLib from PIL import Image, ImageFilter, ImageStat from src import shared @@ -31,8 +31,11 @@ class GameCover: animation = None anim_iter = None - placeholder_pixbuf = GdkPixbuf.Pixbuf.new_from_resource_at_scale( - shared.PREFIX + "/library_placeholder.svg", 400, 600, False + placeholder = Gdk.Texture.new_from_resource( + shared.PREFIX + "/library_placeholder.svg" + ) + placeholder_small = Gdk.Texture.new_from_resource( + shared.PREFIX + "/library_placeholder_small.svg" ) def __init__(self, pictures, path=None): @@ -82,7 +85,7 @@ class GameCover: tmp_path = Gio.File.new_tmp(None)[0].get_path() image.save(tmp_path, "tiff", compression=None) - self.blurred = GdkPixbuf.Pixbuf.new_from_file(tmp_path) + self.blurred = Gdk.Texture.new_from_filename(tmp_path) stat = ImageStat.Stat(image.convert("L")) @@ -92,11 +95,8 @@ class GameCover: (stat.mean[0] + stat.extrema[0][1]) / 510, ) else: - self.blurred = GdkPixbuf.Pixbuf.new_from_resource_at_scale( - shared.PREFIX + "/library_placeholder.svg", 2, 2, False - ) - - self.luminance = (0.1, 0.8) + self.blurred = self.placeholder_small + self.luminance = (0.3, 0.5) return self.blurred @@ -113,9 +113,10 @@ class GameCover: self.animation = None else: for picture in self.pictures: - if not pixbuf: - pixbuf = self.placeholder_pixbuf - picture.set_pixbuf(pixbuf) + if pixbuf: + picture.set_pixbuf(pixbuf) + else: + picture.set_paintable(self.placeholder) def update_animation(self, data): if self.animation == data[1]: diff --git a/src/window.py b/src/window.py index 5e34973..9394020 100644 --- a/src/window.py +++ b/src/window.py @@ -179,7 +179,7 @@ class CartridgesWindow(Adw.ApplicationWindow): self.details_view_game_cover = game.game_cover self.details_view_game_cover.add_picture(self.details_view_cover) - self.details_view_blurred_cover.set_pixbuf( + self.details_view_blurred_cover.set_paintable( self.details_view_game_cover.get_blurred() ) From 286b44360efa6e01723181e1e8f8122c5fe7a375 Mon Sep 17 00:00:00 2001 From: kramo Date: Sun, 18 Jun 2023 13:57:09 +0200 Subject: [PATCH 154/173] Move game_cover away from GdkPixbuf --- src/details_window.py | 4 ++-- src/game_cover.py | 27 ++++++++++++++------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/details_window.py b/src/details_window.py index c75c302..e6d6f1d 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -71,7 +71,7 @@ class DetailsWindow(Adw.Window): self.apply_button.set_label(_("Apply")) self.game_cover.new_cover(self.game.get_cover_path()) - if self.game_cover.get_pixbuf(): + if self.game_cover.get_texture(): self.cover_button_delete_revealer.set_reveal_child(True) else: self.set_title(_("Add New Game")) @@ -205,7 +205,7 @@ class DetailsWindow(Adw.Window): # TODO: this is fucked up (less than before) # Get a cover from SGDB if none is present - if not self.game_cover.get_pixbuf(): + if not self.game_cover.get_texture(): self.game.set_loading(1) sgdb_manager: SGDBManager = shared.store.managers[SGDBManager] sgdb_manager.reset_cancellable() diff --git a/src/game_cover.py b/src/game_cover.py index 5a4621c..662094d 100644 --- a/src/game_cover.py +++ b/src/game_cover.py @@ -24,7 +24,7 @@ from src import shared class GameCover: - pixbuf = None + texture = None blurred = None luminance = None path = None @@ -54,7 +54,7 @@ class GameCover: def new_cover(self, path=None): self.animation = None - self.pixbuf = None + self.texture = None self.blurred = None self.luminance = None self.path = path @@ -64,13 +64,17 @@ class GameCover: task = Gio.Task.new() task.run_in_thread(self.create_func(self.path)) else: - self.pixbuf = GdkPixbuf.Pixbuf.new_from_file(str(path)) + self.texture = Gdk.Texture.new_from_filename(str(path)) if not self.animation: - self.set_pixbuf(self.pixbuf) + self.set_texture(self.texture) - def get_pixbuf(self): - return self.animation.get_static_image() if self.animation else self.pixbuf + def get_texture(self): + return ( + Gdk.Texture.new_for_pixbuf(self.animation.get_static_image()) + if self.animation + else self.texture + ) def get_blurred(self): if not self.blurred: @@ -103,9 +107,9 @@ class GameCover: def add_picture(self, picture): self.pictures.add(picture) if not self.animation: - self.set_pixbuf(self.pixbuf) + self.set_texture(self.texture) - def set_pixbuf(self, pixbuf): + def set_texture(self, texture): self.pictures.discard( picture for picture in self.pictures if not picture.is_visible() ) @@ -113,16 +117,13 @@ class GameCover: self.animation = None else: for picture in self.pictures: - if pixbuf: - picture.set_pixbuf(pixbuf) - else: - picture.set_paintable(self.placeholder) + picture.set_paintable(texture or self.placeholder) def update_animation(self, data): if self.animation == data[1]: self.anim_iter.advance() - self.set_pixbuf(self.anim_iter.get_pixbuf()) + self.set_texture(Gdk.Texture.new_for_pixbuf(self.anim_iter.get_pixbuf())) delay_time = self.anim_iter.get_delay_time() GLib.timeout_add( From a96b989a29af6fc3c09f4837f49cfcde69c51714 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Mon, 19 Jun 2023 12:58:52 +0200 Subject: [PATCH 155/173] Error handling --- src/details_window.py | 8 +++---- src/utils/save_cover.py | 51 ++++++++++++++++++++++------------------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/src/details_window.py b/src/details_window.py index e6d6f1d..b91534a 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -251,11 +251,11 @@ class DetailsWindow(Adw.Window): except GLib.GError: return - self.cover_button_delete_revealer.set_reveal_child(True) - self.cover_changed = True - def resize(): - self.game_cover.new_cover(resize_cover(path)) + if cover := resize_cover(path): + self.game_cover.new_cover(cover) + self.cover_button_delete_revealer.set_reveal_child(True) + self.cover_changed = True self.toggle_loading() self.toggle_loading() diff --git a/src/utils/save_cover.py b/src/utils/save_cover.py index a3e6c67..f405977 100644 --- a/src/utils/save_cover.py +++ b/src/utils/save_cover.py @@ -22,7 +22,7 @@ from pathlib import Path from shutil import copyfile from gi.repository import Gio -from PIL import Image, ImageSequence +from PIL import Image, ImageSequence, UnidentifiedImageError from src import shared @@ -35,32 +35,35 @@ def resize_cover(cover_path=None, pixbuf=None): cover_path = Path(Gio.File.new_tmp("XXXXXX.tiff")[0].get_path()) pixbuf.savev(str(cover_path), "tiff", ["compression"], ["1"]) - with Image.open(cover_path) as image: - if getattr(image, "is_animated", False): - frames = tuple( - frame.resize((200, 300)) for frame in ImageSequence.Iterator(image) - ) + try: + with Image.open(cover_path) as image: + if getattr(image, "is_animated", False): + frames = tuple( + frame.resize((200, 300)) for frame in ImageSequence.Iterator(image) + ) - tmp_path = Path(Gio.File.new_tmp("XXXXXX.gif")[0].get_path()) - frames[0].save( - tmp_path, - save_all=True, - append_images=frames[1:], - ) + tmp_path = Path(Gio.File.new_tmp("XXXXXX.gif")[0].get_path()) + frames[0].save( + tmp_path, + save_all=True, + append_images=frames[1:], + ) - else: - # This might not be necessary in the future - # https://github.com/python-pillow/Pillow/issues/2663 - if image.mode not in ("RGB", "RGBA"): - image = image.convert("RGBA") + else: + # This might not be necessary in the future + # https://github.com/python-pillow/Pillow/issues/2663 + if image.mode not in ("RGB", "RGBA"): + image = image.convert("RGBA") - tmp_path = Path(Gio.File.new_tmp("XXXXXX.tiff")[0].get_path()) - image.resize(shared.image_size).save( - tmp_path, - compression="tiff_adobe_deflate" - if shared.schema.get_boolean("high-quality-images") - else "webp", - ) + tmp_path = Path(Gio.File.new_tmp("XXXXXX.tiff")[0].get_path()) + image.resize(shared.image_size).save( + tmp_path, + compression="tiff_adobe_deflate" + if shared.schema.get_boolean("high-quality-images") + else "webp", + ) + except UnidentifiedImageError: + return None return tmp_path From 4793d50b0af203e7334efed992e2fcf7fe07ef85 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Mon, 19 Jun 2023 13:58:25 +0200 Subject: [PATCH 156/173] Clean up strings in preferences --- data/gtk/preferences.blp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/data/gtk/preferences.blp b/data/gtk/preferences.blp index f7a71b9..9714d93 100644 --- a/data/gtk/preferences.blp +++ b/data/gtk/preferences.blp @@ -77,7 +77,7 @@ template $PreferencesWindow : Adw.PreferencesWindow { show-enable-switch: true; Adw.ActionRow steam_action_row { - title: _("Steam Install Location"); + title: _("Install Location"); Button steam_file_chooser_button { icon-name: "folder-symbolic"; @@ -91,7 +91,7 @@ template $PreferencesWindow : Adw.PreferencesWindow { show-enable-switch: true; Adw.ActionRow lutris_action_row { - title: _("Lutris Install Location"); + title: _("Install Location"); Button lutris_file_chooser_button { icon-name: "folder-symbolic"; @@ -100,7 +100,7 @@ template $PreferencesWindow : Adw.PreferencesWindow { } Adw.ActionRow lutris_cache_action_row { - title: _("Lutris Cache Location"); + title: _("Cache Location"); Button lutris_cache_file_chooser_button { icon-name: "folder-symbolic"; @@ -123,7 +123,7 @@ template $PreferencesWindow : Adw.PreferencesWindow { show-enable-switch: true; Adw.ActionRow heroic_action_row { - title: _("Heroic Install Location"); + title: _("Install Location"); Button heroic_file_chooser_button { icon-name: "folder-symbolic"; @@ -164,7 +164,7 @@ template $PreferencesWindow : Adw.PreferencesWindow { show-enable-switch: true; Adw.ActionRow bottles_action_row { - title: _("Bottles Install Location"); + title: _("Install Location"); Button bottles_file_chooser_button { icon-name: "folder-symbolic"; @@ -178,7 +178,7 @@ template $PreferencesWindow : Adw.PreferencesWindow { show-enable-switch: true; Adw.ActionRow itch_action_row { - title: _("itch Install Location"); + title: _("Install Location"); Button itch_file_chooser_button { icon-name: "folder-symbolic"; @@ -192,7 +192,7 @@ template $PreferencesWindow : Adw.PreferencesWindow { show-enable-switch: true; Adw.ActionRow legendary_action_row { - title: _("Legendary Install Location"); + title: _("Install Location"); Button legendary_file_chooser_button { icon-name: "folder-symbolic"; From 42ba8244d0d3233f3391bfc6c6cbc3c5e55b958e Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Mon, 19 Jun 2023 13:58:43 +0200 Subject: [PATCH 157/173] Add "Reset" button for debugging --- src/preferences.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/preferences.py b/src/preferences.py index 3fb9ea8..d14acb8 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -20,6 +20,7 @@ import os import re from pathlib import Path +from shutil import rmtree from gi.repository import Adw, Gio, GLib, Gtk @@ -87,6 +88,8 @@ class PreferencesWindow(Adw.PreferencesWindow): sgdb_prefer_switch = Gtk.Template.Child() sgdb_animated_switch = Gtk.Template.Child() + danger_zone_group = Gtk.Template.Child() + removed_games = set() def __init__(self, **kwargs): @@ -111,6 +114,32 @@ class PreferencesWindow(Adw.PreferencesWindow): # General self.remove_all_games_button.connect("clicked", self.remove_all_games) + # Debug + if shared.PROFILE == "development": + + def reset_app(*_args): + rmtree(shared.data_dir / "cartridges", True) + rmtree(shared.config_dir / "cartridges", True) + rmtree(shared.cache_dir / "cartridges", True) + shared.win.get_application().quit() + + reset_button = Gtk.Button.new_with_label("Reset") + reset_button.set_valign(Gtk.Align.CENTER) + reset_button.add_css_class("destructive-action") + reset_button.connect("clicked", reset_app) + + self.danger_zone_group.add( + ( + reset_action_row := Adw.ActionRow( + title="Reset App", + subtitle="Completely resets and quits Cartridges", + ) + ) + ) + + reset_action_row.add_suffix(reset_button) + self.set_default_size(-1, 560) + # Steam self.create_preferences(self, "steam", "Steam") From 6ff9039064569de86bc7a098421f12a2f3d0302d Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Mon, 19 Jun 2023 14:58:34 +0200 Subject: [PATCH 158/173] Reset schema --- src/preferences.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/preferences.py b/src/preferences.py index d14acb8..e52b87a 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -121,6 +121,18 @@ class PreferencesWindow(Adw.PreferencesWindow): rmtree(shared.data_dir / "cartridges", True) rmtree(shared.config_dir / "cartridges", True) rmtree(shared.cache_dir / "cartridges", True) + + for key in ( + (settings_schema_source := Gio.SettingsSchemaSource.get_default()) + .lookup(shared.APP_ID, True) + .list_keys() + ): + shared.schema.reset(key) + for key in settings_schema_source.lookup( + shared.APP_ID + ".State", True + ).list_keys(): + shared.state_schema.reset(key) + shared.win.get_application().quit() reset_button = Gtk.Button.new_with_label("Reset") From 9f582dfa3e43b9577e851d002811bc7100ead560 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Mon, 19 Jun 2023 19:31:27 +0200 Subject: [PATCH 159/173] Improve a11y and consistency for info popover --- data/gtk/details_window.blp | 10 ++++++---- src/details_window.py | 6 ++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/data/gtk/details_window.blp b/data/gtk/details_window.blp index 8ca1cab..ecddb13 100644 --- a/data/gtk/details_window.blp +++ b/data/gtk/details_window.blp @@ -129,17 +129,19 @@ template $DetailsWindow : Adw.Window { icon-name: "help-about-symbolic"; tooltip-text: _("More Info"); - popover: Popover { - visible: bind-property exec_info_button.active bidirectional; + popover: Popover exec_info_popover { Label exec_info_label { use-markup: true; wrap: true; - max-width-chars: 30; + max-width-chars: 50; + halign: center; + valign: center; margin-top: 6; - margin-bottom: 12; + margin-bottom: 6; margin-start: 6; margin-end: 6; + selectable: true; } }; diff --git a/src/details_window.py b/src/details_window.py index b91534a..c226a3b 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -48,6 +48,7 @@ class DetailsWindow(Adw.Window): executable = Gtk.Template.Child() exec_info_label = Gtk.Template.Child() + exec_info_popover = Gtk.Template.Child() apply_button = Gtk.Template.Child() @@ -112,6 +113,11 @@ class DetailsWindow(Adw.Window): self.exec_info_label.set_label(exec_info_text) + def clear_info_selection(*_args): + self.exec_info_label.select_region(0, 0) + + self.exec_info_popover.connect("show", clear_info_selection) + self.cover_button_delete.connect("clicked", self.delete_pixbuf) self.cover_button_edit.connect("clicked", self.choose_cover) self.apply_button.connect("clicked", self.apply_preferences) From f9000be27206e8d629154b805f96ca36372f99f3 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 19 Jun 2023 22:47:56 +0200 Subject: [PATCH 160/173] =?UTF-8?q?=F0=9F=9A=A7=20WIP=20new=20location=20s?= =?UTF-8?q?ystem?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TODO - Locations contain the schema key - Schema key overriden at location resolve - No need for callable candidates, but need to represent "this location's key" --- data/gtk/preferences.blp | 40 ++-- src/importer/importer.py | 2 +- src/importer/sources/bottles_source.py | 28 +-- src/importer/sources/heroic_source.py | 31 +-- src/importer/sources/itch_source.py | 22 +- src/importer/sources/legendary_source.py | 24 ++- src/importer/sources/location.py | 64 ++++++ src/importer/sources/lutris_source.py | 40 +++- src/importer/sources/source.py | 49 ++--- src/importer/sources/steam_source.py | 30 +-- src/preferences.py | 259 +++++++++++------------ 11 files changed, 330 insertions(+), 259 deletions(-) create mode 100644 src/importer/sources/location.py diff --git a/data/gtk/preferences.blp b/data/gtk/preferences.blp index 9714d93..cd1f351 100644 --- a/data/gtk/preferences.blp +++ b/data/gtk/preferences.blp @@ -61,6 +61,22 @@ template $PreferencesWindow : Adw.PreferencesWindow { ] } } + + Adw.ActionRow reset_action_row { + title: _("Reset App"); + subtitle: _("Completely resets and quits Cartridges"); + visible: false; + + Button reset_button { + label: _("Reset"); + valign: center; + + styles [ + "destructive-action", + ] + } + } + } } @@ -76,10 +92,10 @@ template $PreferencesWindow : Adw.PreferencesWindow { title: _("Steam"); show-enable-switch: true; - Adw.ActionRow steam_action_row { + Adw.ActionRow steam_data_action_row { title: _("Install Location"); - Button steam_file_chooser_button { + Button steam_data_file_chooser_button { icon-name: "folder-symbolic"; valign: center; } @@ -90,10 +106,10 @@ template $PreferencesWindow : Adw.PreferencesWindow { title: _("Lutris"); show-enable-switch: true; - Adw.ActionRow lutris_action_row { + Adw.ActionRow lutris_data_action_row { title: _("Install Location"); - Button lutris_file_chooser_button { + Button lutris_data_file_chooser_button { icon-name: "folder-symbolic"; valign: center; } @@ -122,10 +138,10 @@ template $PreferencesWindow : Adw.PreferencesWindow { title: _("Heroic"); show-enable-switch: true; - Adw.ActionRow heroic_action_row { + Adw.ActionRow heroic_config_action_row { title: _("Install Location"); - Button heroic_file_chooser_button { + Button heroic_config_file_chooser_button { icon-name: "folder-symbolic"; valign: center; } @@ -163,10 +179,10 @@ template $PreferencesWindow : Adw.PreferencesWindow { title: _("Bottles"); show-enable-switch: true; - Adw.ActionRow bottles_action_row { + Adw.ActionRow bottles_data_action_row { title: _("Install Location"); - Button bottles_file_chooser_button { + Button bottles_data_file_chooser_button { icon-name: "folder-symbolic"; valign: center; } @@ -177,10 +193,10 @@ template $PreferencesWindow : Adw.PreferencesWindow { title: _("itch"); show-enable-switch: true; - Adw.ActionRow itch_action_row { + Adw.ActionRow itch_config_action_row { title: _("Install Location"); - Button itch_file_chooser_button { + Button itch_config_file_chooser_button { icon-name: "folder-symbolic"; valign: center; } @@ -191,10 +207,10 @@ template $PreferencesWindow : Adw.PreferencesWindow { title: _("Legendary"); show-enable-switch: true; - Adw.ActionRow legendary_action_row { + Adw.ActionRow legendary_config_action_row { title: _("Install Location"); - Button legendary_file_chooser_button { + Button legendary_config_file_chooser_button { icon-name: "folder-symbolic"; valign: center; } diff --git a/src/importer/importer.py b/src/importer/importer.py index e08af72..9cbeeb6 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -114,7 +114,7 @@ class Importer: source, *_rest = data # Early exit if not installed - if not source.is_installed: + if not source.is_available: logging.info("Source %s skipped, not installed", source.id) return logging.info("Scanning source %s", source.id) diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index a08c92d..0823c0a 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -25,15 +25,12 @@ import yaml from src import shared from src.game import Game +from src.importer.sources.location import Location from src.importer.sources.source import ( SourceIterationResult, SourceIterator, URLExecutableSource, ) -from src.utils.decorators import ( - replaced_by_path, - replaced_by_schema_key, -) class BottlesSourceIterator(SourceIterator): @@ -42,7 +39,7 @@ class BottlesSourceIterator(SourceIterator): def generator_builder(self) -> SourceIterationResult: """Generator method producing games""" - data = (self.source.location / "library.yml").read_text("utf-8") + data = self.source.data_location["library.yml"].read_text("utf-8") library: dict = yaml.safe_load(data) for entry in library.values(): @@ -65,11 +62,11 @@ class BottlesSourceIterator(SourceIterator): # as Cartridges can't access directories picked via Bottles' file picker portal bottles_location = Path( yaml.safe_load( - (self.source.location / "data.yml").read_text("utf-8") + self.source.data_location["data.yml"].read_text("utf-8") )["custom_bottles_path"] ) except (FileNotFoundError, KeyError): - bottles_location = self.source.location / "bottles" + bottles_location = self.source.data_location.root / "bottles" bottle_path = entry["bottle"]["path"] image_name = entry["thumbnail"].split(":")[1] @@ -88,9 +85,14 @@ class BottlesSource(URLExecutableSource): url_format = 'bottles:run/"{bottle_name}"/"{game_name}"' available_on = set(("linux",)) - @property - @replaced_by_schema_key - @replaced_by_path("~/.var/app/com.usebottles.bottles/data/bottles/") - @replaced_by_path(shared.data_dir / "bottles") - def location(self) -> Path: - raise FileNotFoundError() + data_location = Location( + candidates=( + lambda: shared.schema.get_string("bottles-location"), + "~/.var/app/com.usebottles.bottles/data/bottles/", + shared.data_dir / "bottles/", + ), + paths={ + "library.yml": (False, "library.yml"), + "data.yml": (False, "data.yml"), + }, + ) diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 595afcf..8b828f1 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -22,21 +22,17 @@ import json import logging from hashlib import sha256 from json import JSONDecodeError -from pathlib import Path from time import time from typing import Optional, TypedDict from src import shared from src.game import Game +from src.importer.sources.location import Location from src.importer.sources.source import ( URLExecutableSource, SourceIterationResult, SourceIterator, ) -from src.utils.decorators import ( - replaced_by_path, - replaced_by_schema_key, -) class HeroicLibraryEntry(TypedDict): @@ -103,7 +99,7 @@ class HeroicSourceIterator(SourceIterator): if service == "epic": uri += "?h=400&resize=1&w=300" digest = sha256(uri.encode()).hexdigest() - image_path = self.source.location / "images-cache" / digest + image_path = self.source.config_location.root / "images-cache" / digest additional_data = {"local_image_path": image_path} return (game, additional_data) @@ -116,7 +112,7 @@ class HeroicSourceIterator(SourceIterator): if not shared.schema.get_boolean("heroic-import-" + sub_source["service"]): continue # Load games from JSON - file = self.source.location.joinpath(*sub_source["path"]) + file = self.source.config_location.root.joinpath(*sub_source["path"]) try: library = json.load(file.open())["library"] except (JSONDecodeError, OSError, KeyError): @@ -141,15 +137,20 @@ class HeroicSource(URLExecutableSource): url_format = "heroic://launch/{app_name}" available_on = set(("linux", "win32")) + config_location = Location( + candidates=( + lambda: shared.schema.get_string("heroic-location"), + "~/.var/app/com.heroicgameslauncher.hgl/config/heroic/", + shared.config_dir / "heroic/", + "~/.config/heroic/", + shared.appdata_dir / "heroic/", + ), + paths={ + "config.json": (False, "config.json"), + }, + ) + @property def game_id_format(self) -> str: """The string format used to construct game IDs""" return self.name.lower() + "_{service}_{game_id}" - - @property - @replaced_by_schema_key - @replaced_by_path("~/.var/app/com.heroicgameslauncher.hgl/config/heroic/") - @replaced_by_path(shared.config_dir / "heroic") - @replaced_by_path(shared.appdata_dir / "heroic") - def location(self) -> Path: - raise FileNotFoundError() diff --git a/src/importer/sources/itch_source.py b/src/importer/sources/itch_source.py index ddcb2d0..a36aaa1 100644 --- a/src/importer/sources/itch_source.py +++ b/src/importer/sources/itch_source.py @@ -18,19 +18,18 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -from pathlib import Path from shutil import rmtree from sqlite3 import connect from time import time from src import shared from src.game import Game +from src.importer.sources.location import Location from src.importer.sources.source import ( SourceIterationResult, SourceIterator, URLExecutableSource, ) -from src.utils.decorators import replaced_by_path, replaced_by_schema_key from src.utils.sqlite import copy_db @@ -56,7 +55,7 @@ class ItchSourceIterator(SourceIterator): caves.game_id = games.id ; """ - db_path = copy_db(self.source.location / "db" / "butler.db") + db_path = copy_db(self.source.config_location["butler.db"]) connection = connect(db_path) cursor = connection.execute(db_request) @@ -84,10 +83,13 @@ class ItchSource(URLExecutableSource): url_format = "itch://caves/{cave_id}/launch" available_on = set(("linux", "win32")) - @property - @replaced_by_schema_key - @replaced_by_path("~/.var/app/io.itch.itch/config/itch/") - @replaced_by_path(shared.config_dir / "itch") - @replaced_by_path(shared.appdata_dir / "itch") - def location(self) -> Path: - raise FileNotFoundError() + config_location = Location( + candidates=( + lambda: shared.schema.get_string("itch-location"), + "~/.var/app/io.itch.itch/config/itch/", + shared.config_dir / "itch/", + "~/.config/itch/", + shared.appdata_dir / "itch/", + ), + paths={"butler.db": (False, "db/butler.db")}, + ) diff --git a/src/importer/sources/legendary_source.py b/src/importer/sources/legendary_source.py index 64bd606..7fd2f56 100644 --- a/src/importer/sources/legendary_source.py +++ b/src/importer/sources/legendary_source.py @@ -26,8 +26,8 @@ from typing import Generator from src import shared from src.game import Game +from src.importer.sources.location import Location from src.importer.sources.source import Source, SourceIterationResult, SourceIterator -from src.utils.decorators import replaced_by_path, replaced_by_schema_key class LegendarySourceIterator(SourceIterator): @@ -51,7 +51,7 @@ class LegendarySourceIterator(SourceIterator): data = {} # Get additional metadata from file (optional) - metadata_file = self.source.location / "metadata" / f"{app_name}.json" + metadata_file = self.source.data_location["metadata"] / f"{app_name}.json" try: metadata = json.load(metadata_file.open()) values["developer"] = metadata["metadata"]["developer"] @@ -67,7 +67,7 @@ class LegendarySourceIterator(SourceIterator): def generator_builder(self) -> Generator[SourceIterationResult, None, None]: # Open library - file = self.source.location / "installed.json" + file = self.source.data_location["installed.json"] try: library: dict = json.load(file.open()) except (JSONDecodeError, OSError): @@ -89,11 +89,17 @@ class LegendarySourceIterator(SourceIterator): class LegendarySource(Source): name = "Legendary" executable_format = "legendary launch {app_name}" - iterator_class = LegendarySourceIterator available_on = set(("linux", "win32")) - @property - @replaced_by_schema_key - @replaced_by_path(shared.config_dir / "legendary") - def location(self) -> Path: - raise FileNotFoundError() + iterator_class = LegendarySourceIterator + data_location: Location = Location( + candidates=( + lambda: shared.schema.get_string("legendary-location"), + shared.config_dir / "legendary/", + "~/.config/legendary", + ), + paths={ + "installed.json": (False, "installed.json"), + "metadata": (True, "metadata"), + }, + ) diff --git a/src/importer/sources/location.py b/src/importer/sources/location.py new file mode 100644 index 0000000..651db7b --- /dev/null +++ b/src/importer/sources/location.py @@ -0,0 +1,64 @@ +from pathlib import Path +from typing import Callable, Mapping, Iterable +from os import PathLike + +PathSegment = str | PathLike | Path +PathSegments = Iterable[PathSegment] +Candidate = PathSegments | Callable[[], PathSegments] + + +class UnresolvableLocationError(Exception): + pass + + +class Location: + """ + Class representing a filesystem location + + * A location may have multiple candidate roots + * From its root, multiple subpaths are named and should exist + """ + + candidates: Iterable[Candidate] + paths: Mapping[str, tuple[bool, PathSegments]] + root: Path = None + + def __init__( + self, + candidates: Iterable[Candidate], + paths: Mapping[str, tuple[bool, PathSegments]], + ) -> None: + super().__init__() + self.candidates = candidates + self.paths = paths + + def check_candidate(self, candidate: Path) -> bool: + """Check if a candidate root has the necessary files and directories""" + for type_is_dir, subpath in self.paths.values(): + subpath = Path(candidate) / Path(subpath) + if type_is_dir: + if not subpath.is_dir(): + return False + else: + if not subpath.is_file(): + return False + return True + + def resolve(self) -> None: + """Choose a root path from the candidates for the location. + If none fits, raise a UnresolvableLocationError""" + if self.root is not None: + return + for candidate in self.candidates: + if callable(candidate): + candidate = candidate() + candidate = Path(candidate).expanduser() + if self.check_candidate(candidate): + self.root = candidate + return + raise UnresolvableLocationError() + + def __getitem__(self, key: str): + """Get the computed path from its key for the location""" + self.resolve() + return self.root / self.paths[key][1] diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index 2830bd8..db05a18 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -17,19 +17,18 @@ # along with this program. If not, see . # # SPDX-License-Identifier: GPL-3.0-or-later - from shutil import rmtree from sqlite3 import connect from time import time from src import shared from src.game import Game +from src.importer.sources.location import Location from src.importer.sources.source import ( SourceIterationResult, SourceIterator, URLExecutableSource, ) -from src.utils.decorators import replaced_by_path, replaced_by_schema_key from src.utils.sqlite import copy_db @@ -52,7 +51,7 @@ class LutrisSourceIterator(SourceIterator): ; """ params = {"import_steam": shared.schema.get_boolean("lutris-import-steam")} - db_path = copy_db(self.source.location / "pga.db") + db_path = copy_db(self.source.data_location["pga.db"]) connection = connect(db_path) cursor = connection.execute(request, params) @@ -73,7 +72,7 @@ class LutrisSourceIterator(SourceIterator): game = Game(values, allow_side_effects=False) # Get official image path - image_path = self.source.location / "covers" / "coverart" / f"{row[2]}.jpg" + image_path = self.source.cache_location["coverart"] / f"{row[2]}.jpg" additional_data = {"local_image_path": image_path} # Produce game @@ -91,13 +90,32 @@ class LutrisSource(URLExecutableSource): url_format = "lutris:rungameid/{game_id}" available_on = set(("linux",)) + # FIXME possible bug: location picks ~/.var... and cache_lcoation picks ~/.local... + + data_location = Location( + candidates=( + lambda: shared.schema.get_string("lutris-location"), + "~/.var/app/net.lutris.Lutris/data/lutris/", + shared.data_dir / "lutris/", + "~/.local/share/lutris/", + ), + paths={ + "pga.db": (False, "pga.db"), + }, + ) + + cache_location = Location( + candidates=( + lambda: shared.schema.get_string("lutris-cache-location"), + "~/.var/app/net.lutris.Lutris/cache/lutris/", + shared.cache_dir / "lutris/", + "~/.cache/lutris", + ), + paths={ + "coverart": (True, "coverart"), + }, + ) + @property def game_id_format(self): return super().game_id_format + "_{game_internal_id}" - - @property - @replaced_by_schema_key - @replaced_by_path("~/.var/app/net.lutris.Lutris/data/lutris/") - @replaced_by_path("~/.local/share/lutris/") - def location(self): - raise FileNotFoundError() diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index 376e5d9..1470764 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -20,10 +20,9 @@ import sys from abc import abstractmethod from collections.abc import Iterable, Iterator -from pathlib import Path -from typing import Generator, Any +from typing import Generator, Any, Optional -from src import shared +from src.importer.sources.location import Location from src.game import Game # Type of the data returned by iterating on a Source @@ -62,9 +61,12 @@ class Source(Iterable): """Source of games. E.g an installed app with a config file that lists game directories""" name: str - iterator_class: type[SourceIterator] variant: str = None available_on: set[str] = set() + data_location: Optional[Location] = None + cache_location: Optional[Location] = None + config_location: Optional[Location] = None + iterator_class: type[SourceIterator] @property def full_name(self) -> str: @@ -88,44 +90,21 @@ class Source(Iterable): return self.name.lower() + "_{game_id}" @property - def is_installed(self): - # pylint: disable=pointless-statement - try: - self.location - except FileNotFoundError: - return False + def is_available(self): return sys.platform in self.available_on - @property - def location_key(self) -> str: - """ - The schema key pointing to the user-set location for the source. - May be overriden by inherinting classes. - """ - return f"{self.name.lower()}-location" - - def update_location_schema_key(self): - """Update the schema value for this source's location if possible""" - try: - location = self.location - except FileNotFoundError: - return - shared.schema.set_string(self.location_key, location) - - def __iter__(self) -> SourceIterator: - """Get an iterator for the source""" - return self.iterator_class(self) - - @property - @abstractmethod - def location(self) -> Path: - """The source's location on disk""" - @property @abstractmethod def executable_format(self) -> str: """The executable format used to construct game executables""" + def __iter__(self) -> SourceIterator: + """Get an iterator for the source""" + for location in (self.data_location, self.cache_location, self.config_location): + if location is not None: + location.resolve() + return self.iterator_class(self) + # pylint: disable=abstract-method class URLExecutableSource(Source): diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index f416d06..2110692 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -35,6 +35,7 @@ from src.utils.decorators import ( replaced_by_schema_key, ) from src.utils.steam import SteamFileHelper, SteamInvalidManifestError +from src.importer.sources.location import Location class SteamSourceIterator(SourceIterator): @@ -42,7 +43,7 @@ class SteamSourceIterator(SourceIterator): def get_manifest_dirs(self) -> Iterable[Path]: """Get dirs that contain steam app manifests""" - libraryfolders_path = self.source.location / "steamapps" / "libraryfolders.vdf" + libraryfolders_path = self.source.data_location["libraryfolders.vdf"] with open(libraryfolders_path, "r", encoding="utf-8") as file: contents = file.read() return [ @@ -101,9 +102,7 @@ class SteamSourceIterator(SourceIterator): # Add official cover image image_path = ( - self.source.location - / "appcache" - / "librarycache" + self.source.data_location["librarycache"] / f"{appid}_library_600x900.jpg" ) additional_data = {"local_image_path": image_path, "steam_appid": appid} @@ -114,15 +113,20 @@ class SteamSourceIterator(SourceIterator): class SteamSource(URLExecutableSource): name = "Steam" + available_on = set(("linux", "win32")) iterator_class = SteamSourceIterator url_format = "steam://rungameid/{game_id}" - available_on = set(("linux", "win32")) - @property - @replaced_by_schema_key - @replaced_by_path("~/.var/app/com.valvesoftware.Steam/data/Steam/") - @replaced_by_path(shared.data_dir / "Steam") - @replaced_by_path("~/.steam/") - @replaced_by_path(shared.programfiles32_dir / "Steam") - def location(self): - raise FileNotFoundError() + data_location = Location( + candidates=( + lambda: shared.schema.get_string("steam-location"), + "~/.var/app/com.valvesoftware.Steam/data/Steam/", + shared.data_dir / "Steam/", + "~/.steam/", + shared.programfiles32_dir / "Steam", + ), + paths={ + "libraryfolders.vdf": (False, "steamapps/libraryfolders.vdf"), + "librarycache": (True, "appcache/librarycache"), + }, + ) diff --git a/src/preferences.py b/src/preferences.py index e52b87a..7515c2f 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -24,15 +24,14 @@ from shutil import rmtree from gi.repository import Adw, Gio, GLib, Gtk -# pylint: disable=unused-import from src import shared - -# TODO use the new sources -from src.importers.bottles_importer import bottles_installed -from src.importers.heroic_importer import heroic_installed -from src.importers.itch_importer import itch_installed -from src.importers.lutris_importer import lutris_cache_exists, lutris_installed -from src.importers.steam_importer import steam_installed +from src.importer.sources.bottles_source import BottlesSource +from src.importer.sources.heroic_source import HeroicSource +from src.importer.sources.itch_source import ItchSource +from src.importer.sources.legendary_source import LegendarySource +from src.importer.sources.lutris_source import LutrisSource +from src.importer.sources.source import Source +from src.importer.sources.steam_source import SteamSource from src.utils.create_dialog import create_dialog @@ -49,37 +48,36 @@ class PreferencesWindow(Adw.PreferencesWindow): exit_after_launch_switch = Gtk.Template.Child() cover_launches_game_switch = Gtk.Template.Child() high_quality_images_switch = Gtk.Template.Child() - remove_all_games_button = Gtk.Template.Child() steam_expander_row = Gtk.Template.Child() - steam_action_row = Gtk.Template.Child() - steam_file_chooser_button = Gtk.Template.Child() + steam_data_action_row = Gtk.Template.Child() + steam_data_file_chooser_button = Gtk.Template.Child() lutris_expander_row = Gtk.Template.Child() - lutris_action_row = Gtk.Template.Child() - lutris_file_chooser_button = Gtk.Template.Child() + lutris_data_action_row = Gtk.Template.Child() + lutris_data_file_chooser_button = Gtk.Template.Child() lutris_cache_action_row = Gtk.Template.Child() lutris_cache_file_chooser_button = Gtk.Template.Child() lutris_import_steam_switch = Gtk.Template.Child() heroic_expander_row = Gtk.Template.Child() - heroic_action_row = Gtk.Template.Child() - heroic_file_chooser_button = Gtk.Template.Child() + heroic_config_action_row = Gtk.Template.Child() + heroic_config_file_chooser_button = Gtk.Template.Child() heroic_import_epic_switch = Gtk.Template.Child() heroic_import_gog_switch = Gtk.Template.Child() heroic_import_sideload_switch = Gtk.Template.Child() bottles_expander_row = Gtk.Template.Child() - bottles_action_row = Gtk.Template.Child() - bottles_file_chooser_button = Gtk.Template.Child() + bottles_data_action_row = Gtk.Template.Child() + bottles_data_file_chooser_button = Gtk.Template.Child() itch_expander_row = Gtk.Template.Child() - itch_action_row = Gtk.Template.Child() - itch_file_chooser_button = Gtk.Template.Child() + itch_config_action_row = Gtk.Template.Child() + itch_config_file_chooser_button = Gtk.Template.Child() legendary_expander_row = Gtk.Template.Child() - legendary_action_row = Gtk.Template.Child() - legendary_file_chooser_button = Gtk.Template.Child() + legendary_config_action_row = Gtk.Template.Child() + legendary_config_file_chooser_button = Gtk.Template.Child() sgdb_key_group = Gtk.Template.Child() sgdb_key_entry_row = Gtk.Template.Child() @@ -89,6 +87,9 @@ class PreferencesWindow(Adw.PreferencesWindow): sgdb_animated_switch = Gtk.Template.Child() danger_zone_group = Gtk.Template.Child() + reset_action_row = Gtk.Template.Child() + reset_button = Gtk.Template.Child() + remove_all_games_button = Gtk.Template.Child() removed_games = set() @@ -116,87 +117,25 @@ class PreferencesWindow(Adw.PreferencesWindow): # Debug if shared.PROFILE == "development": - - def reset_app(*_args): - rmtree(shared.data_dir / "cartridges", True) - rmtree(shared.config_dir / "cartridges", True) - rmtree(shared.cache_dir / "cartridges", True) - - for key in ( - (settings_schema_source := Gio.SettingsSchemaSource.get_default()) - .lookup(shared.APP_ID, True) - .list_keys() - ): - shared.schema.reset(key) - for key in settings_schema_source.lookup( - shared.APP_ID + ".State", True - ).list_keys(): - shared.state_schema.reset(key) - - shared.win.get_application().quit() - - reset_button = Gtk.Button.new_with_label("Reset") - reset_button.set_valign(Gtk.Align.CENTER) - reset_button.add_css_class("destructive-action") - reset_button.connect("clicked", reset_app) - - self.danger_zone_group.add( - ( - reset_action_row := Adw.ActionRow( - title="Reset App", - subtitle="Completely resets and quits Cartridges", - ) - ) - ) - - reset_action_row.add_suffix(reset_button) + self.reset_action_row.set_visible(True) + self.reset_button.connect("clicked", self.reset_app) self.set_default_size(-1, 560) - # Steam - self.create_preferences(self, "steam", "Steam") - - # Lutris - self.create_preferences(self, "lutris", "Lutris") - - def set_cache_dir(_source, result, *_args): - try: - path = Path(self.file_chooser.select_folder_finish(result).get_path()) - except GLib.GError: - return - - def response(widget, response): - if response == "choose_folder": - self.choose_folder(widget, set_cache_dir) - - if lutris_cache_exists(path): - self.set_subtitle(self, "lutris-cache") - + # Sources settings + for source_class in ( + BottlesSource, + HeroicSource, + ItchSource, + LegendarySource, + LutrisSource, + SteamSource, + ): + source = source_class() + if not source.is_available: + expander_row = getattr(self, f"{source.id}_expander_row") + expander_row.remove() else: - create_dialog( - self.win, - _("Cache Not Found"), - _("Select the Lutris cache directory."), - "choose_folder", - _("Set Location"), - ).connect("response", response) - - self.set_subtitle(self, "lutris-cache") - - self.lutris_cache_file_chooser_button.connect( - "clicked", self.choose_folder, set_cache_dir - ) - - # Heroic - self.create_preferences(self, "heroic", "Heroic", True) - - # Bottles - self.create_preferences(self, "bottles", "Bottles") - - # itch - self.create_preferences(self, "itch", "itch", True) - - # Legendary - self.create_preferences(self, "legendary", "Legendary", True) + self.init_source_row(source) # SteamGridDB def sgdb_key_changed(*_args): @@ -238,11 +177,6 @@ class PreferencesWindow(Adw.PreferencesWindow): ) ) - # Windows - if os.name == "nt": - self.sources_group.remove(self.lutris_expander_row) - self.sources_group.remove(self.bottles_expander_row) - def get_switch(self, setting): return getattr(self, f'{setting.replace("-", "_")}_switch') @@ -255,8 +189,8 @@ class PreferencesWindow(Adw.PreferencesWindow): Gio.SettingsBindFlags.DEFAULT, ) - def choose_folder(self, _widget, function): - self.file_chooser.select_folder(self.win, None, function, None) + def choose_folder(self, _widget, callback, callback_data=None): + self.file_chooser.select_folder(self.win, None, callback, callback_data) def undo_remove_all(self, *_args): for game in self.removed_games: @@ -281,53 +215,98 @@ class PreferencesWindow(Adw.PreferencesWindow): self.add_toast(self.toast) - def set_subtitle(self, win, source_id): - getattr(win, f'{source_id.replace("-", "_")}_action_row').set_subtitle( - # Remove the path if the dir is picked via the Flatpak portal - re.sub( - "/run/user/\\d*/doc/.*/", - "", - str( - Path(shared.schema.get_string(f"{source_id}-location")).expanduser() - ), - ) - ) + def reset_app(*_args): + rmtree(shared.data_dir / "cartridges", True) + rmtree(shared.config_dir / "cartridges", True) + rmtree(shared.cache_dir / "cartridges", True) + + for key in ( + (settings_schema_source := Gio.SettingsSchemaSource.get_default()) + .lookup(shared.APP_ID, True) + .list_keys() + ): + shared.schema.reset(key) + for key in settings_schema_source.lookup( + shared.APP_ID + ".State", True + ).list_keys(): + shared.state_schema.reset(key) + + shared.win.get_application().quit() + + def update_source_action_row_paths(self, source): + """Set the dir subtitle for a source's action rows""" + for location in ("data", "config", "cache"): + # Get the action row to subtitle + action_row = getattr(self, f"{source.id}_{location}_action_row", None) + if not action_row: + continue + + # Historically "location" meant data or config, so the key stays shared + infix = "-cache" if location == "cache" else "" + key = f"{source.id}{infix}-location" + path = Path(shared.schema.get_string(key)).expanduser() + + # Remove the path if the dir is picked via the Flatpak portal + subtitle = re.sub("/run/user/\\d*/doc/.*/", "", str(path)) + action_row.set_subtitle(subtitle) + + def init_source_row(self, source: Source): + """Initialize a preference row for a source class""" + + def set_dir(_widget, result, location_name): + """Callback called when a dir picker button is clicked""" - def create_preferences(self, win, source_id, name, config=False): - def set_dir(_source, result, *_args): try: - path = Path(win.file_chooser.select_folder_finish(result).get_path()) + path = Path(self.file_chooser.select_folder_finish(result).get_path()) except GLib.GError: return - def response(widget, response): - if response == "choose_folder": - win.choose_folder(widget, set_dir) - - if globals()[f"{source_id}_installed"](path): - self.set_subtitle(win, source_id) + # Good picked location + location = getattr(source, f"{location_name}_location") + if location.check_candidate(path): + # Set the schema + infix = "-cache" if location == "cache" else "" + key = f"{source.id}{infix}-location" + shared.schema.set_string(key, str(path)) + # Update the row + self.update_source_action_row_paths(source) + # Bad picked location, inform user else: - create_dialog( - win, - _("Installation Not Found"), - # The variable is the name of the game launcher - _("Select the {} configuration directory.").format(name) if config - # The variable is the name of the game launcher - else _("Select the {} data directory.").format(name), + if location_name == "cache": + title = "Cache not found" + subtitle_format = "Select the {} cache directory." + else: + title = "Installation not found" + subtitle_format = "Select the {} installation directory." + dialog = create_dialog( + self, + _(title), + _(subtitle_format).format(source.name), "choose_folder", _("Set Location"), - ).connect("response", response) + ) - self.set_subtitle(win, source_id) + def on_response(widget, response): + if response == "choose_folder": + self.choose_folder(widget, set_dir, location_name) + dialog.connect("response", on_response) + + # Bind expander row activation to source being enabled + expander_row = getattr(self, f"{source.id}_expander_row") shared.schema.bind( - source_id, - getattr(win, f"{source_id}_expander_row"), + source.id, + expander_row, "enable-expansion", Gio.SettingsBindFlags.DEFAULT, ) - getattr(win, f"{source_id}_file_chooser_button").connect( - "clicked", win.choose_folder, set_dir - ) + # Connect dir picker buttons + for location in ("data", "config", "cache"): + button = getattr(self, f"{source.id}_{location}_file_chooser_button", None) + if button is not None: + button.connect("clicked", self.choose_folder, set_dir, location) + + # Set the source row subtitles + self.update_source_action_row_paths(source) From e57a2a74df0d47d4d5e14a8cb37d9dd24cf4e542 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 19 Jun 2023 23:11:55 +0200 Subject: [PATCH 161/173] =?UTF-8?q?=F0=9F=9A=A7=20Set=20schema=20on=20loca?= =?UTF-8?q?tion=20resolve?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/bottles_source.py | 2 +- src/importer/sources/heroic_source.py | 2 +- src/importer/sources/itch_source.py | 2 +- src/importer/sources/legendary_source.py | 2 +- src/importer/sources/location.py | 33 ++++++++++++++++++------ src/importer/sources/lutris_source.py | 4 +-- src/importer/sources/steam_source.py | 2 +- 7 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index 0823c0a..d8148d4 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -86,8 +86,8 @@ class BottlesSource(URLExecutableSource): available_on = set(("linux",)) data_location = Location( + schema_key="bottles-location", candidates=( - lambda: shared.schema.get_string("bottles-location"), "~/.var/app/com.usebottles.bottles/data/bottles/", shared.data_dir / "bottles/", ), diff --git a/src/importer/sources/heroic_source.py b/src/importer/sources/heroic_source.py index 8b828f1..17d5da2 100644 --- a/src/importer/sources/heroic_source.py +++ b/src/importer/sources/heroic_source.py @@ -138,8 +138,8 @@ class HeroicSource(URLExecutableSource): available_on = set(("linux", "win32")) config_location = Location( + schema_key="heroic-location", candidates=( - lambda: shared.schema.get_string("heroic-location"), "~/.var/app/com.heroicgameslauncher.hgl/config/heroic/", shared.config_dir / "heroic/", "~/.config/heroic/", diff --git a/src/importer/sources/itch_source.py b/src/importer/sources/itch_source.py index a36aaa1..df861fd 100644 --- a/src/importer/sources/itch_source.py +++ b/src/importer/sources/itch_source.py @@ -84,8 +84,8 @@ class ItchSource(URLExecutableSource): available_on = set(("linux", "win32")) config_location = Location( + schema_key="itch-location", candidates=( - lambda: shared.schema.get_string("itch-location"), "~/.var/app/io.itch.itch/config/itch/", shared.config_dir / "itch/", "~/.config/itch/", diff --git a/src/importer/sources/legendary_source.py b/src/importer/sources/legendary_source.py index 7fd2f56..988122e 100644 --- a/src/importer/sources/legendary_source.py +++ b/src/importer/sources/legendary_source.py @@ -93,8 +93,8 @@ class LegendarySource(Source): iterator_class = LegendarySourceIterator data_location: Location = Location( + schema_key="legendary-location", candidates=( - lambda: shared.schema.get_string("legendary-location"), shared.config_dir / "legendary/", "~/.config/legendary", ), diff --git a/src/importer/sources/location.py b/src/importer/sources/location.py index 651db7b..545fe08 100644 --- a/src/importer/sources/location.py +++ b/src/importer/sources/location.py @@ -2,6 +2,8 @@ from pathlib import Path from typing import Callable, Mapping, Iterable from os import PathLike +from src import shared + PathSegment = str | PathLike | Path PathSegments = Iterable[PathSegment] Candidate = PathSegments | Callable[[], PathSegments] @@ -16,19 +18,24 @@ class Location: Class representing a filesystem location * A location may have multiple candidate roots - * From its root, multiple subpaths are named and should exist + * The path in the schema is always favored + * From the candidate root, multiple subpaths should exist for it to be valid + * When resolved, the schema is updated with the picked chosen """ + schema_key: str candidates: Iterable[Candidate] paths: Mapping[str, tuple[bool, PathSegments]] root: Path = None def __init__( self, + schema_key: str, candidates: Iterable[Candidate], paths: Mapping[str, tuple[bool, PathSegments]], ) -> None: super().__init__() + self.schema_key = schema_key self.candidates = candidates self.paths = paths @@ -47,16 +54,26 @@ class Location: def resolve(self) -> None: """Choose a root path from the candidates for the location. If none fits, raise a UnresolvableLocationError""" + if self.root is not None: return - for candidate in self.candidates: - if callable(candidate): - candidate = candidate() + + # Get the schema candidate + schema_candidate = shared.schema.get_string(self.schema_key) + + # Find the first matching candidate + for candidate in (schema_candidate, *self.candidates): candidate = Path(candidate).expanduser() - if self.check_candidate(candidate): - self.root = candidate - return - raise UnresolvableLocationError() + if not self.check_candidate(candidate): + continue + self.root = candidate + break + else: + # No good candidate found + raise UnresolvableLocationError() + + # Update the schema with the found candidate + shared.schema.set_string(self.schema_key, str(candidate)) def __getitem__(self, key: str): """Get the computed path from its key for the location""" diff --git a/src/importer/sources/lutris_source.py b/src/importer/sources/lutris_source.py index db05a18..1fa0aa0 100644 --- a/src/importer/sources/lutris_source.py +++ b/src/importer/sources/lutris_source.py @@ -93,8 +93,8 @@ class LutrisSource(URLExecutableSource): # FIXME possible bug: location picks ~/.var... and cache_lcoation picks ~/.local... data_location = Location( + schema_key="lutris-location", candidates=( - lambda: shared.schema.get_string("lutris-location"), "~/.var/app/net.lutris.Lutris/data/lutris/", shared.data_dir / "lutris/", "~/.local/share/lutris/", @@ -105,8 +105,8 @@ class LutrisSource(URLExecutableSource): ) cache_location = Location( + schema_key="lutris-cache-location", candidates=( - lambda: shared.schema.get_string("lutris-cache-location"), "~/.var/app/net.lutris.Lutris/cache/lutris/", shared.cache_dir / "lutris/", "~/.cache/lutris", diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 2110692..8adc787 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -118,8 +118,8 @@ class SteamSource(URLExecutableSource): url_format = "steam://rungameid/{game_id}" data_location = Location( + schema_key="steam-location", candidates=( - lambda: shared.schema.get_string("steam-location"), "~/.var/app/com.valvesoftware.Steam/data/Steam/", shared.data_dir / "Steam/", "~/.steam/", From e97c08a42bc1929e95c7f0b9fa971083f07bcab5 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 19 Jun 2023 23:33:18 +0200 Subject: [PATCH 162/173] =?UTF-8?q?=F0=9F=90=9B=20Added=20debug=20info,=20?= =?UTF-8?q?improved=20messages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importer/sources/location.py | 5 ++++- src/preferences.py | 10 ++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/importer/sources/location.py b/src/importer/sources/location.py index 545fe08..8374a20 100644 --- a/src/importer/sources/location.py +++ b/src/importer/sources/location.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path from typing import Callable, Mapping, Iterable from os import PathLike @@ -73,7 +74,9 @@ class Location: raise UnresolvableLocationError() # Update the schema with the found candidate - shared.schema.set_string(self.schema_key, str(candidate)) + value = str(candidate) + shared.schema.set_string(self.schema_key, value) + logging.debug("Resolved value for schema key %s: %s", self.schema_key, value) def __getitem__(self, key: str): """Get the computed path from its key for the location""" diff --git a/src/preferences.py b/src/preferences.py index 7515c2f..2944bc8 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -17,7 +17,7 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -import os +import logging import re from pathlib import Path from shutil import rmtree @@ -267,17 +267,19 @@ class PreferencesWindow(Adw.PreferencesWindow): # Set the schema infix = "-cache" if location == "cache" else "" key = f"{source.id}{infix}-location" - shared.schema.set_string(key, str(path)) + value = str(path) + shared.schema.set_string(key, value) # Update the row self.update_source_action_row_paths(source) + logging.debug("User-set value for schema key %s: %s", key, value) # Bad picked location, inform user else: if location_name == "cache": - title = "Cache not found" + title = "Cache directory not found" subtitle_format = "Select the {} cache directory." else: - title = "Installation not found" + title = "Installation directory not found" subtitle_format = "Select the {} installation directory." dialog = create_dialog( self, From 6408e250eeb8ffba80181289208215482002aeb9 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Mon, 19 Jun 2023 23:33:51 +0200 Subject: [PATCH 163/173] =?UTF-8?q?=F0=9F=90=9B=20Fixed=20breaking=20typo?= =?UTF-8?q?=20in=20location=20picking=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/preferences.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/preferences.py b/src/preferences.py index 2944bc8..dc25957 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -265,7 +265,7 @@ class PreferencesWindow(Adw.PreferencesWindow): location = getattr(source, f"{location_name}_location") if location.check_candidate(path): # Set the schema - infix = "-cache" if location == "cache" else "" + infix = "-cache" if location_name == "cache" else "" key = f"{source.id}{infix}-location" value = str(path) shared.schema.set_string(key, value) From c38a73ea98ee7c3e826665c40a0b65d125c362c5 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Tue, 20 Jun 2023 13:07:49 +0200 Subject: [PATCH 164/173] =?UTF-8?q?=F0=9F=94=A5=20Remove=20old=20importers?= =?UTF-8?q?=20+=20rate=20limiter=20debug=20logs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/importers/bottles_importer.py | 106 ---------------- src/importers/heroic_importer.py | 198 ------------------------------ src/importers/itch_importer.py | 188 ---------------------------- src/importers/lutris_importer.py | 133 -------------------- src/importers/steam_importer.py | 177 -------------------------- src/meson.build | 1 - src/utils/steam.py | 6 - 7 files changed, 809 deletions(-) delete mode 100644 src/importers/bottles_importer.py delete mode 100644 src/importers/heroic_importer.py delete mode 100644 src/importers/itch_importer.py delete mode 100644 src/importers/lutris_importer.py delete mode 100644 src/importers/steam_importer.py diff --git a/src/importers/bottles_importer.py b/src/importers/bottles_importer.py deleted file mode 100644 index dc431af..0000000 --- a/src/importers/bottles_importer.py +++ /dev/null @@ -1,106 +0,0 @@ -# bottles_importer.py -# -# Copyright 2022-2023 kramo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from pathlib import Path -from time import time - -import yaml - -from src import shared -from src.utils.check_install import check_install - - -def bottles_installed(path=None): - location_key = "bottles-location" - check = "library.yml" - - locations = ( - (path,) - if path - else ( - Path(shared.schema.get_string(location_key)).expanduser(), - Path.home() / ".var/app/com.usebottles.bottles/data/bottles", - shared.data_dir / "bottles", - ) - ) - - bottles_dir = check_install(check, locations, (shared.schema, location_key)) - - return bottles_dir - - -def bottles_importer(): - bottles_dir = bottles_installed() - if not bottles_dir: - return - - current_time = int(time()) - - data = (bottles_dir / "library.yml").read_text("utf-8") - - library = yaml.safe_load(data) - - importer = shared.importer - importer.total_queue += len(library) - importer.queue += len(library) - - for game in library: - game = library[game] - values = {} - - values["game_id"] = f'bottles_{game["id"]}' - - if ( - values["game_id"] in shared.win.games - and not shared.win.games[values["game_id"]].removed - ): - importer.save_game() - continue - - values["name"] = game["name"] - values["executable"] = [ - "xdg-open", - f'bottles:run/{game["bottle"]["name"]}/{game["name"]}', - ] - values["hidden"] = False - values["source"] = "bottles" - values["added"] = current_time - - # This will not work if both Cartridges and Bottles are installed via Flatpak - # as Cartridges can't access directories picked via Bottles' file picker portal - try: - bottles_location = Path( - yaml.safe_load((bottles_dir / "data.yml").read_text("utf-8"))[ - "custom_bottles_path" - ] - ) - except (FileNotFoundError, KeyError): - bottles_location = bottles_dir / "bottles" - - grid_path = ( - bottles_location - / game["bottle"]["path"] - / "grids" - / game["thumbnail"].split(":")[1] - ) - - importer.save_game( - values, - grid_path if game["thumbnail"] and grid_path.is_file() else None, - ) diff --git a/src/importers/heroic_importer.py b/src/importers/heroic_importer.py deleted file mode 100644 index 549b4c1..0000000 --- a/src/importers/heroic_importer.py +++ /dev/null @@ -1,198 +0,0 @@ -# heroic_importer.py -# -# Copyright 2022-2023 kramo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# SPDX-License-Identifier: GPL-3.0-or-later - -import json -import os -from hashlib import sha256 -from pathlib import Path -from time import time - -from src import shared -from src.utils.check_install import check_install - - -def heroic_installed(path=None): - location_key = "heroic-location" - check = "config.json" - - locations = ( - (path,) - if path - else ( - Path(shared.schema.get_string(location_key)).expanduser(), - Path.home() / ".var/app/com.heroicgameslauncher.hgl/config/heroic", - shared.config_dir / "heroic", - ) - ) - - if os.name == "nt" and not path: - locations += (Path(os.getenv("appdata")) / "heroic",) - - heroic_dir = check_install(check, locations, (shared.schema, location_key)) - - return heroic_dir - - -def heroic_importer(): - heroic_dir = heroic_installed() - if not heroic_dir: - return - - current_time = int(time()) - importer = shared.importer - - # Import Epic games - if not shared.schema.get_boolean("heroic-import-epic"): - pass - elif (heroic_dir / "store_cache" / "legendary_library.json").is_file(): - library = json.load( - (heroic_dir / "store_cache" / "legendary_library.json").open() - ) - - try: - for game in library["library"]: - if not game["is_installed"]: - continue - - importer.total_queue += 1 - importer.queue += 1 - - values = {} - - app_name = game["app_name"] - values["game_id"] = f"heroic_epic_{app_name}" - - if ( - values["game_id"] in shared.win.games - and not shared.win.games[values["game_id"]].removed - ): - importer.save_game() - continue - - values["name"] = game["title"] - values["developer"] = game["developer"] - values["executable"] = ( - ["start", f"heroic://launch/{app_name}"] - if os.name == "nt" - else ["xdg-open", f"heroic://launch/{app_name}"] - ) - values["hidden"] = False - values["source"] = "heroic_epic" - values["added"] = current_time - - image_path = ( - heroic_dir - / "images-cache" - / sha256( - (f'{game["art_square"]}?h=400&resize=1&w=300').encode() - ).hexdigest() - ) - - importer.save_game(values, image_path if image_path.is_file() else None) - - except KeyError: - pass - - # Import GOG games - if not shared.schema.get_boolean("heroic-import-gog"): - pass - elif (heroic_dir / "gog_store" / "installed.json").is_file() and ( - heroic_dir / "store_cache" / "gog_library.json" - ).is_file(): - installed = json.load((heroic_dir / "gog_store" / "installed.json").open()) - - importer.total_queue += len(installed["installed"]) - importer.queue += len(installed["installed"]) - - for item in installed["installed"]: - values = {} - app_name = item["appName"] - - values["game_id"] = f"heroic_gog_{app_name}" - - if ( - values["game_id"] in shared.win.games - and not shared.win.games[values["game_id"]].removed - ): - importer.save_game() - continue - - # Get game title and developer from gog_library.json as they are not present in installed.json - library = json.load( - (heroic_dir / "store_cache" / "gog_library.json").open() - ) - for game in library["games"]: - if game["app_name"] == app_name: - values["developer"] = game["developer"] - values["name"] = game["title"] - image_path = ( - heroic_dir - / "images-cache" - / sha256(game["art_square"].encode()).hexdigest() - ) - - values["executable"] = ( - ["start", f"heroic://launch/{app_name}"] - if os.name == "nt" - else ["xdg-open", f"heroic://launch/{app_name}"] - ) - values["hidden"] = False - values["source"] = "heroic_gog" - values["added"] = current_time - - importer.save_game(values, image_path if image_path.is_file() else None) - - # Import sideloaded games - if not shared.schema.get_boolean("heroic-import-sideload"): - pass - elif (heroic_dir / "sideload_apps" / "library.json").is_file(): - library = json.load((heroic_dir / "sideload_apps" / "library.json").open()) - - importer.total_queue += len(library["games"]) - importer.queue += len(library["games"]) - - for item in library["games"]: - values = {} - app_name = item["app_name"] - - values["game_id"] = f"heroic_sideload_{app_name}" - - if ( - values["game_id"] in shared.win.games - and not shared.win.games[values["game_id"]].removed - ): - importer.save_game() - continue - - values["name"] = item["title"] - values["executable"] = ( - ["start", f"heroic://launch/{app_name}"] - if os.name == "nt" - else ["xdg-open", f"heroic://launch/{app_name}"] - ) - values["hidden"] = False - values["source"] = "heroic_sideload" - values["added"] = current_time - image_path = ( - heroic_dir - / "images-cache" - / sha256(item["art_square"].encode()).hexdigest() - ) - - importer.save_game(values, image_path if image_path.is_file() else None) diff --git a/src/importers/itch_importer.py b/src/importers/itch_importer.py deleted file mode 100644 index ba8ad6b..0000000 --- a/src/importers/itch_importer.py +++ /dev/null @@ -1,188 +0,0 @@ -# itch_importer.py -# -# Copyright 2022-2023 kramo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# SPDX-License-Identifier: GPL-3.0-or-later - -import os -from pathlib import Path -from shutil import copyfile -from sqlite3 import connect -from time import time - -import requests -from gi.repository import GdkPixbuf, Gio - -from src import shared -from src.utils.check_install import check_install -from src.utils.save_cover import resize_cover - - -def get_game(task, current_time, row): - values = {} - - values["game_id"] = f"itch_{row[0]}" - - if ( - values["game_id"] in shared.win.games - and not shared.win.games[values["game_id"]].removed - ): - task.return_value((None, None)) - return - - values["added"] = current_time - values["executable"] = ( - ["start", f"itch://caves/{row[4]}/launch"] - if os.name == "nt" - else ["xdg-open", f"itch://caves/{row[4]}/launch"] - ) - values["hidden"] = False - values["name"] = row[1] - values["source"] = "itch" - - if row[3] or row[2]: - tmp_file = Gio.File.new_tmp()[0] - try: - with requests.get(row[3] or row[2], timeout=5) as cover: - cover.raise_for_status() - Path(tmp_file.get_path()).write_bytes(cover.content) - except requests.exceptions.RequestException: - task.return_value((values, None)) - return - - game_cover = GdkPixbuf.Pixbuf.new_from_stream_at_scale( - tmp_file.read(), 2, 2, False - ).scale_simple(*shared.image_size, GdkPixbuf.InterpType.BILINEAR) - - itch_pixbuf = GdkPixbuf.Pixbuf.new_from_stream(tmp_file.read()) - itch_pixbuf = itch_pixbuf.scale_simple( - shared.image_size[0], - itch_pixbuf.get_height() * (shared.image_size[0] / itch_pixbuf.get_width()), - GdkPixbuf.InterpType.BILINEAR, - ) - itch_pixbuf.composite( - game_cover, - 0, - (shared.image_size[1] - itch_pixbuf.get_height()) / 2, - itch_pixbuf.get_width(), - itch_pixbuf.get_height(), - 0, - (shared.image_size[1] - itch_pixbuf.get_height()) / 2, - 1.0, - 1.0, - GdkPixbuf.InterpType.BILINEAR, - 255, - ) - - else: - game_cover = None - - task.return_value((values, game_cover)) - - -def get_games_async(rows, importer): - current_time = int(time()) - - # Wrap the function in another one as Gio.Task.run_in_thread does not allow for passing args - def create_func(current_time, row): - def wrapper(task, *_args): - get_game( - task, - current_time, - row, - ) - - return wrapper - - def update_games(_task, result): - final_values = result.propagate_value()[1] - # No need for an if statement as final_value would be None for games we don't want to save - importer.save_game( - final_values[0], - resize_cover(pixbuf=final_values[1]), - ) - - for row in rows: - task = Gio.Task.new(None, None, update_games) - task.run_in_thread(create_func(current_time, row)) - - -def itch_installed(path=None): - location_key = "itch-location" - check = Path("db") / "butler.db" - - locations = ( - (path,) - if path - else ( - Path(shared.schema.get_string(location_key)).expanduser(), - Path.home() / ".var/app/io.itch.itch/config/itch", - shared.config_dir / "itch", - ) - ) - - if os.name == "nt" and not path: - locations += (Path(os.getenv("appdata")) / "itch",) - - itch_dir = check_install(check, locations, (shared.schema, location_key)) - - return itch_dir - - -def itch_importer(): - itch_dir = itch_installed() - if not itch_dir: - return - - database_path = (itch_dir / "db").expanduser() - - db_cache_dir = shared.cache_dir / "cartridges" / "itch" - db_cache_dir.mkdir(parents=True, exist_ok=True) - - # Copy the file because sqlite3 doesn't like databases in /run/user/ - database_tmp_path = db_cache_dir / "butler.db" - - for db_file in database_path.glob("butler.db*"): - copyfile(db_file, (db_cache_dir / db_file.name)) - - db_request = """ - SELECT - games.id, - games.title, - games.cover_url, - games.still_cover_url, - caves.id - FROM - 'caves' - INNER JOIN - 'games' - ON - caves.game_id = games.id - ; - """ - - connection = connect(database_tmp_path) - cursor = connection.execute(db_request) - rows = cursor.fetchall() - connection.close() - # No need to unlink temp files as they disappear when the connection is closed - database_tmp_path.unlink(missing_ok=True) - - importer = shared.importer - importer.total_queue += len(rows) - importer.queue += len(rows) - - get_games_async(rows, importer) diff --git a/src/importers/lutris_importer.py b/src/importers/lutris_importer.py deleted file mode 100644 index a698caf..0000000 --- a/src/importers/lutris_importer.py +++ /dev/null @@ -1,133 +0,0 @@ -# lutris_importer.py -# -# Copyright 2022-2023 kramo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from pathlib import Path -from shutil import copyfile -from sqlite3 import connect -from time import time - -from src import shared -from src.utils.check_install import check_install - - -def lutris_installed(path=None): - location_key = "lutris-location" - check = "pga.db" - - locations = ( - (path,) - if path - else ( - Path(shared.schema.get_string(location_key)).expanduser(), - Path.home() / ".var/app/net.lutris.Lutris/data/lutris", - shared.data_dir / "lutris", - ) - ) - - lutris_dir = check_install(check, locations, (shared.schema, location_key)) - - return lutris_dir - - -def lutris_cache_exists(path=None): - cache_key = "lutris-cache-location" - cache_check = "coverart" - - cache_locations = ( - (path,) - if path - else ( - Path(shared.schema.get_string(cache_key)).expanduser(), - Path.home() / ".var" / "app" / "net.lutris.Lutris" / "cache" / "lutris", - shared.cache_dir / "lutris", - ) - ) - - cache_dir = check_install(cache_check, cache_locations, (shared.schema, cache_key)) - - return cache_dir - - -def lutris_importer(): - lutris_dir = lutris_installed() - if not lutris_dir: - return - - cache_dir = lutris_cache_exists() - if not cache_dir: - return - - db_cache_dir = shared.cache_dir / "cartridges" / "lutris" - db_cache_dir.mkdir(parents=True, exist_ok=True) - - # Copy the file because sqlite3 doesn't like databases in /run/user/ - database_tmp_path = db_cache_dir / "pga.db" - - for db_file in lutris_dir.glob("pga.db*"): - copyfile(db_file, (db_cache_dir / db_file.name)) - - db_request = """ - SELECT - id, name, slug, runner, hidden - FROM - 'games' - WHERE - name IS NOT NULL - AND slug IS NOT NULL - AND configPath IS NOT NULL - AND installed IS TRUE - ; - """ - - connection = connect(database_tmp_path) - cursor = connection.execute(db_request) - rows = cursor.fetchall() - connection.close() - # No need to unlink temp files as they disappear when the connection is closed - database_tmp_path.unlink(missing_ok=True) - - if not shared.schema.get_boolean("lutris-import-steam"): - rows = [row for row in rows if not row[3] == "steam"] - - current_time = int(time()) - - importer = shared.importer - importer.total_queue += len(rows) - importer.queue += len(rows) - - for row in rows: - values = {} - - values["game_id"] = f"lutris_{row[3]}_{row[0]}" - - if ( - values["game_id"] in shared.win.games - and not shared.win.games[values["game_id"]].removed - ): - importer.save_game() - continue - - values["added"] = current_time - values["executable"] = ["xdg-open", f"lutris:rungameid/{row[0]}"] - values["hidden"] = row[4] == 1 - values["name"] = row[1] - values["source"] = f"lutris_{row[3]}" - - image_path = cache_dir / "coverart" / f"{row[2]}.jpg" - importer.save_game(values, image_path if image_path.is_file() else None) diff --git a/src/importers/steam_importer.py b/src/importers/steam_importer.py deleted file mode 100644 index f0e12dd..0000000 --- a/src/importers/steam_importer.py +++ /dev/null @@ -1,177 +0,0 @@ -# steam_importer.py -# -# Copyright 2022-2023 kramo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# SPDX-License-Identifier: GPL-3.0-or-later - -import os -import re -from pathlib import Path -from time import time - -import requests -from gi.repository import Gio - -from src import shared -from src.utils.check_install import check_install - - -def update_values_from_data(content, values): - basic_data = content[values["appid"]] - if not basic_data["success"]: - values["blacklisted"] = True - else: - data = basic_data["data"] - if data.get("developers"): - values["developer"] = ", ".join(data["developers"]) - - if data.get("type") not in {"game", "demo"}: - values["blacklisted"] = True - - return values - - -def get_game(task, datatypes, current_time, appmanifest, steam_dir): - values = {} - - data = appmanifest.read_text("utf-8") - for datatype in datatypes: - value = re.findall(f'"{datatype}"\t\t"(.*)"\n', data, re.IGNORECASE) - try: - values[datatype] = value[0] - except IndexError: - task.return_value((None, None)) - return - - values["game_id"] = f'steam_{values["appid"]}' - - if ( - values["game_id"] in shared.win.games - and not shared.win.games[values["game_id"]].removed - ): - task.return_value((None, None)) - return - - values["executable"] = ( - ["start", f'steam://rungameid/{values["appid"]}'] - if os.name == "nt" - else ["xdg-open", f'steam://rungameid/{values["appid"]}'] - ) - values["hidden"] = False - values["source"] = "steam" - values["added"] = current_time - - image_path = ( - steam_dir - / "appcache" - / "librarycache" - / f'{values["appid"]}_library_600x900.jpg' - ) - - try: - with requests.get( - f'https://store.steampowered.com/api/appdetails?appids={values["appid"]}', - timeout=5, - ) as open_file: - open_file.raise_for_status() - content = open_file.json() - except requests.exceptions.RequestException: - task.return_value((values, image_path if image_path.is_file() else None)) - return - - values = update_values_from_data(content, values) - task.return_value((values, image_path if image_path.is_file() else None)) - - -def get_games_async(appmanifests, steam_dir, importer): - datatypes = ["appid", "name"] - current_time = int(time()) - - # Wrap the function in another one as Gio.Task.run_in_thread does not allow for passing args - def create_func(datatypes, current_time, appmanifest, steam_dir): - def wrapper(task, *_args): - get_game( - task, - datatypes, - current_time, - appmanifest, - steam_dir, - ) - - return wrapper - - def update_games(_task, result): - final_values = result.propagate_value()[1] - # No need for an if statement as final_value would be None for games we don't want to save - importer.save_game(final_values[0], final_values[1]) - - for appmanifest in appmanifests: - task = Gio.Task.new(None, None, update_games) - task.run_in_thread(create_func(datatypes, current_time, appmanifest, steam_dir)) - - -def steam_installed(path=None): - location_key = "steam-location" - check = "steamapps" - - subdirs = ("steam", "Steam") - locations = ( - (path,) - if path - else ( - Path(shared.schema.get_string(location_key)).expanduser(), - Path.home() / ".steam", - shared.data_dir / "Steam", - Path.home() / ".var/app/com.valvesoftware.Steam/data/Steam", - ) - ) - - if os.name == "nt": - locations += (Path(os.getenv("programfiles(x86)")) / "Steam",) - - steam_dir = check_install(check, locations, (shared.schema, location_key), subdirs) - - return steam_dir - - -def steam_importer(): - steam_dir = steam_installed() - if not steam_dir: - return - - appmanifests = [] - - if (lib_file := steam_dir / "steamapps" / "libraryfolders.vdf").is_file(): - libraryfolders = lib_file.open().read() - steam_dirs = [ - Path(path) for path in re.findall('"path"\t\t"(.*)"\n', libraryfolders) - ] - else: - steam_dirs = [steam_dir] - - for directory in steam_dirs: - try: - for open_file in (directory / "steamapps").iterdir(): - if open_file.is_file() and "appmanifest" in open_file.name: - appmanifests.append(open_file) - except FileNotFoundError: - continue - - importer = shared.importer - importer.total_queue += len(appmanifests) - importer.queue += len(appmanifests) - - get_games_async(appmanifests, steam_dir, importer) diff --git a/src/meson.build b/src/meson.build index 6631a62..688ab51 100644 --- a/src/meson.build +++ b/src/meson.build @@ -9,7 +9,6 @@ configure_file( ) install_subdir('importer', install_dir: moduledir) -install_subdir('importers', install_dir: moduledir) install_subdir('utils', install_dir: moduledir) install_subdir('store', install_dir: moduledir) install_subdir('logging', install_dir: moduledir) diff --git a/src/utils/steam.py b/src/utils/steam.py index a15548e..25a1584 100644 --- a/src/utils/steam.py +++ b/src/utils/steam.py @@ -80,12 +80,6 @@ class SteamRateLimiter(RateLimiter): self.pick_history.remove_old_entries() super().__init__() - @property - def refill_spacing(self) -> float: - spacing = super().refill_spacing - logging.debug("Next Steam API request token in %f seconds", spacing) - return spacing - def acquire(self): """Get a token from the bucket and store the pick history in the schema""" super().acquire() From 41c2a1023a662686f45e1c773f5c7a3d08912e64 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Thu, 22 Jun 2023 21:13:27 +0200 Subject: [PATCH 165/173] Backport Bottles thumbnail fix --- src/importer/sources/bottles_source.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/importer/sources/bottles_source.py b/src/importer/sources/bottles_source.py index d8148d4..1136334 100644 --- a/src/importer/sources/bottles_source.py +++ b/src/importer/sources/bottles_source.py @@ -69,9 +69,12 @@ class BottlesSourceIterator(SourceIterator): bottles_location = self.source.data_location.root / "bottles" bottle_path = entry["bottle"]["path"] - image_name = entry["thumbnail"].split(":")[1] - image_path = bottles_location / bottle_path / "grids" / image_name - additional_data = {"local_image_path": image_path} + + additional_data = {} + if entry["thumbnail"]: + image_name = entry["thumbnail"].split(":")[1] + image_path = bottles_location / bottle_path / "grids" / image_name + additional_data = {"local_image_path": image_path} # Produce game yield (game, additional_data) From 3fa80a53c65623684269a74a8658adbb9ab7381c Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 24 Jun 2023 15:13:35 +0200 Subject: [PATCH 166/173] =?UTF-8?q?=F0=9F=8E=A8=20Work=20on=20import=20err?= =?UTF-8?q?or=20handling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Generic ErrorProducer class - Importer and managers are error producers - SGDB Auth friendly error - Bad source location friendly errors (data, config, cache) - Removed unused decorators --- src/errors/error_producer.py | 28 ++++++++++++ src/errors/friendly_error.py | 47 ++++++++++++++++++++ src/importer/importer.py | 35 +++++++++++---- src/importer/sources/legendary_source.py | 1 - src/importer/sources/source.py | 20 +++++++-- src/importer/sources/steam_source.py | 4 -- src/meson.build | 1 + src/store/managers/manager.py | 34 +++++--------- src/store/managers/sgdb_manager.py | 10 +++-- src/utils/decorators.py | 56 ------------------------ 10 files changed, 136 insertions(+), 100 deletions(-) create mode 100644 src/errors/error_producer.py create mode 100644 src/errors/friendly_error.py delete mode 100644 src/utils/decorators.py diff --git a/src/errors/error_producer.py b/src/errors/error_producer.py new file mode 100644 index 0000000..7f31fea --- /dev/null +++ b/src/errors/error_producer.py @@ -0,0 +1,28 @@ +from threading import Lock + + +class ErrorProducer: + """ + A mixin for objects that produce errors. + + Specifies the report_error and collect_errors methods in a thread-safe manner. + """ + + errors: list[Exception] = None + errors_lock: Lock = None + + def __init__(self) -> None: + self.errors = [] + self.errors_lock = Lock() + + def report_error(self, error: Exception): + """Report an error""" + with self.errors_lock: + self.errors.append(error) + + def collect_errors(self) -> list[Exception]: + """Collect and remove the errors produced by the object""" + with self.errors_lock: + errors = self.errors.copy() + self.errors.clear() + return errors diff --git a/src/errors/friendly_error.py b/src/errors/friendly_error.py new file mode 100644 index 0000000..9fef536 --- /dev/null +++ b/src/errors/friendly_error.py @@ -0,0 +1,47 @@ +from typing import Iterable + + +class FriendlyError(Exception): + """ + An error that is supposed to be shown to the user in a nice format + + Use `raise ... from ...` to preserve context. + """ + + title_format: str + title_args: Iterable[str] + subtitle_format: str + subtitle_args: Iterable[str] + + @property + def title(self) -> str: + """Get the gettext translated error title""" + return _(self.title_format).format(self.title_args) + + @property + def subtitle(self) -> str: + """Get the gettext translated error subtitle""" + return _(self.subtitle_format).format(self.subtitle_args) + + def __init__( + self, + title: str, + subtitle: str, + title_args: Iterable[str] = None, + subtitle_args: Iterable[str] = None, + ) -> None: + """Create a friendly error + + :param str title: The error's title, translatable with gettext + :param str subtitle: The error's subtitle, translatable with gettext + """ + super().__init__() + if title is not None: + self.title_format = title + if subtitle is not None: + self.subtitle_format = subtitle + self.title_args = title_args if title_args else () + self.subtitle_args = subtitle_args if subtitle_args else () + + def __str__(self) -> str: + return f"{self.title} - {self.subtitle}" diff --git a/src/importer/importer.py b/src/importer/importer.py index 9cbeeb6..b684bcb 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -20,18 +20,21 @@ import logging -from gi.repository import Adw, Gtk, GLib +from gi.repository import Adw, GLib, Gtk from src import shared +from src.errors.error_producer import ErrorProducer +from src.errors.friendly_error import FriendlyError from src.game import Game from src.importer.sources.source import Source +from src.store.managers.async_manager import AsyncManager from src.store.pipeline import Pipeline from src.utils.create_dialog import create_dialog from src.utils.task import Task # pylint: disable=too-many-instance-attributes -class Importer: +class Importer(ErrorProducer): """A class in charge of scanning sources for games""" progressbar = None @@ -47,6 +50,7 @@ class Importer: game_pipelines: set[Pipeline] = None def __init__(self): + super().__init__() self.game_pipelines = set() self.sources = set() @@ -81,6 +85,15 @@ class Importer: self.create_dialog() + # Collect all errors and reset the cancellables for the managers + # - Only one importer exists at any given time + # - Every import starts fresh + self.collect_errors() + for manager in shared.store.managers.values(): + manager.collect_errors() + if isinstance(manager, AsyncManager): + manager.reset_cancellable() + for source in self.sources: logging.debug("Importing games from source %s", source.id) task = Task.new(None, None, self.source_callback, (source,)) @@ -129,10 +142,9 @@ class Importer: iteration_result = next(iterator) except StopIteration: break - except Exception as exception: # pylint: disable=broad-exception-caught - logging.exception( - "Exception in source %s", source.id, exc_info=exception - ) + except Exception as error: # pylint: disable=broad-exception-caught + logging.exception("%s in %s", type(error).__name__, source.id) + self.report_error(error) continue # Handle the result depending on its type @@ -202,9 +214,16 @@ class Importer: string = _("The following errors occured during import:") errors = "" + # Collect all errors that happened in the importer and the managers + collected_errors: list[Exception] = [] + collected_errors.extend(self.collect_errors()) for manager in shared.store.managers.values(): - for error in manager.collect_errors(): - errors += "\n\n" + str(error) + collected_errors.extend(manager.collect_errors()) + for error in collected_errors: + # Only display friendly errors + if not isinstance(error, FriendlyError): + continue + errors += "\n\n" + str(error) if errors: create_dialog( diff --git a/src/importer/sources/legendary_source.py b/src/importer/sources/legendary_source.py index 988122e..4806c24 100644 --- a/src/importer/sources/legendary_source.py +++ b/src/importer/sources/legendary_source.py @@ -20,7 +20,6 @@ import json import logging from json import JSONDecodeError -from pathlib import Path from time import time from typing import Generator diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index 1470764..80c62a1 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -20,10 +20,11 @@ import sys from abc import abstractmethod from collections.abc import Iterable, Iterator -from typing import Generator, Any, Optional +from typing import Any, Generator, Optional -from src.importer.sources.location import Location +from src.errors.friendly_error import FriendlyError from src.game import Game +from src.importer.sources.location import Location, UnresolvableLocationError # Type of the data returned by iterating on a Source SourceIterationResult = None | Game | tuple[Game, tuple[Any]] @@ -100,9 +101,20 @@ class Source(Iterable): def __iter__(self) -> SourceIterator: """Get an iterator for the source""" - for location in (self.data_location, self.cache_location, self.config_location): - if location is not None: + for location_name in ("data", "cache", "config"): + location = getattr(self, f"{location_name}_location", None) + if location is None: + continue + try: location.resolve() + except UnresolvableLocationError as error: + raise FriendlyError( + # The variable is the source's name + f"Invalid {location_name} location for {{}}", + "Change it or disable the source in the preferences", + (self.name,), + (self.name,), + ) from error return self.iterator_class(self) diff --git a/src/importer/sources/steam_source.py b/src/importer/sources/steam_source.py index 8adc787..aed6328 100644 --- a/src/importer/sources/steam_source.py +++ b/src/importer/sources/steam_source.py @@ -30,10 +30,6 @@ from src.importer.sources.source import ( SourceIterator, URLExecutableSource, ) -from src.utils.decorators import ( - replaced_by_path, - replaced_by_schema_key, -) from src.utils.steam import SteamFileHelper, SteamInvalidManifestError from src.importer.sources.location import Location diff --git a/src/meson.build b/src/meson.build index 688ab51..5bb4a75 100644 --- a/src/meson.build +++ b/src/meson.build @@ -12,6 +12,7 @@ install_subdir('importer', install_dir: moduledir) install_subdir('utils', install_dir: moduledir) install_subdir('store', install_dir: moduledir) install_subdir('logging', install_dir: moduledir) +install_subdir('errors', install_dir: moduledir) install_data( [ 'main.py', diff --git a/src/store/managers/manager.py b/src/store/managers/manager.py index c6b8ea9..fbf5556 100644 --- a/src/store/managers/manager.py +++ b/src/store/managers/manager.py @@ -19,14 +19,15 @@ import logging from abc import abstractmethod -from threading import Lock from time import sleep from typing import Any, Callable, Container +from src.errors.error_producer import ErrorProducer +from src.errors.friendly_error import FriendlyError from src.game import Game -class Manager: +class Manager(ErrorProducer): """Class in charge of handling a post creation action for games. * May connect to signals on the game to handle them. @@ -44,30 +45,10 @@ class Manager: retry_delay: int = 3 max_tries: int = 3 - errors: list[Exception] - errors_lock: Lock = None - @property def name(self): return type(self).__name__ - def __init__(self) -> None: - super().__init__() - self.errors = [] - self.errors_lock = Lock() - - def report_error(self, error: Exception): - """Report an error that happened in Manager.process_game""" - with self.errors_lock: - self.errors.append(error) - - def collect_errors(self) -> list[Exception]: - """Get the errors produced by the manager and remove them from self.errors""" - with self.errors_lock: - errors = self.errors.copy() - self.errors.clear() - return errors - @abstractmethod def manager_logic(self, game: Game, additional_data: dict) -> None: """ @@ -87,6 +68,11 @@ class Manager: def handle_error(error: Exception): nonlocal tries + # If FriendlyError, handle its cause instead + base_error = error + if isinstance(error, FriendlyError): + error = error.__cause__ + log_args = ( type(error).__name__, self.name, @@ -105,7 +91,7 @@ class Manager: if tries > self.max_tries: # Handle being out of retries logging.error(out_of_retries_format, *log_args) - self.report_error(error) + self.report_error(base_error) else: # Handle retryable errors logging.error(retrying_format, *log_args) @@ -116,7 +102,7 @@ class Manager: else: # Handle unretryable errors logging.error(unretryable_format, *log_args, exc_info=error) - self.report_error(error) + self.report_error(base_error) def try_manager_logic(): try: diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index ac59592..e4571a2 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -21,10 +21,11 @@ from json import JSONDecodeError from requests.exceptions import HTTPError, SSLError +from src.errors.friendly_error import FriendlyError from src.game import Game from src.store.managers.async_manager import AsyncManager -from src.store.managers.online_cover_manager import OnlineCoverManager from src.store.managers.local_cover_manager import LocalCoverManager +from src.store.managers.online_cover_manager import OnlineCoverManager from src.store.managers.steam_api_manager import SteamAPIManager from src.utils.steamgriddb import SGDBAuthError, SGDBHelper @@ -39,7 +40,10 @@ class SGDBManager(AsyncManager): try: sgdb = SGDBHelper() sgdb.conditionaly_update_cover(game) - except SGDBAuthError: + except SGDBAuthError as error: # If invalid auth, cancel all SGDBManager tasks self.cancellable.cancel() - raise + raise FriendlyError( + "Couldn't authenticate to SGDB", + "Verify your API key in the preferences", + ) from error diff --git a/src/utils/decorators.py b/src/utils/decorators.py deleted file mode 100644 index fcb3cff..0000000 --- a/src/utils/decorators.py +++ /dev/null @@ -1,56 +0,0 @@ -# decorators.py -# -# Copyright 2023 Geoffrey Coulaud -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# SPDX-License-Identifier: GPL-3.0-or-later - -from pathlib import Path -from os import PathLike -from functools import wraps - -from src import shared - - -def replaced_by_path(override: PathLike): # Decorator builder - """Replace the method's returned path with the override - if the override exists on disk""" - - def decorator(original_function): # Built decorator (closure) - @wraps(original_function) - def wrapper(*args, **kwargs): # func's override - path = Path(override).expanduser() - if path.exists(): - return path - return original_function(*args, **kwargs) - - return wrapper - - return decorator - - -def replaced_by_schema_key(original_method): # Built decorator (closure) - """ - Replace the original method's value by the path pointed at in the schema - by the class' location key (if that override exists) - """ - - @wraps(original_method) - def wrapper(*args, **kwargs): # func's override - source = args[0] - override = shared.schema.get_string(source.location_key) - return replaced_by_path(override)(original_method)(*args, **kwargs) - - return wrapper From 5e5a2fe7462e151a52ef1f79073fc87bed6520ea Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 24 Jun 2023 16:13:41 +0200 Subject: [PATCH 167/173] =?UTF-8?q?=F0=9F=8E=A8=20Improved=20importer=20er?= =?UTF-8?q?ror=20dialog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Kept simple for single errors - Made more readable for multiple errors --- src/importer/importer.py | 60 ++++++++++++++++++++---------- src/store/managers/sgdb_manager.py | 2 +- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index b684bcb..bca9d18 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -211,30 +211,50 @@ class Importer(ErrorProducer): def create_error_dialog(self): """Dialog containing all errors raised by importers""" - string = _("The following errors occured during import:") - errors = "" # Collect all errors that happened in the importer and the managers - collected_errors: list[Exception] = [] - collected_errors.extend(self.collect_errors()) + errors: list[Exception] = [] + errors.extend(self.collect_errors()) for manager in shared.store.managers.values(): - collected_errors.extend(manager.collect_errors()) - for error in collected_errors: - # Only display friendly errors - if not isinstance(error, FriendlyError): - continue - errors += "\n\n" + str(error) + errors.extend(manager.collect_errors()) - if errors: - create_dialog( - shared.win, - "Warning", - string + errors, - "open_preferences_import", - _("Preferences"), - ).connect("response", self.dialog_response_callback) - else: + # Filter out non friendly errors + errors = list(filter(lambda error: isinstance(error, FriendlyError), errors)) + + # No error to display + if not errors: self.timeout_toast() + return + + # Create error dialog + dialog = Adw.MessageDialog() + dialog.set_heading(_("Warning")) + dialog.add_response("close", _("Dismiss")) + dialog.add_response("open_preferences_import", _("Preferences")) + dialog.set_default_response("open_preferences_import") + dialog.connect("response", self.dialog_response_callback) + dialog.set_transient_for(shared.win) + + if len(errors) == 1: + # Display the single error in the dialog body + error = errors[0] + dialog.set_body(f"{error.title}\n{error.subtitle}") + else: + # Display the errors in a list + list_box = Gtk.ListBox() + list_box.set_selection_mode(Gtk.SelectionMode.NONE) + list_box.set_css_classes(["boxed-list"]) + list_box.set_margin_top(16) + for error in errors: + row = Adw.ActionRow() + row.set_title(error.title) + row.set_subtitle(error.subtitle) + list_box.append(row) + dialog.set_body(_("The following errors occured during import")) + dialog.set_extra_child(list_box) + + # Dialog is ready, present it + dialog.present() def create_summary_toast(self): """N games imported toast""" @@ -272,9 +292,9 @@ class Importer(ErrorProducer): def dialog_response_callback(self, _widget, response, *args): """Handle after-import dialogs callback""" + logging.debug("After-import dialog response: %s (%s)", response, str(args)) if response == "open_preferences": self.open_preferences(*args) - elif response == "open_preferences_import": self.open_preferences(*args).connect("close-request", self.timeout_toast) else: diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index e4571a2..20e339a 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -44,6 +44,6 @@ class SGDBManager(AsyncManager): # If invalid auth, cancel all SGDBManager tasks self.cancellable.cancel() raise FriendlyError( - "Couldn't authenticate to SGDB", + "Couldn't authenticate to SteamGridDB", "Verify your API key in the preferences", ) from error From f1acb55ece5da83503aeac71a6d3ce9a4277520e Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 24 Jun 2023 16:14:30 +0200 Subject: [PATCH 168/173] Removed unused import --- src/importer/importer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index bca9d18..ac71dca 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -29,7 +29,6 @@ from src.game import Game from src.importer.sources.source import Source from src.store.managers.async_manager import AsyncManager from src.store.pipeline import Pipeline -from src.utils.create_dialog import create_dialog from src.utils.task import Task From e73bc5507c3c08363896a111a4508d546ee7b53b Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Mon, 26 Jun 2023 10:56:01 +0200 Subject: [PATCH 169/173] Change display of errors --- src/importer/importer.py | 21 +++++++++++++-------- src/store/managers/sgdb_manager.py | 4 ++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index ac71dca..6fb41b3 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -218,7 +218,14 @@ class Importer(ErrorProducer): errors.extend(manager.collect_errors()) # Filter out non friendly errors - errors = list(filter(lambda error: isinstance(error, FriendlyError), errors)) + errors = set( + tuple( + (error.title, error.subtitle) + for error in ( + filter(lambda error: isinstance(error, FriendlyError), errors) + ) + ) + ) # No error to display if not errors: @@ -235,9 +242,8 @@ class Importer(ErrorProducer): dialog.set_transient_for(shared.win) if len(errors) == 1: - # Display the single error in the dialog body - error = errors[0] - dialog.set_body(f"{error.title}\n{error.subtitle}") + dialog.set_heading((error := next(iter(errors)))[0]) + dialog.set_body(error[1]) else: # Display the errors in a list list_box = Gtk.ListBox() @@ -246,13 +252,12 @@ class Importer(ErrorProducer): list_box.set_margin_top(16) for error in errors: row = Adw.ActionRow() - row.set_title(error.title) - row.set_subtitle(error.subtitle) + row.set_title(error[0]) + row.set_subtitle(error[1]) list_box.append(row) - dialog.set_body(_("The following errors occured during import")) + dialog.set_body(_("The following errors occured during import:")) dialog.set_extra_child(list_box) - # Dialog is ready, present it dialog.present() def create_summary_toast(self): diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index 20e339a..b535381 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -44,6 +44,6 @@ class SGDBManager(AsyncManager): # If invalid auth, cancel all SGDBManager tasks self.cancellable.cancel() raise FriendlyError( - "Couldn't authenticate to SteamGridDB", - "Verify your API key in the preferences", + "Couldn't authenticate SteamGridDB", + "Verify your API key in preferences", ) from error From 92add88c5d1d5e5818870bd9b3fffcff298c50dc Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Mon, 26 Jun 2023 11:00:59 +0200 Subject: [PATCH 170/173] Change error dialog margin --- src/importer/importer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/importer/importer.py b/src/importer/importer.py index 6fb41b3..0c7b309 100644 --- a/src/importer/importer.py +++ b/src/importer/importer.py @@ -249,9 +249,9 @@ class Importer(ErrorProducer): list_box = Gtk.ListBox() list_box.set_selection_mode(Gtk.SelectionMode.NONE) list_box.set_css_classes(["boxed-list"]) - list_box.set_margin_top(16) + list_box.set_margin_top(8) for error in errors: - row = Adw.ActionRow() + row = Adw.ActionRow.new() row.set_title(error[0]) row.set_subtitle(error[1]) list_box.append(row) From 4f7dc8716ac74b9c89b0bf150080cdbf99c47a4d Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Mon, 26 Jun 2023 11:08:33 +0200 Subject: [PATCH 171/173] Make errors properly translatable --- src/errors/friendly_error.py | 4 ++-- src/importer/sources/source.py | 14 ++++++++++---- src/store/managers/sgdb_manager.py | 4 ++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/errors/friendly_error.py b/src/errors/friendly_error.py index 9fef536..4c9ca7f 100644 --- a/src/errors/friendly_error.py +++ b/src/errors/friendly_error.py @@ -16,12 +16,12 @@ class FriendlyError(Exception): @property def title(self) -> str: """Get the gettext translated error title""" - return _(self.title_format).format(self.title_args) + return self.title_format.format(self.title_args) @property def subtitle(self) -> str: """Get the gettext translated error subtitle""" - return _(self.subtitle_format).format(self.subtitle_args) + return self.subtitle_format.format(self.subtitle_args) def __init__( self, diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index 80c62a1..89475a4 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -101,7 +101,13 @@ class Source(Iterable): def __iter__(self) -> SourceIterator: """Get an iterator for the source""" - for location_name in ("data", "cache", "config"): + for location_name in ( + locations := { + "data": _("data"), + "cache": _("cache"), + "config": _("configuration"), + }.keys() + ): location = getattr(self, f"{location_name}_location", None) if location is None: continue @@ -109,9 +115,9 @@ class Source(Iterable): location.resolve() except UnresolvableLocationError as error: raise FriendlyError( - # The variable is the source's name - f"Invalid {location_name} location for {{}}", - "Change it or disable the source in the preferences", + # The variables are the type of location (eg. cache) and the source's name + _("Invalid {} location for {{}}").format(locations[location_name]), + _("Change it or disable the source in the preferences"), (self.name,), (self.name,), ) from error diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index b535381..64a6d83 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -44,6 +44,6 @@ class SGDBManager(AsyncManager): # If invalid auth, cancel all SGDBManager tasks self.cancellable.cancel() raise FriendlyError( - "Couldn't authenticate SteamGridDB", - "Verify your API key in preferences", + _("Couldn't authenticate SteamGridDB"), + _("Verify your API key in preferences"), ) from error From 523aa8a82c562515666a292b75c9054fad80cdab Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Mon, 26 Jun 2023 11:21:42 +0200 Subject: [PATCH 172/173] Cleanups --- data/cartridges.gresource.xml.in | 2 +- data/gtk/{details_window.blp => details-window.blp} | 0 data/meson.build | 2 +- po/POTFILES | 4 ++-- src/details_window.py | 10 +++++----- 5 files changed, 9 insertions(+), 9 deletions(-) rename data/gtk/{details_window.blp => details-window.blp} (100%) diff --git a/data/cartridges.gresource.xml.in b/data/cartridges.gresource.xml.in index 90d5c3b..472f6fb 100644 --- a/data/cartridges.gresource.xml.in +++ b/data/cartridges.gresource.xml.in @@ -5,7 +5,7 @@ gtk/help-overlay.ui gtk/game.ui gtk/preferences.ui - gtk/details_window.ui + gtk/details-window.ui gtk/style.css gtk/style-dark.css library_placeholder.svg diff --git a/data/gtk/details_window.blp b/data/gtk/details-window.blp similarity index 100% rename from data/gtk/details_window.blp rename to data/gtk/details-window.blp diff --git a/data/meson.build b/data/meson.build index 071fb69..20512b3 100644 --- a/data/meson.build +++ b/data/meson.build @@ -4,7 +4,7 @@ blueprints = custom_target('blueprints', 'gtk/window.blp', 'gtk/game.blp', 'gtk/preferences.blp', - 'gtk/details_window.blp' + 'gtk/details-window.blp' ), output: '.', command: [find_program('blueprint-compiler'), 'batch-compile', '@OUTPUT@', '@CURRENT_SOURCE_DIR@', '@INPUT@'], diff --git a/po/POTFILES b/po/POTFILES index 271d33c..b835079 100644 --- a/po/POTFILES +++ b/po/POTFILES @@ -15,5 +15,5 @@ src/game.py src/preferences.py src/utils/create_dialog.py -src/utils/importer.py -src/utils/steamgriddb.py +src/importer/sources/source.py +src/store/managers/sgdb_manager.py \ No newline at end of file diff --git a/src/details_window.py b/src/details_window.py index c226a3b..dd7267b 100644 --- a/src/details_window.py +++ b/src/details_window.py @@ -24,15 +24,15 @@ from gi.repository import Adw, Gio, GLib, Gtk from PIL import Image from src import shared +from src.errors.friendly_error import FriendlyError from src.game import Game from src.game_cover import GameCover from src.store.managers.sgdb_manager import SGDBManager from src.utils.create_dialog import create_dialog from src.utils.save_cover import resize_cover, save_cover -from src.utils.steamgriddb import SGDBAuthError -@Gtk.Template(resource_path=shared.PREFIX + "/gtk/details_window.ui") +@Gtk.Template(resource_path=shared.PREFIX + "/gtk/details-window.ui") class DetailsWindow(Adw.Window): __gtype_name__ = "DetailsWindow" @@ -230,11 +230,11 @@ class DetailsWindow(Adw.Window): # Handle errors that occured for error in manager.collect_errors(): # On auth error, inform the user - if isinstance(error, SGDBAuthError): + if isinstance(error, FriendlyError): create_dialog( shared.win, - _("Couldn't Connect to SteamGridDB"), - str(error), + error.title, + error.subtitle, "open_preferences", _("Preferences"), ).connect("response", self.update_cover_error_response) From cbd0b3f2873897ece24d50736826bce8578fdbe1 Mon Sep 17 00:00:00 2001 From: kramo <93832451+kra-mo@users.noreply.github.com> Date: Mon, 26 Jun 2023 11:37:39 +0200 Subject: [PATCH 173/173] Update translations --- data/gtk/preferences.blp | 6 +- po/POTFILES | 2 +- po/cartridges.pot | 264 ++++++++++++----------------- src/importer/sources/source.py | 10 +- src/store/managers/sgdb_manager.py | 2 +- 5 files changed, 123 insertions(+), 161 deletions(-) diff --git a/data/gtk/preferences.blp b/data/gtk/preferences.blp index cd1f351..eacd33c 100644 --- a/data/gtk/preferences.blp +++ b/data/gtk/preferences.blp @@ -63,12 +63,12 @@ template $PreferencesWindow : Adw.PreferencesWindow { } Adw.ActionRow reset_action_row { - title: _("Reset App"); - subtitle: _("Completely resets and quits Cartridges"); + title: "Reset App"; + subtitle: "Completely resets and quits Cartridges"; visible: false; Button reset_button { - label: _("Reset"); + label: "Reset"; valign: center; styles [ diff --git a/po/POTFILES b/po/POTFILES index b835079..6b25bb0 100644 --- a/po/POTFILES +++ b/po/POTFILES @@ -2,7 +2,7 @@ data/hu.kramo.Cartridges.desktop.in data/hu.kramo.Cartridges.gschema.xml.in data/hu.kramo.Cartridges.metainfo.xml.in -data/gtk/details_window.blp +data/gtk/details-window.blp data/gtk/game.blp data/gtk/help-overlay.blp data/gtk/preferences.blp diff --git a/po/cartridges.pot b/po/cartridges.pot index 992daec..7117cf0 100644 --- a/po/cartridges.pot +++ b/po/cartridges.pot @@ -8,18 +8,18 @@ msgid "" msgstr "" "Project-Id-Version: Cartridges\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2023-05-25 17:48+0200\n" +"POT-Creation-Date: 2023-06-26 11:37+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" "Language: \n" "MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=UTF-8\n" +"Content-Type: text/plain; charset=CHARSET\n" "Content-Transfer-Encoding: 8bit\n" #: data/hu.kramo.Cartridges.desktop.in:3 -#: data/hu.kramo.Cartridges.metainfo.xml.in:6 data/gtk/window.blp:48 -#: src/main.py:109 +#: data/hu.kramo.Cartridges.metainfo.xml.in:6 data/gtk/window.blp:47 +#: src/main.py:146 msgid "Cartridges" msgstr "" @@ -48,75 +48,74 @@ msgstr "" msgid "Library" msgstr "" -#: data/hu.kramo.Cartridges.metainfo.xml.in:34 src/details_window.py:66 +#: data/hu.kramo.Cartridges.metainfo.xml.in:34 src/details_window.py:67 msgid "Edit Game Details" msgstr "" -#: data/hu.kramo.Cartridges.metainfo.xml.in:38 data/gtk/window.blp:72 +#: data/hu.kramo.Cartridges.metainfo.xml.in:38 data/gtk/window.blp:71 msgid "Game Details" msgstr "" -#: data/hu.kramo.Cartridges.metainfo.xml.in:42 data/gtk/window.blp:417 -#: src/utils/importer.py:92 src/utils/importer.py:124 -#: src/utils/steamgriddb.py:115 +#: data/hu.kramo.Cartridges.metainfo.xml.in:42 data/gtk/window.blp:416 +#: src/details_window.py:239 msgid "Preferences" msgstr "" -#: data/gtk/details_window.blp:25 +#: data/gtk/details-window.blp:25 msgid "Cancel" msgstr "" -#: data/gtk/details_window.blp:57 +#: data/gtk/details-window.blp:57 msgid "New Cover" msgstr "" -#: data/gtk/details_window.blp:75 +#: data/gtk/details-window.blp:75 msgid "Delete Cover" msgstr "" -#: data/gtk/details_window.blp:101 data/gtk/details_window.blp:106 +#: data/gtk/details-window.blp:101 data/gtk/details-window.blp:106 #: data/gtk/game.blp:80 msgid "Title" msgstr "" -#: data/gtk/details_window.blp:102 +#: data/gtk/details-window.blp:102 msgid "The title of the game" msgstr "" -#: data/gtk/details_window.blp:112 data/gtk/details_window.blp:117 +#: data/gtk/details-window.blp:112 data/gtk/details-window.blp:117 msgid "Developer" msgstr "" -#: data/gtk/details_window.blp:113 +#: data/gtk/details-window.blp:113 msgid "The developer or publisher (optional)" msgstr "" -#: data/gtk/details_window.blp:123 data/gtk/details_window.blp:153 +#: data/gtk/details-window.blp:123 data/gtk/details-window.blp:155 msgid "Executable" msgstr "" -#: data/gtk/details_window.blp:124 +#: data/gtk/details-window.blp:124 msgid "File to open or command to run when launching the game" msgstr "" -#: data/gtk/details_window.blp:130 +#: data/gtk/details-window.blp:130 msgid "More Info" msgstr "" -#: data/gtk/game.blp:102 data/gtk/game.blp:121 data/gtk/window.blp:196 +#: data/gtk/game.blp:102 data/gtk/game.blp:121 data/gtk/window.blp:195 msgid "Edit" msgstr "" -#: data/gtk/game.blp:107 src/window.py:205 +#: data/gtk/game.blp:107 src/window.py:169 msgid "Hide" msgstr "" #: data/gtk/game.blp:112 data/gtk/game.blp:131 data/gtk/preferences.blp:56 -#: data/gtk/window.blp:210 +#: data/gtk/window.blp:209 msgid "Remove" msgstr "" -#: data/gtk/game.blp:126 src/window.py:207 +#: data/gtk/game.blp:126 src/window.py:171 msgid "Unhide" msgstr "" @@ -128,8 +127,8 @@ msgstr "" msgid "Quit" msgstr "" -#: data/gtk/help-overlay.blp:19 data/gtk/window.blp:218 data/gtk/window.blp:258 -#: data/gtk/window.blp:324 +#: data/gtk/help-overlay.blp:19 data/gtk/window.blp:217 data/gtk/window.blp:257 +#: data/gtk/window.blp:323 msgid "Search" msgstr "" @@ -141,7 +140,7 @@ msgstr "" msgid "Shortcuts" msgstr "" -#: data/gtk/help-overlay.blp:34 src/game.py:169 src/preferences.py:98 +#: data/gtk/help-overlay.blp:34 src/game.py:105 src/preferences.py:103 msgid "Undo" msgstr "" @@ -169,7 +168,7 @@ msgstr "" msgid "Remove game" msgstr "" -#: data/gtk/preferences.blp:13 data/gtk/preferences.blp:206 +#: data/gtk/preferences.blp:13 data/gtk/preferences.blp:236 msgid "Behavior" msgstr "" @@ -185,7 +184,7 @@ msgstr "" msgid "Swaps the behavior of the cover image and the play button" msgstr "" -#: data/gtk/preferences.blp:36 src/details_window.py:84 +#: data/gtk/preferences.blp:36 src/details_window.py:81 msgid "Images" msgstr "" @@ -205,99 +204,89 @@ msgstr "" msgid "Remove All Games" msgstr "" -#: data/gtk/preferences.blp:69 data/gtk/window.blp:28 data/gtk/window.blp:443 +#: data/gtk/preferences.blp:85 data/gtk/window.blp:27 data/gtk/window.blp:442 msgid "Import" msgstr "" -#: data/gtk/preferences.blp:73 +#: data/gtk/preferences.blp:89 msgid "Sources" msgstr "" -#: data/gtk/preferences.blp:76 +#: data/gtk/preferences.blp:92 msgid "Steam" msgstr "" -#: data/gtk/preferences.blp:80 -msgid "Steam Install Location" +#: data/gtk/preferences.blp:96 data/gtk/preferences.blp:110 +#: data/gtk/preferences.blp:142 data/gtk/preferences.blp:183 +#: data/gtk/preferences.blp:197 data/gtk/preferences.blp:211 +msgid "Install Location" msgstr "" -#: data/gtk/preferences.blp:90 +#: data/gtk/preferences.blp:106 msgid "Lutris" msgstr "" -#: data/gtk/preferences.blp:94 -msgid "Lutris Install Location" +#: data/gtk/preferences.blp:119 +msgid "Cache Location" msgstr "" -#: data/gtk/preferences.blp:103 -msgid "Lutris Cache Location" -msgstr "" - -#: data/gtk/preferences.blp:112 +#: data/gtk/preferences.blp:128 msgid "Import Steam Games" msgstr "" -#: data/gtk/preferences.blp:122 +#: data/gtk/preferences.blp:138 msgid "Heroic" msgstr "" -#: data/gtk/preferences.blp:126 -msgid "Heroic Install Location" -msgstr "" - -#: data/gtk/preferences.blp:135 +#: data/gtk/preferences.blp:151 msgid "Import Epic Games" msgstr "" -#: data/gtk/preferences.blp:144 +#: data/gtk/preferences.blp:160 msgid "Import GOG Games" msgstr "" -#: data/gtk/preferences.blp:153 +#: data/gtk/preferences.blp:169 msgid "Import Sideloaded Games" msgstr "" -#: data/gtk/preferences.blp:163 +#: data/gtk/preferences.blp:179 msgid "Bottles" msgstr "" -#: data/gtk/preferences.blp:167 -msgid "Bottles Install Location" -msgstr "" - -#: data/gtk/preferences.blp:177 +#: data/gtk/preferences.blp:193 msgid "itch" msgstr "" -#: data/gtk/preferences.blp:181 -msgid "itch Install Location" +#: data/gtk/preferences.blp:207 +msgid "Legendary" msgstr "" -#: data/gtk/preferences.blp:194 +#: data/gtk/preferences.blp:224 msgid "SteamGridDB" msgstr "" -#: data/gtk/preferences.blp:198 +#: data/gtk/preferences.blp:228 msgid "Authentication" msgstr "" -#: data/gtk/preferences.blp:201 +#: data/gtk/preferences.blp:231 msgid "API Key" msgstr "" -#: data/gtk/preferences.blp:209 +#: data/gtk/preferences.blp:239 msgid "Use SteamGridDB" msgstr "" -#: data/gtk/preferences.blp:210 +#: data/gtk/preferences.blp:240 msgid "Download images when adding or importing games" msgstr "" -#: data/gtk/preferences.blp:219 +#: data/gtk/preferences.blp:249 msgid "Prefer Over Official Images" msgstr "" -#: data/gtk/preferences.blp:228 +#: data/gtk/preferences.blp:258 msgid "Prefer Animated Images" msgstr "" @@ -309,142 +298,134 @@ msgstr "" msgid "Try a different search." msgstr "" -#: data/gtk/window.blp:22 +#: data/gtk/window.blp:21 msgid "No Games" msgstr "" -#: data/gtk/window.blp:23 +#: data/gtk/window.blp:22 msgid "Use the + button to add games." msgstr "" -#: data/gtk/window.blp:41 +#: data/gtk/window.blp:40 msgid "No Hidden Games" msgstr "" -#: data/gtk/window.blp:42 +#: data/gtk/window.blp:41 msgid "Games you hide will appear here." msgstr "" -#: data/gtk/window.blp:65 data/gtk/window.blp:305 +#: data/gtk/window.blp:64 data/gtk/window.blp:304 msgid "Back" msgstr "" -#: data/gtk/window.blp:122 +#: data/gtk/window.blp:121 msgid "Game Title" msgstr "" -#: data/gtk/window.blp:177 +#: data/gtk/window.blp:176 msgid "Play" msgstr "" -#: data/gtk/window.blp:244 data/gtk/window.blp:436 +#: data/gtk/window.blp:243 data/gtk/window.blp:435 msgid "Add Game" msgstr "" -#: data/gtk/window.blp:251 data/gtk/window.blp:317 +#: data/gtk/window.blp:250 data/gtk/window.blp:316 msgid "Main Menu" msgstr "" -#: data/gtk/window.blp:312 +#: data/gtk/window.blp:311 msgid "Hidden Games" msgstr "" -#: data/gtk/window.blp:375 +#: data/gtk/window.blp:374 msgid "Sort" msgstr "" -#: data/gtk/window.blp:378 +#: data/gtk/window.blp:377 msgid "A-Z" msgstr "" -#: data/gtk/window.blp:384 +#: data/gtk/window.blp:383 msgid "Z-A" msgstr "" -#: data/gtk/window.blp:390 +#: data/gtk/window.blp:389 msgid "Newest" msgstr "" -#: data/gtk/window.blp:396 +#: data/gtk/window.blp:395 msgid "Oldest" msgstr "" -#: data/gtk/window.blp:402 +#: data/gtk/window.blp:401 msgid "Last Played" msgstr "" -#: data/gtk/window.blp:409 +#: data/gtk/window.blp:408 msgid "Show Hidden" msgstr "" -#: data/gtk/window.blp:422 +#: data/gtk/window.blp:421 msgid "Keyboard Shortcuts" msgstr "" -#: data/gtk/window.blp:427 +#: data/gtk/window.blp:426 msgid "About Cartridges" msgstr "" #. Translators: Replace this with your name for it to show up in the about window -#: src/main.py:127 +#: src/main.py:164 msgid "translator_credits" msgstr "" -#: src/window.py:187 -msgid "Today" -msgstr "" - -#: src/window.py:189 -msgid "Yesterday" -msgstr "" - #. The variable is the date when the game was added -#: src/window.py:228 +#: src/window.py:192 msgid "Added: {}" msgstr "" -#: src/window.py:231 +#: src/window.py:195 msgid "Never" msgstr "" #. The variable is the date when the game was last played -#: src/window.py:235 +#: src/window.py:199 msgid "Last played: {}" msgstr "" -#: src/details_window.py:75 +#: src/details_window.py:72 msgid "Apply" msgstr "" -#: src/details_window.py:81 +#: src/details_window.py:78 msgid "Add New Game" msgstr "" -#: src/details_window.py:82 +#: src/details_window.py:79 msgid "Confirm" msgstr "" #. Translate this string as you would translate "file" -#: src/details_window.py:94 +#: src/details_window.py:91 msgid "file.txt" msgstr "" #. As in software -#: src/details_window.py:96 +#: src/details_window.py:93 msgid "program" msgstr "" #. Translate this string as you would translate "path to {}" -#: src/details_window.py:101 src/details_window.py:103 +#: src/details_window.py:98 src/details_window.py:100 msgid "C:\\path\\to\\{}" msgstr "" #. Translate this string as you would translate "path to {}" -#: src/details_window.py:107 src/details_window.py:109 +#: src/details_window.py:104 src/details_window.py:106 msgid "/path/to/{}" msgstr "" -#: src/details_window.py:113 +#: src/details_window.py:111 msgid "" "To launch the executable \"{}\", use the command:\n" "\n" @@ -457,101 +438,82 @@ msgid "" "If the path contains spaces, make sure to wrap it in double quotes!" msgstr "" -#: src/details_window.py:143 src/details_window.py:149 +#: src/details_window.py:146 src/details_window.py:152 msgid "Couldn't Add Game" msgstr "" -#: src/details_window.py:143 src/details_window.py:176 +#: src/details_window.py:146 src/details_window.py:181 msgid "Game title cannot be empty." msgstr "" -#: src/details_window.py:149 src/details_window.py:184 +#: src/details_window.py:152 src/details_window.py:189 msgid "Executable cannot be empty." msgstr "" -#: src/details_window.py:175 src/details_window.py:183 +#: src/details_window.py:180 src/details_window.py:188 msgid "Couldn't Apply Preferences" msgstr "" #. The variable is the title of the game -#: src/game.py:208 +#: src/game.py:141 msgid "{} launched" msgstr "" #. The variable is the title of the game -#: src/game.py:220 +#: src/game.py:154 msgid "{} hidden" msgstr "" -#: src/game.py:220 +#: src/game.py:154 msgid "{} unhidden" msgstr "" -#. The variable is the title of the game -#: src/game.py:233 +#: src/game.py:171 msgid "{} removed" msgstr "" -#: src/preferences.py:97 +#: src/preferences.py:102 msgid "All games removed" msgstr "" -#: src/preferences.py:136 -msgid "Cache Not Found" -msgstr "" - -#: src/preferences.py:137 -msgid "Select the Lutris cache directory." -msgstr "" - -#: src/preferences.py:139 src/preferences.py:292 -msgid "Set Location" -msgstr "" - -#: src/preferences.py:166 +#: src/preferences.py:149 msgid "" "An API key is required to use SteamGridDB. You can generate one {}here{}." msgstr "" -#: src/preferences.py:286 -msgid "Installation Not Found" -msgstr "" - -#. The variable is the name of the game launcher -#: src/preferences.py:288 -msgid "Select the {} configuration directory." -msgstr "" - -#. The variable is the name of the game launcher -#: src/preferences.py:290 -msgid "Select the {} data directory." +#: src/preferences.py:289 +msgid "Set Location" msgstr "" #: src/utils/create_dialog.py:25 msgid "Dismiss" msgstr "" -#: src/utils/importer.py:41 -msgid "Importing Games…" +#: src/importer/sources/source.py:106 +msgid "Data" msgstr "" -#: src/utils/importer.py:76 -msgid "Importing Covers…" +#: src/importer/sources/source.py:107 +msgid "Cache" msgstr "" -#: src/utils/importer.py:91 -msgid "No new games found" +#: src/importer/sources/source.py:108 +msgid "Configuration" msgstr "" -#: src/utils/importer.py:98 -msgid "1 game imported" +#. The variables are the type of location (eg. cache) and the source's name +#: src/importer/sources/source.py:119 +msgid "Invalid {} Location for {{}}" msgstr "" -#. The variable is the number of games -#: src/utils/importer.py:104 -msgid "{} games imported" +#: src/importer/sources/source.py:120 +msgid "Change it or disable the source in preferences" msgstr "" -#: src/utils/importer.py:121 src/utils/steamgriddb.py:112 -msgid "Couldn't Connect to SteamGridDB" +#: src/store/managers/sgdb_manager.py:47 +msgid "Couldn't Authenticate SteamGridDB" +msgstr "" + +#: src/store/managers/sgdb_manager.py:48 +msgid "Verify your API key in preferences" msgstr "" diff --git a/src/importer/sources/source.py b/src/importer/sources/source.py index 89475a4..a26b487 100644 --- a/src/importer/sources/source.py +++ b/src/importer/sources/source.py @@ -103,9 +103,9 @@ class Source(Iterable): """Get an iterator for the source""" for location_name in ( locations := { - "data": _("data"), - "cache": _("cache"), - "config": _("configuration"), + "data": _("Data"), + "cache": _("Cache"), + "config": _("Configuration"), }.keys() ): location = getattr(self, f"{location_name}_location", None) @@ -116,8 +116,8 @@ class Source(Iterable): except UnresolvableLocationError as error: raise FriendlyError( # The variables are the type of location (eg. cache) and the source's name - _("Invalid {} location for {{}}").format(locations[location_name]), - _("Change it or disable the source in the preferences"), + _("Invalid {} Location for {{}}").format(locations[location_name]), + _("Change it or disable the source in preferences"), (self.name,), (self.name,), ) from error diff --git a/src/store/managers/sgdb_manager.py b/src/store/managers/sgdb_manager.py index 64a6d83..142495f 100644 --- a/src/store/managers/sgdb_manager.py +++ b/src/store/managers/sgdb_manager.py @@ -44,6 +44,6 @@ class SGDBManager(AsyncManager): # If invalid auth, cancel all SGDBManager tasks self.cancellable.cancel() raise FriendlyError( - _("Couldn't authenticate SteamGridDB"), + _("Couldn't Authenticate SteamGridDB"), _("Verify your API key in preferences"), ) from error