🚧 More work on importer and source

src/importer/importer.py

@@ -20,7 +20,7 @@ class Importer:
 
     def __init__(self, win) -> None:
         self.games = set()
-        self.sources = list()
+        self.sources = set()
         self.counts = dict()
         self.games_lock = Lock()
         self.progress_lock = Lock()
@@ -31,9 +31,10 @@ class Importer:
         # Compute overall values
         done = 0
         total = 0
-        for source in self.sources:
+        with self.progress_lock:
-            done += self.counts[source.id]["done"]
+            for source in self.sources:
-            total += self.counts[source.id]["total"]
+                done += self.counts[source.id]["done"]
+                total += self.counts[source.id]["total"]
         # Compute progress
         progress = 1
         if total > 0:
@@ -58,59 +59,55 @@ class Importer:
         self.import_dialog.present()
 
     def close_dialog(self):
-        """Close the import dialog"""
         self.import_dialog.close()
 
     def update_progressbar(self):
-        """Update the progress bar"""
+        self.progressbar.set_fraction(self.progress)
-        progress = self.progress()
-        self.progressbar.set_fraction(progress)
 
     def add_source(self, source):
-        """Add a source to import games from"""
+        self.sources.add(source)
-        self.sources.append(source)
         self.counts[source.id] = {"done": 0, "total": 0}
 
     def import_games(self):
-        """Import games from the specified sources"""
-
         self.create_dialog()
 
-        # Scan all sources
         threads = []
+
+        # Scan all sources
         for source in self.sources:
-            t = Thread(
+            t = Thread(target=self.__import_source, args=tuple(source,))  # fmt: skip
-                None,
-                self.__import_from_source,
-                args=tuple(
-                    source,
-                ),
-            )
             threads.append(t)
             t.start()
-
-        # Wait for all of them to finish
         for t in threads:
             t.join()
 
+        # Add SGDB images
+        # TODO isolate SGDB in a game manager
+        threads.clear()
+        for game in self.games:
+            t = Thread(target=self.__add_sgdb_image, args=tuple(game,))  # fmt: skip
+            threads.append(t)
+            t.start()
+        for t in threads:
+            t.join()
         self.close_dialog()
 
-    def __import_from_source(self, *args, **kwargs):
+    def __import_source(self, *args, **kwargs):
         """Source import thread entry point"""
         source, *rest = args
-
         iterator = source.__iter__()
-        for game in iterator:
+        with self.progress_lock:
-            self.games_lock.acquire()
-            self.games.add(game)
-            self.games_lock.release()
-
-            # TODO SGDB image
-            # Who's in charge of image adding ?
-
-            self.progress_lock.acquire()
             self.counts[source.id]["total"] = len(iterator)
-            if not game.blacklisted:
+        for game in iterator:
-                self.counts[source.id]["done"] += 1
+            with self.games_lock:
+                self.games.add(game)
+            with self.progress_lock:
+                if not game.blacklisted:
+                    self.counts[source.id]["done"] += 1
             self.update_progressbar()
-            self.progress_lock.release()
+        exit(0)
+
+    def __add_sgdb_image(self, *args, **kwargs):
+        """SGDB import thread entry point"""
+        # TODO get id, then save image
+        exit(0)

src/importer/source.py

@@ -6,11 +6,6 @@ from enum import IntEnum, auto
 class SourceIterator(Iterator):
     """Data producer for a source of games"""
 
-    class States(IntEnum):
-        DEFAULT = auto()
-        READY = auto()
-
-    state = States.DEFAULT
     source = None
 
     def __init__(self, source) -> None:
@@ -20,6 +15,10 @@ class SourceIterator(Iterator):
     def __iter__(self):
         return self
 
+    @abstractmethod
+    def __len__(self):
+        pass
+
     @abstractmethod
     def __next__(self):
         pass

src/importer/sources/lutris_source.py

@@ -1,4 +1,4 @@
-from functools import cached_property
+from functools import cached_property, cache
 from sqlite3 import connect
 
 from src.game import Game
@@ -8,77 +8,82 @@ from src.importer.decorators import replaced_by_schema_key, replaced_by_path
 
 
 class LutrisSourceIterator(SourceIterator):
-    ignore_steam_games = False  # TODO get that value
+    import_steam = False
-
     db_connection = None
     db_cursor = None
     db_location = None
-    db_request = None
+    db_len_request = """
+        SELECT count(*)
+        FROM 'games'
+        WHERE
+            name IS NOT NULL
+            AND slug IS NOT NULL
+            AND configPath IS NOT NULL
+            AND installed
+            AND (runner IS NOT "steam" OR :import_steam)
+        ;
+    """
+    db_games_request = """
+        SELECT id, name, slug, runner, hidden
+        FROM 'games'
+        WHERE
+            name IS NOT NULL
+            AND slug IS NOT NULL
+            AND configPath IS NOT NULL
+            AND installed
+            AND (runner IS NOT "steam" OR :import_steam)
+        ;
+    """
+    db_request_params = None
 
-    def __init__(self, ignore_steam_games):
+    def __init__(self, *args, **kwargs):
-        super().__init__()
+        super().__init__(*args, **kwargs)
-        self.ignore_steam_games = ignore_steam_games
+        self.import_steam = self.source.win.schema.get_boolean("lutris-import-steam")
-        self.db_connection = None
-        self.db_cursor = None
         self.db_location = self.source.location / "pga.db"
-        self.db_request = """
+        self.db_connection = connect(self.db_location)
-            SELECT
+        self.db_request_params = {"import_steam": self.import_steam}
-                id, name, slug, runner, hidden
+        self.__len__()  # Init iterator length
-            FROM
+        self.db_cursor = self.db_connection.execute(
-                'games'
+            self.db_games_request, self.db_request_params
-            WHERE
+        )
-                name IS NOT NULL
+
-                AND slug IS NOT NULL
+    @cache
-                AND configPath IS NOT NULL
+    def __len__(self):
-                AND installed IS TRUE
+        cursor = self.db_connection.execute(self.db_len_request, self.db_request_params)
-            ;
+        return cursor.fetchone()[0]
-        """
 
     def __next__(self):
         """Produce games. Behaviour depends on the state of the iterator."""
+        # TODO decouple game creation from the window object
 
-        # Get database contents iterator
+        row = None
-        if self.state == self.States.DEFAULT:
+        try:
-            self.db_connection = connect(self.db_location)
+            row = self.db_cursor.__next__()
-            self.db_cursor = self.db_connection.execute(self.db_request)
+        except StopIteration as e:
-            self.state = self.States.READY
+            self.db_connection.close()
+            raise e
 
-        while True:
+        # Create game
-            # Get next DB value
+        row = self.__next_row()
-            try:
+        values = {
-                row = self.db_cursor.__next__()
+            "hidden": row[4],
-            except StopIteration as e:
+            "name": row[1],
-                self.db_connection.close()
+            "source": f"{self.source.id}_{row[3]}",
-                raise e
+            "game_id": self.source.game_id_format.format(
+                game_id=row[2], game_internal_id=row[0]
+            ),
+            "executable": self.source.executable_format.format(game_id=row[2]),
+            "developer": None,  # TODO get developer metadata on Lutris
+        }
+        game = Game(self.source.win, values)
 
-            # Ignore steam games if requested
+        # Save official image
-            if row[3] == "steam" and self.ignore_steam_games:
+        image_path = self.source.cache_location / "coverart" / f"{row[2]}.jpg"
-                continue
+        if image_path.exists():
+            resized = resize_cover(self.source.win, image_path)
+            save_cover(self.source.win, values["game_id"], resized)
 
-            # Build basic game
+        return game
-            # TODO decouple game creation from the window object (later)
-            values = {
-                "hidden": row[4],
-                "name": row[1],
-                "source": f"{self.source.id}_{row[3]}",
-                "game_id": self.source.game_id_format.format(
-                    game_id=row[2], game_internal_id=row[0]
-                ),
-                "executable": self.source.executable_format.format(game_id=row[2]),
-                "developer": None,  # TODO get developer metadata on Lutris
-            }
-            game = Game(self.source.win, values)
-
-            # Save official image
-            image_path = self.source.cache_location / "coverart" / f"{row[2]}.jpg"
-            if image_path.exists():
-                resized = resize_cover(self.source.win, image_path)
-                save_cover(self.source.win, values["game_id"], resized)
-
-            # TODO Save SGDB
-            SGDBSave(self.win, self.games, self)
-
-            return values
 
 
 class LutrisSource(Source):