🚧 More work on importer and source

src/importer/importer.py

@@ -20,7 +20,7 @@ class Importer:
 
     def __init__(self, win) -> None:
         self.games = set()
-        self.sources = list()
+        self.sources = set()
         self.counts = dict()
         self.games_lock = Lock()
         self.progress_lock = Lock()
@@ -31,6 +31,7 @@ class Importer:
         # Compute overall values
         done = 0
         total = 0
+        with self.progress_lock:
             for source in self.sources:
                 done += self.counts[source.id]["done"]
                 total += self.counts[source.id]["total"]
@@ -58,59 +59,55 @@ class Importer:
         self.import_dialog.present()
 
     def close_dialog(self):
-        """Close the import dialog"""
         self.import_dialog.close()
 
     def update_progressbar(self):
-        """Update the progress bar"""
+        self.progressbar.set_fraction(self.progress)
-        progress = self.progress()
-        self.progressbar.set_fraction(progress)
 
     def add_source(self, source):
-        """Add a source to import games from"""
+        self.sources.add(source)
-        self.sources.append(source)
         self.counts[source.id] = {"done": 0, "total": 0}
 
     def import_games(self):
-        """Import games from the specified sources"""
-
         self.create_dialog()
 
-        # Scan all sources
         threads = []
+
+        # Scan all sources
         for source in self.sources:
-            t = Thread(
+            t = Thread(target=self.__import_source, args=tuple(source,))  # fmt: skip
-                None,
-                self.__import_from_source,
-                args=tuple(
-                    source,
-                ),
-            )
             threads.append(t)
             t.start()
-
-        # Wait for all of them to finish
         for t in threads:
             t.join()
 
+        # Add SGDB images
+        # TODO isolate SGDB in a game manager
+        threads.clear()
+        for game in self.games:
+            t = Thread(target=self.__add_sgdb_image, args=tuple(game,))  # fmt: skip
+            threads.append(t)
+            t.start()
+        for t in threads:
+            t.join()
         self.close_dialog()
 
-    def __import_from_source(self, *args, **kwargs):
+    def __import_source(self, *args, **kwargs):
         """Source import thread entry point"""
         source, *rest = args
-
         iterator = source.__iter__()
-        for game in iterator:
+        with self.progress_lock:
-            self.games_lock.acquire()
-            self.games.add(game)
-            self.games_lock.release()
-
-            # TODO SGDB image
-            # Who's in charge of image adding ?
-
-            self.progress_lock.acquire()
             self.counts[source.id]["total"] = len(iterator)
+        for game in iterator:
+            with self.games_lock:
+                self.games.add(game)
+            with self.progress_lock:
                 if not game.blacklisted:
                     self.counts[source.id]["done"] += 1
             self.update_progressbar()
-            self.progress_lock.release()
+        exit(0)
+
+    def __add_sgdb_image(self, *args, **kwargs):
+        """SGDB import thread entry point"""
+        # TODO get id, then save image
+        exit(0)

src/importer/source.py

@@ -6,11 +6,6 @@ from enum import IntEnum, auto
 class SourceIterator(Iterator):
     """Data producer for a source of games"""
 
-    class States(IntEnum):
-        DEFAULT = auto()
-        READY = auto()
-
-    state = States.DEFAULT
     source = None
 
     def __init__(self, source) -> None:
@@ -20,6 +15,10 @@ class SourceIterator(Iterator):
     def __iter__(self):
         return self
 
+    @abstractmethod
+    def __len__(self):
+        pass
+
     @abstractmethod
     def __next__(self):
         pass

src/importer/sources/lutris_source.py

@@ -1,4 +1,4 @@
-from functools import cached_property
+from functools import cached_property, cache
 from sqlite3 import connect
 
 from src.game import Game
@@ -8,55 +8,63 @@ from src.importer.decorators import replaced_by_schema_key, replaced_by_path
 
 
 class LutrisSourceIterator(SourceIterator):
-    ignore_steam_games = False  # TODO get that value
+    import_steam = False
-
     db_connection = None
     db_cursor = None
     db_location = None
-    db_request = None
+    db_len_request = """
-
+        SELECT count(*)
-    def __init__(self, ignore_steam_games):
+        FROM 'games'
-        super().__init__()
-        self.ignore_steam_games = ignore_steam_games
-        self.db_connection = None
-        self.db_cursor = None
-        self.db_location = self.source.location / "pga.db"
-        self.db_request = """
-            SELECT
-                id, name, slug, runner, hidden
-            FROM
-                'games'
         WHERE
             name IS NOT NULL
             AND slug IS NOT NULL
             AND configPath IS NOT NULL
-                AND installed IS TRUE
+            AND installed
+            AND (runner IS NOT "steam" OR :import_steam)
         ;
     """
+    db_games_request = """
+        SELECT id, name, slug, runner, hidden
+        FROM 'games'
+        WHERE
+            name IS NOT NULL
+            AND slug IS NOT NULL
+            AND configPath IS NOT NULL
+            AND installed
+            AND (runner IS NOT "steam" OR :import_steam)
+        ;
+    """
+    db_request_params = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.import_steam = self.source.win.schema.get_boolean("lutris-import-steam")
+        self.db_location = self.source.location / "pga.db"
+        self.db_connection = connect(self.db_location)
+        self.db_request_params = {"import_steam": self.import_steam}
+        self.__len__()  # Init iterator length
+        self.db_cursor = self.db_connection.execute(
+            self.db_games_request, self.db_request_params
+        )
+
+    @cache
+    def __len__(self):
+        cursor = self.db_connection.execute(self.db_len_request, self.db_request_params)
+        return cursor.fetchone()[0]
 
     def __next__(self):
         """Produce games. Behaviour depends on the state of the iterator."""
+        # TODO decouple game creation from the window object
 
-        # Get database contents iterator
+        row = None
-        if self.state == self.States.DEFAULT:
-            self.db_connection = connect(self.db_location)
-            self.db_cursor = self.db_connection.execute(self.db_request)
-            self.state = self.States.READY
-
-        while True:
-            # Get next DB value
         try:
             row = self.db_cursor.__next__()
         except StopIteration as e:
             self.db_connection.close()
             raise e
 
-            # Ignore steam games if requested
+        # Create game
-            if row[3] == "steam" and self.ignore_steam_games:
+        row = self.__next_row()
-                continue
-
-            # Build basic game
-            # TODO decouple game creation from the window object (later)
         values = {
             "hidden": row[4],
             "name": row[1],
@@ -75,10 +83,7 @@ class LutrisSourceIterator(SourceIterator):
             resized = resize_cover(self.source.win, image_path)
             save_cover(self.source.win, values["game_id"], resized)
 
-            # TODO Save SGDB
+        return game
-            SGDBSave(self.win, self.games, self)
-
-            return values
 
 
 class LutrisSource(Source):