Various fixes

- Improved pipeline performance
- Improver importer progress
- Steam API slow down to not get 429-ed
(but still allow bursts on smaller steam libraries)

src/importer/importer.py

@@ -39,8 +39,9 @@ class Importer:
 
     @property
     def pipelines_progress(self):
+        progress = sum([pipeline.progress for pipeline in self.game_pipelines])
         try:
-            progress = self.n_pipelines_done / len(self.game_pipelines)
+            progress = progress / len(self.game_pipelines)
         except ZeroDivisionError:
             progress = 1
         return progress

src/store/pipeline.py

@@ -30,7 +30,7 @@ class Pipeline(GObject.Object):
 
     @property
     def is_done(self) -> bool:
-        return len(self.not_done) == 0
+        return len(self.waiting) == 0 and len(self.running) == 0
 
     @property
     def blocked(self) -> set[Manager]:
@@ -49,6 +49,17 @@ class Pipeline(GObject.Object):
         """Get the managers that can be run"""
         return self.waiting - self.blocked
 
+    @property
+    def progress(self) -> float:
+        """Get the pipeline progress. Should only be a rough idea."""
+        n_done = len(self.done)
+        n_total = len(self.waiting) + len(self.running) + n_done
+        try:
+            progress = n_done / n_total
+        except ZeroDivisionError:
+            progress = 1
+        return progress
+
     def advance(self):
         """Spawn tasks for managers that are able to run for a game"""
 

src/utils/steam.py

@@ -48,8 +48,8 @@ class SteamRateLimiter(TokenBucketRateLimiter):
     # 200 requests per 5 min seems to be the limit
     # https://stackoverflow.com/questions/76047820/how-am-i-exceeding-steam-apis-rate-limit
     # https://stackoverflow.com/questions/51795457/avoiding-error-429-too-many-requests-steam-web-api
-    REFILL_SPACING_SECONDS = 1.5
+    REFILL_SPACING_SECONDS = 5 * 60 / 100
-    MAX_TOKENS = 200
+    MAX_TOKENS = 100
 
     def __init__(self) -> None:
         # Load initial tokens from schema
@@ -98,8 +98,9 @@ class SteamHelper:
     def get_api_data(self, appid) -> SteamAPIData:
         """
         Get online data for a game from its appid.
-
         May block to satisfy the Steam web API limitations.
+
+        See https://wiki.teamfortress.com/wiki/User:RJackson/StorefrontAPI#appdetails
         """
 
         # Get data from the API (way block to satisfy its limits)