From ebd22e27da01bd77e5e7ef9490a57a2f935ca590 Mon Sep 17 00:00:00 2001
From: GeoffreyCoulaud <geoffrey.coulaud@gmail.com>
Date: Sun, 4 Jun 2023 02:45:52 +0200
Subject: [PATCH] Various fixes

- Improved pipeline performance
- Improver importer progress
- Steam API slow down to not get 429-ed
(but still allow bursts on smaller steam libraries)
---
 src/importer/importer.py |  3 ++-
 src/store/pipeline.py    | 13 ++++++++++++-
 src/utils/steam.py       |  7 ++++---
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/importer/importer.py b/src/importer/importer.py
index 1cdf2cd..92e41f1 100644
--- a/src/importer/importer.py
+++ b/src/importer/importer.py
@@ -39,8 +39,9 @@ class Importer:
 
     @property
     def pipelines_progress(self):
+        progress = sum([pipeline.progress for pipeline in self.game_pipelines])
         try:
-            progress = self.n_pipelines_done / len(self.game_pipelines)
+            progress = progress / len(self.game_pipelines)
         except ZeroDivisionError:
             progress = 1
         return progress
diff --git a/src/store/pipeline.py b/src/store/pipeline.py
index 4372709..63fc7d3 100644
--- a/src/store/pipeline.py
+++ b/src/store/pipeline.py
@@ -30,7 +30,7 @@ class Pipeline(GObject.Object):
 
     @property
     def is_done(self) -> bool:
-        return len(self.not_done) == 0
+        return len(self.waiting) == 0 and len(self.running) == 0
 
     @property
     def blocked(self) -> set[Manager]:
@@ -49,6 +49,17 @@ class Pipeline(GObject.Object):
         """Get the managers that can be run"""
         return self.waiting - self.blocked
 
+    @property
+    def progress(self) -> float:
+        """Get the pipeline progress. Should only be a rough idea."""
+        n_done = len(self.done)
+        n_total = len(self.waiting) + len(self.running) + n_done
+        try:
+            progress = n_done / n_total
+        except ZeroDivisionError:
+            progress = 1
+        return progress
+
     def advance(self):
         """Spawn tasks for managers that are able to run for a game"""
 
diff --git a/src/utils/steam.py b/src/utils/steam.py
index 58d8ee1..5389382 100644
--- a/src/utils/steam.py
+++ b/src/utils/steam.py
@@ -48,8 +48,8 @@ class SteamRateLimiter(TokenBucketRateLimiter):
     # 200 requests per 5 min seems to be the limit
     # https://stackoverflow.com/questions/76047820/how-am-i-exceeding-steam-apis-rate-limit
     # https://stackoverflow.com/questions/51795457/avoiding-error-429-too-many-requests-steam-web-api
-    REFILL_SPACING_SECONDS = 1.5
-    MAX_TOKENS = 200
+    REFILL_SPACING_SECONDS = 5 * 60 / 100
+    MAX_TOKENS = 100
 
     def __init__(self) -> None:
         # Load initial tokens from schema
@@ -98,8 +98,9 @@ class SteamHelper:
     def get_api_data(self, appid) -> SteamAPIData:
         """
         Get online data for a game from its appid.
-
         May block to satisfy the Steam web API limitations.
+
+        See https://wiki.teamfortress.com/wiki/User:RJackson/StorefrontAPI#appdetails
         """
 
         # Get data from the API (way block to satisfy its limits)