From c96b64f72ed17fa39157289d5af0ff616e34388b Mon Sep 17 00:00:00 2001 From: Rilic Date: Sat, 15 Jul 2023 14:28:04 +0100 Subject: [PATCH 01/20] Implement Retroarch Importer I'm quite rusty with Python, so please let me know if there are any (or many) grave mistakes! --- data/gtk/preferences.blp | 14 +++ data/hu.kramo.Cartridges.gschema.xml.in | 8 +- flatpak/hu.kramo.Cartridges.Devel.json | 1 + src/importer/sources/retroarch_source.py | 119 +++++++++++++++++++++++ src/main.py | 4 + src/preferences.py | 6 ++ 6 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 src/importer/sources/retroarch_source.py diff --git a/data/gtk/preferences.blp b/data/gtk/preferences.blp index fbc2dee..8547e44 100644 --- a/data/gtk/preferences.blp +++ b/data/gtk/preferences.blp @@ -226,6 +226,20 @@ template $PreferencesWindow : Adw.PreferencesWindow { } } + Adw.ExpanderRow retroarch_expander_row { + title: _("Retroarch"); + show-enable-switch: true; + + Adw.ActionRow retroarch_config_action_row { + title: _("Install Location"); + + Button retroarch_config_file_chooser_button { + icon-name: "folder-symbolic"; + valign: center; + } + } + } + Adw.ExpanderRow flatpak_expander_row { title: _("Flatpak"); show-enable-switch: true; diff --git a/data/hu.kramo.Cartridges.gschema.xml.in b/data/hu.kramo.Cartridges.gschema.xml.in index 4fffaad..9086a2c 100644 --- a/data/hu.kramo.Cartridges.gschema.xml.in +++ b/data/hu.kramo.Cartridges.gschema.xml.in @@ -64,6 +64,12 @@ "~/.config/legendary/" + + true + + + "~/.var/app/org.libretro.RetroArch/config/retroarch/" + true @@ -110,4 +116,4 @@ "[]" - \ No newline at end of file + diff --git a/flatpak/hu.kramo.Cartridges.Devel.json b/flatpak/hu.kramo.Cartridges.Devel.json index 1c2ecea..e74107f 100644 --- a/flatpak/hu.kramo.Cartridges.Devel.json +++ b/flatpak/hu.kramo.Cartridges.Devel.json @@ -17,6 +17,7 @@ "--filesystem=~/.var/app/com.heroicgameslauncher.hgl/config/heroic/:ro", "--filesystem=~/.var/app/com.usebottles.bottles/data/bottles/:ro", "--filesystem=~/.var/app/io.itch.itch/config/itch/:ro", + "--filesystem=~/.var/app/org.libretro.RetroArch/config/retroarch/:ro", "--filesystem=/var/lib/flatpak:ro" ], "cleanup" : [ diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py new file mode 100644 index 0000000..b11cdcd --- /dev/null +++ b/src/importer/sources/retroarch_source.py @@ -0,0 +1,119 @@ +# retroarch_source.py +# +# Copyright 2023 Rilic +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + +from pathlib import Path +from time import time + +import json +import os +import logging +from json import JSONDecodeError + +from src import shared +from src.game import Game +from src.importer.sources.location import Location +from src.importer.sources.source import ( + SourceIterationResult, + SourceIterator, + Source +) + + +class RetroarchSourceIterator(SourceIterator): + source: "RetroarchSource" + + def generator_builder(self) -> SourceIterationResult: + playlist_files = [] + for file in os.listdir(self.source.config_location["playlists"]): + if file.endswith('.lpl'): + playlist_files.append(file) + + playlist_items = [] + for playlist_file in playlist_files: + open_file = open(str(self.source.config_location["playlists"]) + "/" + playlist_file) + try: + playlist_json = json.load(open_file) + except (JSONDecodeError, OSError, KeyError): + logging.warning("Cannot read playlist file: %s", str(playlist_file)) + continue + + for item in playlist_json["items"]: + # Select the core. Try the content's core first, then the playlist's + # default core. + core_path = item["core_path"] + if core_path == "DETECT": + default_core = playlist_json["default_core_path"] + if default_core: + core_path = default_core + else: + logging.warning("Cannot find core for: %s", str(item["path"])) + continue + + # Build game + game_title = item["label"].split("(", 1)[0] + values = { + "source": self.source.id, + "added": int(time()), + "name": game_title, + "game_id": self.source.game_id_format.format(game_id=item["crc32"][:8]), + "executable": self.source.executable_format.format( + rom_path = item["path"], + core_path = core_path, + ) + } + + game = Game(values) + additional_data = {} + + # Get boxart + boxart_image_name = item["label"].split(".", 1)[0] + ".png" + boxart_folder_name = playlist_file.split(".", 1)[0] + image_path = self.source.config_location["thumbnails"] / boxart_folder_name / "Named_Boxarts" / boxart_image_name + additional_data = {"local_image_path": image_path} + + yield(game, additional_data) + + +class RetroarchSource(Source): + args = ' -L "{core_path}" "{rom_path}"' + + name = "Retroarch" + available_on = {"linux"} + iterator_class = RetroarchSourceIterator + executable_format = 'retroarch' + args + + config_location = Location( + schema_key="retroarch-location", + candidates=( + shared.flatpak_dir / "org.libretro.RetroArch" / "config" / "retroarch", + shared.config_dir / "retroarch", + shared.home / ".config" / "retroarch", + ), + paths={ + "playlists": (True, "playlists"), + "thumbnails": (True, "thumbnails"), + }, + ) + + # Check if installation is flatpak'd. + # TODO: There's probably a MUCH better way of doing this. + # There's *is* a URI format, but it doesn't seem to work on the flatpak + # version of Retroarch. https://github.com/libretro/RetroArch/pull/13563 + if str(shared.flatpak_dir) in str(config_location["playlists"]): + executable_format = 'flatpak run org.libretro.RetroArch' + args diff --git a/src/main.py b/src/main.py index d0afd3e..ff6ac83 100644 --- a/src/main.py +++ b/src/main.py @@ -33,6 +33,7 @@ from src import shared from src.details_window import DetailsWindow from src.game import Game from src.importer.importer import Importer +from src.importer.sources.retroarch_source import RetroarchSource from src.importer.sources.bottles_source import BottlesSource from src.importer.sources.flatpak_source import FlatpakSource from src.importer.sources.heroic_source import HeroicSource @@ -230,6 +231,9 @@ class CartridgesApplication(Adw.Application): if shared.schema.get_boolean("legendary"): importer.add_source(LegendarySource()) + if shared.schema.get_boolean("retroarch"): + importer.add_source(RetroarchSource()) + importer.run() def on_remove_game_action(self, *_args): diff --git a/src/preferences.py b/src/preferences.py index f090b4a..15bfbed 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -30,6 +30,7 @@ from src.importer.sources.flatpak_source import FlatpakSource from src.importer.sources.heroic_source import HeroicSource from src.importer.sources.itch_source import ItchSource from src.importer.sources.legendary_source import LegendarySource +from src.importer.sources.retroarch_source import RetroarchSource from src.importer.sources.location import UnresolvableLocationError from src.importer.sources.lutris_source import LutrisSource from src.importer.sources.source import Source @@ -82,6 +83,9 @@ class PreferencesWindow(Adw.PreferencesWindow): legendary_config_action_row = Gtk.Template.Child() legendary_config_file_chooser_button = Gtk.Template.Child() + retroarch_expander_row = Gtk.Template.Child() + retroarch_config_file_chooser_button = Gtk.Template.Child() + flatpak_expander_row = Gtk.Template.Child() flatpak_data_action_row = Gtk.Template.Child() flatpak_data_file_chooser_button = Gtk.Template.Child() @@ -138,6 +142,7 @@ class PreferencesWindow(Adw.PreferencesWindow): ItchSource, LegendarySource, LutrisSource, + RetroarchSource, SteamSource, ): source = source_class() @@ -385,3 +390,4 @@ class PreferencesWindow(Adw.PreferencesWindow): # Set the source row subtitles self.resolve_locations(source) self.update_source_action_row_paths(source) + From 9ccb315a2d32689bca9852db74b620ec90640691 Mon Sep 17 00:00:00 2001 From: Rilic Date: Sun, 16 Jul 2023 15:50:28 +0100 Subject: [PATCH 02/20] Make Retroarch executable_format a property --- src/importer/sources/retroarch_source.py | 57 +++++++++++++----------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index b11cdcd..872d985 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -17,22 +17,18 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +import json +import logging +import os +from json import JSONDecodeError from pathlib import Path from time import time -import json -import os -import logging -from json import JSONDecodeError - from src import shared from src.game import Game from src.importer.sources.location import Location -from src.importer.sources.source import ( - SourceIterationResult, - SourceIterator, - Source -) +from src.importer.sources.source import (Source, SourceIterationResult, + SourceIterator) class RetroarchSourceIterator(SourceIterator): @@ -41,12 +37,14 @@ class RetroarchSourceIterator(SourceIterator): def generator_builder(self) -> SourceIterationResult: playlist_files = [] for file in os.listdir(self.source.config_location["playlists"]): - if file.endswith('.lpl'): + if file.endswith(".lpl"): playlist_files.append(file) playlist_items = [] for playlist_file in playlist_files: - open_file = open(str(self.source.config_location["playlists"]) + "/" + playlist_file) + open_file = open( + str(self.source.config_location["playlists"]) + "/" + playlist_file + ) try: playlist_json = json.load(open_file) except (JSONDecodeError, OSError, KeyError): @@ -71,11 +69,13 @@ class RetroarchSourceIterator(SourceIterator): "source": self.source.id, "added": int(time()), "name": game_title, - "game_id": self.source.game_id_format.format(game_id=item["crc32"][:8]), + "game_id": self.source.game_id_format.format( + game_id=item["crc32"][:8] + ), "executable": self.source.executable_format.format( - rom_path = item["path"], - core_path = core_path, - ) + rom_path=item["path"], + core_path=core_path, + ), } game = Game(values) @@ -84,10 +84,15 @@ class RetroarchSourceIterator(SourceIterator): # Get boxart boxart_image_name = item["label"].split(".", 1)[0] + ".png" boxart_folder_name = playlist_file.split(".", 1)[0] - image_path = self.source.config_location["thumbnails"] / boxart_folder_name / "Named_Boxarts" / boxart_image_name + image_path = ( + self.source.config_location["thumbnails"] + / boxart_folder_name + / "Named_Boxarts" + / boxart_image_name + ) additional_data = {"local_image_path": image_path} - yield(game, additional_data) + yield (game, additional_data) class RetroarchSource(Source): @@ -96,7 +101,6 @@ class RetroarchSource(Source): name = "Retroarch" available_on = {"linux"} iterator_class = RetroarchSourceIterator - executable_format = 'retroarch' + args config_location = Location( schema_key="retroarch-location", @@ -110,10 +114,11 @@ class RetroarchSource(Source): "thumbnails": (True, "thumbnails"), }, ) - - # Check if installation is flatpak'd. - # TODO: There's probably a MUCH better way of doing this. - # There's *is* a URI format, but it doesn't seem to work on the flatpak - # version of Retroarch. https://github.com/libretro/RetroArch/pull/13563 - if str(shared.flatpak_dir) in str(config_location["playlists"]): - executable_format = 'flatpak run org.libretro.RetroArch' + args + + @property + def executable_format(self): + self.config_location.resolve() + is_flatpak = self.config_location.root.is_relative_to(shared.flatpak_dir) + base = "flatpak run org.libretro.RetroArch" if is_flatpak else "retroarch" + args = '-L "{core_path}" "{rom_path}"' + return f"{base} {args}" From 86a34f15964ea01e33b1462242009cec4f03da69 Mon Sep 17 00:00:00 2001 From: Rilic Date: Sun, 16 Jul 2023 15:58:50 +0100 Subject: [PATCH 03/20] Improve playlist file opening check --- src/importer/sources/retroarch_source.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index 872d985..e9a460e 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -27,8 +27,7 @@ from time import time from src import shared from src.game import Game from src.importer.sources.location import Location -from src.importer.sources.source import (Source, SourceIterationResult, - SourceIterator) +from src.importer.sources.source import Source, SourceIterationResult, SourceIterator class RetroarchSourceIterator(SourceIterator): @@ -40,13 +39,13 @@ class RetroarchSourceIterator(SourceIterator): if file.endswith(".lpl"): playlist_files.append(file) - playlist_items = [] for playlist_file in playlist_files: - open_file = open( - str(self.source.config_location["playlists"]) + "/" + playlist_file - ) try: - playlist_json = json.load(open_file) + with open( + self.source.config_location["playlists"] / playlist_file, + encoding="utf-8", + ) as open_file: + playlist_json = json.load(open_file) except (JSONDecodeError, OSError, KeyError): logging.warning("Cannot read playlist file: %s", str(playlist_file)) continue @@ -114,11 +113,11 @@ class RetroarchSource(Source): "thumbnails": (True, "thumbnails"), }, ) - + @property def executable_format(self): self.config_location.resolve() is_flatpak = self.config_location.root.is_relative_to(shared.flatpak_dir) base = "flatpak run org.libretro.RetroArch" if is_flatpak else "retroarch" args = '-L "{core_path}" "{rom_path}"' - return f"{base} {args}" + return f"{base} {args}" From 45884d5c11928e858b9ea783ac360ba027565d0e Mon Sep 17 00:00:00 2001 From: Rilic Date: Sun, 16 Jul 2023 16:05:35 +0100 Subject: [PATCH 04/20] Add missing Retroarch action row --- src/preferences.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/preferences.py b/src/preferences.py index 15bfbed..9a3a002 100644 --- a/src/preferences.py +++ b/src/preferences.py @@ -30,9 +30,9 @@ from src.importer.sources.flatpak_source import FlatpakSource from src.importer.sources.heroic_source import HeroicSource from src.importer.sources.itch_source import ItchSource from src.importer.sources.legendary_source import LegendarySource -from src.importer.sources.retroarch_source import RetroarchSource from src.importer.sources.location import UnresolvableLocationError from src.importer.sources.lutris_source import LutrisSource +from src.importer.sources.retroarch_source import RetroarchSource from src.importer.sources.source import Source from src.importer.sources.steam_source import SteamSource from src.utils.create_dialog import create_dialog @@ -84,6 +84,7 @@ class PreferencesWindow(Adw.PreferencesWindow): legendary_config_file_chooser_button = Gtk.Template.Child() retroarch_expander_row = Gtk.Template.Child() + retroarch_config_action_row = Gtk.Template.Child() retroarch_config_file_chooser_button = Gtk.Template.Child() flatpak_expander_row = Gtk.Template.Child() @@ -390,4 +391,3 @@ class PreferencesWindow(Adw.PreferencesWindow): # Set the source row subtitles self.resolve_locations(source) self.update_source_action_row_paths(source) - From f9cfc311fa8adfeeee6d2f464c1ae59f4c669b4e Mon Sep 17 00:00:00 2001 From: Rilic Date: Sun, 16 Jul 2023 16:22:28 +0100 Subject: [PATCH 05/20] Simplify playlist file location code --- src/importer/sources/retroarch_source.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index e9a460e..ec78a52 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -34,10 +34,7 @@ class RetroarchSourceIterator(SourceIterator): source: "RetroarchSource" def generator_builder(self) -> SourceIterationResult: - playlist_files = [] - for file in os.listdir(self.source.config_location["playlists"]): - if file.endswith(".lpl"): - playlist_files.append(file) + playlist_files = self.source.config_location["playlists"].glob("*.lpl") for playlist_file in playlist_files: try: @@ -82,7 +79,7 @@ class RetroarchSourceIterator(SourceIterator): # Get boxart boxart_image_name = item["label"].split(".", 1)[0] + ".png" - boxart_folder_name = playlist_file.split(".", 1)[0] + boxart_folder_name = playlist_file.name.split(".", 1)[0] image_path = ( self.source.config_location["thumbnails"] / boxart_folder_name From 5a89f8a542b57945ba18c52b79f629b35dca8563 Mon Sep 17 00:00:00 2001 From: Rilic Date: Sun, 16 Jul 2023 16:33:58 +0100 Subject: [PATCH 06/20] Improve core selection logic --- src/importer/sources/retroarch_source.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index ec78a52..189a1e8 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -34,6 +34,7 @@ class RetroarchSourceIterator(SourceIterator): source: "RetroarchSource" def generator_builder(self) -> SourceIterationResult: + # Get all playlist files, ending in .lpl playlist_files = self.source.config_location["playlists"].glob("*.lpl") for playlist_file in playlist_files: @@ -48,13 +49,15 @@ class RetroarchSourceIterator(SourceIterator): continue for item in playlist_json["items"]: - # Select the core. Try the content's core first, then the playlist's - # default core. - core_path = item["core_path"] - if core_path == "DETECT": - default_core = playlist_json["default_core_path"] - if default_core: - core_path = default_core + # Select the core. + # Try the content's core first, then the playlist's default core. + # If none can be used, warn the user and continue. + for core_path in ( + item["core_path"], + playlist_json["default_core_path"], + ): + if core_path != "DETECT": + break else: logging.warning("Cannot find core for: %s", str(item["path"])) continue From 9ffbcc73ae6767ab30c5db767d0231316d9163b0 Mon Sep 17 00:00:00 2001 From: Rilic Date: Sun, 16 Jul 2023 16:47:16 +0100 Subject: [PATCH 07/20] De-indent else in core selection loop --- src/importer/sources/retroarch_source.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index 189a1e8..11da224 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -58,9 +58,9 @@ class RetroarchSourceIterator(SourceIterator): ): if core_path != "DETECT": break - else: - logging.warning("Cannot find core for: %s", str(item["path"])) - continue + else: + logging.warning("Cannot find core for: %s", str(item["path"])) + continue # Build game game_title = item["label"].split("(", 1)[0] From 06de79ad54e0fdb196f4861a50aa1831733cafd8 Mon Sep 17 00:00:00 2001 From: Rilic Date: Sun, 16 Jul 2023 16:57:11 +0100 Subject: [PATCH 08/20] Use native game label instead of truncating --- src/importer/sources/retroarch_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index 11da224..e3992f6 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -63,7 +63,7 @@ class RetroarchSourceIterator(SourceIterator): continue # Build game - game_title = item["label"].split("(", 1)[0] + game_title = item["label"] values = { "source": self.source.id, "added": int(time()), From 19b0737715539a8fa080032267fd81ad4de8515a Mon Sep 17 00:00:00 2001 From: Rilic Date: Sun, 16 Jul 2023 18:29:42 +0100 Subject: [PATCH 09/20] Fix the boxart not being found for some games. I overthought this problem way too hard lmao --- src/importer/sources/retroarch_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index e3992f6..fe2fb11 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -81,7 +81,8 @@ class RetroarchSourceIterator(SourceIterator): additional_data = {} # Get boxart - boxart_image_name = item["label"].split(".", 1)[0] + ".png" + boxart_image_name = item["label"] + ".png" + boxart_image_name = boxart_image_name.replace("&", "_") boxart_folder_name = playlist_file.name.split(".", 1)[0] image_path = ( self.source.config_location["thumbnails"] From 6b26076b92c36d04be831fb50751835482e55c58 Mon Sep 17 00:00:00 2001 From: Rilic Date: Sun, 16 Jul 2023 18:32:14 +0100 Subject: [PATCH 10/20] Use pre-existing variable for game name aaaaaaaa --- src/importer/sources/retroarch_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index fe2fb11..8c6e78f 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -81,7 +81,7 @@ class RetroarchSourceIterator(SourceIterator): additional_data = {} # Get boxart - boxart_image_name = item["label"] + ".png" + boxart_image_name = game_title + ".png" boxart_image_name = boxart_image_name.replace("&", "_") boxart_folder_name = playlist_file.name.split(".", 1)[0] image_path = ( From 9a7875eb87000c98f36acc9a91f12474f88e6ab8 Mon Sep 17 00:00:00 2001 From: Rilic Date: Sun, 16 Jul 2023 22:45:27 +0100 Subject: [PATCH 11/20] Replace crc32 game_id with md5 hash from path --- src/importer/sources/retroarch_source.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index 8c6e78f..1f1c656 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -17,9 +17,9 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +import hashlib import json import logging -import os from json import JSONDecodeError from pathlib import Path from time import time @@ -63,14 +63,13 @@ class RetroarchSourceIterator(SourceIterator): continue # Build game - game_title = item["label"] + game_id = hashlib.md5(item["path"].encode("utf-8")).hexdigest() + values = { "source": self.source.id, "added": int(time()), - "name": game_title, - "game_id": self.source.game_id_format.format( - game_id=item["crc32"][:8] - ), + "name": item["label"], + "game_id": self.source.game_id_format.format(game_id=game_id), "executable": self.source.executable_format.format( rom_path=item["path"], core_path=core_path, @@ -81,7 +80,7 @@ class RetroarchSourceIterator(SourceIterator): additional_data = {} # Get boxart - boxart_image_name = game_title + ".png" + boxart_image_name = item["label"] + ".png" boxart_image_name = boxart_image_name.replace("&", "_") boxart_folder_name = playlist_file.name.split(".", 1)[0] image_path = ( @@ -98,7 +97,7 @@ class RetroarchSourceIterator(SourceIterator): class RetroarchSource(Source): args = ' -L "{core_path}" "{rom_path}"' - name = "Retroarch" + name = _("RetroArch") available_on = {"linux"} iterator_class = RetroarchSourceIterator From 0865d4e133a4ed611cdfceec71deb279fe9c6362 Mon Sep 17 00:00:00 2001 From: Rilic Date: Sun, 16 Jul 2023 22:57:14 +0100 Subject: [PATCH 12/20] Make RetroArch capitalisation consistent --- data/gtk/preferences.blp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/gtk/preferences.blp b/data/gtk/preferences.blp index 8547e44..51302b1 100644 --- a/data/gtk/preferences.blp +++ b/data/gtk/preferences.blp @@ -227,7 +227,7 @@ template $PreferencesWindow : Adw.PreferencesWindow { } Adw.ExpanderRow retroarch_expander_row { - title: _("Retroarch"); + title: _("RetroArch"); show-enable-switch: true; Adw.ActionRow retroarch_config_action_row { From 0a2051f5c7753b69efa3a477272ef62da1d194eb Mon Sep 17 00:00:00 2001 From: Rilic Date: Mon, 17 Jul 2023 12:52:38 +0100 Subject: [PATCH 13/20] Improve error handling for unselected cores --- src/importer/sources/retroarch_source.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index 1f1c656..a790ae2 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -25,6 +25,7 @@ from pathlib import Path from time import time from src import shared +from src.errors.friendly_error import FriendlyError from src.game import Game from src.importer.sources.location import Location from src.importer.sources.source import Source, SourceIterationResult, SourceIterator @@ -37,6 +38,8 @@ class RetroarchSourceIterator(SourceIterator): # Get all playlist files, ending in .lpl playlist_files = self.source.config_location["playlists"].glob("*.lpl") + bad_playlists = set() + for playlist_file in playlist_files: try: with open( @@ -56,10 +59,11 @@ class RetroarchSourceIterator(SourceIterator): item["core_path"], playlist_json["default_core_path"], ): - if core_path != "DETECT": + if core_path not in ("DETECT", ""): break else: logging.warning("Cannot find core for: %s", str(item["path"])) + bad_playlists.add(playlist_file.stem) continue # Build game @@ -93,6 +97,15 @@ class RetroarchSourceIterator(SourceIterator): yield (game, additional_data) + if bad_playlists: + raise FriendlyError( + _("No RetroArch Core Selected"), + # The variable is a newline separated list of playlists + _("The following playlists have no default core:") + + "\n\n{}\n\n".format("\n".join(bad_playlists)) + + _("Games with no core selected were not imported"), + ) + class RetroarchSource(Source): args = ' -L "{core_path}" "{rom_path}"' From faee57a42a95f423e7c97f839b147ef26032b2b4 Mon Sep 17 00:00:00 2001 From: Rilic Date: Mon, 17 Jul 2023 12:55:03 +0100 Subject: [PATCH 14/20] Clean up code and unused imports --- src/importer/sources/retroarch_source.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index a790ae2..c0ac356 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -21,7 +21,6 @@ import hashlib import json import logging from json import JSONDecodeError -from pathlib import Path from time import time from src import shared @@ -35,6 +34,8 @@ class RetroarchSourceIterator(SourceIterator): source: "RetroarchSource" def generator_builder(self) -> SourceIterationResult: + added_time = int(time()) + # Get all playlist files, ending in .lpl playlist_files = self.source.config_location["playlists"].glob("*.lpl") @@ -71,7 +72,7 @@ class RetroarchSourceIterator(SourceIterator): values = { "source": self.source.id, - "added": int(time()), + "added": added_time, "name": item["label"], "game_id": self.source.game_id_format.format(game_id=game_id), "executable": self.source.executable_format.format( @@ -108,8 +109,6 @@ class RetroarchSourceIterator(SourceIterator): class RetroarchSource(Source): - args = ' -L "{core_path}" "{rom_path}"' - name = _("RetroArch") available_on = {"linux"} iterator_class = RetroarchSourceIterator From 386120a50580d23820a39a0aec793b90ff75fdb9 Mon Sep 17 00:00:00 2001 From: Rilic Date: Mon, 17 Jul 2023 16:26:08 +0100 Subject: [PATCH 15/20] Code cleanup and fix cover image rename regex --- src/importer/sources/retroarch_source.py | 18 ++++++++---------- src/main.py | 2 +- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index c0ac356..0e7c3a9 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -17,10 +17,11 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -import hashlib import json import logging +from hashlib import md5 from json import JSONDecodeError +from re import sub from time import time from src import shared @@ -35,20 +36,18 @@ class RetroarchSourceIterator(SourceIterator): def generator_builder(self) -> SourceIterationResult: added_time = int(time()) + bad_playlists = set() # Get all playlist files, ending in .lpl playlist_files = self.source.config_location["playlists"].glob("*.lpl") - bad_playlists = set() - for playlist_file in playlist_files: try: - with open( - self.source.config_location["playlists"] / playlist_file, + with playlist_file.open( encoding="utf-8", ) as open_file: playlist_json = json.load(open_file) - except (JSONDecodeError, OSError, KeyError): + except (JSONDecodeError, OSError): logging.warning("Cannot read playlist file: %s", str(playlist_file)) continue @@ -68,7 +67,7 @@ class RetroarchSourceIterator(SourceIterator): continue # Build game - game_id = hashlib.md5(item["path"].encode("utf-8")).hexdigest() + game_id = md5(item["path"].encode("utf-8")).hexdigest() values = { "source": self.source.id, @@ -82,12 +81,11 @@ class RetroarchSourceIterator(SourceIterator): } game = Game(values) - additional_data = {} # Get boxart boxart_image_name = item["label"] + ".png" - boxart_image_name = boxart_image_name.replace("&", "_") - boxart_folder_name = playlist_file.name.split(".", 1)[0] + boxart_image_name = sub(r"[&\*\/:`<>\?\\\|]", "_", boxart_image_name) + boxart_folder_name = playlist_file.stem image_path = ( self.source.config_location["thumbnails"] / boxart_folder_name diff --git a/src/main.py b/src/main.py index ff6ac83..4280ac6 100644 --- a/src/main.py +++ b/src/main.py @@ -33,13 +33,13 @@ from src import shared from src.details_window import DetailsWindow from src.game import Game from src.importer.importer import Importer -from src.importer.sources.retroarch_source import RetroarchSource from src.importer.sources.bottles_source import BottlesSource from src.importer.sources.flatpak_source import FlatpakSource from src.importer.sources.heroic_source import HeroicSource from src.importer.sources.itch_source import ItchSource from src.importer.sources.legendary_source import LegendarySource from src.importer.sources.lutris_source import LutrisSource +from src.importer.sources.retroarch_source import RetroarchSource from src.importer.sources.steam_source import SteamSource from src.logging.setup import log_system_info, setup_logging from src.preferences import PreferencesWindow From 8eca19d9a19a2d586e5faa75c7b1828ba8ff23b6 Mon Sep 17 00:00:00 2001 From: Rilic Date: Sun, 23 Jul 2023 17:11:14 +0100 Subject: [PATCH 16/20] Changes - Add Windows support - Add Steam RetroArch support - Add support for custom playlist and thumbnail directories --- src/importer/sources/retroarch_source.py | 64 ++++++++++++++++++++---- src/shared.py.in | 1 + 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index 0e7c3a9..0960368 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -19,9 +19,10 @@ import json import logging +import re from hashlib import md5 from json import JSONDecodeError -from re import sub +from pathlib import Path from time import time from src import shared @@ -29,19 +30,39 @@ from src.errors.friendly_error import FriendlyError from src.game import Game from src.importer.sources.location import Location from src.importer.sources.source import Source, SourceIterationResult, SourceIterator +from src.importer.sources.steam_source import SteamSource class RetroarchSourceIterator(SourceIterator): source: "RetroarchSource" + def get_config_value(self, key: str, config_data: str): + for item in re.findall(f'{key}\\s*=\\s*"(.*)"\n', config_data, re.IGNORECASE): + logging.debug(str(item)) + return item + + raise KeyError(f"Key not found in RetroArch config: {key}") + def generator_builder(self) -> SourceIterationResult: added_time = int(time()) bad_playlists = set() + config_file = self.source.config_location["config"] + with config_file.open(encoding="utf-8") as open_file: + config_data = open_file.read() + + playlist_folder = Path( + self.get_config_value("playlist_directory", config_data) + ).expanduser() + thumbnail_folder = Path( + self.get_config_value("thumbnails_directory", config_data) + ).expanduser() + # Get all playlist files, ending in .lpl - playlist_files = self.source.config_location["playlists"].glob("*.lpl") + playlist_files = playlist_folder.glob("*.lpl") for playlist_file in playlist_files: + logging.debug(playlist_file) try: with playlist_file.open( encoding="utf-8", @@ -84,10 +105,10 @@ class RetroarchSourceIterator(SourceIterator): # Get boxart boxart_image_name = item["label"] + ".png" - boxart_image_name = sub(r"[&\*\/:`<>\?\\\|]", "_", boxart_image_name) + boxart_image_name = re.sub(r"[&\*\/:`<>\?\\\|]", "_", boxart_image_name) boxart_folder_name = playlist_file.stem image_path = ( - self.source.config_location["thumbnails"] + thumbnail_folder / boxart_folder_name / "Named_Boxarts" / boxart_image_name @@ -107,20 +128,45 @@ class RetroarchSourceIterator(SourceIterator): class RetroarchSource(Source): + def __init__(self) -> None: + super().__init__() + # Find steam location + libraryfolders = SteamSource().data_location["libraryfolders.vdf"] + library_path = "" + + with open(libraryfolders, "r", encoding="utf-8") as open_file: + for line in open_file: + if '"path"' in line: + library_path = re.findall( + '"path"\\s+"(.*)"\n', line, re.IGNORECASE + )[0] + elif "1118310" in line: + break + + if library_path: + self.config_location.candidates.append( + Path(f"{library_path}/steamapps/common/RetroArch") + ) + name = _("RetroArch") - available_on = {"linux"} + available_on = {"linux", "windows"} iterator_class = RetroarchSourceIterator config_location = Location( schema_key="retroarch-location", - candidates=( + candidates=[ shared.flatpak_dir / "org.libretro.RetroArch" / "config" / "retroarch", shared.config_dir / "retroarch", shared.home / ".config" / "retroarch", - ), + Path("C:\\RetroArch-Win64"), + Path("C:\\RetroArch-Win32"), + shared.local_appdata_dir + / "Packages" + / "1e4cf179-f3c2-404f-b9f3-cb2070a5aad8_8ngdn9a6dx1ma" + / "LocalState", + ], paths={ - "playlists": (True, "playlists"), - "thumbnails": (True, "thumbnails"), + "config": (False, "retroarch.cfg"), }, ) diff --git a/src/shared.py.in b/src/shared.py.in index 92fc574..91e7b6d 100644 --- a/src/shared.py.in +++ b/src/shared.py.in @@ -41,6 +41,7 @@ games_dir = data_dir / "cartridges" / "games" covers_dir = data_dir / "cartridges" / "covers" appdata_dir = Path(os.getenv("appdata") or "C:\\Users\\Default\\AppData\\Roaming") +local_appdata_dir = Path(os.getenv("csidl_local_appdata") or "C:\\Users\\Default\\AppData\\Local") programfiles32_dir = Path(os.getenv("programfiles(x86)") or "C:\\Program Files (x86)") scale_factor = max( From 311ed3b09c872929adebf184de46f2879a2be655 Mon Sep 17 00:00:00 2001 From: Rilic Date: Sun, 23 Jul 2023 17:21:24 +0100 Subject: [PATCH 17/20] Refactoring and error checks --- src/importer/sources/retroarch_source.py | 49 ++++++++++++++---------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index 0960368..6481e90 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -28,7 +28,7 @@ from time import time from src import shared from src.errors.friendly_error import FriendlyError from src.game import Game -from src.importer.sources.location import Location +from src.importer.sources.location import Location, UnresolvableLocationError from src.importer.sources.source import Source, SourceIterationResult, SourceIterator from src.importer.sources.steam_source import SteamSource @@ -128,26 +128,6 @@ class RetroarchSourceIterator(SourceIterator): class RetroarchSource(Source): - def __init__(self) -> None: - super().__init__() - # Find steam location - libraryfolders = SteamSource().data_location["libraryfolders.vdf"] - library_path = "" - - with open(libraryfolders, "r", encoding="utf-8") as open_file: - for line in open_file: - if '"path"' in line: - library_path = re.findall( - '"path"\\s+"(.*)"\n', line, re.IGNORECASE - )[0] - elif "1118310" in line: - break - - if library_path: - self.config_location.candidates.append( - Path(f"{library_path}/steamapps/common/RetroArch") - ) - name = _("RetroArch") available_on = {"linux", "windows"} iterator_class = RetroarchSourceIterator @@ -177,3 +157,30 @@ class RetroarchSource(Source): base = "flatpak run org.libretro.RetroArch" if is_flatpak else "retroarch" args = '-L "{core_path}" "{rom_path}"' return f"{base} {args}" + + def __init__(self) -> None: + super().__init__() + + try: + self.config_location.candidates.append(self.get_steam_location()) + except (OSError, KeyError, UnresolvableLocationError): + pass + + def get_steam_location(self) -> str: + # Find steam location + libraryfolders = SteamSource().data_location["libraryfolders.vdf"] + library_path = "" + + with open(libraryfolders, "r", encoding="utf-8") as open_file: + for line in open_file: + if '"path"' in line: + library_path = re.findall( + '"path"\\s+"(.*)"\n', line, re.IGNORECASE + )[0] + elif "1118310" in line: + break + + if library_path: + return Path(f"{library_path}/steamapps/common/RetroArch") + + raise ValueError("No Steam RetroArch installed.") From 7598f1ea713a54da24144f37d94fb317ec026dd9 Mon Sep 17 00:00:00 2001 From: Rilic Date: Tue, 1 Aug 2023 11:30:39 +0100 Subject: [PATCH 18/20] Convert RetroArch importer to new format --- src/importer/sources/retroarch_source.py | 67 ++++++++++++++---------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index 6481e90..ccdb467 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -24,16 +24,21 @@ from hashlib import md5 from json import JSONDecodeError from pathlib import Path from time import time +from typing import NamedTuple from src import shared from src.errors.friendly_error import FriendlyError from src.game import Game -from src.importer.sources.location import Location, UnresolvableLocationError -from src.importer.sources.source import Source, SourceIterationResult, SourceIterator +from src.importer.sources.location import ( + Location, + LocationSubPath, + UnresolvableLocationError, +) +from src.importer.sources.source import Source, SourceIterable from src.importer.sources.steam_source import SteamSource -class RetroarchSourceIterator(SourceIterator): +class RetroarchSourceIterable(SourceIterable): source: "RetroarchSource" def get_config_value(self, key: str, config_data: str): @@ -43,11 +48,11 @@ class RetroarchSourceIterator(SourceIterator): raise KeyError(f"Key not found in RetroArch config: {key}") - def generator_builder(self) -> SourceIterationResult: + def __iter__(self): added_time = int(time()) bad_playlists = set() - config_file = self.source.config_location["config"] + config_file = self.source.locations.config["retroarch.cfg"] with config_file.open(encoding="utf-8") as open_file: config_data = open_file.read() @@ -91,7 +96,7 @@ class RetroarchSourceIterator(SourceIterator): game_id = md5(item["path"].encode("utf-8")).hexdigest() values = { - "source": self.source.id, + "source": self.source.source_id, "added": added_time, "name": item["label"], "game_id": self.source.game_id_format.format(game_id=game_id), @@ -127,33 +132,41 @@ class RetroarchSourceIterator(SourceIterator): ) +class RetroarchLocations(NamedTuple): + config: Location + + class RetroarchSource(Source): name = _("RetroArch") + source_id = "retroarch" available_on = {"linux", "windows"} - iterator_class = RetroarchSourceIterator + iterable_class = RetroarchSourceIterable - config_location = Location( - schema_key="retroarch-location", - candidates=[ - shared.flatpak_dir / "org.libretro.RetroArch" / "config" / "retroarch", - shared.config_dir / "retroarch", - shared.home / ".config" / "retroarch", - Path("C:\\RetroArch-Win64"), - Path("C:\\RetroArch-Win32"), - shared.local_appdata_dir - / "Packages" - / "1e4cf179-f3c2-404f-b9f3-cb2070a5aad8_8ngdn9a6dx1ma" - / "LocalState", - ], - paths={ - "config": (False, "retroarch.cfg"), - }, + locations = RetroarchLocations( + Location( + schema_key="retroarch-location", + candidates=[ + shared.flatpak_dir / "org.libretro.RetroArch" / "config" / "retroarch", + shared.config_dir / "retroarch", + shared.home / ".config" / "retroarch", + Path("C:\\RetroArch-Win64"), + Path("C:\\RetroArch-Win32"), + shared.local_appdata_dir + / "Packages" + / "1e4cf179-f3c2-404f-b9f3-cb2070a5aad8_8ngdn9a6dx1ma" + / "LocalState", + ], + paths={ + "retroarch.cfg": LocationSubPath("retroarch.cfg"), + }, + invalid_subtitle=Location.CONFIG_INVALID_SUBTITLE, + ) ) @property def executable_format(self): - self.config_location.resolve() - is_flatpak = self.config_location.root.is_relative_to(shared.flatpak_dir) + self.locations.config.resolve() + is_flatpak = self.locations.config.root.is_relative_to(shared.flatpak_dir) base = "flatpak run org.libretro.RetroArch" if is_flatpak else "retroarch" args = '-L "{core_path}" "{rom_path}"' return f"{base} {args}" @@ -162,13 +175,13 @@ class RetroarchSource(Source): super().__init__() try: - self.config_location.candidates.append(self.get_steam_location()) + self.locations.config.candidates.append(self.get_steam_location()) except (OSError, KeyError, UnresolvableLocationError): pass def get_steam_location(self) -> str: # Find steam location - libraryfolders = SteamSource().data_location["libraryfolders.vdf"] + libraryfolders = SteamSource().locations.data["libraryfolders.vdf"] library_path = "" with open(libraryfolders, "r", encoding="utf-8") as open_file: From 8de7226a2f044c64674557ba8bd000964e59250f Mon Sep 17 00:00:00 2001 From: Rilic Date: Tue, 1 Aug 2023 11:56:15 +0100 Subject: [PATCH 19/20] Change "windows" to "win32" --- src/importer/sources/retroarch_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index ccdb467..a64c906 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -139,7 +139,7 @@ class RetroarchLocations(NamedTuple): class RetroarchSource(Source): name = _("RetroArch") source_id = "retroarch" - available_on = {"linux", "windows"} + available_on = {"linux", "win32"} iterable_class = RetroarchSourceIterable locations = RetroarchLocations( From 2e844b2d06e220e75522a7b956a8867e871d3f37 Mon Sep 17 00:00:00 2001 From: Rilic Date: Thu, 3 Aug 2023 19:27:55 +0100 Subject: [PATCH 20/20] Improve RetroArch Steam support And remove Windows support. For now. --- src/importer/sources/retroarch_source.py | 39 ++++++++++++++---------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/importer/sources/retroarch_source.py b/src/importer/sources/retroarch_source.py index a64c906..578d9b9 100644 --- a/src/importer/sources/retroarch_source.py +++ b/src/importer/sources/retroarch_source.py @@ -43,6 +43,9 @@ class RetroarchSourceIterable(SourceIterable): def get_config_value(self, key: str, config_data: str): for item in re.findall(f'{key}\\s*=\\s*"(.*)"\n', config_data, re.IGNORECASE): + if item.startswith(":"): + item = item.replace(":", str(self.source.locations.config.root)) + logging.debug(str(item)) return item @@ -139,7 +142,7 @@ class RetroarchLocations(NamedTuple): class RetroarchSource(Source): name = _("RetroArch") source_id = "retroarch" - available_on = {"linux", "win32"} + available_on = {"linux"} iterable_class = RetroarchSourceIterable locations = RetroarchLocations( @@ -149,12 +152,14 @@ class RetroarchSource(Source): shared.flatpak_dir / "org.libretro.RetroArch" / "config" / "retroarch", shared.config_dir / "retroarch", shared.home / ".config" / "retroarch", - Path("C:\\RetroArch-Win64"), - Path("C:\\RetroArch-Win32"), - shared.local_appdata_dir - / "Packages" - / "1e4cf179-f3c2-404f-b9f3-cb2070a5aad8_8ngdn9a6dx1ma" - / "LocalState", + # TODO: Windows support, waiting for executable path setting improvement + # Path("C:\\RetroArch-Win64"), + # Path("C:\\RetroArch-Win32"), + # TODO: UWP support (URL handler - https://github.com/libretro/RetroArch/pull/13563) + # shared.local_appdata_dir + # / "Packages" + # / "1e4cf179-f3c2-404f-b9f3-cb2070a5aad8_8ngdn9a6dx1ma" + # / "LocalState", ], paths={ "retroarch.cfg": LocationSubPath("retroarch.cfg"), @@ -182,18 +187,20 @@ class RetroarchSource(Source): def get_steam_location(self) -> str: # Find steam location libraryfolders = SteamSource().locations.data["libraryfolders.vdf"] - library_path = "" - + parse_apps = False with open(libraryfolders, "r", encoding="utf-8") as open_file: + # Search each line for a library path and store it each time a new one is found. for line in open_file: if '"path"' in line: library_path = re.findall( '"path"\\s+"(.*)"\n', line, re.IGNORECASE )[0] - elif "1118310" in line: - break - - if library_path: - return Path(f"{library_path}/steamapps/common/RetroArch") - - raise ValueError("No Steam RetroArch installed.") + elif '"apps"' in line: + parse_apps = True + elif parse_apps and "}" in line: + parse_apps = False + # Stop searching, as the library path directly above the appid has been found. + elif parse_apps and '"1118310"' in line: + return Path(f"{library_path}/steamapps/common/RetroArch") + else: + logging.debug("Steam RetroArch not installed.")