From 3c019796c26876569b0de8ffd21828693c77cbc8 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 8 Jul 2023 13:54:43 +0200 Subject: [PATCH 1/5] Initial code --- src/main.py | 2 + src/utils/migrate_files_v1_to_v2.py | 88 +++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 src/utils/migrate_files_v1_to_v2.py diff --git a/src/main.py b/src/main.py index e7c7553..bc8f0e6 100644 --- a/src/main.py +++ b/src/main.py @@ -49,6 +49,7 @@ from src.store.managers.online_cover_manager import OnlineCoverManager from src.store.managers.sgdb_manager import SGDBManager from src.store.managers.steam_api_manager import SteamAPIManager from src.store.store import Store +from src.utils.migrate_files_v1_to_v2 import migrate_files_v1_to_v2 from src.window import CartridgesWindow @@ -281,5 +282,6 @@ def main(_version): """App entry point""" setup_logging() log_system_info() + migrate_files_v1_to_v2() app = CartridgesApplication() return app.run(sys.argv) diff --git a/src/utils/migrate_files_v1_to_v2.py b/src/utils/migrate_files_v1_to_v2.py new file mode 100644 index 0000000..87c09a1 --- /dev/null +++ b/src/utils/migrate_files_v1_to_v2.py @@ -0,0 +1,88 @@ +# window.py +# +# Copyright 2023 Geoffrey Coulaud +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# SPDX-License-Identifier: GPL-3.0-or-later + +import logging +from pathlib import Path +from shutil import copyfile +import os + +from src import shared + + +def migrate_files_v1_to_v2(): + """ + Migrate user data from the v1.X locations to the latest location. + + Fix for commit 4a204442b5d8ba2e918f8c2605d72e483bf35efd + where the windows directories for data, config and cache changed. + """ + + old_data_dir: Path = ( + Path(os.getenv("XDG_DATA_HOME")) + if "XDG_DATA_HOME" in os.environ + else Path.home() / ".local" / "share" + ) + + # Skip if there is no old dir + # Skip if old == new + # Skip if already migrated + migrated_file = old_data_dir / ".migrated" + if ( + not old_data_dir.is_dir() + or str(old_data_dir) == str(shared.data_dir) + or migrated_file.is_file() + ): + return + + logging.info("Migrating data dir %s", str(old_data_dir)) + + # Migrate games if they don't exist in the current data dir. + # If a game is migrated, its covers should be too. + old_games_dir = old_data_dir / "games" + old_covers_dir = old_data_dir / "covers" + current_games_dir = shared.data_dir / "games" + current_covers_dir = shared.data_dir / "covers" + for game_file in old_games_dir.iterdir(): + # Ignore non game files + if not game_file.is_file() or game_file.suffix != ".json": + continue + + # Do nothing if already in games dir + destination_game_file = current_games_dir / game_file.name + if destination_game_file.exists(): + continue + + # Else, migrate the game + copyfile(game_file, destination_game_file) + logging.info("Copied %s -> %s", str(game_file), str(destination_game_file)) + + # Migrate covers + for suffix in (".tiff", ".gif"): + cover_file = old_covers_dir / game_file.with_suffix(suffix).name + if not cover_file.is_file(): + continue + destination_cover_file = current_covers_dir / cover_file.name + copyfile(cover_file, destination_cover_file) + logging.info( + "Copied %s -> %s", str(cover_file), str(destination_cover_file) + ) + + # Signal that this dir is migrated + migrated_file.touch() + logging.info("Migration done") From 867e4d3cce7182bb0b05a28c665ad13ca07e8c5a Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 8 Jul 2023 14:35:43 +0200 Subject: [PATCH 2/5] Fixes to initial code --- src/utils/migrate_files_v1_to_v2.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/utils/migrate_files_v1_to_v2.py b/src/utils/migrate_files_v1_to_v2.py index 87c09a1..bebb4a0 100644 --- a/src/utils/migrate_files_v1_to_v2.py +++ b/src/utils/migrate_files_v1_to_v2.py @@ -38,11 +38,12 @@ def migrate_files_v1_to_v2(): if "XDG_DATA_HOME" in os.environ else Path.home() / ".local" / "share" ) + old_cartridges_data_dir = old_data_dir / "cartridges" # Skip if there is no old dir - # Skip if old == new + # Skip if old == current # Skip if already migrated - migrated_file = old_data_dir / ".migrated" + migrated_file = old_cartridges_data_dir / ".migrated" if ( not old_data_dir.is_dir() or str(old_data_dir) == str(shared.data_dir) @@ -52,19 +53,21 @@ def migrate_files_v1_to_v2(): logging.info("Migrating data dir %s", str(old_data_dir)) + # Create the current data dir if needed + if not shared.data_dir.is_dir(): + shared.data_dir.mkdir(parents=True) + # Migrate games if they don't exist in the current data dir. # If a game is migrated, its covers should be too. - old_games_dir = old_data_dir / "games" - old_covers_dir = old_data_dir / "covers" - current_games_dir = shared.data_dir / "games" - current_covers_dir = shared.data_dir / "covers" + old_games_dir = old_cartridges_data_dir / "games" + old_covers_dir = old_cartridges_data_dir / "covers" for game_file in old_games_dir.iterdir(): # Ignore non game files if not game_file.is_file() or game_file.suffix != ".json": continue # Do nothing if already in games dir - destination_game_file = current_games_dir / game_file.name + destination_game_file = shared.games_dir / game_file.name if destination_game_file.exists(): continue @@ -77,7 +80,7 @@ def migrate_files_v1_to_v2(): cover_file = old_covers_dir / game_file.with_suffix(suffix).name if not cover_file.is_file(): continue - destination_cover_file = current_covers_dir / cover_file.name + destination_cover_file = shared.covers_dir / cover_file.name copyfile(cover_file, destination_cover_file) logging.info( "Copied %s -> %s", str(cover_file), str(destination_cover_file) From 618a98ee894fc7154d5038f555b0205be2a17d65 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 8 Jul 2023 15:09:30 +0200 Subject: [PATCH 3/5] Handle manually added games separately --- src/utils/migrate_files_v1_to_v2.py | 53 +++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/src/utils/migrate_files_v1_to_v2.py b/src/utils/migrate_files_v1_to_v2.py index bebb4a0..9c5dbae 100644 --- a/src/utils/migrate_files_v1_to_v2.py +++ b/src/utils/migrate_files_v1_to_v2.py @@ -17,10 +17,11 @@ # # SPDX-License-Identifier: GPL-3.0-or-later +import json import logging +import os from pathlib import Path from shutil import copyfile -import os from src import shared @@ -57,27 +58,59 @@ def migrate_files_v1_to_v2(): if not shared.data_dir.is_dir(): shared.data_dir.mkdir(parents=True) - # Migrate games if they don't exist in the current data dir. - # If a game is migrated, its covers should be too. old_games_dir = old_cartridges_data_dir / "games" old_covers_dir = old_cartridges_data_dir / "covers" - for game_file in old_games_dir.iterdir(): - # Ignore non game files - if not game_file.is_file() or game_file.suffix != ".json": + + old_games = set(old_games_dir.glob("*.json")) + old_imported_games = set( + filter(lambda path: path.name.startswith("imported_"), old_games) + ) + old_other_games = old_games - old_imported_games + + # Discover current imported games + imported_game_number = 0 + imported_execs = set() + for game in shared.games_dir.glob("imported_*.json"): + try: + game_data = json.load(game.open("r")) + except (OSError, json.JSONDecodeError): + continue + number = int(game_data["game_id"].replace("imported_", "")) + imported_game_number = max(number, imported_game_number) + imported_execs.add(game_data["executable"]) + + # Migrate imported game files + for game in old_imported_games: + try: + game_data = json.load(game.open("r")) + except (OSError, json.JSONDecodeError): continue + # Don't migrate if there's a game with the same exec + if game_data["executable"] in imported_execs: + continue + + # Migrate with updated index + imported_game_number += 1 + game_id = f"imported_{imported_game_number}" + game_data["game_id"] = game_id + destination_game_file = shared.games_dir / f"{game_id}.json" + json.dump(game_data, destination_game_file.open("w")) + + # Migrate all other games + for game in old_other_games: # Do nothing if already in games dir - destination_game_file = shared.games_dir / game_file.name + destination_game_file = shared.games_dir / game.name if destination_game_file.exists(): continue # Else, migrate the game - copyfile(game_file, destination_game_file) - logging.info("Copied %s -> %s", str(game_file), str(destination_game_file)) + copyfile(game, destination_game_file) + logging.info("Copied %s -> %s", str(game), str(destination_game_file)) # Migrate covers for suffix in (".tiff", ".gif"): - cover_file = old_covers_dir / game_file.with_suffix(suffix).name + cover_file = old_covers_dir / game.with_suffix(suffix).name if not cover_file.is_file(): continue destination_cover_file = shared.covers_dir / cover_file.name From 3f5f8b71e88d83279798e389c36fce94599ae1cf Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 8 Jul 2023 15:25:59 +0200 Subject: [PATCH 4/5] Manually imported - Migrate covers, log message --- src/utils/migrate_files_v1_to_v2.py | 66 ++++++++++++++++------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/src/utils/migrate_files_v1_to_v2.py b/src/utils/migrate_files_v1_to_v2.py index 9c5dbae..3d2a7b4 100644 --- a/src/utils/migrate_files_v1_to_v2.py +++ b/src/utils/migrate_files_v1_to_v2.py @@ -25,6 +25,29 @@ from shutil import copyfile from src import shared +old_data_dir: Path = ( + Path(os.getenv("XDG_DATA_HOME")) + if "XDG_DATA_HOME" in os.environ + else Path.home() / ".local" / "share" +) +old_cartridges_data_dir = old_data_dir / "cartridges" +migrated_file = old_cartridges_data_dir / ".migrated" +old_games_dir = old_cartridges_data_dir / "games" +old_covers_dir = old_cartridges_data_dir / "covers" + + +def migrate_game_covers(game_file: Path): + """Migrate a game covers from a source game file (json) to current dir""" + + # Migrate covers + for suffix in (".tiff", ".gif"): + cover_file = old_covers_dir / game_file.with_suffix(suffix).name + if not cover_file.is_file(): + continue + destination_cover_file = shared.covers_dir / cover_file.name + copyfile(cover_file, destination_cover_file) + logging.info("Copied %s -> %s", str(cover_file), str(destination_cover_file)) + def migrate_files_v1_to_v2(): """ @@ -34,17 +57,9 @@ def migrate_files_v1_to_v2(): where the windows directories for data, config and cache changed. """ - old_data_dir: Path = ( - Path(os.getenv("XDG_DATA_HOME")) - if "XDG_DATA_HOME" in os.environ - else Path.home() / ".local" / "share" - ) - old_cartridges_data_dir = old_data_dir / "cartridges" - # Skip if there is no old dir # Skip if old == current # Skip if already migrated - migrated_file = old_cartridges_data_dir / ".migrated" if ( not old_data_dir.is_dir() or str(old_data_dir) == str(shared.data_dir) @@ -58,9 +73,6 @@ def migrate_files_v1_to_v2(): if not shared.data_dir.is_dir(): shared.data_dir.mkdir(parents=True) - old_games_dir = old_cartridges_data_dir / "games" - old_covers_dir = old_cartridges_data_dir / "covers" - old_games = set(old_games_dir.glob("*.json")) old_imported_games = set( filter(lambda path: path.name.startswith("imported_"), old_games) @@ -70,9 +82,9 @@ def migrate_files_v1_to_v2(): # Discover current imported games imported_game_number = 0 imported_execs = set() - for game in shared.games_dir.glob("imported_*.json"): + for game_file in shared.games_dir.glob("imported_*.json"): try: - game_data = json.load(game.open("r")) + game_data = json.load(game_file.open("r")) except (OSError, json.JSONDecodeError): continue number = int(game_data["game_id"].replace("imported_", "")) @@ -80,9 +92,9 @@ def migrate_files_v1_to_v2(): imported_execs.add(game_data["executable"]) # Migrate imported game files - for game in old_imported_games: + for game_file in old_imported_games: try: - game_data = json.load(game.open("r")) + game_data = json.load(game_file.open("r")) except (OSError, json.JSONDecodeError): continue @@ -96,28 +108,22 @@ def migrate_files_v1_to_v2(): game_data["game_id"] = game_id destination_game_file = shared.games_dir / f"{game_id}.json" json.dump(game_data, destination_game_file.open("w")) + logging.info( + "Copied (updated id) %s -> %s", str(game_file), str(destination_game_file) + ) + migrate_game_covers(game_file) # Migrate all other games - for game in old_other_games: + for game_file in old_other_games: # Do nothing if already in games dir - destination_game_file = shared.games_dir / game.name + destination_game_file = shared.games_dir / game_file.name if destination_game_file.exists(): continue # Else, migrate the game - copyfile(game, destination_game_file) - logging.info("Copied %s -> %s", str(game), str(destination_game_file)) - - # Migrate covers - for suffix in (".tiff", ".gif"): - cover_file = old_covers_dir / game.with_suffix(suffix).name - if not cover_file.is_file(): - continue - destination_cover_file = shared.covers_dir / cover_file.name - copyfile(cover_file, destination_cover_file) - logging.info( - "Copied %s -> %s", str(cover_file), str(destination_cover_file) - ) + copyfile(game_file, destination_game_file) + logging.info("Copied %s -> %s", str(game_file), str(destination_game_file)) + migrate_game_covers(game_file) # Signal that this dir is migrated migrated_file.touch() From 34863901fd733cc5ba6ef7d199b1ef0bf6a8f879 Mon Sep 17 00:00:00 2001 From: GeoffreyCoulaud Date: Sat, 8 Jul 2023 15:38:38 +0200 Subject: [PATCH 5/5] Various tweaks - Removed unnecessary comments - Renamed path variables from file to path - Don't check for XDG_DATA_HOME - Move files instead of copying them - Format json dump --- src/utils/migrate_files_v1_to_v2.py | 74 ++++++++++++++--------------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/src/utils/migrate_files_v1_to_v2.py b/src/utils/migrate_files_v1_to_v2.py index 3d2a7b4..ae68741 100644 --- a/src/utils/migrate_files_v1_to_v2.py +++ b/src/utils/migrate_files_v1_to_v2.py @@ -1,4 +1,4 @@ -# window.py +# migrate_files_v1_to_v2.py # # Copyright 2023 Geoffrey Coulaud # @@ -19,34 +19,26 @@ import json import logging -import os from pathlib import Path -from shutil import copyfile from src import shared -old_data_dir: Path = ( - Path(os.getenv("XDG_DATA_HOME")) - if "XDG_DATA_HOME" in os.environ - else Path.home() / ".local" / "share" -) +old_data_dir = Path.home() / ".local" / "share" old_cartridges_data_dir = old_data_dir / "cartridges" -migrated_file = old_cartridges_data_dir / ".migrated" +migrated_file_path = old_cartridges_data_dir / ".migrated" old_games_dir = old_cartridges_data_dir / "games" old_covers_dir = old_cartridges_data_dir / "covers" -def migrate_game_covers(game_file: Path): - """Migrate a game covers from a source game file (json) to current dir""" - - # Migrate covers +def migrate_game_covers(game_path: Path): + """Migrate a game covers from a source game path to the current dir""" for suffix in (".tiff", ".gif"): - cover_file = old_covers_dir / game_file.with_suffix(suffix).name - if not cover_file.is_file(): + cover_path = old_covers_dir / game_path.with_suffix(suffix).name + if not cover_path.is_file(): continue - destination_cover_file = shared.covers_dir / cover_file.name - copyfile(cover_file, destination_cover_file) - logging.info("Copied %s -> %s", str(cover_file), str(destination_cover_file)) + destination_cover_path = shared.covers_dir / cover_path.name + logging.info("Moving %s -> %s", str(cover_path), str(destination_cover_path)) + cover_path.rename(destination_cover_path) def migrate_files_v1_to_v2(): @@ -63,7 +55,7 @@ def migrate_files_v1_to_v2(): if ( not old_data_dir.is_dir() or str(old_data_dir) == str(shared.data_dir) - or migrated_file.is_file() + or migrated_file_path.is_file() ): return @@ -73,18 +65,18 @@ def migrate_files_v1_to_v2(): if not shared.data_dir.is_dir(): shared.data_dir.mkdir(parents=True) - old_games = set(old_games_dir.glob("*.json")) - old_imported_games = set( - filter(lambda path: path.name.startswith("imported_"), old_games) + old_game_paths = set(old_games_dir.glob("*.json")) + old_imported_game_paths = set( + filter(lambda path: path.name.startswith("imported_"), old_game_paths) ) - old_other_games = old_games - old_imported_games + old_other_game_paths = old_game_paths - old_imported_game_paths # Discover current imported games imported_game_number = 0 imported_execs = set() - for game_file in shared.games_dir.glob("imported_*.json"): + for game_path in shared.games_dir.glob("imported_*.json"): try: - game_data = json.load(game_file.open("r")) + game_data = json.load(game_path.open("r")) except (OSError, json.JSONDecodeError): continue number = int(game_data["game_id"].replace("imported_", "")) @@ -92,9 +84,9 @@ def migrate_files_v1_to_v2(): imported_execs.add(game_data["executable"]) # Migrate imported game files - for game_file in old_imported_games: + for game_path in old_imported_game_paths: try: - game_data = json.load(game_file.open("r")) + game_data = json.load(game_path.open("r")) except (OSError, json.JSONDecodeError): continue @@ -106,25 +98,31 @@ def migrate_files_v1_to_v2(): imported_game_number += 1 game_id = f"imported_{imported_game_number}" game_data["game_id"] = game_id - destination_game_file = shared.games_dir / f"{game_id}.json" - json.dump(game_data, destination_game_file.open("w")) + destination_game_path = shared.games_dir / f"{game_id}.json" logging.info( - "Copied (updated id) %s -> %s", str(game_file), str(destination_game_file) + "Moving (updated id) %s -> %s", str(game_path), str(destination_game_path) ) - migrate_game_covers(game_file) + json.dump( + game_data, + destination_game_path.open("w"), + indent=4, + sort_keys=True, + ) + game_path.unlink() + migrate_game_covers(game_path) # Migrate all other games - for game_file in old_other_games: + for game_path in old_other_game_paths: # Do nothing if already in games dir - destination_game_file = shared.games_dir / game_file.name - if destination_game_file.exists(): + destination_game_path = shared.games_dir / game_path.name + if destination_game_path.exists(): continue # Else, migrate the game - copyfile(game_file, destination_game_file) - logging.info("Copied %s -> %s", str(game_file), str(destination_game_file)) - migrate_game_covers(game_file) + logging.info("Moving %s -> %s", str(game_path), str(destination_game_path)) + game_path.rename(destination_game_path) + migrate_game_covers(game_path) # Signal that this dir is migrated - migrated_file.touch() + migrated_file_path.touch() logging.info("Migration done")