From 0991108b11ab34aeae033132c369b523c9290eac Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 15 Feb 2020 20:16:16 +0100 Subject: [PATCH 01/43] updated .gitignore and added readme to scripts folder --- .gitignore | 3 +++ scripts/readme.txt | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 scripts/readme.txt diff --git a/.gitignore b/.gitignore index e691145..e74fd7e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ __pycache__/ *$py.class *.swp virtualenv/ +scripts/* +!scripts/readme.txt +.idea/ diff --git a/scripts/readme.txt b/scripts/readme.txt new file mode 100644 index 0000000..a8fad1a --- /dev/null +++ b/scripts/readme.txt @@ -0,0 +1,29 @@ +Various files need to be in this directory. +If you don't have them, ask someone who does. + +The required files are: + +Z shell: +- update_bgm.zsh +- update_movie.zsh +- update_se.zsh +- update_voice.zsh + +Python scripts: +- adx_parse.py +- bingo_fuck.py +- chara_room_mission.py +- custom_mission.py +- event_bonus.py +- orphan_quest.py +- pad_2_csv.py +- pad_2_json.py +- pad_2_pad.py +- pad_2_pad_louisstyle.py +- pad_stat.py +- pad_update.py +- store_parser.py +- tsv_2_pad.py +- tsv_parse.py +- txt_2_pad.py +- utage_decrypt.py From c5736014db0b1546d1cf32378432e9db0b3b44ce Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 16 Feb 2020 00:15:29 +0100 Subject: [PATCH 02/43] added "is_global" flag --- divatool.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/divatool.py b/divatool.py index cd5f342..f759b4f 100755 --- a/divatool.py +++ b/divatool.py @@ -15,6 +15,16 @@ LANGUAGES = ["jpn", "eng", "rus"] def main(): parser = argparse.ArgumentParser(description="Unpack and process XDU data") + global_flag_parser = argparse.ArgumentParser(add_help=False) + global_flag_parser.add_argument( + "-g", + "--global", + help="Whether we are running on Global XDU.", + action="store_true", + dest="is_global", + default=False, + ) + subparsers = parser.add_subparsers( metavar="", title="subcommands", dest="command" ) @@ -81,7 +91,9 @@ def main(): parser_utage_crypt.add_argument("TARGET", help="Input", type=str) # utage translate parser_utage_translate = parser_utage_subparsers.add_parser( - "translate", help="Generate keyed tsv and json files" + "translate", + help="Generate keyed tsv and json files", + parents=[global_flag_parser], ) parser_utage_translate.add_argument("INPUT", help="Input", type=str) parser_utage_translate.add_argument( @@ -92,7 +104,7 @@ def main(): ) # utage names parser_utage_names = parser_utage_subparsers.add_parser( - "names", help="Generate and update name files" + "names", help="Generate and update name files", parents=[global_flag_parser] ) parser_utage_names.add_argument("INPUT", help="Input", type=str) parser_utage_names.add_argument( @@ -124,7 +136,7 @@ def main(): metavar="", title="subcommand", dest="subcommand" ) parser_xdudata_update = parser_xdudata_subparsers.add_parser( - "update", help="Update an XDUData directory" + "update", help="Update an XDUData directory", parents=[global_flag_parser] ) parser_xdudata_update.add_argument( "CACHE", help="files/ directory from XDU cache", type=str @@ -152,14 +164,20 @@ def main(): raise FileNotFoundError("Could not find {}".format(args.TARGET)) elif args.subcommand == "translate": if os.path.isfile(args.INPUT): - utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR) + utage.translate.translate_file( + args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global + ) elif os.path.isdir(args.INPUT): - utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR) + utage.translate.translate_dir( + args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global + ) else: raise FileNotFoundError("Could not find {}".format(args.INPUT)) elif args.subcommand == "names": if os.path.isdir(args.INPUT): - utage.names.update_names(args.INPUT, args.OLD, LANGUAGES) + utage.names.update_names( + args.INPUT, args.OLD, LANGUAGES, args.is_global + ) else: raise FileNotFoundError(args.INPUT) elif args.command == "diva": @@ -168,7 +186,12 @@ def main(): elif args.command == "xdudata": if args.subcommand == "update": xdudata.update_all( - args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES + args.CACHE, + args.XDUDATA, + args.TRANS, + args.EXTRACT, + LANGUAGES, + args.is_global, ) return 0 From ecde988fd948c78bfc7d27a184efb8ec56f5d5e2 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 16 Feb 2020 00:55:31 +0100 Subject: [PATCH 03/43] sorting out zsh script calling and folder names --- .gitignore | 4 +- divatool.py | 2 + {scripts => tools}/readme.txt | 7 ++- utage/names.py | 12 ++--- utage/translate.py | 22 +++------- xdudata/update.py | 83 +++++++++++++++++++++++++---------- 6 files changed, 79 insertions(+), 51 deletions(-) rename {scripts => tools}/readme.txt (87%) diff --git a/.gitignore b/.gitignore index e74fd7e..f8d0fa1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ __pycache__/ *$py.class *.swp virtualenv/ -scripts/* -!scripts/readme.txt +tools/* +!tools/readme.txt .idea/ diff --git a/divatool.py b/divatool.py index f759b4f..caf2bef 100755 --- a/divatool.py +++ b/divatool.py @@ -184,12 +184,14 @@ def main(): if args.subcommand == "quest": diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) elif args.command == "xdudata": + script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools")) if args.subcommand == "update": xdudata.update_all( args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, + script_dir, LANGUAGES, args.is_global, ) diff --git a/scripts/readme.txt b/tools/readme.txt similarity index 87% rename from scripts/readme.txt rename to tools/readme.txt index a8fad1a..f10fdc0 100644 --- a/scripts/readme.txt +++ b/tools/readme.txt @@ -3,7 +3,12 @@ If you don't have them, ask someone who does. The required files are: -Z shell: +Binaries: +- acb_extract.exe +- usm_extract.exe +- clHCA + +Z shell scripts: - update_bgm.zsh - update_movie.zsh - update_se.zsh diff --git a/utage/names.py b/utage/names.py index 56ef85c..5c5f2e2 100644 --- a/utage/names.py +++ b/utage/names.py @@ -9,7 +9,7 @@ from glob import glob from multiprocessing import Pool -def update_names(dir_in, old_path, languages): +def update_names(dir_in, old_path, languages, is_global): names = extract_names(dir_in) # we have to update japanese like the rest because CustomData exists @@ -30,11 +30,7 @@ def update_names(dir_in, old_path, languages): langfile, "w", newline="\n" ) as lang_file: # you're using git right json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -101,8 +97,6 @@ def read_mission(tsv_path, char_names, char_sets): if row["Command"] and row["Command"].startswith("//"): continue if row["Text"] and row["Arg1"]: - if (row["Arg1"] not in char_names) and ( - row["Arg1"] not in char_sets - ): + if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets): new_names.add(row["Arg1"]) return new_names diff --git a/utage/translate.py b/utage/translate.py index 6c47289..f70fc07 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -10,26 +10,22 @@ from glob import glob from multiprocessing.pool import Pool -def translate_dir(dir_in, tsv_out_dir, json_out_dir): +def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global): files = [ f - for f in glob( - os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True - ) + for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True) if "{os.path.sep}Diva{os.path.sep}" not in f ] if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) - ptrans = partial( - translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir - ) + ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir) with Pool() as p: p.map(ptrans, files) -def translate_file(file_in, tsv_out_dir, json_out_dir): +def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): try: if not file_in.endswith(".tsv"): raise ValueError("Invalid File Type for {}".format(file_in)) @@ -49,9 +45,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir): tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num) ) json_output_path = os.path.join( - json_out_dir, - event_folder, - "{}_translations_jpn.json".format(id_num), + json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num), ) # need to create output paths and avoid races when threading @@ -90,11 +84,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir): with io.open(json_output_path, "w", newline="\n") as json_out: # we don't want to sort these because they're _1, ..., _10, etc json.dump( - json_str, - json_out, - ensure_ascii=False, - indent="\t", - sort_keys=False, + json_str, json_out, ensure_ascii=False, indent="\t", sort_keys=False, ) except Exception: diff --git a/xdudata/update.py b/xdudata/update.py index fd747f9..6924b13 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -1,4 +1,7 @@ +from contextlib import suppress + import diva +import shutil import os import subprocess import tempfile @@ -8,35 +11,59 @@ from distutils.dir_util import copy_tree, remove_tree from glob import glob -def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): +def update_all( + cache_dir, + update_dir, + translation_dir, + extract_dir, + script_dir, + languages, + is_global, +): + print("Cleaning cache") + cleanup_cache(cache_dir) + print("Copying images") - copy_images(cache_dir, update_dir) + # copy_images(cache_dir, update_dir) print("Processing TSV files") - update_tsv(cache_dir, update_dir, translation_dir, languages) + # TODO + # update_tsv(cache_dir, update_dir, translation_dir, languages, is_global) print("Processing Quest Json files") - diva.quest.update_missions(cache_dir, translation_dir, languages) + # diva.quest.update_missions(cache_dir, translation_dir, languages) - # this code isn't going to run on windows because i need to rewrite it - # it just calls my zsh scripts and generates it that way, need to replace them + # this code isn't going to run on windows (it will in WSL though) because i need to rewrite it + # it just calls my zsh tools and generates it that way, need to replace them # with real python or something + + # first copy over the necessary avtools to the extract dir + # TODO make it so that you don't need to copy them over (for example call them with python) + print("Copying necessary tools into extracting directory") + shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir) + shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir) + shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir) + print("Processing Sound Effects") copy_tree( os.path.join(cache_dir, "Android/Asset/Sound/Se"), os.path.join(extract_dir, "Se"), ) - subprocess.run("./update_se.zsh", shell=True, cwd=extract_dir) - copy_tree(os.path.join(extract_dir, "se"), os.path.join(update_dir, "Se")) + subprocess.run( + os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir + ) + copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se")) print("Processing BGM - don't forget to copy that loop file") copy_tree( os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), os.path.join(extract_dir, "Bgm"), ) - subprocess.run("./update_bgm.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "bgm"), os.path.join(update_dir, "Bgm") + os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") ) print("Processing voices") @@ -44,9 +71,11 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): os.path.join(cache_dir, "Android/Asset/Sound/Voice"), os.path.join(extract_dir, "Voice"), ) - subprocess.run("./update_voice.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "voice"), os.path.join(update_dir, "Voice") + os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") ) print("Processing movies - this might take forever") @@ -54,22 +83,22 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): os.path.join(cache_dir, "Common/Asset/Movie"), os.path.join(extract_dir, "Movie"), ) - subprocess.run("./update_movie.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "movie"), + os.path.join(extract_dir, "movie_extracted"), os.path.join(update_dir, "Asset/Movie"), ) -def update_tsv(cache_dir, update_dir, translation_dir, languages): +def update_tsv(cache_dir, update_dir, translation_dir, languages, is_global): utage_tmp = tempfile.mkdtemp() copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp) utage.crypt.crypt_dir( utage_tmp, bytearray("SampleSecretKey", "utf-8"), False, False ) - for encrypted in glob( - os.path.join(utage_tmp, "**/*.utage"), recursive=True - ): + for encrypted in glob(os.path.join(utage_tmp, "**/*.utage"), recursive=True): try: os.remove(encrypted) except: @@ -79,11 +108,10 @@ def update_tsv(cache_dir, update_dir, translation_dir, languages): utage_tmp, os.path.join(update_dir, "Utage"), os.path.join(translation_dir, "Missions"), + is_global, ) utage.names.update_names(utage_tmp, translation_dir, languages) - copy_tree( - os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva") - ) + copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) remove_tree(utage_tmp) @@ -93,6 +121,15 @@ def copy_images(cache_dir, update_dir): os.path.join(update_dir, "Asset/Image"), ) copy_tree( - os.path.join(cache_dir, "Common/Sample"), - os.path.join(update_dir, "Sample"), + os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), ) + + +def cleanup_cache(cache_dir): + with suppress(FileNotFoundError): + shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) + os.remove( + os.path.join( + cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" + ) + ) From fe9741a20a9240de60ee238db7d16db81b7d9d5a Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 16 Feb 2020 02:24:03 +0100 Subject: [PATCH 04/43] fixed up updater --- diva/quest.py | 26 +++++++------------------- divatool.py | 3 +-- utage/translate.py | 7 ++++++- xdudata/update.py | 30 +++++++++++++++--------------- 4 files changed, 29 insertions(+), 37 deletions(-) diff --git a/diva/quest.py b/diva/quest.py index d84e3b2..5a6d27d 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -18,6 +18,8 @@ def scene_mst(qdb, old_path, dir_in, languages): locations = load_locations(dir_in) + print(locations) + for id, scene in scenes.items(): i = 0 while True: @@ -26,18 +28,14 @@ def scene_mst(qdb, old_path, dir_in, languages): except KeyError as exc: i += 1 if i == len(scene["Parts"]): - raise KeyError( - f"Could not find folder for scene {id}" - ) from exc + raise KeyError(f"Could not find folder for scene {id}") from exc continue break with io.open( os.path.join(old_path, "XduScene.json"), "w", newline="\n" ) as json_file: - json.dump( - scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True - ) + json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True) for lang in languages: out_dict = {} @@ -66,11 +64,7 @@ def scene_mst(qdb, old_path, dir_in, languages): with io.open(langfile, "w", newline="\n") as lang_file: json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -80,9 +74,7 @@ def quest_mst(qdb, old_path, languages): with io.open( os.path.join(old_path, "XduQuest.json"), "w", newline="\n" ) as json_file: - json.dump( - quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True - ) + json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True) for lang in languages: out_dict = {} @@ -101,11 +93,7 @@ def quest_mst(qdb, old_path, languages): with io.open(langfile, "w", newline="\n") as lang_file: json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) return quests diff --git a/divatool.py b/divatool.py index caf2bef..ec36fed 100755 --- a/divatool.py +++ b/divatool.py @@ -136,7 +136,7 @@ def main(): metavar="", title="subcommand", dest="subcommand" ) parser_xdudata_update = parser_xdudata_subparsers.add_parser( - "update", help="Update an XDUData directory", parents=[global_flag_parser] + "update", help="Update an XDUData directory" ) parser_xdudata_update.add_argument( "CACHE", help="files/ directory from XDU cache", type=str @@ -193,7 +193,6 @@ def main(): args.EXTRACT, script_dir, LANGUAGES, - args.is_global, ) return 0 diff --git a/utage/translate.py b/utage/translate.py index f70fc07..8b70398 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -20,7 +20,12 @@ def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global): if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) - ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir) + ptrans = partial( + translate_file, + tsv_out_dir=tsv_out_dir, + json_out_dir=json_out_dir, + is_global=is_global, + ) with Pool() as p: p.map(ptrans, files) diff --git a/xdudata/update.py b/xdudata/update.py index 6924b13..b659c3f 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -12,28 +12,21 @@ from glob import glob def update_all( - cache_dir, - update_dir, - translation_dir, - extract_dir, - script_dir, - languages, - is_global, + cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages, ): print("Cleaning cache") cleanup_cache(cache_dir) print("Copying images") - # copy_images(cache_dir, update_dir) + copy_images(cache_dir, update_dir) print("Processing TSV files") - # TODO - # update_tsv(cache_dir, update_dir, translation_dir, languages, is_global) + update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) print("Processing Quest Json files") - # diva.quest.update_missions(cache_dir, translation_dir, languages) + diva.quest.update_missions(extract_dir, translation_dir, languages) - # this code isn't going to run on windows (it will in WSL though) because i need to rewrite it + # this code isn't going to run on windows (it is in WSL though) because i need to rewrite it # it just calls my zsh tools and generates it that way, need to replace them # with real python or something @@ -92,7 +85,7 @@ def update_all( ) -def update_tsv(cache_dir, update_dir, translation_dir, languages, is_global): +def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages): utage_tmp = tempfile.mkdtemp() copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp) utage.crypt.crypt_dir( @@ -104,13 +97,20 @@ def update_tsv(cache_dir, update_dir, translation_dir, languages, is_global): except: pass + # copy over files for the quest updater + copy_tree(utage_tmp, os.path.join(extract_dir, "Common/Asset/Utage")) + shutil.copy2(os.path.join(cache_dir, "QuestMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "QuestSceneMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "QuestPartMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "ResourceEntry.db"), extract_dir) + utage.translate.translate_dir( utage_tmp, os.path.join(update_dir, "Utage"), os.path.join(translation_dir, "Missions"), - is_global, + False, ) - utage.names.update_names(utage_tmp, translation_dir, languages) + utage.names.update_names(utage_tmp, translation_dir, languages, False) copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) remove_tree(utage_tmp) From 1ef9a26aeea11a790fc346a275694591c889f036 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 16 Feb 2020 03:05:24 +0100 Subject: [PATCH 05/43] added basic nametranslations for global languages --- utage/names.py | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/utage/names.py b/utage/names.py index 5c5f2e2..b6633f0 100644 --- a/utage/names.py +++ b/utage/names.py @@ -10,13 +10,24 @@ from multiprocessing import Pool def update_names(dir_in, old_path, languages, is_global): - names = extract_names(dir_in) + names = extract_names(dir_in, is_global) + + global_languages = {"enm": 1, "zh": 2, "ko": 3} + if is_global: + languages = global_languages.keys() # we have to update japanese like the rest because CustomData exists # and POKELABO deleted the wedding gear event stuff from the game files for l in languages: if l == "jpn": out_dict = dict(zip(names, names)) + elif l in global_languages.keys(): + out_dict = {} + for name in names: + spl = name.split("\t") + idx = global_languages[l] + val = spl[idx] if spl[idx] != "None" else "" + out_dict[spl[0]] = val else: out_dict = dict.fromkeys(names, "") @@ -34,13 +45,13 @@ def update_names(dir_in, old_path, languages, is_global): ) -def extract_names(dir_in): +def extract_names(dir_in, is_global): char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv") if not os.path.isfile(char_tsv_path): raise FileNotFoundError("Character.tsv not found") - names, sets = read_char_tsv(char_tsv_path) + names, sets = read_char_tsv(char_tsv_path, is_global) files = [ f @@ -50,18 +61,20 @@ def extract_names(dir_in): if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) - # tfw gil - # it's still faster on my machine so i'm keeping it - read_partial = partial(read_mission, char_names=names, char_sets=sets) - with Pool() as p: - new_names = p.map(read_partial, files) - for x in new_names: - names.update(x) + # TODO unsupported on global for now + if not is_global: + # tfw gil + # it's still faster on my machine so i'm keeping it + read_partial = partial(read_mission, char_names=names, char_sets=sets) + with Pool() as p: + new_names = p.map(read_partial, files) + for x in new_names: + names.update(x) return names -def read_char_tsv(char_tsv_path): +def read_char_tsv(char_tsv_path, is_global): char_names = set() char_sets = set() @@ -77,7 +90,12 @@ def read_char_tsv(char_tsv_path): and row["CharacterName"].strip() and row["NameText"].strip() ): - char_names.add(row["NameText"]) + if is_global: + char_names.add( + f"{row['NameText']}\t{row['CharaEnglish']}\t{row['CharaChinese']}\t{row['CharaKorean']}" + ) + else: + char_names.add(row["NameText"]) char_sets.add(row["CharacterName"]) return char_names, char_sets From 05a29b6ee655e963c42363bccf7d8ba44e59779f Mon Sep 17 00:00:00 2001 From: argoneus Date: Thu, 20 Feb 2020 23:57:09 +0100 Subject: [PATCH 06/43] translate command now works with global languages --- diva/quest.py | 2 -- utage/translate.py | 61 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/diva/quest.py b/diva/quest.py index 5a6d27d..f2342a1 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -18,8 +18,6 @@ def scene_mst(qdb, old_path, dir_in, languages): locations = load_locations(dir_in) - print(locations) - for id, scene in scenes.items(): i = 0 while True: diff --git a/utage/translate.py b/utage/translate.py index 8b70398..6ea96b4 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -4,6 +4,7 @@ import io import json import os import traceback +from contextlib import suppress from functools import partial from glob import glob @@ -49,9 +50,17 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): tsv_output_path = os.path.join( tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num) ) - json_output_path = os.path.join( - json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num), - ) + languages = ["jpn"] + if is_global: + languages += ["enm", "zh", "ko"] + json_output_paths = { + lang: os.path.join( + json_out_dir, + event_folder, + "{}_translations_{}.json".format(id_num, lang), + ) + for lang in languages + } # need to create output paths and avoid races when threading if not os.path.exists(os.path.dirname(tsv_output_path)): @@ -61,16 +70,21 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): if e.errno != errno.EEXIST: raise - if not os.path.exists(os.path.dirname(json_output_path)): + if not os.path.exists(os.path.dirname(json_output_paths["jpn"])): try: - os.makedirs(os.path.dirname(json_output_path)) + os.makedirs(os.path.dirname(json_output_paths["jpn"])) except OSError as e: if e.errno != errno.EEXIST: raise with open(file_in, "r") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') - tsv_keyed, json_str = process_tsv(tsv, id_num) + + t_fieldnames = tsv.fieldnames + if is_global: + t_fieldnames = tsv.fieldnames[: tsv.fieldnames.index("English") + 1] + + tsv_keyed, json_str = process_tsv(tsv, id_num, is_global) # csv handles newlines, don't set it in io.open with io.open(tsv_output_path, "w", newline="") as tsv_out: @@ -79,28 +93,39 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): delimiter="\t", quotechar='"', lineterminator="\n", - fieldnames=tsv.fieldnames, + fieldnames=t_fieldnames, extrasaction="ignore", ) writer.writeheader() for row in tsv_keyed: writer.writerow(row) - with io.open(json_output_path, "w", newline="\n") as json_out: - # we don't want to sort these because they're _1, ..., _10, etc - json.dump( - json_str, json_out, ensure_ascii=False, indent="\t", sort_keys=False, - ) + for lang in languages: + if not json_str[lang]: + continue + with io.open(json_output_paths[lang], "w", newline="\n") as json_out: + # we don't want to sort these because they're _1, ..., _10, etc + json.dump( + json_str[lang], + json_out, + ensure_ascii=False, + indent="\t", + sort_keys=False, + ) except Exception: traceback.print_exc() print("Error processing file: " + file_in) -def process_tsv(tsv, id_num): +def process_tsv(tsv, id_num, is_global): i = 0 tsv_keyed = [] - key_dict = {} + key_dict = {"jpn": {}} + if is_global: + key_dict["enm"] = {} + key_dict["zh"] = {} + key_dict["ko"] = {} for row in tsv: try: @@ -110,8 +135,14 @@ def process_tsv(tsv, id_num): if row["Text"] and row["Text"].strip(): key = "{}_{}".format(id_num, i) i += 1 + key_dict["jpn"][key] = row["Text"] + if is_global: + with suppress(KeyError): + if row["English"]: + key_dict["enm"][key] = row["English"] + key_dict["zh"][key] = row["Chinese"] + key_dict["ko"][key] = row["Korean"] row["English"] = key - key_dict[key] = row["Text"] tsv_keyed.append(row) except Exception: traceback.print_exc() From 01fb281481459e6d07808f241ffbc74b73ef3183 Mon Sep 17 00:00:00 2001 From: argoneus Date: Fri, 21 Feb 2020 00:06:16 +0100 Subject: [PATCH 07/43] fixed quotes thanks j --- utage/translate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utage/translate.py b/utage/translate.py index 6ea96b4..2344b0a 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -78,7 +78,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): raise with open(file_in, "r") as tsv_file: - tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') + tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE) t_fieldnames = tsv.fieldnames if is_global: From bf14c96338a5752d1722decd3110b944dfd3198d Mon Sep 17 00:00:00 2001 From: argoneus Date: Fri, 6 Mar 2020 19:29:33 +0100 Subject: [PATCH 08/43] updated readme --- .gitignore | 4 ++-- README.md | 21 +++++++++++++++++++++ tools/.gitkeep | 0 tools/readme.txt | 34 ---------------------------------- 4 files changed, 23 insertions(+), 36 deletions(-) create mode 100644 README.md create mode 100644 tools/.gitkeep delete mode 100644 tools/readme.txt diff --git a/.gitignore b/.gitignore index f8d0fa1..cd455e6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ __pycache__/ *$py.class *.swp virtualenv/ -tools/* -!tools/readme.txt .idea/ +tools/* +!tools/.gitkeep diff --git a/README.md b/README.md new file mode 100644 index 0000000..ba0de08 --- /dev/null +++ b/README.md @@ -0,0 +1,21 @@ +# divatool + +A tool for that one game everyone hates but works on anyway. + +## Requirements + +Divatool is written in and uses [Python 3](https://www.python.org/downloads/). + +There are also several third party dependencies not provided as part of this repository, +mostly used for the `xdudata` command. You need to have the binaries of the following tools +either in your `$PATH` variable or in the [`tools`](./tools) folder. These tools are: + +* [clHCA](https://github.com/KinoMyu/FastHCADecoder) +* acb_extract +* usm_extract + +Make sure these executables are called the same way on your machine. + +## Usage + +TODO diff --git a/tools/.gitkeep b/tools/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tools/readme.txt b/tools/readme.txt deleted file mode 100644 index f10fdc0..0000000 --- a/tools/readme.txt +++ /dev/null @@ -1,34 +0,0 @@ -Various files need to be in this directory. -If you don't have them, ask someone who does. - -The required files are: - -Binaries: -- acb_extract.exe -- usm_extract.exe -- clHCA - -Z shell scripts: -- update_bgm.zsh -- update_movie.zsh -- update_se.zsh -- update_voice.zsh - -Python scripts: -- adx_parse.py -- bingo_fuck.py -- chara_room_mission.py -- custom_mission.py -- event_bonus.py -- orphan_quest.py -- pad_2_csv.py -- pad_2_json.py -- pad_2_pad.py -- pad_2_pad_louisstyle.py -- pad_stat.py -- pad_update.py -- store_parser.py -- tsv_2_pad.py -- tsv_parse.py -- txt_2_pad.py -- utage_decrypt.py From 9bede047a546c92aef79c8bfa1e63e0a114de8b4 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 7 Mar 2020 00:38:40 +0100 Subject: [PATCH 09/43] updated readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ba0de08..54fe834 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ either in your `$PATH` variable or in the [`tools`](./tools) folder. These tools * [clHCA](https://github.com/KinoMyu/FastHCADecoder) * acb_extract * usm_extract +* [ffmpeg](https://www.ffmpeg.org/download.html) +* [mono](https://www.mono-project.com/download/stable/) (if using Linux) Make sure these executables are called the same way on your machine. From 404de876241904a06ce6edaae198167c0ad8c605 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 7 Mar 2020 05:15:21 +0100 Subject: [PATCH 10/43] ported update_se --- divatool.py | 14 ++-- xdudata/asset_extract.py | 150 +++++++++++++++++++++++++++++++++++++++ xdudata/update.py | 128 ++++++++++++--------------------- 3 files changed, 203 insertions(+), 89 deletions(-) create mode 100644 xdudata/asset_extract.py diff --git a/divatool.py b/divatool.py index ec36fed..e528b57 100755 --- a/divatool.py +++ b/divatool.py @@ -184,15 +184,15 @@ def main(): if args.subcommand == "quest": diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) elif args.command == "xdudata": - script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools")) if args.subcommand == "update": + # add the tools folder to PATH + os.environ["PATH"] += os.pathsep + os.path.join( + os.path.abspath(os.path.dirname(__file__)), "tools" + ) + + # update the data xdudata.update_all( - args.CACHE, - args.XDUDATA, - args.TRANS, - args.EXTRACT, - script_dir, - LANGUAGES, + args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES, ) return 0 diff --git a/xdudata/asset_extract.py b/xdudata/asset_extract.py new file mode 100644 index 0000000..da1ac44 --- /dev/null +++ b/xdudata/asset_extract.py @@ -0,0 +1,150 @@ +import functools +import multiprocessing +import os +import shutil +import subprocess +from contextlib import contextmanager +from distutils.dir_util import copy_tree +from glob import glob + +OPERATING_SYSTEM = os.name + + +def process_se(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Se"), + os.path.join(extract_dir, "Se"), + ) + + from_dir = "Se" + to_dir = "Se_Extracted" + acb_prefix = "_vgmt_acb_ext" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.wav".format(from_dir), recursive=True) + to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True)) + for file in to_del: + os.remove(file) + + acb_files = [file for file in glob("{}/*.acb".format(from_dir))] + acb_cmd = get_dotnet_cmd("acb_extract.exe") + run_program(acb_files, cmd=acb_cmd) + + hca_files = [ + file + for file in glob( + "{}/{}*/**/*.hca".format(from_dir, acb_prefix), recursive=True + ) + ] + run_program(hca_files, cmd="clHCA") + + basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] + char_secs = [basename.split("_", maxsplit=1) for basename in basenames] + for cs in char_secs: + char = cs[0] + sec = str(cs[1]) if len(cs) == 2 else "" + os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True) + wav_files = glob( + "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)), + recursive=True, + ) + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + wav_cmds = [ + [ + "-i", + os.path.abspath(file), + "-acodec", + "libopus", + "-ab", + "192k", + "-af", + "aresample=48000", + os.path.abspath( + "{}/{}/{}/{}.opus".format( + to_dir, + char, + sec, + os.path.splitext(os.path.basename(file))[0], + ) + ), + "-n", + ] + for file in wav_files + ] + pool.map(partial_ffmpeg_cmd, wav_cmds) + + copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Se")) + + +def process_bgm(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), + os.path.join(extract_dir, "Bgm"), + ) + subprocess.run(os.path.join("TODO", "update_bgm.zsh"), shell=True, cwd=extract_dir) + copy_tree( + os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") + ) + + +def process_voice(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Voice"), + os.path.join(extract_dir, "Voice"), + ) + subprocess.run( + os.path.join("TODO", "update_voice.zsh"), shell=True, cwd=extract_dir + ) + copy_tree( + os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") + ) + + +def process_movie(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Common/Asset/Movie"), + os.path.join(extract_dir, "Movie"), + ) + subprocess.run( + os.path.join("TODO", "update_movie.zsh"), shell=True, cwd=extract_dir + ) + copy_tree( + os.path.join(extract_dir, "movie_extracted"), + os.path.join(update_dir, "Asset/Movie"), + ) + + +@contextmanager +def change_cwd(new_path): + old_cwd = os.getcwd() + os.chdir(new_path) + try: + yield + finally: + os.chdir(old_cwd) + + +def get_dotnet_cmd(name): + ext = os.path.splitext(name)[1] + if ext != ".exe": + raise RuntimeError("This doesn't seem like a .NET program.") + path = shutil.which(name) + if OPERATING_SYSTEM == "nt": + return [path] + return ["mono", path] + + +def run_program(args, cmd=None): + if isinstance(cmd, str): + cmd = [cmd] + full_cmd = cmd + args + subprocess.run(full_cmd) + + +def get_pool(size=multiprocessing.cpu_count()): + pool = multiprocessing.Pool(size) + return pool diff --git a/xdudata/update.py b/xdudata/update.py index b659c3f..0d693d3 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -3,85 +3,69 @@ from contextlib import suppress import diva import shutil import os -import subprocess import tempfile import utage from distutils.dir_util import copy_tree, remove_tree from glob import glob +from xdudata.asset_extract import process_se, process_bgm, process_voice, process_movie + def update_all( - cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages, + cache_dir, update_dir, translation_dir, extract_dir, languages, ): - print("Cleaning cache") - cleanup_cache(cache_dir) - - print("Copying images") - copy_images(cache_dir, update_dir) - - print("Processing TSV files") - update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) - - print("Processing Quest Json files") - diva.quest.update_missions(extract_dir, translation_dir, languages) - - # this code isn't going to run on windows (it is in WSL though) because i need to rewrite it - # it just calls my zsh tools and generates it that way, need to replace them - # with real python or something - - # first copy over the necessary avtools to the extract dir - # TODO make it so that you don't need to copy them over (for example call them with python) - print("Copying necessary tools into extracting directory") - shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir) - shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir) - shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir) + # TODO uncomment + # print("Checking tool availability") + # validate_tools() + # + # print("Cleaning cache") + # cleanup_cache(cache_dir) + # + # print("Copying images") + # copy_images(cache_dir, update_dir) + # + # print("Processing TSV files") + # update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) + # + # print("Processing Quest Json files") + # diva.quest.update_missions(extract_dir, translation_dir, languages) print("Processing Sound Effects") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Se"), - os.path.join(extract_dir, "Se"), - ) - subprocess.run( - os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir - ) - copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se")) + process_se(cache_dir, extract_dir, update_dir) - print("Processing BGM - don't forget to copy that loop file") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), - os.path.join(extract_dir, "Bgm"), - ) - subprocess.run( - os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir - ) - copy_tree( - os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") - ) + # print("Processing BGM") + # process_bgm(cache_dir, extract_dir, update_dir) + # + # print("Processing voices") + # process_voice(cache_dir, extract_dir, update_dir) + # + # print("Processing movies - this might take forever") + # process_movie(cache_dir, extract_dir, update_dir) - print("Processing voices") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Voice"), - os.path.join(extract_dir, "Voice"), - ) - subprocess.run( - os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir - ) - copy_tree( - os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") - ) - print("Processing movies - this might take forever") +def validate_tools(): + # TODO validate all the tools are in PATH + ... + + +def cleanup_cache(cache_dir): + with suppress(FileNotFoundError): + shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) + os.remove( + os.path.join( + cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" + ) + ) + + +def copy_images(cache_dir, update_dir): copy_tree( - os.path.join(cache_dir, "Common/Asset/Movie"), - os.path.join(extract_dir, "Movie"), - ) - subprocess.run( - os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir + os.path.join(cache_dir, "Common/Asset/Image"), + os.path.join(update_dir, "Asset/Image"), ) copy_tree( - os.path.join(extract_dir, "movie_extracted"), - os.path.join(update_dir, "Asset/Movie"), + os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), ) @@ -113,23 +97,3 @@ def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages): utage.names.update_names(utage_tmp, translation_dir, languages, False) copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) remove_tree(utage_tmp) - - -def copy_images(cache_dir, update_dir): - copy_tree( - os.path.join(cache_dir, "Common/Asset/Image"), - os.path.join(update_dir, "Asset/Image"), - ) - copy_tree( - os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), - ) - - -def cleanup_cache(cache_dir): - with suppress(FileNotFoundError): - shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) - os.remove( - os.path.join( - cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" - ) - ) From cb0a4d8bf6195c3d459f8fada4adfa73a0bcfa04 Mon Sep 17 00:00:00 2001 From: argoneus Date: Thu, 23 Apr 2020 21:44:50 +0200 Subject: [PATCH 11/43] ported update_bgm to python --- adx/adx_parser.py | 10 +++---- xdudata/asset_extract.py | 58 ++++++++++++++++++++++++++++++++++------ 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/adx/adx_parser.py b/adx/adx_parser.py index 993fc3c..d6d4c5d 100644 --- a/adx/adx_parser.py +++ b/adx/adx_parser.py @@ -74,7 +74,7 @@ class adx_file(object): self.encrypted = self.get_val(CRYPT_OFF, CRYPT_LEN) self.d_offset = self.get_val(DATA_OFF_OFF, DATA_OFF_LEN) if self.d_offset < DATA_OFF_MIN: - return None + return self.l_exists = 1 # now load things for math self.sam_rate = self.get_val(SRATE_OFF, SRATE_LEN) @@ -106,13 +106,11 @@ class adx_file(object): return None if self.l_start > self.sam_count or self.l_end > self.sam_count: raise ValueError("Invalid Loop Data") - return None if self.l_style not in LOOP_TYPE_SUP: raise NotImplementedError( "Loop Style {} Not Implemented".format(str(self.l_style)) ) - return None - ret = {} + ret = dict() ret["duration"] = self.sam_count / self.sam_rate ret["loop_start"] = {} ret["loop_start"]["seconds"] = self.l_start / self.sam_rate @@ -123,7 +121,5 @@ class adx_file(object): ret["loop_end"] = {} ret["loop_end"]["seconds"] = self.l_end / self.sam_rate ret["loop_end"]["samples_native"] = self.l_end - ret["loop_end"]["samples_48k"] = dumb_round( - self.l_end / self.sam_rate * 48000 - ) + ret["loop_end"]["samples_48k"] = dumb_round(self.l_end / self.sam_rate * 48000) return ret diff --git a/xdudata/asset_extract.py b/xdudata/asset_extract.py index da1ac44..014b762 100644 --- a/xdudata/asset_extract.py +++ b/xdudata/asset_extract.py @@ -7,6 +7,8 @@ from contextlib import contextmanager from distutils.dir_util import copy_tree from glob import glob +from adx import extract_loop_data_from_dir + OPERATING_SYSTEM = os.name @@ -35,10 +37,7 @@ def process_se(cache_dir, extract_dir, update_dir): run_program(acb_files, cmd=acb_cmd) hca_files = [ - file - for file in glob( - "{}/{}*/**/*.hca".format(from_dir, acb_prefix), recursive=True - ) + file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) ] run_program(hca_files, cmd="clHCA") @@ -85,10 +84,53 @@ def process_bgm(cache_dir, extract_dir, update_dir): os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), os.path.join(extract_dir, "Bgm"), ) - subprocess.run(os.path.join("TODO", "update_bgm.zsh"), shell=True, cwd=extract_dir) - copy_tree( - os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") - ) + + from_dir = "Bgm" + to_dir = "Bgm_Extracted" + loop_file = "loop.json" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.adx".format(from_dir), recursive=True) + for file in to_del: + os.remove(file) + + acb_files = [file for file in glob("{}/*.acb".format(from_dir))] + acb_cmd = get_dotnet_cmd("acb_extract.exe") + run_program(acb_files, cmd=acb_cmd) + + adx_files = [ + file for file in glob("{}/**/*.adx".format(from_dir), recursive=True) + ] + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + wav_cmds = [ + [ + "-i", + os.path.abspath(file), + "-acodec", + "libopus", + "-ab", + "192k", + "-af", + "aresample=48000", + os.path.abspath( + "{}/{}.opus".format( + to_dir, os.path.splitext(os.path.basename(file))[0], + ) + ), + "-n", + ] + for file in adx_files + ] + pool.map(partial_ffmpeg_cmd, wav_cmds) + + extract_loop_data_from_dir(from_dir, loop_file) + shutil.copy2(loop_file, os.path.join(to_dir, "BgmLoop.json")) + + copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Bgm")) def process_voice(cache_dir, extract_dir, update_dir): From 1f26e8a9cedbd3e285e022939dbe7a9c978089d9 Mon Sep 17 00:00:00 2001 From: argoneus Date: Fri, 15 May 2020 15:44:18 +0200 Subject: [PATCH 12/43] rewrote zsh scripts for asset extract --- xdudata/asset_extract.py | 124 +++++++++++++++++++++++++++++++++++---- xdudata/update.py | 45 +++++++------- 2 files changed, 135 insertions(+), 34 deletions(-) diff --git a/xdudata/asset_extract.py b/xdudata/asset_extract.py index 014b762..8237162 100644 --- a/xdudata/asset_extract.py +++ b/xdudata/asset_extract.py @@ -138,12 +138,67 @@ def process_voice(cache_dir, extract_dir, update_dir): os.path.join(cache_dir, "Android/Asset/Sound/Voice"), os.path.join(extract_dir, "Voice"), ) - subprocess.run( - os.path.join("TODO", "update_voice.zsh"), shell=True, cwd=extract_dir - ) - copy_tree( - os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") - ) + + from_dir = "Voice" + to_dir = "Voice_Extracted" + acb_prefix = "_vgmt_acb_ext" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.wav".format(from_dir), recursive=True) + to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True)) + for file in to_del: + os.remove(file) + + acb_files = [file for file in glob("{}/*.acb".format(from_dir))] + acb_cmd = get_dotnet_cmd("acb_extract.exe") + run_program(acb_files, cmd=acb_cmd) + + hca_files = [ + file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) + ] + run_program(hca_files, cmd="clHCA") + + basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] + char_secs = [basename.split("_", maxsplit=1) for basename in basenames] + for cs in char_secs: + char = cs[0] + sec = str(cs[1]) if len(cs) == 2 else "" + os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True) + wav_files = glob( + "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)), + recursive=True, + ) + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + + wav_cmds = [ + [ + "-i", + os.path.abspath(file), + "-acodec", + "libopus", + "-ab", + "192k", + "-af", + "aresample=48000", + os.path.abspath( + "{}/{}/{}/{}.opus".format( + to_dir, + char, + sec, + os.path.splitext(os.path.basename(file))[0], + ) + ), + "-n", + ] + for file in wav_files + ] + pool.map(partial_ffmpeg_cmd, wav_cmds) + + copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Voice")) def process_movie(cache_dir, extract_dir, update_dir): @@ -151,12 +206,59 @@ def process_movie(cache_dir, extract_dir, update_dir): os.path.join(cache_dir, "Common/Asset/Movie"), os.path.join(extract_dir, "Movie"), ) - subprocess.run( - os.path.join("TODO", "update_movie.zsh"), shell=True, cwd=extract_dir - ) + + from_dir = "Movie" + to_dir = "Movie_Extracted" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.adx".format(from_dir), recursive=True) + to_del.extend(glob("{}/**/*.m2v".format(from_dir), recursive=True)) + for file in to_del: + os.remove(file) + + movie_paths = [ + os.path.abspath(os.path.splitext(file)[0]) + for file in glob("{}/*.usm".format(from_dir)) + if not os.path.basename(file).startswith("mini_radio") + ] + + usm_files = [[path + ".usm"] for path in movie_paths] + usm_cmd = get_dotnet_cmd("usm_extract.exe") + partial_usm_cmd = functools.partial(run_program, cmd=usm_cmd) + pool.map(partial_usm_cmd, usm_files) + + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + mux_cmds = [ + [ + "-i", + path + ".adx", + "-i", + path + ".m2v", + "-c:a", + "libopus", + "-b:a", + "196k", + "-af", + "aresample=48000", + "-c:v", + "libvpx", + "-b:v", + "1M", + "-threads", + "1", + os.path.abspath("{}/{}.webm".format(to_dir, os.path.basename(path))), + "-n", + ] + for path in movie_paths + ] + pool.map(partial_ffmpeg_cmd, mux_cmds) + copy_tree( - os.path.join(extract_dir, "movie_extracted"), - os.path.join(update_dir, "Asset/Movie"), + os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Asset", "Movie") ) diff --git a/xdudata/update.py b/xdudata/update.py index 0d693d3..a59d30c 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -15,33 +15,32 @@ from xdudata.asset_extract import process_se, process_bgm, process_voice, proces def update_all( cache_dir, update_dir, translation_dir, extract_dir, languages, ): - # TODO uncomment - # print("Checking tool availability") - # validate_tools() - # - # print("Cleaning cache") - # cleanup_cache(cache_dir) - # - # print("Copying images") - # copy_images(cache_dir, update_dir) - # - # print("Processing TSV files") - # update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) - # - # print("Processing Quest Json files") - # diva.quest.update_missions(extract_dir, translation_dir, languages) + print("Checking tool availability") + validate_tools() + + print("Cleaning cache") + cleanup_cache(cache_dir) + + print("Copying images") + copy_images(cache_dir, update_dir) + + print("Processing TSV files") + update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) + + print("Processing Quest Json files") + diva.quest.update_missions(extract_dir, translation_dir, languages) print("Processing Sound Effects") process_se(cache_dir, extract_dir, update_dir) - # print("Processing BGM") - # process_bgm(cache_dir, extract_dir, update_dir) - # - # print("Processing voices") - # process_voice(cache_dir, extract_dir, update_dir) - # - # print("Processing movies - this might take forever") - # process_movie(cache_dir, extract_dir, update_dir) + print("Processing BGM") + process_bgm(cache_dir, extract_dir, update_dir) + + print("Processing voices") + process_voice(cache_dir, extract_dir, update_dir) + + print("Processing movies - this might take forever") + process_movie(cache_dir, extract_dir, update_dir) def validate_tools(): From c15343b4b8cf63ca0c6068718a09cbf83f18249e Mon Sep 17 00:00:00 2001 From: argoneus Date: Fri, 15 May 2020 23:13:38 +0200 Subject: [PATCH 13/43] fixed newlines and other windows stuff --- diva/quest.py | 14 +++++++------- utage/names.py | 12 ++++++------ utage/translate.py | 17 ++++++++--------- xdudata/asset_extract.py | 13 +++++++++++-- 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/diva/quest.py b/diva/quest.py index f2342a1..ed4d092 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -31,7 +31,7 @@ def scene_mst(qdb, old_path, dir_in, languages): break with io.open( - os.path.join(old_path, "XduScene.json"), "w", newline="\n" + os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8" ) as json_file: json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True) @@ -56,11 +56,11 @@ def scene_mst(qdb, old_path, dir_in, languages): langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json") if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) - with io.open(langfile, "w", newline="\n") as lang_file: + with io.open(langfile, "w", encoding="utf-8") as lang_file: json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -70,7 +70,7 @@ def quest_mst(qdb, old_path, languages): quests = qdb.get_quests() with io.open( - os.path.join(old_path, "XduQuest.json"), "w", newline="\n" + os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8" ) as json_file: json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True) @@ -85,11 +85,11 @@ def quest_mst(qdb, old_path, languages): langfile = os.path.join(old_path, f"XduQuestNames_{lang}.json") if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) - with io.open(langfile, "w", newline="\n") as lang_file: + with io.open(langfile, "w", encoding="utf-8") as lang_file: json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -108,7 +108,7 @@ def load_locations(dir_in): raise FileNotFoundError("Ya didn't decrypt the files dummy") for scenario in settings: - with open(scenario, "r") as tsv_file: + with open(scenario, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') for row in tsv: tokens = row["FileName"].split("/") diff --git a/utage/names.py b/utage/names.py index b6633f0..b47f3b8 100644 --- a/utage/names.py +++ b/utage/names.py @@ -12,7 +12,7 @@ from multiprocessing import Pool def update_names(dir_in, old_path, languages, is_global): names = extract_names(dir_in, is_global) - global_languages = {"enm": 1, "zh": 2, "ko": 3} + global_languages = {"enm": 1, "zho": 2, "kor": 3} if is_global: languages = global_languages.keys() @@ -33,12 +33,12 @@ def update_names(dir_in, old_path, languages, is_global): langfile = os.path.join(old_path, "nametranslations_{}.json".format(l)) if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) with io.open( - langfile, "w", newline="\n" + langfile, "w", encoding="utf-8" ) as lang_file: # you're using git right json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, @@ -56,7 +56,7 @@ def extract_names(dir_in, is_global): files = [ f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) - if re.search(r"/[0-9]{9}\.tsv$", f) + if re.search(r"[0-9]{9}\.tsv$", f) ] if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) @@ -78,7 +78,7 @@ def read_char_tsv(char_tsv_path, is_global): char_names = set() char_sets = set() - with open(char_tsv_path, "r") as char_tsv_file: + with open(char_tsv_path, "r", encoding="utf-8") as char_tsv_file: char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"') for row in char_tsv: @@ -103,7 +103,7 @@ def read_char_tsv(char_tsv_path, is_global): def read_mission(tsv_path, char_names, char_sets): new_names = set() - with open(tsv_path, "r") as tsv_file: + with open(tsv_path, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') if ( ("Arg1" not in tsv.fieldnames) diff --git a/utage/translate.py b/utage/translate.py index 2344b0a..29dcd39 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -52,7 +52,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): ) languages = ["jpn"] if is_global: - languages += ["enm", "zh", "ko"] + languages += ["enm", "zho", "kor"] json_output_paths = { lang: os.path.join( json_out_dir, @@ -77,7 +77,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): if e.errno != errno.EEXIST: raise - with open(file_in, "r") as tsv_file: + with open(file_in, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE) t_fieldnames = tsv.fieldnames @@ -87,12 +87,11 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): tsv_keyed, json_str = process_tsv(tsv, id_num, is_global) # csv handles newlines, don't set it in io.open - with io.open(tsv_output_path, "w", newline="") as tsv_out: + with io.open(tsv_output_path, "w", newline="", encoding="utf-8") as tsv_out: writer = csv.DictWriter( tsv_out, delimiter="\t", quotechar='"', - lineterminator="\n", fieldnames=t_fieldnames, extrasaction="ignore", ) @@ -103,7 +102,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): for lang in languages: if not json_str[lang]: continue - with io.open(json_output_paths[lang], "w", newline="\n") as json_out: + with io.open(json_output_paths[lang], "w", encoding="utf-8") as json_out: # we don't want to sort these because they're _1, ..., _10, etc json.dump( json_str[lang], @@ -124,8 +123,8 @@ def process_tsv(tsv, id_num, is_global): key_dict = {"jpn": {}} if is_global: key_dict["enm"] = {} - key_dict["zh"] = {} - key_dict["ko"] = {} + key_dict["zho"] = {} + key_dict["kor"] = {} for row in tsv: try: @@ -140,8 +139,8 @@ def process_tsv(tsv, id_num, is_global): with suppress(KeyError): if row["English"]: key_dict["enm"][key] = row["English"] - key_dict["zh"][key] = row["Chinese"] - key_dict["ko"][key] = row["Korean"] + key_dict["zho"][key] = row["Chinese"] + key_dict["kor"][key] = row["Korean"] row["English"] = key tsv_keyed.append(row) except Exception: diff --git a/xdudata/asset_extract.py b/xdudata/asset_extract.py index 8237162..04e088c 100644 --- a/xdudata/asset_extract.py +++ b/xdudata/asset_extract.py @@ -10,6 +10,7 @@ from glob import glob from adx import extract_loop_data_from_dir OPERATING_SYSTEM = os.name +CHUNK_SIZE = 100 def process_se(cache_dir, extract_dir, update_dir): @@ -39,7 +40,8 @@ def process_se(cache_dir, extract_dir, update_dir): hca_files = [ file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) ] - run_program(hca_files, cmd="clHCA") + for hca_sublist in split_list(hca_files, CHUNK_SIZE): + run_program(hca_sublist, cmd="clHCA") basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] char_secs = [basename.split("_", maxsplit=1) for basename in basenames] @@ -160,7 +162,8 @@ def process_voice(cache_dir, extract_dir, update_dir): hca_files = [ file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) ] - run_program(hca_files, cmd="clHCA") + for hca_sublist in split_list(hca_files, CHUNK_SIZE): + run_program(hca_sublist, cmd="clHCA") basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] char_secs = [basename.split("_", maxsplit=1) for basename in basenames] @@ -292,3 +295,9 @@ def run_program(args, cmd=None): def get_pool(size=multiprocessing.cpu_count()): pool = multiprocessing.Pool(size) return pool + + +def split_list(input_list, chunk_size): + return [ + input_list[i : i + chunk_size] for i in range(0, len(input_list), chunk_size) + ] From b541f43804391c4f4cf1e68f2c4c18c9d7219081 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 16 May 2020 13:15:58 +0200 Subject: [PATCH 14/43] updated -g to not replace jp translations --- utage/translate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utage/translate.py b/utage/translate.py index 29dcd39..93ae072 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -52,7 +52,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): ) languages = ["jpn"] if is_global: - languages += ["enm", "zho", "kor"] + languages = ["enm", "zho", "kor"] json_output_paths = { lang: os.path.join( json_out_dir, @@ -70,9 +70,9 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): if e.errno != errno.EEXIST: raise - if not os.path.exists(os.path.dirname(json_output_paths["jpn"])): + if not os.path.exists(os.path.dirname(json_output_paths[languages[0]])): try: - os.makedirs(os.path.dirname(json_output_paths["jpn"])) + os.makedirs(os.path.dirname(json_output_paths[languages[0]])) except OSError as e: if e.errno != errno.EEXIST: raise From a2308379f2d70e1192166a3c8393704033d29880 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 16 May 2020 13:47:41 +0200 Subject: [PATCH 15/43] added --utage switch to diva quest --- diva/quest.py | 17 ++++++++--------- divatool.py | 14 ++++++++++++-- xdudata/update.py | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/diva/quest.py b/diva/quest.py index ed4d092..a7d2a72 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -7,16 +7,18 @@ import os from glob import glob -def update_missions(dir_in, old_path, languages): +def update_missions(dir_in, old_path, languages, is_global, utage_in=None): + if not utage_in: + utage_in = os.path.join(dir_in, "Common/Asset/Utage") qdb = db.QuestDB(dir_in) quest_mst(qdb, old_path, languages) - scene_mst(qdb, old_path, dir_in, languages) + scene_mst(qdb, old_path, dir_in, languages, utage_in) -def scene_mst(qdb, old_path, dir_in, languages): +def scene_mst(qdb, old_path, dir_in, languages, utage_in): scenes = qdb.get_scenes() - locations = load_locations(dir_in) + locations = load_locations(dir_in, utage_in) for id, scene in scenes.items(): i = 0 @@ -97,12 +99,9 @@ def quest_mst(qdb, old_path, languages): return quests -def load_locations(dir_in): +def load_locations(dir_in, utage_in): locations = {} - settings = glob( - os.path.join(dir_in, "Common/Asset/Utage/**/Settings/Scenario.tsv"), - recursive=True, - ) + settings = glob("{}/**/Settings/Scenario.tsv".format(utage_in), recursive=True,) if len(settings) == 0: raise FileNotFoundError("Ya didn't decrypt the files dummy") diff --git a/divatool.py b/divatool.py index e528b57..17d32ef 100755 --- a/divatool.py +++ b/divatool.py @@ -117,13 +117,16 @@ def main(): metavar="", title="subcommand", dest="subcommand" ) parser_diva_quest = parser_diva_subparsers.add_parser( - "quest", help="Generate quest JSON files" + "quest", help="Generate quest JSON files", parents=[global_flag_parser], ) parser_diva_quest.add_argument( "INPUT", help="Directory of databases containing Quest*.db and ResourceEntry.db", type=str, ) + parser_diva_quest.add_argument( + "--utage", help="Directory of Utage folders and files.", type=str, default=None, + ) parser_diva_quest.add_argument( "OLD", help="Directory with old quest files", nargs="?", type=str, default="." ) @@ -182,7 +185,14 @@ def main(): raise FileNotFoundError(args.INPUT) elif args.command == "diva": if args.subcommand == "quest": - diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) + if not args.utage: + diva.quest.update_missions( + args.INPUT, args.OLD, LANGUAGES, args.is_global + ) + else: + diva.quest.update_missions( + args.INPUT, args.OLD, LANGUAGES, args.is_global, utage_in=args.utage + ) elif args.command == "xdudata": if args.subcommand == "update": # add the tools folder to PATH diff --git a/xdudata/update.py b/xdudata/update.py index a59d30c..8dd8af4 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -28,7 +28,7 @@ def update_all( update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) print("Processing Quest Json files") - diva.quest.update_missions(extract_dir, translation_dir, languages) + diva.quest.update_missions(extract_dir, translation_dir, languages, False) print("Processing Sound Effects") process_se(cache_dir, extract_dir, update_dir) From 1f5ff96b8efaa8e609a160ef39061a1f3fee5ce3 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 16 May 2020 15:38:56 +0200 Subject: [PATCH 16/43] fixed diva quest --- db/base.py | 2 +- db/quest.py | 24 ++++++++++++++++-------- diva/quest.py | 49 ++++++++++++++++++++++++++++++++----------------- 3 files changed, 49 insertions(+), 26 deletions(-) diff --git a/db/base.py b/db/base.py index afa5817..80132be 100644 --- a/db/base.py +++ b/db/base.py @@ -23,7 +23,7 @@ class BaseDB(object): return self def __exit__(self, exc_type, exc_value, traceback): - self.quest_conn.close() + self.conn.close() def attach_dbs(self, databases, path): for d in databases: diff --git a/db/quest.py b/db/quest.py index c4b13f8..7054e94 100644 --- a/db/quest.py +++ b/db/quest.py @@ -1,16 +1,24 @@ from .base import BaseDB -DATABASES = { - "QuestMst.db": "Quest", - "QuestSceneMst.db": "QuestScene", - "QuestPartMst.db": "QuestPart", - "ResourceEntry.db": "ResourceEntry", -} + +def get_dbs_by_lang(language=None): + databases = { + "QuestMst": "Quest", + "QuestSceneMst": "QuestScene", + "QuestPartMst": "QuestPart", + "ResourceEntry": "ResourceEntry", + } + for key in list(databases): + if not language: + databases[key + ".db"] = databases.pop(key) + else: + databases[key + "_" + language + ".db"] = databases.pop(key) + return databases class QuestDB(BaseDB): - def __init__(self, path): - super().__init__(path, DATABASES) + def __init__(self, path, language=None): + super().__init__(path, get_dbs_by_lang(language)) def get_quests(self): self.cursor.execute( diff --git a/diva/quest.py b/diva/quest.py index a7d2a72..7af5214 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -10,15 +10,22 @@ from glob import glob def update_missions(dir_in, old_path, languages, is_global, utage_in=None): if not utage_in: utage_in = os.path.join(dir_in, "Common/Asset/Utage") - qdb = db.QuestDB(dir_in) - quest_mst(qdb, old_path, languages) - scene_mst(qdb, old_path, dir_in, languages, utage_in) + if is_global: + languages = ["enm", "zho", "kor"] + for lang in languages: + qdb = db.QuestDB(dir_in, lang) + quest_mst(qdb, old_path, [lang], is_global) + scene_mst(qdb, old_path, [lang], utage_in, is_global) + else: + qdb = db.QuestDB(dir_in) + quest_mst(qdb, old_path, languages, is_global) + scene_mst(qdb, old_path, languages, utage_in, is_global) -def scene_mst(qdb, old_path, dir_in, languages, utage_in): +def scene_mst(qdb, old_path, languages, utage_in, is_global): scenes = qdb.get_scenes() - locations = load_locations(dir_in, utage_in) + locations = load_locations(utage_in) for id, scene in scenes.items(): i = 0 @@ -28,18 +35,23 @@ def scene_mst(qdb, old_path, dir_in, languages, utage_in): except KeyError as exc: i += 1 if i == len(scene["Parts"]): + if is_global: + break raise KeyError(f"Could not find folder for scene {id}") from exc continue break - with io.open( - os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8" - ) as json_file: - json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True) + if not is_global: + with io.open( + os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8" + ) as json_file: + json.dump( + scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True + ) for lang in languages: out_dict = {} - if lang == "jpn": + if lang == "jpn" or is_global: for key, value in scenes.items(): out_dict[key] = { "Name": value["Name"], @@ -68,17 +80,20 @@ def scene_mst(qdb, old_path, dir_in, languages, utage_in): ) -def quest_mst(qdb, old_path, languages): +def quest_mst(qdb, old_path, languages, is_global): quests = qdb.get_quests() - with io.open( - os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8" - ) as json_file: - json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True) + if not is_global: + with io.open( + os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8" + ) as json_file: + json.dump( + quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True + ) for lang in languages: out_dict = {} - if lang == "jpn": + if lang == "jpn" or is_global: for key, value in quests.items(): out_dict[key] = {"Name": value["Name"], "Enabled": False} else: @@ -99,7 +114,7 @@ def quest_mst(qdb, old_path, languages): return quests -def load_locations(dir_in, utage_in): +def load_locations(utage_in): locations = {} settings = glob("{}/**/Settings/Scenario.tsv".format(utage_in), recursive=True,) From 2cc7eba22bb3b5df665cf650687cf83984f6abf8 Mon Sep 17 00:00:00 2001 From: argoneus Date: Thu, 9 Jul 2020 19:07:45 +0200 Subject: [PATCH 17/43] removed todo --- xdudata/update.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/xdudata/update.py b/xdudata/update.py index 8dd8af4..8f61b6f 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -15,9 +15,6 @@ from xdudata.asset_extract import process_se, process_bgm, process_voice, proces def update_all( cache_dir, update_dir, translation_dir, extract_dir, languages, ): - print("Checking tool availability") - validate_tools() - print("Cleaning cache") cleanup_cache(cache_dir) @@ -43,11 +40,6 @@ def update_all( process_movie(cache_dir, extract_dir, update_dir) -def validate_tools(): - # TODO validate all the tools are in PATH - ... - - def cleanup_cache(cache_dir): with suppress(FileNotFoundError): shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) From 12051045cc921fd73a12ec18e07fd4f99a5c8b7d Mon Sep 17 00:00:00 2001 From: argoneus Date: Wed, 21 Oct 2020 23:40:00 +0200 Subject: [PATCH 18/43] added scene images to scene json --- db/quest.py | 6 +++++- diva/quest.py | 33 +++++++++++++++++---------------- utage/translate.py | 2 +- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/db/quest.py b/db/quest.py index 7054e94..bedc636 100644 --- a/db/quest.py +++ b/db/quest.py @@ -52,12 +52,14 @@ class QuestDB(BaseDB): questSceneMstId, name, summaryText, + img, group_concat(partIds) as parts FROM ( SELECT qs.questSceneMstId, qs.name, qs.summaryText, + qs.img, CASE WHEN qp.afterTalkName == '' AND qp.beforeTalkName == '' THEN NULL WHEN qp.afterTalkName == '' AND qp.beforeTalkName != '' THEN qp.beforeTalkName @@ -74,9 +76,11 @@ class QuestDB(BaseDB): ) res = self.cursor.fetchall() scenes = {} - for scene_id, name, summary, parts in res: + for scene_id, name, summary, img, parts in res: scenes[str(scene_id)] = {} scenes[str(scene_id)]["Name"] = name scenes[str(scene_id)]["SummaryText"] = summary + if img: + scenes[str(scene_id)]["Image"] = img scenes[str(scene_id)]["Parts"] = parts.split(",") return scenes diff --git a/diva/quest.py b/diva/quest.py index 7af5214..2b0cb41 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -51,22 +51,23 @@ def scene_mst(qdb, old_path, languages, utage_in, is_global): for lang in languages: out_dict = {} - if lang == "jpn" or is_global: - for key, value in scenes.items(): - out_dict[key] = { - "Name": value["Name"], - "SummaryText": value["SummaryText"], - "Credits": "POKELABO", - "Enabled": False, - } - else: - for key in scenes.keys(): - out_dict[key] = { - "Name": "", - "SummaryText": "", - "Credits": "", - "Enabled": False, - } + for key, value in scenes.items(): + name = "" + summary = "" + credit = "" + enabled = False + + if lang == "jpn" or is_global: + name = value["Name"] + summary = value["SummaryText"] + credit = "POKELABO" + + out_dict[key] = { + "Name": name, + "SummaryText": summary, + "Credits": credit, + "Enabled": enabled, + } langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json") if os.path.isfile(langfile): diff --git a/utage/translate.py b/utage/translate.py index 93ae072..cd3b875 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -15,7 +15,7 @@ def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global): files = [ f for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True) - if "{os.path.sep}Diva{os.path.sep}" not in f + if f"{os.path.sep}Diva{os.path.sep}" not in f ] if len(files) == 0: From 7ef9b77264673d1c5b46a5d074ac3bd4f5432f61 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 31 Oct 2020 12:43:18 +0100 Subject: [PATCH 19/43] excluding Diva files from name parsing --- utage/names.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utage/names.py b/utage/names.py index b47f3b8..0ea03f8 100644 --- a/utage/names.py +++ b/utage/names.py @@ -57,6 +57,7 @@ def extract_names(dir_in, is_global): f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r"[0-9]{9}\.tsv$", f) + and f"{os.path.sep}Diva{os.path.sep}" not in f ] if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) From 2638b72b6b148b56f670b44d152ec777b98a27bf Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 14 Feb 2021 20:41:04 +0100 Subject: [PATCH 20/43] added cze as language --- divatool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/divatool.py b/divatool.py index 17d32ef..f43a927 100755 --- a/divatool.py +++ b/divatool.py @@ -9,7 +9,7 @@ import utage import db import xdudata -LANGUAGES = ["jpn", "eng", "rus"] +LANGUAGES = ["jpn", "eng", "rus", "cze"] def main(): From a9b671eb12ca2c0102ec0f70fb08b8c71befe4e1 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 22 May 2021 15:14:38 +0200 Subject: [PATCH 21/43] ignoring scripts in side03 --- diva/quest.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/diva/quest.py b/diva/quest.py index 2b0cb41..a2306f0 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -129,8 +129,14 @@ def load_locations(utage_in): tokens = row["FileName"].split("/") if len(tokens) < 3: continue - if tokens[2] in locations: - raise ValueError("Conflicting paths for file {tokens[2]}") - locations[tokens[2]] = tokens[0] + folder, _scenario, part_id = tokens + # side03 seems to only contain prototype scripts so we ignore it + if folder == "side03": + continue + if part_id in locations: + raise ValueError( + f"Conflicting paths for scene part {part_id}, existing folder: {locations[part_id]}" + ) + locations[part_id] = folder return locations From 70321bcbf27c12fb5c683b613841af706669b179 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 15 Feb 2020 20:16:16 +0100 Subject: [PATCH 22/43] updated .gitignore and added readme to scripts folder --- .gitignore | 3 +++ scripts/readme.txt | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 scripts/readme.txt diff --git a/.gitignore b/.gitignore index e691145..e74fd7e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ __pycache__/ *$py.class *.swp virtualenv/ +scripts/* +!scripts/readme.txt +.idea/ diff --git a/scripts/readme.txt b/scripts/readme.txt new file mode 100644 index 0000000..a8fad1a --- /dev/null +++ b/scripts/readme.txt @@ -0,0 +1,29 @@ +Various files need to be in this directory. +If you don't have them, ask someone who does. + +The required files are: + +Z shell: +- update_bgm.zsh +- update_movie.zsh +- update_se.zsh +- update_voice.zsh + +Python scripts: +- adx_parse.py +- bingo_fuck.py +- chara_room_mission.py +- custom_mission.py +- event_bonus.py +- orphan_quest.py +- pad_2_csv.py +- pad_2_json.py +- pad_2_pad.py +- pad_2_pad_louisstyle.py +- pad_stat.py +- pad_update.py +- store_parser.py +- tsv_2_pad.py +- tsv_parse.py +- txt_2_pad.py +- utage_decrypt.py From d761d12c63ed85310378a9f28c5b54c82a85584a Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 16 Feb 2020 00:15:29 +0100 Subject: [PATCH 23/43] added "is_global" flag --- divatool.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/divatool.py b/divatool.py index cd5f342..f759b4f 100755 --- a/divatool.py +++ b/divatool.py @@ -15,6 +15,16 @@ LANGUAGES = ["jpn", "eng", "rus"] def main(): parser = argparse.ArgumentParser(description="Unpack and process XDU data") + global_flag_parser = argparse.ArgumentParser(add_help=False) + global_flag_parser.add_argument( + "-g", + "--global", + help="Whether we are running on Global XDU.", + action="store_true", + dest="is_global", + default=False, + ) + subparsers = parser.add_subparsers( metavar="", title="subcommands", dest="command" ) @@ -81,7 +91,9 @@ def main(): parser_utage_crypt.add_argument("TARGET", help="Input", type=str) # utage translate parser_utage_translate = parser_utage_subparsers.add_parser( - "translate", help="Generate keyed tsv and json files" + "translate", + help="Generate keyed tsv and json files", + parents=[global_flag_parser], ) parser_utage_translate.add_argument("INPUT", help="Input", type=str) parser_utage_translate.add_argument( @@ -92,7 +104,7 @@ def main(): ) # utage names parser_utage_names = parser_utage_subparsers.add_parser( - "names", help="Generate and update name files" + "names", help="Generate and update name files", parents=[global_flag_parser] ) parser_utage_names.add_argument("INPUT", help="Input", type=str) parser_utage_names.add_argument( @@ -124,7 +136,7 @@ def main(): metavar="", title="subcommand", dest="subcommand" ) parser_xdudata_update = parser_xdudata_subparsers.add_parser( - "update", help="Update an XDUData directory" + "update", help="Update an XDUData directory", parents=[global_flag_parser] ) parser_xdudata_update.add_argument( "CACHE", help="files/ directory from XDU cache", type=str @@ -152,14 +164,20 @@ def main(): raise FileNotFoundError("Could not find {}".format(args.TARGET)) elif args.subcommand == "translate": if os.path.isfile(args.INPUT): - utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR) + utage.translate.translate_file( + args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global + ) elif os.path.isdir(args.INPUT): - utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR) + utage.translate.translate_dir( + args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global + ) else: raise FileNotFoundError("Could not find {}".format(args.INPUT)) elif args.subcommand == "names": if os.path.isdir(args.INPUT): - utage.names.update_names(args.INPUT, args.OLD, LANGUAGES) + utage.names.update_names( + args.INPUT, args.OLD, LANGUAGES, args.is_global + ) else: raise FileNotFoundError(args.INPUT) elif args.command == "diva": @@ -168,7 +186,12 @@ def main(): elif args.command == "xdudata": if args.subcommand == "update": xdudata.update_all( - args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES + args.CACHE, + args.XDUDATA, + args.TRANS, + args.EXTRACT, + LANGUAGES, + args.is_global, ) return 0 From 77e78a73eb3268099b81ae968e949a3b68cd8e0c Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 16 Feb 2020 00:55:31 +0100 Subject: [PATCH 24/43] sorting out zsh script calling and folder names --- .gitignore | 4 +- divatool.py | 2 + {scripts => tools}/readme.txt | 7 ++- utage/names.py | 12 ++--- utage/translate.py | 22 +++------- xdudata/update.py | 83 +++++++++++++++++++++++++---------- 6 files changed, 79 insertions(+), 51 deletions(-) rename {scripts => tools}/readme.txt (87%) diff --git a/.gitignore b/.gitignore index e74fd7e..f8d0fa1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ __pycache__/ *$py.class *.swp virtualenv/ -scripts/* -!scripts/readme.txt +tools/* +!tools/readme.txt .idea/ diff --git a/divatool.py b/divatool.py index f759b4f..caf2bef 100755 --- a/divatool.py +++ b/divatool.py @@ -184,12 +184,14 @@ def main(): if args.subcommand == "quest": diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) elif args.command == "xdudata": + script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools")) if args.subcommand == "update": xdudata.update_all( args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, + script_dir, LANGUAGES, args.is_global, ) diff --git a/scripts/readme.txt b/tools/readme.txt similarity index 87% rename from scripts/readme.txt rename to tools/readme.txt index a8fad1a..f10fdc0 100644 --- a/scripts/readme.txt +++ b/tools/readme.txt @@ -3,7 +3,12 @@ If you don't have them, ask someone who does. The required files are: -Z shell: +Binaries: +- acb_extract.exe +- usm_extract.exe +- clHCA + +Z shell scripts: - update_bgm.zsh - update_movie.zsh - update_se.zsh diff --git a/utage/names.py b/utage/names.py index 56ef85c..5c5f2e2 100644 --- a/utage/names.py +++ b/utage/names.py @@ -9,7 +9,7 @@ from glob import glob from multiprocessing import Pool -def update_names(dir_in, old_path, languages): +def update_names(dir_in, old_path, languages, is_global): names = extract_names(dir_in) # we have to update japanese like the rest because CustomData exists @@ -30,11 +30,7 @@ def update_names(dir_in, old_path, languages): langfile, "w", newline="\n" ) as lang_file: # you're using git right json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -101,8 +97,6 @@ def read_mission(tsv_path, char_names, char_sets): if row["Command"] and row["Command"].startswith("//"): continue if row["Text"] and row["Arg1"]: - if (row["Arg1"] not in char_names) and ( - row["Arg1"] not in char_sets - ): + if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets): new_names.add(row["Arg1"]) return new_names diff --git a/utage/translate.py b/utage/translate.py index 6c47289..f70fc07 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -10,26 +10,22 @@ from glob import glob from multiprocessing.pool import Pool -def translate_dir(dir_in, tsv_out_dir, json_out_dir): +def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global): files = [ f - for f in glob( - os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True - ) + for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True) if "{os.path.sep}Diva{os.path.sep}" not in f ] if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) - ptrans = partial( - translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir - ) + ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir) with Pool() as p: p.map(ptrans, files) -def translate_file(file_in, tsv_out_dir, json_out_dir): +def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): try: if not file_in.endswith(".tsv"): raise ValueError("Invalid File Type for {}".format(file_in)) @@ -49,9 +45,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir): tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num) ) json_output_path = os.path.join( - json_out_dir, - event_folder, - "{}_translations_jpn.json".format(id_num), + json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num), ) # need to create output paths and avoid races when threading @@ -90,11 +84,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir): with io.open(json_output_path, "w", newline="\n") as json_out: # we don't want to sort these because they're _1, ..., _10, etc json.dump( - json_str, - json_out, - ensure_ascii=False, - indent="\t", - sort_keys=False, + json_str, json_out, ensure_ascii=False, indent="\t", sort_keys=False, ) except Exception: diff --git a/xdudata/update.py b/xdudata/update.py index fd747f9..6924b13 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -1,4 +1,7 @@ +from contextlib import suppress + import diva +import shutil import os import subprocess import tempfile @@ -8,35 +11,59 @@ from distutils.dir_util import copy_tree, remove_tree from glob import glob -def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): +def update_all( + cache_dir, + update_dir, + translation_dir, + extract_dir, + script_dir, + languages, + is_global, +): + print("Cleaning cache") + cleanup_cache(cache_dir) + print("Copying images") - copy_images(cache_dir, update_dir) + # copy_images(cache_dir, update_dir) print("Processing TSV files") - update_tsv(cache_dir, update_dir, translation_dir, languages) + # TODO + # update_tsv(cache_dir, update_dir, translation_dir, languages, is_global) print("Processing Quest Json files") - diva.quest.update_missions(cache_dir, translation_dir, languages) + # diva.quest.update_missions(cache_dir, translation_dir, languages) - # this code isn't going to run on windows because i need to rewrite it - # it just calls my zsh scripts and generates it that way, need to replace them + # this code isn't going to run on windows (it will in WSL though) because i need to rewrite it + # it just calls my zsh tools and generates it that way, need to replace them # with real python or something + + # first copy over the necessary avtools to the extract dir + # TODO make it so that you don't need to copy them over (for example call them with python) + print("Copying necessary tools into extracting directory") + shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir) + shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir) + shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir) + print("Processing Sound Effects") copy_tree( os.path.join(cache_dir, "Android/Asset/Sound/Se"), os.path.join(extract_dir, "Se"), ) - subprocess.run("./update_se.zsh", shell=True, cwd=extract_dir) - copy_tree(os.path.join(extract_dir, "se"), os.path.join(update_dir, "Se")) + subprocess.run( + os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir + ) + copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se")) print("Processing BGM - don't forget to copy that loop file") copy_tree( os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), os.path.join(extract_dir, "Bgm"), ) - subprocess.run("./update_bgm.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "bgm"), os.path.join(update_dir, "Bgm") + os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") ) print("Processing voices") @@ -44,9 +71,11 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): os.path.join(cache_dir, "Android/Asset/Sound/Voice"), os.path.join(extract_dir, "Voice"), ) - subprocess.run("./update_voice.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "voice"), os.path.join(update_dir, "Voice") + os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") ) print("Processing movies - this might take forever") @@ -54,22 +83,22 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): os.path.join(cache_dir, "Common/Asset/Movie"), os.path.join(extract_dir, "Movie"), ) - subprocess.run("./update_movie.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "movie"), + os.path.join(extract_dir, "movie_extracted"), os.path.join(update_dir, "Asset/Movie"), ) -def update_tsv(cache_dir, update_dir, translation_dir, languages): +def update_tsv(cache_dir, update_dir, translation_dir, languages, is_global): utage_tmp = tempfile.mkdtemp() copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp) utage.crypt.crypt_dir( utage_tmp, bytearray("SampleSecretKey", "utf-8"), False, False ) - for encrypted in glob( - os.path.join(utage_tmp, "**/*.utage"), recursive=True - ): + for encrypted in glob(os.path.join(utage_tmp, "**/*.utage"), recursive=True): try: os.remove(encrypted) except: @@ -79,11 +108,10 @@ def update_tsv(cache_dir, update_dir, translation_dir, languages): utage_tmp, os.path.join(update_dir, "Utage"), os.path.join(translation_dir, "Missions"), + is_global, ) utage.names.update_names(utage_tmp, translation_dir, languages) - copy_tree( - os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva") - ) + copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) remove_tree(utage_tmp) @@ -93,6 +121,15 @@ def copy_images(cache_dir, update_dir): os.path.join(update_dir, "Asset/Image"), ) copy_tree( - os.path.join(cache_dir, "Common/Sample"), - os.path.join(update_dir, "Sample"), + os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), ) + + +def cleanup_cache(cache_dir): + with suppress(FileNotFoundError): + shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) + os.remove( + os.path.join( + cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" + ) + ) From 057273552b7db1024260e9198862ef4ef66155b5 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 16 Feb 2020 02:24:03 +0100 Subject: [PATCH 25/43] fixed up updater --- diva/quest.py | 26 +++++++------------------- divatool.py | 3 +-- utage/translate.py | 7 ++++++- xdudata/update.py | 30 +++++++++++++++--------------- 4 files changed, 29 insertions(+), 37 deletions(-) diff --git a/diva/quest.py b/diva/quest.py index d84e3b2..5a6d27d 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -18,6 +18,8 @@ def scene_mst(qdb, old_path, dir_in, languages): locations = load_locations(dir_in) + print(locations) + for id, scene in scenes.items(): i = 0 while True: @@ -26,18 +28,14 @@ def scene_mst(qdb, old_path, dir_in, languages): except KeyError as exc: i += 1 if i == len(scene["Parts"]): - raise KeyError( - f"Could not find folder for scene {id}" - ) from exc + raise KeyError(f"Could not find folder for scene {id}") from exc continue break with io.open( os.path.join(old_path, "XduScene.json"), "w", newline="\n" ) as json_file: - json.dump( - scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True - ) + json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True) for lang in languages: out_dict = {} @@ -66,11 +64,7 @@ def scene_mst(qdb, old_path, dir_in, languages): with io.open(langfile, "w", newline="\n") as lang_file: json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -80,9 +74,7 @@ def quest_mst(qdb, old_path, languages): with io.open( os.path.join(old_path, "XduQuest.json"), "w", newline="\n" ) as json_file: - json.dump( - quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True - ) + json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True) for lang in languages: out_dict = {} @@ -101,11 +93,7 @@ def quest_mst(qdb, old_path, languages): with io.open(langfile, "w", newline="\n") as lang_file: json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) return quests diff --git a/divatool.py b/divatool.py index caf2bef..ec36fed 100755 --- a/divatool.py +++ b/divatool.py @@ -136,7 +136,7 @@ def main(): metavar="", title="subcommand", dest="subcommand" ) parser_xdudata_update = parser_xdudata_subparsers.add_parser( - "update", help="Update an XDUData directory", parents=[global_flag_parser] + "update", help="Update an XDUData directory" ) parser_xdudata_update.add_argument( "CACHE", help="files/ directory from XDU cache", type=str @@ -193,7 +193,6 @@ def main(): args.EXTRACT, script_dir, LANGUAGES, - args.is_global, ) return 0 diff --git a/utage/translate.py b/utage/translate.py index f70fc07..8b70398 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -20,7 +20,12 @@ def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global): if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) - ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir) + ptrans = partial( + translate_file, + tsv_out_dir=tsv_out_dir, + json_out_dir=json_out_dir, + is_global=is_global, + ) with Pool() as p: p.map(ptrans, files) diff --git a/xdudata/update.py b/xdudata/update.py index 6924b13..b659c3f 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -12,28 +12,21 @@ from glob import glob def update_all( - cache_dir, - update_dir, - translation_dir, - extract_dir, - script_dir, - languages, - is_global, + cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages, ): print("Cleaning cache") cleanup_cache(cache_dir) print("Copying images") - # copy_images(cache_dir, update_dir) + copy_images(cache_dir, update_dir) print("Processing TSV files") - # TODO - # update_tsv(cache_dir, update_dir, translation_dir, languages, is_global) + update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) print("Processing Quest Json files") - # diva.quest.update_missions(cache_dir, translation_dir, languages) + diva.quest.update_missions(extract_dir, translation_dir, languages) - # this code isn't going to run on windows (it will in WSL though) because i need to rewrite it + # this code isn't going to run on windows (it is in WSL though) because i need to rewrite it # it just calls my zsh tools and generates it that way, need to replace them # with real python or something @@ -92,7 +85,7 @@ def update_all( ) -def update_tsv(cache_dir, update_dir, translation_dir, languages, is_global): +def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages): utage_tmp = tempfile.mkdtemp() copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp) utage.crypt.crypt_dir( @@ -104,13 +97,20 @@ def update_tsv(cache_dir, update_dir, translation_dir, languages, is_global): except: pass + # copy over files for the quest updater + copy_tree(utage_tmp, os.path.join(extract_dir, "Common/Asset/Utage")) + shutil.copy2(os.path.join(cache_dir, "QuestMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "QuestSceneMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "QuestPartMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "ResourceEntry.db"), extract_dir) + utage.translate.translate_dir( utage_tmp, os.path.join(update_dir, "Utage"), os.path.join(translation_dir, "Missions"), - is_global, + False, ) - utage.names.update_names(utage_tmp, translation_dir, languages) + utage.names.update_names(utage_tmp, translation_dir, languages, False) copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) remove_tree(utage_tmp) From 9b998704e2fd9b13a7e60b3533ab46c74a273f1a Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 16 Feb 2020 03:05:24 +0100 Subject: [PATCH 26/43] added basic nametranslations for global languages --- utage/names.py | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/utage/names.py b/utage/names.py index 5c5f2e2..b6633f0 100644 --- a/utage/names.py +++ b/utage/names.py @@ -10,13 +10,24 @@ from multiprocessing import Pool def update_names(dir_in, old_path, languages, is_global): - names = extract_names(dir_in) + names = extract_names(dir_in, is_global) + + global_languages = {"enm": 1, "zh": 2, "ko": 3} + if is_global: + languages = global_languages.keys() # we have to update japanese like the rest because CustomData exists # and POKELABO deleted the wedding gear event stuff from the game files for l in languages: if l == "jpn": out_dict = dict(zip(names, names)) + elif l in global_languages.keys(): + out_dict = {} + for name in names: + spl = name.split("\t") + idx = global_languages[l] + val = spl[idx] if spl[idx] != "None" else "" + out_dict[spl[0]] = val else: out_dict = dict.fromkeys(names, "") @@ -34,13 +45,13 @@ def update_names(dir_in, old_path, languages, is_global): ) -def extract_names(dir_in): +def extract_names(dir_in, is_global): char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv") if not os.path.isfile(char_tsv_path): raise FileNotFoundError("Character.tsv not found") - names, sets = read_char_tsv(char_tsv_path) + names, sets = read_char_tsv(char_tsv_path, is_global) files = [ f @@ -50,18 +61,20 @@ def extract_names(dir_in): if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) - # tfw gil - # it's still faster on my machine so i'm keeping it - read_partial = partial(read_mission, char_names=names, char_sets=sets) - with Pool() as p: - new_names = p.map(read_partial, files) - for x in new_names: - names.update(x) + # TODO unsupported on global for now + if not is_global: + # tfw gil + # it's still faster on my machine so i'm keeping it + read_partial = partial(read_mission, char_names=names, char_sets=sets) + with Pool() as p: + new_names = p.map(read_partial, files) + for x in new_names: + names.update(x) return names -def read_char_tsv(char_tsv_path): +def read_char_tsv(char_tsv_path, is_global): char_names = set() char_sets = set() @@ -77,7 +90,12 @@ def read_char_tsv(char_tsv_path): and row["CharacterName"].strip() and row["NameText"].strip() ): - char_names.add(row["NameText"]) + if is_global: + char_names.add( + f"{row['NameText']}\t{row['CharaEnglish']}\t{row['CharaChinese']}\t{row['CharaKorean']}" + ) + else: + char_names.add(row["NameText"]) char_sets.add(row["CharacterName"]) return char_names, char_sets From 15afb39759bfdd71a450998ff28ad0b018681e07 Mon Sep 17 00:00:00 2001 From: argoneus Date: Thu, 20 Feb 2020 23:57:09 +0100 Subject: [PATCH 27/43] translate command now works with global languages --- diva/quest.py | 2 -- utage/translate.py | 61 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/diva/quest.py b/diva/quest.py index 5a6d27d..f2342a1 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -18,8 +18,6 @@ def scene_mst(qdb, old_path, dir_in, languages): locations = load_locations(dir_in) - print(locations) - for id, scene in scenes.items(): i = 0 while True: diff --git a/utage/translate.py b/utage/translate.py index 8b70398..6ea96b4 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -4,6 +4,7 @@ import io import json import os import traceback +from contextlib import suppress from functools import partial from glob import glob @@ -49,9 +50,17 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): tsv_output_path = os.path.join( tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num) ) - json_output_path = os.path.join( - json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num), - ) + languages = ["jpn"] + if is_global: + languages += ["enm", "zh", "ko"] + json_output_paths = { + lang: os.path.join( + json_out_dir, + event_folder, + "{}_translations_{}.json".format(id_num, lang), + ) + for lang in languages + } # need to create output paths and avoid races when threading if not os.path.exists(os.path.dirname(tsv_output_path)): @@ -61,16 +70,21 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): if e.errno != errno.EEXIST: raise - if not os.path.exists(os.path.dirname(json_output_path)): + if not os.path.exists(os.path.dirname(json_output_paths["jpn"])): try: - os.makedirs(os.path.dirname(json_output_path)) + os.makedirs(os.path.dirname(json_output_paths["jpn"])) except OSError as e: if e.errno != errno.EEXIST: raise with open(file_in, "r") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') - tsv_keyed, json_str = process_tsv(tsv, id_num) + + t_fieldnames = tsv.fieldnames + if is_global: + t_fieldnames = tsv.fieldnames[: tsv.fieldnames.index("English") + 1] + + tsv_keyed, json_str = process_tsv(tsv, id_num, is_global) # csv handles newlines, don't set it in io.open with io.open(tsv_output_path, "w", newline="") as tsv_out: @@ -79,28 +93,39 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): delimiter="\t", quotechar='"', lineterminator="\n", - fieldnames=tsv.fieldnames, + fieldnames=t_fieldnames, extrasaction="ignore", ) writer.writeheader() for row in tsv_keyed: writer.writerow(row) - with io.open(json_output_path, "w", newline="\n") as json_out: - # we don't want to sort these because they're _1, ..., _10, etc - json.dump( - json_str, json_out, ensure_ascii=False, indent="\t", sort_keys=False, - ) + for lang in languages: + if not json_str[lang]: + continue + with io.open(json_output_paths[lang], "w", newline="\n") as json_out: + # we don't want to sort these because they're _1, ..., _10, etc + json.dump( + json_str[lang], + json_out, + ensure_ascii=False, + indent="\t", + sort_keys=False, + ) except Exception: traceback.print_exc() print("Error processing file: " + file_in) -def process_tsv(tsv, id_num): +def process_tsv(tsv, id_num, is_global): i = 0 tsv_keyed = [] - key_dict = {} + key_dict = {"jpn": {}} + if is_global: + key_dict["enm"] = {} + key_dict["zh"] = {} + key_dict["ko"] = {} for row in tsv: try: @@ -110,8 +135,14 @@ def process_tsv(tsv, id_num): if row["Text"] and row["Text"].strip(): key = "{}_{}".format(id_num, i) i += 1 + key_dict["jpn"][key] = row["Text"] + if is_global: + with suppress(KeyError): + if row["English"]: + key_dict["enm"][key] = row["English"] + key_dict["zh"][key] = row["Chinese"] + key_dict["ko"][key] = row["Korean"] row["English"] = key - key_dict[key] = row["Text"] tsv_keyed.append(row) except Exception: traceback.print_exc() From cdf57ce5831cb396f257e765c994bd7390ea8651 Mon Sep 17 00:00:00 2001 From: argoneus Date: Fri, 21 Feb 2020 00:06:16 +0100 Subject: [PATCH 28/43] fixed quotes thanks j --- utage/translate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utage/translate.py b/utage/translate.py index 6ea96b4..2344b0a 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -78,7 +78,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): raise with open(file_in, "r") as tsv_file: - tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') + tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE) t_fieldnames = tsv.fieldnames if is_global: From 30af5fbe9abbba7e36a822b4c2609ab1df013d7b Mon Sep 17 00:00:00 2001 From: argoneus Date: Fri, 6 Mar 2020 19:29:33 +0100 Subject: [PATCH 29/43] updated readme --- .gitignore | 4 ++-- README.md | 21 +++++++++++++++++++++ tools/.gitkeep | 0 tools/readme.txt | 34 ---------------------------------- 4 files changed, 23 insertions(+), 36 deletions(-) create mode 100644 README.md create mode 100644 tools/.gitkeep delete mode 100644 tools/readme.txt diff --git a/.gitignore b/.gitignore index f8d0fa1..cd455e6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ __pycache__/ *$py.class *.swp virtualenv/ -tools/* -!tools/readme.txt .idea/ +tools/* +!tools/.gitkeep diff --git a/README.md b/README.md new file mode 100644 index 0000000..ba0de08 --- /dev/null +++ b/README.md @@ -0,0 +1,21 @@ +# divatool + +A tool for that one game everyone hates but works on anyway. + +## Requirements + +Divatool is written in and uses [Python 3](https://www.python.org/downloads/). + +There are also several third party dependencies not provided as part of this repository, +mostly used for the `xdudata` command. You need to have the binaries of the following tools +either in your `$PATH` variable or in the [`tools`](./tools) folder. These tools are: + +* [clHCA](https://github.com/KinoMyu/FastHCADecoder) +* acb_extract +* usm_extract + +Make sure these executables are called the same way on your machine. + +## Usage + +TODO diff --git a/tools/.gitkeep b/tools/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tools/readme.txt b/tools/readme.txt deleted file mode 100644 index f10fdc0..0000000 --- a/tools/readme.txt +++ /dev/null @@ -1,34 +0,0 @@ -Various files need to be in this directory. -If you don't have them, ask someone who does. - -The required files are: - -Binaries: -- acb_extract.exe -- usm_extract.exe -- clHCA - -Z shell scripts: -- update_bgm.zsh -- update_movie.zsh -- update_se.zsh -- update_voice.zsh - -Python scripts: -- adx_parse.py -- bingo_fuck.py -- chara_room_mission.py -- custom_mission.py -- event_bonus.py -- orphan_quest.py -- pad_2_csv.py -- pad_2_json.py -- pad_2_pad.py -- pad_2_pad_louisstyle.py -- pad_stat.py -- pad_update.py -- store_parser.py -- tsv_2_pad.py -- tsv_parse.py -- txt_2_pad.py -- utage_decrypt.py From 2da7834c61de22b16a9558e59b03f742e0e6dab0 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 7 Mar 2020 00:38:40 +0100 Subject: [PATCH 30/43] updated readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ba0de08..54fe834 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ either in your `$PATH` variable or in the [`tools`](./tools) folder. These tools * [clHCA](https://github.com/KinoMyu/FastHCADecoder) * acb_extract * usm_extract +* [ffmpeg](https://www.ffmpeg.org/download.html) +* [mono](https://www.mono-project.com/download/stable/) (if using Linux) Make sure these executables are called the same way on your machine. From 560e00764b50cd2bc867b2e3889c7928de7c8ce9 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 7 Mar 2020 05:15:21 +0100 Subject: [PATCH 31/43] ported update_se --- divatool.py | 14 ++-- xdudata/asset_extract.py | 150 +++++++++++++++++++++++++++++++++++++++ xdudata/update.py | 128 ++++++++++++--------------------- 3 files changed, 203 insertions(+), 89 deletions(-) create mode 100644 xdudata/asset_extract.py diff --git a/divatool.py b/divatool.py index ec36fed..e528b57 100755 --- a/divatool.py +++ b/divatool.py @@ -184,15 +184,15 @@ def main(): if args.subcommand == "quest": diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) elif args.command == "xdudata": - script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools")) if args.subcommand == "update": + # add the tools folder to PATH + os.environ["PATH"] += os.pathsep + os.path.join( + os.path.abspath(os.path.dirname(__file__)), "tools" + ) + + # update the data xdudata.update_all( - args.CACHE, - args.XDUDATA, - args.TRANS, - args.EXTRACT, - script_dir, - LANGUAGES, + args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES, ) return 0 diff --git a/xdudata/asset_extract.py b/xdudata/asset_extract.py new file mode 100644 index 0000000..da1ac44 --- /dev/null +++ b/xdudata/asset_extract.py @@ -0,0 +1,150 @@ +import functools +import multiprocessing +import os +import shutil +import subprocess +from contextlib import contextmanager +from distutils.dir_util import copy_tree +from glob import glob + +OPERATING_SYSTEM = os.name + + +def process_se(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Se"), + os.path.join(extract_dir, "Se"), + ) + + from_dir = "Se" + to_dir = "Se_Extracted" + acb_prefix = "_vgmt_acb_ext" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.wav".format(from_dir), recursive=True) + to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True)) + for file in to_del: + os.remove(file) + + acb_files = [file for file in glob("{}/*.acb".format(from_dir))] + acb_cmd = get_dotnet_cmd("acb_extract.exe") + run_program(acb_files, cmd=acb_cmd) + + hca_files = [ + file + for file in glob( + "{}/{}*/**/*.hca".format(from_dir, acb_prefix), recursive=True + ) + ] + run_program(hca_files, cmd="clHCA") + + basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] + char_secs = [basename.split("_", maxsplit=1) for basename in basenames] + for cs in char_secs: + char = cs[0] + sec = str(cs[1]) if len(cs) == 2 else "" + os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True) + wav_files = glob( + "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)), + recursive=True, + ) + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + wav_cmds = [ + [ + "-i", + os.path.abspath(file), + "-acodec", + "libopus", + "-ab", + "192k", + "-af", + "aresample=48000", + os.path.abspath( + "{}/{}/{}/{}.opus".format( + to_dir, + char, + sec, + os.path.splitext(os.path.basename(file))[0], + ) + ), + "-n", + ] + for file in wav_files + ] + pool.map(partial_ffmpeg_cmd, wav_cmds) + + copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Se")) + + +def process_bgm(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), + os.path.join(extract_dir, "Bgm"), + ) + subprocess.run(os.path.join("TODO", "update_bgm.zsh"), shell=True, cwd=extract_dir) + copy_tree( + os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") + ) + + +def process_voice(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Voice"), + os.path.join(extract_dir, "Voice"), + ) + subprocess.run( + os.path.join("TODO", "update_voice.zsh"), shell=True, cwd=extract_dir + ) + copy_tree( + os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") + ) + + +def process_movie(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Common/Asset/Movie"), + os.path.join(extract_dir, "Movie"), + ) + subprocess.run( + os.path.join("TODO", "update_movie.zsh"), shell=True, cwd=extract_dir + ) + copy_tree( + os.path.join(extract_dir, "movie_extracted"), + os.path.join(update_dir, "Asset/Movie"), + ) + + +@contextmanager +def change_cwd(new_path): + old_cwd = os.getcwd() + os.chdir(new_path) + try: + yield + finally: + os.chdir(old_cwd) + + +def get_dotnet_cmd(name): + ext = os.path.splitext(name)[1] + if ext != ".exe": + raise RuntimeError("This doesn't seem like a .NET program.") + path = shutil.which(name) + if OPERATING_SYSTEM == "nt": + return [path] + return ["mono", path] + + +def run_program(args, cmd=None): + if isinstance(cmd, str): + cmd = [cmd] + full_cmd = cmd + args + subprocess.run(full_cmd) + + +def get_pool(size=multiprocessing.cpu_count()): + pool = multiprocessing.Pool(size) + return pool diff --git a/xdudata/update.py b/xdudata/update.py index b659c3f..0d693d3 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -3,85 +3,69 @@ from contextlib import suppress import diva import shutil import os -import subprocess import tempfile import utage from distutils.dir_util import copy_tree, remove_tree from glob import glob +from xdudata.asset_extract import process_se, process_bgm, process_voice, process_movie + def update_all( - cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages, + cache_dir, update_dir, translation_dir, extract_dir, languages, ): - print("Cleaning cache") - cleanup_cache(cache_dir) - - print("Copying images") - copy_images(cache_dir, update_dir) - - print("Processing TSV files") - update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) - - print("Processing Quest Json files") - diva.quest.update_missions(extract_dir, translation_dir, languages) - - # this code isn't going to run on windows (it is in WSL though) because i need to rewrite it - # it just calls my zsh tools and generates it that way, need to replace them - # with real python or something - - # first copy over the necessary avtools to the extract dir - # TODO make it so that you don't need to copy them over (for example call them with python) - print("Copying necessary tools into extracting directory") - shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir) - shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir) - shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir) + # TODO uncomment + # print("Checking tool availability") + # validate_tools() + # + # print("Cleaning cache") + # cleanup_cache(cache_dir) + # + # print("Copying images") + # copy_images(cache_dir, update_dir) + # + # print("Processing TSV files") + # update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) + # + # print("Processing Quest Json files") + # diva.quest.update_missions(extract_dir, translation_dir, languages) print("Processing Sound Effects") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Se"), - os.path.join(extract_dir, "Se"), - ) - subprocess.run( - os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir - ) - copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se")) + process_se(cache_dir, extract_dir, update_dir) - print("Processing BGM - don't forget to copy that loop file") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), - os.path.join(extract_dir, "Bgm"), - ) - subprocess.run( - os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir - ) - copy_tree( - os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") - ) + # print("Processing BGM") + # process_bgm(cache_dir, extract_dir, update_dir) + # + # print("Processing voices") + # process_voice(cache_dir, extract_dir, update_dir) + # + # print("Processing movies - this might take forever") + # process_movie(cache_dir, extract_dir, update_dir) - print("Processing voices") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Voice"), - os.path.join(extract_dir, "Voice"), - ) - subprocess.run( - os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir - ) - copy_tree( - os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") - ) - print("Processing movies - this might take forever") +def validate_tools(): + # TODO validate all the tools are in PATH + ... + + +def cleanup_cache(cache_dir): + with suppress(FileNotFoundError): + shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) + os.remove( + os.path.join( + cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" + ) + ) + + +def copy_images(cache_dir, update_dir): copy_tree( - os.path.join(cache_dir, "Common/Asset/Movie"), - os.path.join(extract_dir, "Movie"), - ) - subprocess.run( - os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir + os.path.join(cache_dir, "Common/Asset/Image"), + os.path.join(update_dir, "Asset/Image"), ) copy_tree( - os.path.join(extract_dir, "movie_extracted"), - os.path.join(update_dir, "Asset/Movie"), + os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), ) @@ -113,23 +97,3 @@ def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages): utage.names.update_names(utage_tmp, translation_dir, languages, False) copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) remove_tree(utage_tmp) - - -def copy_images(cache_dir, update_dir): - copy_tree( - os.path.join(cache_dir, "Common/Asset/Image"), - os.path.join(update_dir, "Asset/Image"), - ) - copy_tree( - os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), - ) - - -def cleanup_cache(cache_dir): - with suppress(FileNotFoundError): - shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) - os.remove( - os.path.join( - cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" - ) - ) From ddca11509bc9bba90479f5954468e1f64b2f6abc Mon Sep 17 00:00:00 2001 From: argoneus Date: Thu, 23 Apr 2020 21:44:50 +0200 Subject: [PATCH 32/43] ported update_bgm to python --- adx/adx_parser.py | 10 +++---- xdudata/asset_extract.py | 58 ++++++++++++++++++++++++++++++++++------ 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/adx/adx_parser.py b/adx/adx_parser.py index 993fc3c..d6d4c5d 100644 --- a/adx/adx_parser.py +++ b/adx/adx_parser.py @@ -74,7 +74,7 @@ class adx_file(object): self.encrypted = self.get_val(CRYPT_OFF, CRYPT_LEN) self.d_offset = self.get_val(DATA_OFF_OFF, DATA_OFF_LEN) if self.d_offset < DATA_OFF_MIN: - return None + return self.l_exists = 1 # now load things for math self.sam_rate = self.get_val(SRATE_OFF, SRATE_LEN) @@ -106,13 +106,11 @@ class adx_file(object): return None if self.l_start > self.sam_count or self.l_end > self.sam_count: raise ValueError("Invalid Loop Data") - return None if self.l_style not in LOOP_TYPE_SUP: raise NotImplementedError( "Loop Style {} Not Implemented".format(str(self.l_style)) ) - return None - ret = {} + ret = dict() ret["duration"] = self.sam_count / self.sam_rate ret["loop_start"] = {} ret["loop_start"]["seconds"] = self.l_start / self.sam_rate @@ -123,7 +121,5 @@ class adx_file(object): ret["loop_end"] = {} ret["loop_end"]["seconds"] = self.l_end / self.sam_rate ret["loop_end"]["samples_native"] = self.l_end - ret["loop_end"]["samples_48k"] = dumb_round( - self.l_end / self.sam_rate * 48000 - ) + ret["loop_end"]["samples_48k"] = dumb_round(self.l_end / self.sam_rate * 48000) return ret diff --git a/xdudata/asset_extract.py b/xdudata/asset_extract.py index da1ac44..014b762 100644 --- a/xdudata/asset_extract.py +++ b/xdudata/asset_extract.py @@ -7,6 +7,8 @@ from contextlib import contextmanager from distutils.dir_util import copy_tree from glob import glob +from adx import extract_loop_data_from_dir + OPERATING_SYSTEM = os.name @@ -35,10 +37,7 @@ def process_se(cache_dir, extract_dir, update_dir): run_program(acb_files, cmd=acb_cmd) hca_files = [ - file - for file in glob( - "{}/{}*/**/*.hca".format(from_dir, acb_prefix), recursive=True - ) + file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) ] run_program(hca_files, cmd="clHCA") @@ -85,10 +84,53 @@ def process_bgm(cache_dir, extract_dir, update_dir): os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), os.path.join(extract_dir, "Bgm"), ) - subprocess.run(os.path.join("TODO", "update_bgm.zsh"), shell=True, cwd=extract_dir) - copy_tree( - os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") - ) + + from_dir = "Bgm" + to_dir = "Bgm_Extracted" + loop_file = "loop.json" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.adx".format(from_dir), recursive=True) + for file in to_del: + os.remove(file) + + acb_files = [file for file in glob("{}/*.acb".format(from_dir))] + acb_cmd = get_dotnet_cmd("acb_extract.exe") + run_program(acb_files, cmd=acb_cmd) + + adx_files = [ + file for file in glob("{}/**/*.adx".format(from_dir), recursive=True) + ] + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + wav_cmds = [ + [ + "-i", + os.path.abspath(file), + "-acodec", + "libopus", + "-ab", + "192k", + "-af", + "aresample=48000", + os.path.abspath( + "{}/{}.opus".format( + to_dir, os.path.splitext(os.path.basename(file))[0], + ) + ), + "-n", + ] + for file in adx_files + ] + pool.map(partial_ffmpeg_cmd, wav_cmds) + + extract_loop_data_from_dir(from_dir, loop_file) + shutil.copy2(loop_file, os.path.join(to_dir, "BgmLoop.json")) + + copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Bgm")) def process_voice(cache_dir, extract_dir, update_dir): From 9f73c50d56cd32782d7abfedcbef84d7bd635736 Mon Sep 17 00:00:00 2001 From: argoneus Date: Fri, 15 May 2020 15:44:18 +0200 Subject: [PATCH 33/43] rewrote zsh scripts for asset extract --- xdudata/asset_extract.py | 124 +++++++++++++++++++++++++++++++++++---- xdudata/update.py | 45 +++++++------- 2 files changed, 135 insertions(+), 34 deletions(-) diff --git a/xdudata/asset_extract.py b/xdudata/asset_extract.py index 014b762..8237162 100644 --- a/xdudata/asset_extract.py +++ b/xdudata/asset_extract.py @@ -138,12 +138,67 @@ def process_voice(cache_dir, extract_dir, update_dir): os.path.join(cache_dir, "Android/Asset/Sound/Voice"), os.path.join(extract_dir, "Voice"), ) - subprocess.run( - os.path.join("TODO", "update_voice.zsh"), shell=True, cwd=extract_dir - ) - copy_tree( - os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") - ) + + from_dir = "Voice" + to_dir = "Voice_Extracted" + acb_prefix = "_vgmt_acb_ext" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.wav".format(from_dir), recursive=True) + to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True)) + for file in to_del: + os.remove(file) + + acb_files = [file for file in glob("{}/*.acb".format(from_dir))] + acb_cmd = get_dotnet_cmd("acb_extract.exe") + run_program(acb_files, cmd=acb_cmd) + + hca_files = [ + file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) + ] + run_program(hca_files, cmd="clHCA") + + basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] + char_secs = [basename.split("_", maxsplit=1) for basename in basenames] + for cs in char_secs: + char = cs[0] + sec = str(cs[1]) if len(cs) == 2 else "" + os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True) + wav_files = glob( + "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)), + recursive=True, + ) + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + + wav_cmds = [ + [ + "-i", + os.path.abspath(file), + "-acodec", + "libopus", + "-ab", + "192k", + "-af", + "aresample=48000", + os.path.abspath( + "{}/{}/{}/{}.opus".format( + to_dir, + char, + sec, + os.path.splitext(os.path.basename(file))[0], + ) + ), + "-n", + ] + for file in wav_files + ] + pool.map(partial_ffmpeg_cmd, wav_cmds) + + copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Voice")) def process_movie(cache_dir, extract_dir, update_dir): @@ -151,12 +206,59 @@ def process_movie(cache_dir, extract_dir, update_dir): os.path.join(cache_dir, "Common/Asset/Movie"), os.path.join(extract_dir, "Movie"), ) - subprocess.run( - os.path.join("TODO", "update_movie.zsh"), shell=True, cwd=extract_dir - ) + + from_dir = "Movie" + to_dir = "Movie_Extracted" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.adx".format(from_dir), recursive=True) + to_del.extend(glob("{}/**/*.m2v".format(from_dir), recursive=True)) + for file in to_del: + os.remove(file) + + movie_paths = [ + os.path.abspath(os.path.splitext(file)[0]) + for file in glob("{}/*.usm".format(from_dir)) + if not os.path.basename(file).startswith("mini_radio") + ] + + usm_files = [[path + ".usm"] for path in movie_paths] + usm_cmd = get_dotnet_cmd("usm_extract.exe") + partial_usm_cmd = functools.partial(run_program, cmd=usm_cmd) + pool.map(partial_usm_cmd, usm_files) + + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + mux_cmds = [ + [ + "-i", + path + ".adx", + "-i", + path + ".m2v", + "-c:a", + "libopus", + "-b:a", + "196k", + "-af", + "aresample=48000", + "-c:v", + "libvpx", + "-b:v", + "1M", + "-threads", + "1", + os.path.abspath("{}/{}.webm".format(to_dir, os.path.basename(path))), + "-n", + ] + for path in movie_paths + ] + pool.map(partial_ffmpeg_cmd, mux_cmds) + copy_tree( - os.path.join(extract_dir, "movie_extracted"), - os.path.join(update_dir, "Asset/Movie"), + os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Asset", "Movie") ) diff --git a/xdudata/update.py b/xdudata/update.py index 0d693d3..a59d30c 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -15,33 +15,32 @@ from xdudata.asset_extract import process_se, process_bgm, process_voice, proces def update_all( cache_dir, update_dir, translation_dir, extract_dir, languages, ): - # TODO uncomment - # print("Checking tool availability") - # validate_tools() - # - # print("Cleaning cache") - # cleanup_cache(cache_dir) - # - # print("Copying images") - # copy_images(cache_dir, update_dir) - # - # print("Processing TSV files") - # update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) - # - # print("Processing Quest Json files") - # diva.quest.update_missions(extract_dir, translation_dir, languages) + print("Checking tool availability") + validate_tools() + + print("Cleaning cache") + cleanup_cache(cache_dir) + + print("Copying images") + copy_images(cache_dir, update_dir) + + print("Processing TSV files") + update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) + + print("Processing Quest Json files") + diva.quest.update_missions(extract_dir, translation_dir, languages) print("Processing Sound Effects") process_se(cache_dir, extract_dir, update_dir) - # print("Processing BGM") - # process_bgm(cache_dir, extract_dir, update_dir) - # - # print("Processing voices") - # process_voice(cache_dir, extract_dir, update_dir) - # - # print("Processing movies - this might take forever") - # process_movie(cache_dir, extract_dir, update_dir) + print("Processing BGM") + process_bgm(cache_dir, extract_dir, update_dir) + + print("Processing voices") + process_voice(cache_dir, extract_dir, update_dir) + + print("Processing movies - this might take forever") + process_movie(cache_dir, extract_dir, update_dir) def validate_tools(): From 5a9859cb560e18a44ba795e8c5aadc66604797f2 Mon Sep 17 00:00:00 2001 From: argoneus Date: Fri, 15 May 2020 23:13:38 +0200 Subject: [PATCH 34/43] fixed newlines and other windows stuff --- diva/quest.py | 14 +++++++------- utage/names.py | 12 ++++++------ utage/translate.py | 17 ++++++++--------- xdudata/asset_extract.py | 13 +++++++++++-- 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/diva/quest.py b/diva/quest.py index f2342a1..ed4d092 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -31,7 +31,7 @@ def scene_mst(qdb, old_path, dir_in, languages): break with io.open( - os.path.join(old_path, "XduScene.json"), "w", newline="\n" + os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8" ) as json_file: json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True) @@ -56,11 +56,11 @@ def scene_mst(qdb, old_path, dir_in, languages): langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json") if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) - with io.open(langfile, "w", newline="\n") as lang_file: + with io.open(langfile, "w", encoding="utf-8") as lang_file: json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -70,7 +70,7 @@ def quest_mst(qdb, old_path, languages): quests = qdb.get_quests() with io.open( - os.path.join(old_path, "XduQuest.json"), "w", newline="\n" + os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8" ) as json_file: json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True) @@ -85,11 +85,11 @@ def quest_mst(qdb, old_path, languages): langfile = os.path.join(old_path, f"XduQuestNames_{lang}.json") if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) - with io.open(langfile, "w", newline="\n") as lang_file: + with io.open(langfile, "w", encoding="utf-8") as lang_file: json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -108,7 +108,7 @@ def load_locations(dir_in): raise FileNotFoundError("Ya didn't decrypt the files dummy") for scenario in settings: - with open(scenario, "r") as tsv_file: + with open(scenario, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') for row in tsv: tokens = row["FileName"].split("/") diff --git a/utage/names.py b/utage/names.py index b6633f0..b47f3b8 100644 --- a/utage/names.py +++ b/utage/names.py @@ -12,7 +12,7 @@ from multiprocessing import Pool def update_names(dir_in, old_path, languages, is_global): names = extract_names(dir_in, is_global) - global_languages = {"enm": 1, "zh": 2, "ko": 3} + global_languages = {"enm": 1, "zho": 2, "kor": 3} if is_global: languages = global_languages.keys() @@ -33,12 +33,12 @@ def update_names(dir_in, old_path, languages, is_global): langfile = os.path.join(old_path, "nametranslations_{}.json".format(l)) if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) with io.open( - langfile, "w", newline="\n" + langfile, "w", encoding="utf-8" ) as lang_file: # you're using git right json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, @@ -56,7 +56,7 @@ def extract_names(dir_in, is_global): files = [ f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) - if re.search(r"/[0-9]{9}\.tsv$", f) + if re.search(r"[0-9]{9}\.tsv$", f) ] if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) @@ -78,7 +78,7 @@ def read_char_tsv(char_tsv_path, is_global): char_names = set() char_sets = set() - with open(char_tsv_path, "r") as char_tsv_file: + with open(char_tsv_path, "r", encoding="utf-8") as char_tsv_file: char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"') for row in char_tsv: @@ -103,7 +103,7 @@ def read_char_tsv(char_tsv_path, is_global): def read_mission(tsv_path, char_names, char_sets): new_names = set() - with open(tsv_path, "r") as tsv_file: + with open(tsv_path, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') if ( ("Arg1" not in tsv.fieldnames) diff --git a/utage/translate.py b/utage/translate.py index 2344b0a..29dcd39 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -52,7 +52,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): ) languages = ["jpn"] if is_global: - languages += ["enm", "zh", "ko"] + languages += ["enm", "zho", "kor"] json_output_paths = { lang: os.path.join( json_out_dir, @@ -77,7 +77,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): if e.errno != errno.EEXIST: raise - with open(file_in, "r") as tsv_file: + with open(file_in, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE) t_fieldnames = tsv.fieldnames @@ -87,12 +87,11 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): tsv_keyed, json_str = process_tsv(tsv, id_num, is_global) # csv handles newlines, don't set it in io.open - with io.open(tsv_output_path, "w", newline="") as tsv_out: + with io.open(tsv_output_path, "w", newline="", encoding="utf-8") as tsv_out: writer = csv.DictWriter( tsv_out, delimiter="\t", quotechar='"', - lineterminator="\n", fieldnames=t_fieldnames, extrasaction="ignore", ) @@ -103,7 +102,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): for lang in languages: if not json_str[lang]: continue - with io.open(json_output_paths[lang], "w", newline="\n") as json_out: + with io.open(json_output_paths[lang], "w", encoding="utf-8") as json_out: # we don't want to sort these because they're _1, ..., _10, etc json.dump( json_str[lang], @@ -124,8 +123,8 @@ def process_tsv(tsv, id_num, is_global): key_dict = {"jpn": {}} if is_global: key_dict["enm"] = {} - key_dict["zh"] = {} - key_dict["ko"] = {} + key_dict["zho"] = {} + key_dict["kor"] = {} for row in tsv: try: @@ -140,8 +139,8 @@ def process_tsv(tsv, id_num, is_global): with suppress(KeyError): if row["English"]: key_dict["enm"][key] = row["English"] - key_dict["zh"][key] = row["Chinese"] - key_dict["ko"][key] = row["Korean"] + key_dict["zho"][key] = row["Chinese"] + key_dict["kor"][key] = row["Korean"] row["English"] = key tsv_keyed.append(row) except Exception: diff --git a/xdudata/asset_extract.py b/xdudata/asset_extract.py index 8237162..04e088c 100644 --- a/xdudata/asset_extract.py +++ b/xdudata/asset_extract.py @@ -10,6 +10,7 @@ from glob import glob from adx import extract_loop_data_from_dir OPERATING_SYSTEM = os.name +CHUNK_SIZE = 100 def process_se(cache_dir, extract_dir, update_dir): @@ -39,7 +40,8 @@ def process_se(cache_dir, extract_dir, update_dir): hca_files = [ file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) ] - run_program(hca_files, cmd="clHCA") + for hca_sublist in split_list(hca_files, CHUNK_SIZE): + run_program(hca_sublist, cmd="clHCA") basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] char_secs = [basename.split("_", maxsplit=1) for basename in basenames] @@ -160,7 +162,8 @@ def process_voice(cache_dir, extract_dir, update_dir): hca_files = [ file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) ] - run_program(hca_files, cmd="clHCA") + for hca_sublist in split_list(hca_files, CHUNK_SIZE): + run_program(hca_sublist, cmd="clHCA") basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] char_secs = [basename.split("_", maxsplit=1) for basename in basenames] @@ -292,3 +295,9 @@ def run_program(args, cmd=None): def get_pool(size=multiprocessing.cpu_count()): pool = multiprocessing.Pool(size) return pool + + +def split_list(input_list, chunk_size): + return [ + input_list[i : i + chunk_size] for i in range(0, len(input_list), chunk_size) + ] From 17dc031b9ecb7f6090028cbccf1f412acee98af8 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 16 May 2020 13:15:58 +0200 Subject: [PATCH 35/43] updated -g to not replace jp translations --- utage/translate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utage/translate.py b/utage/translate.py index 29dcd39..93ae072 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -52,7 +52,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): ) languages = ["jpn"] if is_global: - languages += ["enm", "zho", "kor"] + languages = ["enm", "zho", "kor"] json_output_paths = { lang: os.path.join( json_out_dir, @@ -70,9 +70,9 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): if e.errno != errno.EEXIST: raise - if not os.path.exists(os.path.dirname(json_output_paths["jpn"])): + if not os.path.exists(os.path.dirname(json_output_paths[languages[0]])): try: - os.makedirs(os.path.dirname(json_output_paths["jpn"])) + os.makedirs(os.path.dirname(json_output_paths[languages[0]])) except OSError as e: if e.errno != errno.EEXIST: raise From c811af1d08ddb8526d9a9ac1be7597d57eb77437 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 16 May 2020 13:47:41 +0200 Subject: [PATCH 36/43] added --utage switch to diva quest --- diva/quest.py | 17 ++++++++--------- divatool.py | 14 ++++++++++++-- xdudata/update.py | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/diva/quest.py b/diva/quest.py index ed4d092..a7d2a72 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -7,16 +7,18 @@ import os from glob import glob -def update_missions(dir_in, old_path, languages): +def update_missions(dir_in, old_path, languages, is_global, utage_in=None): + if not utage_in: + utage_in = os.path.join(dir_in, "Common/Asset/Utage") qdb = db.QuestDB(dir_in) quest_mst(qdb, old_path, languages) - scene_mst(qdb, old_path, dir_in, languages) + scene_mst(qdb, old_path, dir_in, languages, utage_in) -def scene_mst(qdb, old_path, dir_in, languages): +def scene_mst(qdb, old_path, dir_in, languages, utage_in): scenes = qdb.get_scenes() - locations = load_locations(dir_in) + locations = load_locations(dir_in, utage_in) for id, scene in scenes.items(): i = 0 @@ -97,12 +99,9 @@ def quest_mst(qdb, old_path, languages): return quests -def load_locations(dir_in): +def load_locations(dir_in, utage_in): locations = {} - settings = glob( - os.path.join(dir_in, "Common/Asset/Utage/**/Settings/Scenario.tsv"), - recursive=True, - ) + settings = glob("{}/**/Settings/Scenario.tsv".format(utage_in), recursive=True,) if len(settings) == 0: raise FileNotFoundError("Ya didn't decrypt the files dummy") diff --git a/divatool.py b/divatool.py index e528b57..17d32ef 100755 --- a/divatool.py +++ b/divatool.py @@ -117,13 +117,16 @@ def main(): metavar="", title="subcommand", dest="subcommand" ) parser_diva_quest = parser_diva_subparsers.add_parser( - "quest", help="Generate quest JSON files" + "quest", help="Generate quest JSON files", parents=[global_flag_parser], ) parser_diva_quest.add_argument( "INPUT", help="Directory of databases containing Quest*.db and ResourceEntry.db", type=str, ) + parser_diva_quest.add_argument( + "--utage", help="Directory of Utage folders and files.", type=str, default=None, + ) parser_diva_quest.add_argument( "OLD", help="Directory with old quest files", nargs="?", type=str, default="." ) @@ -182,7 +185,14 @@ def main(): raise FileNotFoundError(args.INPUT) elif args.command == "diva": if args.subcommand == "quest": - diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) + if not args.utage: + diva.quest.update_missions( + args.INPUT, args.OLD, LANGUAGES, args.is_global + ) + else: + diva.quest.update_missions( + args.INPUT, args.OLD, LANGUAGES, args.is_global, utage_in=args.utage + ) elif args.command == "xdudata": if args.subcommand == "update": # add the tools folder to PATH diff --git a/xdudata/update.py b/xdudata/update.py index a59d30c..8dd8af4 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -28,7 +28,7 @@ def update_all( update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) print("Processing Quest Json files") - diva.quest.update_missions(extract_dir, translation_dir, languages) + diva.quest.update_missions(extract_dir, translation_dir, languages, False) print("Processing Sound Effects") process_se(cache_dir, extract_dir, update_dir) From 7f33acf8ea0f98ef60920e596774d199a21e3034 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 16 May 2020 15:38:56 +0200 Subject: [PATCH 37/43] fixed diva quest --- db/base.py | 2 +- db/quest.py | 24 ++++++++++++++++-------- diva/quest.py | 49 ++++++++++++++++++++++++++++++++----------------- 3 files changed, 49 insertions(+), 26 deletions(-) diff --git a/db/base.py b/db/base.py index afa5817..80132be 100644 --- a/db/base.py +++ b/db/base.py @@ -23,7 +23,7 @@ class BaseDB(object): return self def __exit__(self, exc_type, exc_value, traceback): - self.quest_conn.close() + self.conn.close() def attach_dbs(self, databases, path): for d in databases: diff --git a/db/quest.py b/db/quest.py index c4b13f8..7054e94 100644 --- a/db/quest.py +++ b/db/quest.py @@ -1,16 +1,24 @@ from .base import BaseDB -DATABASES = { - "QuestMst.db": "Quest", - "QuestSceneMst.db": "QuestScene", - "QuestPartMst.db": "QuestPart", - "ResourceEntry.db": "ResourceEntry", -} + +def get_dbs_by_lang(language=None): + databases = { + "QuestMst": "Quest", + "QuestSceneMst": "QuestScene", + "QuestPartMst": "QuestPart", + "ResourceEntry": "ResourceEntry", + } + for key in list(databases): + if not language: + databases[key + ".db"] = databases.pop(key) + else: + databases[key + "_" + language + ".db"] = databases.pop(key) + return databases class QuestDB(BaseDB): - def __init__(self, path): - super().__init__(path, DATABASES) + def __init__(self, path, language=None): + super().__init__(path, get_dbs_by_lang(language)) def get_quests(self): self.cursor.execute( diff --git a/diva/quest.py b/diva/quest.py index a7d2a72..7af5214 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -10,15 +10,22 @@ from glob import glob def update_missions(dir_in, old_path, languages, is_global, utage_in=None): if not utage_in: utage_in = os.path.join(dir_in, "Common/Asset/Utage") - qdb = db.QuestDB(dir_in) - quest_mst(qdb, old_path, languages) - scene_mst(qdb, old_path, dir_in, languages, utage_in) + if is_global: + languages = ["enm", "zho", "kor"] + for lang in languages: + qdb = db.QuestDB(dir_in, lang) + quest_mst(qdb, old_path, [lang], is_global) + scene_mst(qdb, old_path, [lang], utage_in, is_global) + else: + qdb = db.QuestDB(dir_in) + quest_mst(qdb, old_path, languages, is_global) + scene_mst(qdb, old_path, languages, utage_in, is_global) -def scene_mst(qdb, old_path, dir_in, languages, utage_in): +def scene_mst(qdb, old_path, languages, utage_in, is_global): scenes = qdb.get_scenes() - locations = load_locations(dir_in, utage_in) + locations = load_locations(utage_in) for id, scene in scenes.items(): i = 0 @@ -28,18 +35,23 @@ def scene_mst(qdb, old_path, dir_in, languages, utage_in): except KeyError as exc: i += 1 if i == len(scene["Parts"]): + if is_global: + break raise KeyError(f"Could not find folder for scene {id}") from exc continue break - with io.open( - os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8" - ) as json_file: - json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True) + if not is_global: + with io.open( + os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8" + ) as json_file: + json.dump( + scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True + ) for lang in languages: out_dict = {} - if lang == "jpn": + if lang == "jpn" or is_global: for key, value in scenes.items(): out_dict[key] = { "Name": value["Name"], @@ -68,17 +80,20 @@ def scene_mst(qdb, old_path, dir_in, languages, utage_in): ) -def quest_mst(qdb, old_path, languages): +def quest_mst(qdb, old_path, languages, is_global): quests = qdb.get_quests() - with io.open( - os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8" - ) as json_file: - json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True) + if not is_global: + with io.open( + os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8" + ) as json_file: + json.dump( + quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True + ) for lang in languages: out_dict = {} - if lang == "jpn": + if lang == "jpn" or is_global: for key, value in quests.items(): out_dict[key] = {"Name": value["Name"], "Enabled": False} else: @@ -99,7 +114,7 @@ def quest_mst(qdb, old_path, languages): return quests -def load_locations(dir_in, utage_in): +def load_locations(utage_in): locations = {} settings = glob("{}/**/Settings/Scenario.tsv".format(utage_in), recursive=True,) From 421e313f10f8bf5688e14395b82272de5a4d3b3a Mon Sep 17 00:00:00 2001 From: argoneus Date: Thu, 9 Jul 2020 19:07:45 +0200 Subject: [PATCH 38/43] removed todo --- xdudata/update.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/xdudata/update.py b/xdudata/update.py index 8dd8af4..8f61b6f 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -15,9 +15,6 @@ from xdudata.asset_extract import process_se, process_bgm, process_voice, proces def update_all( cache_dir, update_dir, translation_dir, extract_dir, languages, ): - print("Checking tool availability") - validate_tools() - print("Cleaning cache") cleanup_cache(cache_dir) @@ -43,11 +40,6 @@ def update_all( process_movie(cache_dir, extract_dir, update_dir) -def validate_tools(): - # TODO validate all the tools are in PATH - ... - - def cleanup_cache(cache_dir): with suppress(FileNotFoundError): shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) From b42c731c171c443e70b9639abfc62b4f905f8d94 Mon Sep 17 00:00:00 2001 From: argoneus Date: Wed, 21 Oct 2020 23:40:00 +0200 Subject: [PATCH 39/43] added scene images to scene json --- db/quest.py | 6 +++++- diva/quest.py | 33 +++++++++++++++++---------------- utage/translate.py | 2 +- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/db/quest.py b/db/quest.py index 7054e94..bedc636 100644 --- a/db/quest.py +++ b/db/quest.py @@ -52,12 +52,14 @@ class QuestDB(BaseDB): questSceneMstId, name, summaryText, + img, group_concat(partIds) as parts FROM ( SELECT qs.questSceneMstId, qs.name, qs.summaryText, + qs.img, CASE WHEN qp.afterTalkName == '' AND qp.beforeTalkName == '' THEN NULL WHEN qp.afterTalkName == '' AND qp.beforeTalkName != '' THEN qp.beforeTalkName @@ -74,9 +76,11 @@ class QuestDB(BaseDB): ) res = self.cursor.fetchall() scenes = {} - for scene_id, name, summary, parts in res: + for scene_id, name, summary, img, parts in res: scenes[str(scene_id)] = {} scenes[str(scene_id)]["Name"] = name scenes[str(scene_id)]["SummaryText"] = summary + if img: + scenes[str(scene_id)]["Image"] = img scenes[str(scene_id)]["Parts"] = parts.split(",") return scenes diff --git a/diva/quest.py b/diva/quest.py index 7af5214..2b0cb41 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -51,22 +51,23 @@ def scene_mst(qdb, old_path, languages, utage_in, is_global): for lang in languages: out_dict = {} - if lang == "jpn" or is_global: - for key, value in scenes.items(): - out_dict[key] = { - "Name": value["Name"], - "SummaryText": value["SummaryText"], - "Credits": "POKELABO", - "Enabled": False, - } - else: - for key in scenes.keys(): - out_dict[key] = { - "Name": "", - "SummaryText": "", - "Credits": "", - "Enabled": False, - } + for key, value in scenes.items(): + name = "" + summary = "" + credit = "" + enabled = False + + if lang == "jpn" or is_global: + name = value["Name"] + summary = value["SummaryText"] + credit = "POKELABO" + + out_dict[key] = { + "Name": name, + "SummaryText": summary, + "Credits": credit, + "Enabled": enabled, + } langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json") if os.path.isfile(langfile): diff --git a/utage/translate.py b/utage/translate.py index 93ae072..cd3b875 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -15,7 +15,7 @@ def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global): files = [ f for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True) - if "{os.path.sep}Diva{os.path.sep}" not in f + if f"{os.path.sep}Diva{os.path.sep}" not in f ] if len(files) == 0: From 3319ba7098c0ae1bc204108040dd4aecdcb129ea Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 31 Oct 2020 12:43:18 +0100 Subject: [PATCH 40/43] excluding Diva files from name parsing --- utage/names.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utage/names.py b/utage/names.py index b47f3b8..0ea03f8 100644 --- a/utage/names.py +++ b/utage/names.py @@ -57,6 +57,7 @@ def extract_names(dir_in, is_global): f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r"[0-9]{9}\.tsv$", f) + and f"{os.path.sep}Diva{os.path.sep}" not in f ] if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) From 6f83c7c8a7dfcc5b95f91a716c460733c3ad3e91 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 14 Feb 2021 20:41:04 +0100 Subject: [PATCH 41/43] added cze as language --- divatool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/divatool.py b/divatool.py index 17d32ef..f43a927 100755 --- a/divatool.py +++ b/divatool.py @@ -9,7 +9,7 @@ import utage import db import xdudata -LANGUAGES = ["jpn", "eng", "rus"] +LANGUAGES = ["jpn", "eng", "rus", "cze"] def main(): From d062c94d469f7a784238921f109f1b2127dc094f Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 22 May 2021 15:14:38 +0200 Subject: [PATCH 42/43] ignoring scripts in side03 --- diva/quest.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/diva/quest.py b/diva/quest.py index 2b0cb41..a2306f0 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -129,8 +129,14 @@ def load_locations(utage_in): tokens = row["FileName"].split("/") if len(tokens) < 3: continue - if tokens[2] in locations: - raise ValueError("Conflicting paths for file {tokens[2]}") - locations[tokens[2]] = tokens[0] + folder, _scenario, part_id = tokens + # side03 seems to only contain prototype scripts so we ignore it + if folder == "side03": + continue + if part_id in locations: + raise ValueError( + f"Conflicting paths for scene part {part_id}, existing folder: {locations[part_id]}" + ) + locations[part_id] = folder return locations From 8b1bb1e3b507f18f92491872acab31137d878f03 Mon Sep 17 00:00:00 2001 From: johnnyzb Date: Mon, 10 Aug 2026 20:23:14 +0000 Subject: [PATCH 43/43] Update README with a basic example --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 54fe834..81ee494 100644 --- a/README.md +++ b/README.md @@ -20,4 +20,17 @@ Make sure these executables are called the same way on your machine. ## Usage -TODO +TODO: add detailed descriptions for all parameters. + +Using the tool to update XDUData for the player: + +``` +divatool.py xdudata update [path to the 'files' folder from the game] [path to the XDUData folder for the player] [path to the 'xdutranslations' repo folder] [path to the temp files folder] +``` + +Example: +``` +divatool.py xdudata update /home/user/gameapp/files /home/user/player/XDUData /home/user/git/xdutranslations /var/tmp +``` + +Note that the 'files' folder from the app must contain the db files for this to work.