From ecde988fd948c78bfc7d27a184efb8ec56f5d5e2 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 16 Feb 2020 00:55:31 +0100 Subject: [PATCH] sorting out zsh script calling and folder names --- .gitignore | 4 +- divatool.py | 2 + {scripts => tools}/readme.txt | 7 ++- utage/names.py | 12 ++--- utage/translate.py | 22 +++------- xdudata/update.py | 83 +++++++++++++++++++++++++---------- 6 files changed, 79 insertions(+), 51 deletions(-) rename {scripts => tools}/readme.txt (87%) diff --git a/.gitignore b/.gitignore index e74fd7e..f8d0fa1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ __pycache__/ *$py.class *.swp virtualenv/ -scripts/* -!scripts/readme.txt +tools/* +!tools/readme.txt .idea/ diff --git a/divatool.py b/divatool.py index f759b4f..caf2bef 100755 --- a/divatool.py +++ b/divatool.py @@ -184,12 +184,14 @@ def main(): if args.subcommand == "quest": diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) elif args.command == "xdudata": + script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools")) if args.subcommand == "update": xdudata.update_all( args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, + script_dir, LANGUAGES, args.is_global, ) diff --git a/scripts/readme.txt b/tools/readme.txt similarity index 87% rename from scripts/readme.txt rename to tools/readme.txt index a8fad1a..f10fdc0 100644 --- a/scripts/readme.txt +++ b/tools/readme.txt @@ -3,7 +3,12 @@ If you don't have them, ask someone who does. The required files are: -Z shell: +Binaries: +- acb_extract.exe +- usm_extract.exe +- clHCA + +Z shell scripts: - update_bgm.zsh - update_movie.zsh - update_se.zsh diff --git a/utage/names.py b/utage/names.py index 56ef85c..5c5f2e2 100644 --- a/utage/names.py +++ b/utage/names.py @@ -9,7 +9,7 @@ from glob import glob from multiprocessing import Pool -def update_names(dir_in, old_path, languages): +def update_names(dir_in, old_path, languages, is_global): names = extract_names(dir_in) # we have to update japanese like the rest because CustomData exists @@ -30,11 +30,7 @@ def update_names(dir_in, old_path, languages): langfile, "w", newline="\n" ) as lang_file: # you're using git right json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -101,8 +97,6 @@ def read_mission(tsv_path, char_names, char_sets): if row["Command"] and row["Command"].startswith("//"): continue if row["Text"] and row["Arg1"]: - if (row["Arg1"] not in char_names) and ( - row["Arg1"] not in char_sets - ): + if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets): new_names.add(row["Arg1"]) return new_names diff --git a/utage/translate.py b/utage/translate.py index 6c47289..f70fc07 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -10,26 +10,22 @@ from glob import glob from multiprocessing.pool import Pool -def translate_dir(dir_in, tsv_out_dir, json_out_dir): +def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global): files = [ f - for f in glob( - os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True - ) + for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True) if "{os.path.sep}Diva{os.path.sep}" not in f ] if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) - ptrans = partial( - translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir - ) + ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir) with Pool() as p: p.map(ptrans, files) -def translate_file(file_in, tsv_out_dir, json_out_dir): +def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): try: if not file_in.endswith(".tsv"): raise ValueError("Invalid File Type for {}".format(file_in)) @@ -49,9 +45,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir): tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num) ) json_output_path = os.path.join( - json_out_dir, - event_folder, - "{}_translations_jpn.json".format(id_num), + json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num), ) # need to create output paths and avoid races when threading @@ -90,11 +84,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir): with io.open(json_output_path, "w", newline="\n") as json_out: # we don't want to sort these because they're _1, ..., _10, etc json.dump( - json_str, - json_out, - ensure_ascii=False, - indent="\t", - sort_keys=False, + json_str, json_out, ensure_ascii=False, indent="\t", sort_keys=False, ) except Exception: diff --git a/xdudata/update.py b/xdudata/update.py index fd747f9..6924b13 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -1,4 +1,7 @@ +from contextlib import suppress + import diva +import shutil import os import subprocess import tempfile @@ -8,35 +11,59 @@ from distutils.dir_util import copy_tree, remove_tree from glob import glob -def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): +def update_all( + cache_dir, + update_dir, + translation_dir, + extract_dir, + script_dir, + languages, + is_global, +): + print("Cleaning cache") + cleanup_cache(cache_dir) + print("Copying images") - copy_images(cache_dir, update_dir) + # copy_images(cache_dir, update_dir) print("Processing TSV files") - update_tsv(cache_dir, update_dir, translation_dir, languages) + # TODO + # update_tsv(cache_dir, update_dir, translation_dir, languages, is_global) print("Processing Quest Json files") - diva.quest.update_missions(cache_dir, translation_dir, languages) + # diva.quest.update_missions(cache_dir, translation_dir, languages) - # this code isn't going to run on windows because i need to rewrite it - # it just calls my zsh scripts and generates it that way, need to replace them + # this code isn't going to run on windows (it will in WSL though) because i need to rewrite it + # it just calls my zsh tools and generates it that way, need to replace them # with real python or something + + # first copy over the necessary avtools to the extract dir + # TODO make it so that you don't need to copy them over (for example call them with python) + print("Copying necessary tools into extracting directory") + shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir) + shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir) + shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir) + print("Processing Sound Effects") copy_tree( os.path.join(cache_dir, "Android/Asset/Sound/Se"), os.path.join(extract_dir, "Se"), ) - subprocess.run("./update_se.zsh", shell=True, cwd=extract_dir) - copy_tree(os.path.join(extract_dir, "se"), os.path.join(update_dir, "Se")) + subprocess.run( + os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir + ) + copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se")) print("Processing BGM - don't forget to copy that loop file") copy_tree( os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), os.path.join(extract_dir, "Bgm"), ) - subprocess.run("./update_bgm.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "bgm"), os.path.join(update_dir, "Bgm") + os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") ) print("Processing voices") @@ -44,9 +71,11 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): os.path.join(cache_dir, "Android/Asset/Sound/Voice"), os.path.join(extract_dir, "Voice"), ) - subprocess.run("./update_voice.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "voice"), os.path.join(update_dir, "Voice") + os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") ) print("Processing movies - this might take forever") @@ -54,22 +83,22 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): os.path.join(cache_dir, "Common/Asset/Movie"), os.path.join(extract_dir, "Movie"), ) - subprocess.run("./update_movie.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "movie"), + os.path.join(extract_dir, "movie_extracted"), os.path.join(update_dir, "Asset/Movie"), ) -def update_tsv(cache_dir, update_dir, translation_dir, languages): +def update_tsv(cache_dir, update_dir, translation_dir, languages, is_global): utage_tmp = tempfile.mkdtemp() copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp) utage.crypt.crypt_dir( utage_tmp, bytearray("SampleSecretKey", "utf-8"), False, False ) - for encrypted in glob( - os.path.join(utage_tmp, "**/*.utage"), recursive=True - ): + for encrypted in glob(os.path.join(utage_tmp, "**/*.utage"), recursive=True): try: os.remove(encrypted) except: @@ -79,11 +108,10 @@ def update_tsv(cache_dir, update_dir, translation_dir, languages): utage_tmp, os.path.join(update_dir, "Utage"), os.path.join(translation_dir, "Missions"), + is_global, ) utage.names.update_names(utage_tmp, translation_dir, languages) - copy_tree( - os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva") - ) + copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) remove_tree(utage_tmp) @@ -93,6 +121,15 @@ def copy_images(cache_dir, update_dir): os.path.join(update_dir, "Asset/Image"), ) copy_tree( - os.path.join(cache_dir, "Common/Sample"), - os.path.join(update_dir, "Sample"), + os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), ) + + +def cleanup_cache(cache_dir): + with suppress(FileNotFoundError): + shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) + os.remove( + os.path.join( + cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" + ) + )