From 560e00764b50cd2bc867b2e3889c7928de7c8ce9 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 7 Mar 2020 05:15:21 +0100 Subject: [PATCH] ported update_se --- divatool.py | 14 ++-- xdudata/asset_extract.py | 150 +++++++++++++++++++++++++++++++++++++++ xdudata/update.py | 128 ++++++++++++--------------------- 3 files changed, 203 insertions(+), 89 deletions(-) create mode 100644 xdudata/asset_extract.py diff --git a/divatool.py b/divatool.py index ec36fed..e528b57 100755 --- a/divatool.py +++ b/divatool.py @@ -184,15 +184,15 @@ def main(): if args.subcommand == "quest": diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) elif args.command == "xdudata": - script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools")) if args.subcommand == "update": + # add the tools folder to PATH + os.environ["PATH"] += os.pathsep + os.path.join( + os.path.abspath(os.path.dirname(__file__)), "tools" + ) + + # update the data xdudata.update_all( - args.CACHE, - args.XDUDATA, - args.TRANS, - args.EXTRACT, - script_dir, - LANGUAGES, + args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES, ) return 0 diff --git a/xdudata/asset_extract.py b/xdudata/asset_extract.py new file mode 100644 index 0000000..da1ac44 --- /dev/null +++ b/xdudata/asset_extract.py @@ -0,0 +1,150 @@ +import functools +import multiprocessing +import os +import shutil +import subprocess +from contextlib import contextmanager +from distutils.dir_util import copy_tree +from glob import glob + +OPERATING_SYSTEM = os.name + + +def process_se(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Se"), + os.path.join(extract_dir, "Se"), + ) + + from_dir = "Se" + to_dir = "Se_Extracted" + acb_prefix = "_vgmt_acb_ext" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.wav".format(from_dir), recursive=True) + to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True)) + for file in to_del: + os.remove(file) + + acb_files = [file for file in glob("{}/*.acb".format(from_dir))] + acb_cmd = get_dotnet_cmd("acb_extract.exe") + run_program(acb_files, cmd=acb_cmd) + + hca_files = [ + file + for file in glob( + "{}/{}*/**/*.hca".format(from_dir, acb_prefix), recursive=True + ) + ] + run_program(hca_files, cmd="clHCA") + + basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] + char_secs = [basename.split("_", maxsplit=1) for basename in basenames] + for cs in char_secs: + char = cs[0] + sec = str(cs[1]) if len(cs) == 2 else "" + os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True) + wav_files = glob( + "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)), + recursive=True, + ) + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + wav_cmds = [ + [ + "-i", + os.path.abspath(file), + "-acodec", + "libopus", + "-ab", + "192k", + "-af", + "aresample=48000", + os.path.abspath( + "{}/{}/{}/{}.opus".format( + to_dir, + char, + sec, + os.path.splitext(os.path.basename(file))[0], + ) + ), + "-n", + ] + for file in wav_files + ] + pool.map(partial_ffmpeg_cmd, wav_cmds) + + copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Se")) + + +def process_bgm(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), + os.path.join(extract_dir, "Bgm"), + ) + subprocess.run(os.path.join("TODO", "update_bgm.zsh"), shell=True, cwd=extract_dir) + copy_tree( + os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") + ) + + +def process_voice(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Voice"), + os.path.join(extract_dir, "Voice"), + ) + subprocess.run( + os.path.join("TODO", "update_voice.zsh"), shell=True, cwd=extract_dir + ) + copy_tree( + os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") + ) + + +def process_movie(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Common/Asset/Movie"), + os.path.join(extract_dir, "Movie"), + ) + subprocess.run( + os.path.join("TODO", "update_movie.zsh"), shell=True, cwd=extract_dir + ) + copy_tree( + os.path.join(extract_dir, "movie_extracted"), + os.path.join(update_dir, "Asset/Movie"), + ) + + +@contextmanager +def change_cwd(new_path): + old_cwd = os.getcwd() + os.chdir(new_path) + try: + yield + finally: + os.chdir(old_cwd) + + +def get_dotnet_cmd(name): + ext = os.path.splitext(name)[1] + if ext != ".exe": + raise RuntimeError("This doesn't seem like a .NET program.") + path = shutil.which(name) + if OPERATING_SYSTEM == "nt": + return [path] + return ["mono", path] + + +def run_program(args, cmd=None): + if isinstance(cmd, str): + cmd = [cmd] + full_cmd = cmd + args + subprocess.run(full_cmd) + + +def get_pool(size=multiprocessing.cpu_count()): + pool = multiprocessing.Pool(size) + return pool diff --git a/xdudata/update.py b/xdudata/update.py index b659c3f..0d693d3 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -3,85 +3,69 @@ from contextlib import suppress import diva import shutil import os -import subprocess import tempfile import utage from distutils.dir_util import copy_tree, remove_tree from glob import glob +from xdudata.asset_extract import process_se, process_bgm, process_voice, process_movie + def update_all( - cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages, + cache_dir, update_dir, translation_dir, extract_dir, languages, ): - print("Cleaning cache") - cleanup_cache(cache_dir) - - print("Copying images") - copy_images(cache_dir, update_dir) - - print("Processing TSV files") - update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) - - print("Processing Quest Json files") - diva.quest.update_missions(extract_dir, translation_dir, languages) - - # this code isn't going to run on windows (it is in WSL though) because i need to rewrite it - # it just calls my zsh tools and generates it that way, need to replace them - # with real python or something - - # first copy over the necessary avtools to the extract dir - # TODO make it so that you don't need to copy them over (for example call them with python) - print("Copying necessary tools into extracting directory") - shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir) - shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir) - shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir) + # TODO uncomment + # print("Checking tool availability") + # validate_tools() + # + # print("Cleaning cache") + # cleanup_cache(cache_dir) + # + # print("Copying images") + # copy_images(cache_dir, update_dir) + # + # print("Processing TSV files") + # update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) + # + # print("Processing Quest Json files") + # diva.quest.update_missions(extract_dir, translation_dir, languages) print("Processing Sound Effects") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Se"), - os.path.join(extract_dir, "Se"), - ) - subprocess.run( - os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir - ) - copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se")) + process_se(cache_dir, extract_dir, update_dir) - print("Processing BGM - don't forget to copy that loop file") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), - os.path.join(extract_dir, "Bgm"), - ) - subprocess.run( - os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir - ) - copy_tree( - os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") - ) + # print("Processing BGM") + # process_bgm(cache_dir, extract_dir, update_dir) + # + # print("Processing voices") + # process_voice(cache_dir, extract_dir, update_dir) + # + # print("Processing movies - this might take forever") + # process_movie(cache_dir, extract_dir, update_dir) - print("Processing voices") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Voice"), - os.path.join(extract_dir, "Voice"), - ) - subprocess.run( - os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir - ) - copy_tree( - os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") - ) - print("Processing movies - this might take forever") +def validate_tools(): + # TODO validate all the tools are in PATH + ... + + +def cleanup_cache(cache_dir): + with suppress(FileNotFoundError): + shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) + os.remove( + os.path.join( + cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" + ) + ) + + +def copy_images(cache_dir, update_dir): copy_tree( - os.path.join(cache_dir, "Common/Asset/Movie"), - os.path.join(extract_dir, "Movie"), - ) - subprocess.run( - os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir + os.path.join(cache_dir, "Common/Asset/Image"), + os.path.join(update_dir, "Asset/Image"), ) copy_tree( - os.path.join(extract_dir, "movie_extracted"), - os.path.join(update_dir, "Asset/Movie"), + os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), ) @@ -113,23 +97,3 @@ def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages): utage.names.update_names(utage_tmp, translation_dir, languages, False) copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) remove_tree(utage_tmp) - - -def copy_images(cache_dir, update_dir): - copy_tree( - os.path.join(cache_dir, "Common/Asset/Image"), - os.path.join(update_dir, "Asset/Image"), - ) - copy_tree( - os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), - ) - - -def cleanup_cache(cache_dir): - with suppress(FileNotFoundError): - shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) - os.remove( - os.path.join( - cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" - ) - )