ported update_se

divatool.py

@@ -184,15 +184,15 @@ def main():
         if args.subcommand == "quest":
             diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES)
     elif args.command == "xdudata":
-        script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools"))
         if args.subcommand == "update":
+            # add the tools folder to PATH
+            os.environ["PATH"] += os.pathsep + os.path.join(
+                os.path.abspath(os.path.dirname(__file__)), "tools"
+            )
+
+            # update the data
             xdudata.update_all(
-                args.CACHE,
+                args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES,
-                args.XDUDATA,
-                args.TRANS,
-                args.EXTRACT,
-                script_dir,
-                LANGUAGES,
             )
 
     return 0

xdudata/asset_extract.py

@@ -0,0 +1,150 @@
+import functools
+import multiprocessing
+import os
+import shutil
+import subprocess
+from contextlib import contextmanager
+from distutils.dir_util import copy_tree
+from glob import glob
+
+OPERATING_SYSTEM = os.name
+
+
+def process_se(cache_dir, extract_dir, update_dir):
+    copy_tree(
+        os.path.join(cache_dir, "Android/Asset/Sound/Se"),
+        os.path.join(extract_dir, "Se"),
+    )
+
+    from_dir = "Se"
+    to_dir = "Se_Extracted"
+    acb_prefix = "_vgmt_acb_ext"
+
+    pool = get_pool()
+
+    with change_cwd(extract_dir):
+        os.makedirs(to_dir, exist_ok=True)
+
+        to_del = glob("{}/**/*.wav".format(from_dir), recursive=True)
+        to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True))
+        for file in to_del:
+            os.remove(file)
+
+        acb_files = [file for file in glob("{}/*.acb".format(from_dir))]
+        acb_cmd = get_dotnet_cmd("acb_extract.exe")
+        run_program(acb_files, cmd=acb_cmd)
+
+        hca_files = [
+            file
+            for file in glob(
+                "{}/{}*/**/*.hca".format(from_dir, acb_prefix), recursive=True
+            )
+        ]
+        run_program(hca_files, cmd="clHCA")
+
+        basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files]
+        char_secs = [basename.split("_", maxsplit=1) for basename in basenames]
+        for cs in char_secs:
+            char = cs[0]
+            sec = str(cs[1]) if len(cs) == 2 else ""
+            os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True)
+            wav_files = glob(
+                "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)),
+                recursive=True,
+            )
+            partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg")
+            wav_cmds = [
+                [
+                    "-i",
+                    os.path.abspath(file),
+                    "-acodec",
+                    "libopus",
+                    "-ab",
+                    "192k",
+                    "-af",
+                    "aresample=48000",
+                    os.path.abspath(
+                        "{}/{}/{}/{}.opus".format(
+                            to_dir,
+                            char,
+                            sec,
+                            os.path.splitext(os.path.basename(file))[0],
+                        )
+                    ),
+                    "-n",
+                ]
+                for file in wav_files
+            ]
+            pool.map(partial_ffmpeg_cmd, wav_cmds)
+
+    copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Se"))
+
+
+def process_bgm(cache_dir, extract_dir, update_dir):
+    copy_tree(
+        os.path.join(cache_dir, "Android/Asset/Sound/Bgm"),
+        os.path.join(extract_dir, "Bgm"),
+    )
+    subprocess.run(os.path.join("TODO", "update_bgm.zsh"), shell=True, cwd=extract_dir)
+    copy_tree(
+        os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm")
+    )
+
+
+def process_voice(cache_dir, extract_dir, update_dir):
+    copy_tree(
+        os.path.join(cache_dir, "Android/Asset/Sound/Voice"),
+        os.path.join(extract_dir, "Voice"),
+    )
+    subprocess.run(
+        os.path.join("TODO", "update_voice.zsh"), shell=True, cwd=extract_dir
+    )
+    copy_tree(
+        os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice")
+    )
+
+
+def process_movie(cache_dir, extract_dir, update_dir):
+    copy_tree(
+        os.path.join(cache_dir, "Common/Asset/Movie"),
+        os.path.join(extract_dir, "Movie"),
+    )
+    subprocess.run(
+        os.path.join("TODO", "update_movie.zsh"), shell=True, cwd=extract_dir
+    )
+    copy_tree(
+        os.path.join(extract_dir, "movie_extracted"),
+        os.path.join(update_dir, "Asset/Movie"),
+    )
+
+
+@contextmanager
+def change_cwd(new_path):
+    old_cwd = os.getcwd()
+    os.chdir(new_path)
+    try:
+        yield
+    finally:
+        os.chdir(old_cwd)
+
+
+def get_dotnet_cmd(name):
+    ext = os.path.splitext(name)[1]
+    if ext != ".exe":
+        raise RuntimeError("This doesn't seem like a .NET program.")
+    path = shutil.which(name)
+    if OPERATING_SYSTEM == "nt":
+        return [path]
+    return ["mono", path]
+
+
+def run_program(args, cmd=None):
+    if isinstance(cmd, str):
+        cmd = [cmd]
+    full_cmd = cmd + args
+    subprocess.run(full_cmd)
+
+
+def get_pool(size=multiprocessing.cpu_count()):
+    pool = multiprocessing.Pool(size)
+    return pool

xdudata/update.py

@@ -3,85 +3,69 @@ from contextlib import suppress
 import diva
 import shutil
 import os
-import subprocess
 import tempfile
 import utage
 
 from distutils.dir_util import copy_tree, remove_tree
 from glob import glob
 
+from xdudata.asset_extract import process_se, process_bgm, process_voice, process_movie
+
 
 def update_all(
-    cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages,
+    cache_dir, update_dir, translation_dir, extract_dir, languages,
 ):
-    print("Cleaning cache")
+    # TODO uncomment
-    cleanup_cache(cache_dir)
+    # print("Checking tool availability")
-
+    # validate_tools()
-    print("Copying images")
+    #
-    copy_images(cache_dir, update_dir)
+    # print("Cleaning cache")
-
+    # cleanup_cache(cache_dir)
-    print("Processing TSV files")
+    #
-    update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages)
+    # print("Copying images")
-
+    # copy_images(cache_dir, update_dir)
-    print("Processing Quest Json files")
+    #
-    diva.quest.update_missions(extract_dir, translation_dir, languages)
+    # print("Processing TSV files")
-
+    # update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages)
-    # this code isn't going to run on windows (it is in WSL though) because i need to rewrite it
+    #
-    # it just calls my zsh tools and generates it that way, need to replace them
+    # print("Processing Quest Json files")
-    # with real python or something
+    # diva.quest.update_missions(extract_dir, translation_dir, languages)
-
-    # first copy over the necessary avtools to the extract dir
-    # TODO make it so that you don't need to copy them over (for example call them with python)
-    print("Copying necessary tools into extracting directory")
-    shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir)
-    shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir)
-    shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir)
 
     print("Processing Sound Effects")
-    copy_tree(
+    process_se(cache_dir, extract_dir, update_dir)
-        os.path.join(cache_dir, "Android/Asset/Sound/Se"),
-        os.path.join(extract_dir, "Se"),
-    )
-    subprocess.run(
-        os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir
-    )
-    copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se"))
 
-    print("Processing BGM - don't forget to copy that loop file")
+    # print("Processing BGM")
-    copy_tree(
+    # process_bgm(cache_dir, extract_dir, update_dir)
-        os.path.join(cache_dir, "Android/Asset/Sound/Bgm"),
+    #
-        os.path.join(extract_dir, "Bgm"),
+    # print("Processing voices")
+    # process_voice(cache_dir, extract_dir, update_dir)
+    #
+    # print("Processing movies - this might take forever")
+    # process_movie(cache_dir, extract_dir, update_dir)
+
+
+def validate_tools():
+    # TODO validate all the tools are in PATH
+    ...
+
+
+def cleanup_cache(cache_dir):
+    with suppress(FileNotFoundError):
+        shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32"))
+        os.remove(
+            os.path.join(
+                cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage"
             )
-    subprocess.run(
-        os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir
-    )
-    copy_tree(
-        os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm")
         )
 
-    print("Processing voices")
-    copy_tree(
-        os.path.join(cache_dir, "Android/Asset/Sound/Voice"),
-        os.path.join(extract_dir, "Voice"),
-    )
-    subprocess.run(
-        os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir
-    )
-    copy_tree(
-        os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice")
-    )
 
-    print("Processing movies - this might take forever")
+def copy_images(cache_dir, update_dir):
     copy_tree(
-        os.path.join(cache_dir, "Common/Asset/Movie"),
+        os.path.join(cache_dir, "Common/Asset/Image"),
-        os.path.join(extract_dir, "Movie"),
+        os.path.join(update_dir, "Asset/Image"),
-    )
-    subprocess.run(
-        os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir
     )
     copy_tree(
-        os.path.join(extract_dir, "movie_extracted"),
+        os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"),
-        os.path.join(update_dir, "Asset/Movie"),
     )
 
 
@@ -113,23 +97,3 @@ def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages):
     utage.names.update_names(utage_tmp, translation_dir, languages, False)
     copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva"))
     remove_tree(utage_tmp)
-
-
-def copy_images(cache_dir, update_dir):
-    copy_tree(
-        os.path.join(cache_dir, "Common/Asset/Image"),
-        os.path.join(update_dir, "Asset/Image"),
-    )
-    copy_tree(
-        os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"),
-    )
-
-
-def cleanup_cache(cache_dir):
-    with suppress(FileNotFoundError):
-        shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32"))
-        os.remove(
-            os.path.join(
-                cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage"
-            )
-        )