sorting out zsh script calling and folder names

.gitignore

@@ -3,6 +3,6 @@ __pycache__/
 *$py.class
 *.swp
 virtualenv/
-scripts/*
+tools/*
-!scripts/readme.txt
+!tools/readme.txt
 .idea/

divatool.py

@@ -184,12 +184,14 @@ def main():
         if args.subcommand == "quest":
             diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES)
     elif args.command == "xdudata":
+        script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools"))
         if args.subcommand == "update":
             xdudata.update_all(
                 args.CACHE,
                 args.XDUDATA,
                 args.TRANS,
                 args.EXTRACT,
+                script_dir,
                 LANGUAGES,
                 args.is_global,
             )

tools/readme.txt

@@ -3,7 +3,12 @@ If you don't have them, ask someone who does.
 
 The required files are:
 
-Z shell:
+Binaries:
+- acb_extract.exe
+- usm_extract.exe
+- clHCA
+
+Z shell scripts:
 - update_bgm.zsh
 - update_movie.zsh
 - update_se.zsh

utage/names.py

@@ -9,7 +9,7 @@ from glob import glob
 from multiprocessing import Pool
 
 
-def update_names(dir_in, old_path, languages):
+def update_names(dir_in, old_path, languages, is_global):
     names = extract_names(dir_in)
 
     # we have to update japanese like the rest because CustomData exists
@@ -30,11 +30,7 @@ def update_names(dir_in, old_path, languages):
             langfile, "w", newline="\n"
         ) as lang_file:  # you're using git right
             json.dump(
-                out_dict,
+                out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
-                lang_file,
-                ensure_ascii=False,
-                indent="\t",
-                sort_keys=True,
             )
 
 
@@ -101,8 +97,6 @@ def read_mission(tsv_path, char_names, char_sets):
             if row["Command"] and row["Command"].startswith("//"):
                 continue
             if row["Text"] and row["Arg1"]:
-                if (row["Arg1"] not in char_names) and (
+                if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets):
-                    row["Arg1"] not in char_sets
-                ):
                     new_names.add(row["Arg1"])
     return new_names

utage/translate.py

@@ -10,26 +10,22 @@ from glob import glob
 from multiprocessing.pool import Pool
 
 
-def translate_dir(dir_in, tsv_out_dir, json_out_dir):
+def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global):
     files = [
         f
-        for f in glob(
+        for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True)
-            os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True
-        )
         if "{os.path.sep}Diva{os.path.sep}" not in f
     ]
 
     if len(files) == 0:
         raise FileNotFoundError("No valid files found in directory: " + dir_in)
 
-    ptrans = partial(
+    ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir)
-        translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir
-    )
     with Pool() as p:
         p.map(ptrans, files)
 
 
-def translate_file(file_in, tsv_out_dir, json_out_dir):
+def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
     try:
         if not file_in.endswith(".tsv"):
             raise ValueError("Invalid File Type for {}".format(file_in))
@@ -49,9 +45,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
             tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num)
         )
         json_output_path = os.path.join(
-            json_out_dir,
+            json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num),
-            event_folder,
-            "{}_translations_jpn.json".format(id_num),
         )
 
         # need to create output paths and avoid races when threading
@@ -90,11 +84,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
         with io.open(json_output_path, "w", newline="\n") as json_out:
             # we don't want to sort these because they're _1, ..., _10, etc
             json.dump(
-                json_str,
+                json_str, json_out, ensure_ascii=False, indent="\t", sort_keys=False,
-                json_out,
-                ensure_ascii=False,
-                indent="\t",
-                sort_keys=False,
             )
 
     except Exception:

xdudata/update.py

@@ -1,4 +1,7 @@
+from contextlib import suppress
+
 import diva
+import shutil
 import os
 import subprocess
 import tempfile
@@ -8,35 +11,59 @@ from distutils.dir_util import copy_tree, remove_tree
 from glob import glob
 
 
-def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages):
+def update_all(
+    cache_dir,
+    update_dir,
+    translation_dir,
+    extract_dir,
+    script_dir,
+    languages,
+    is_global,
+):
+    print("Cleaning cache")
+    cleanup_cache(cache_dir)
+
     print("Copying images")
-    copy_images(cache_dir, update_dir)
+    # copy_images(cache_dir, update_dir)
 
     print("Processing TSV files")
-    update_tsv(cache_dir, update_dir, translation_dir, languages)
+    # TODO
+    # update_tsv(cache_dir, update_dir, translation_dir, languages, is_global)
 
     print("Processing Quest Json files")
-    diva.quest.update_missions(cache_dir, translation_dir, languages)
+    # diva.quest.update_missions(cache_dir, translation_dir, languages)
 
-    # this code isn't going to run on windows because i need to rewrite it
+    # this code isn't going to run on windows (it will in WSL though) because i need to rewrite it
-    # it just calls my zsh scripts and generates it that way, need to replace them
+    # it just calls my zsh tools and generates it that way, need to replace them
     # with real python or something
+
+    # first copy over the necessary avtools to the extract dir
+    # TODO make it so that you don't need to copy them over (for example call them with python)
+    print("Copying necessary tools into extracting directory")
+    shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir)
+    shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir)
+    shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir)
+
     print("Processing Sound Effects")
     copy_tree(
         os.path.join(cache_dir, "Android/Asset/Sound/Se"),
         os.path.join(extract_dir, "Se"),
     )
-    subprocess.run("./update_se.zsh", shell=True, cwd=extract_dir)
+    subprocess.run(
-    copy_tree(os.path.join(extract_dir, "se"), os.path.join(update_dir, "Se"))
+        os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir
+    )
+    copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se"))
 
     print("Processing BGM - don't forget to copy that loop file")
     copy_tree(
         os.path.join(cache_dir, "Android/Asset/Sound/Bgm"),
         os.path.join(extract_dir, "Bgm"),
     )
-    subprocess.run("./update_bgm.zsh", shell=True, cwd=extract_dir)
+    subprocess.run(
+        os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir
+    )
     copy_tree(
-        os.path.join(extract_dir, "bgm"), os.path.join(update_dir, "Bgm")
+        os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm")
     )
 
     print("Processing voices")
@@ -44,9 +71,11 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages):
         os.path.join(cache_dir, "Android/Asset/Sound/Voice"),
         os.path.join(extract_dir, "Voice"),
     )
-    subprocess.run("./update_voice.zsh", shell=True, cwd=extract_dir)
+    subprocess.run(
+        os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir
+    )
     copy_tree(
-        os.path.join(extract_dir, "voice"), os.path.join(update_dir, "Voice")
+        os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice")
     )
 
     print("Processing movies - this might take forever")
@@ -54,22 +83,22 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages):
         os.path.join(cache_dir, "Common/Asset/Movie"),
         os.path.join(extract_dir, "Movie"),
     )
-    subprocess.run("./update_movie.zsh", shell=True, cwd=extract_dir)
+    subprocess.run(
+        os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir
+    )
     copy_tree(
-        os.path.join(extract_dir, "movie"),
+        os.path.join(extract_dir, "movie_extracted"),
         os.path.join(update_dir, "Asset/Movie"),
     )
 
 
-def update_tsv(cache_dir, update_dir, translation_dir, languages):
+def update_tsv(cache_dir, update_dir, translation_dir, languages, is_global):
     utage_tmp = tempfile.mkdtemp()
     copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp)
     utage.crypt.crypt_dir(
         utage_tmp, bytearray("SampleSecretKey", "utf-8"), False, False
     )
-    for encrypted in glob(
+    for encrypted in glob(os.path.join(utage_tmp, "**/*.utage"), recursive=True):
-        os.path.join(utage_tmp, "**/*.utage"), recursive=True
-    ):
         try:
             os.remove(encrypted)
         except:
@@ -79,11 +108,10 @@ def update_tsv(cache_dir, update_dir, translation_dir, languages):
         utage_tmp,
         os.path.join(update_dir, "Utage"),
         os.path.join(translation_dir, "Missions"),
+        is_global,
     )
     utage.names.update_names(utage_tmp, translation_dir, languages)
-    copy_tree(
+    copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva"))
-        os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")
-    )
     remove_tree(utage_tmp)
 
 
@@ -93,6 +121,15 @@ def copy_images(cache_dir, update_dir):
         os.path.join(update_dir, "Asset/Image"),
     )
     copy_tree(
-        os.path.join(cache_dir, "Common/Sample"),
+        os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"),
-        os.path.join(update_dir, "Sample"),
     )
+
+
+def cleanup_cache(cache_dir):
+    with suppress(FileNotFoundError):
+        shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32"))
+        os.remove(
+            os.path.join(
+                cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage"
+            )
+        )