Merge pull request 'Added --global flag for Global XDU' (#2) from feature/global_update into develop

.gitignore

@@ -3,3 +3,6 @@ __pycache__/
 *$py.class
 *.swp
 virtualenv/
+tools/*
+!tools/readme.txt
+.idea/

diva/quest.py

@@ -26,18 +26,14 @@ def scene_mst(qdb, old_path, dir_in, languages):
             except KeyError as exc:
                 i += 1
                 if i == len(scene["Parts"]):
-                    raise KeyError(
+                    raise KeyError(f"Could not find folder for scene {id}") from exc
-                        f"Could not find folder for scene {id}"
-                    ) from exc
                 continue
             break
 
     with io.open(
         os.path.join(old_path, "XduScene.json"), "w", newline="\n"
     ) as json_file:
-        json.dump(
+        json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
-            scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True
-        )
 
     for lang in languages:
         out_dict = {}
@@ -66,11 +62,7 @@ def scene_mst(qdb, old_path, dir_in, languages):
 
         with io.open(langfile, "w", newline="\n") as lang_file:
             json.dump(
-                out_dict,
+                out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
-                lang_file,
-                ensure_ascii=False,
-                indent="\t",
-                sort_keys=True,
             )
 
 
@@ -80,9 +72,7 @@ def quest_mst(qdb, old_path, languages):
     with io.open(
         os.path.join(old_path, "XduQuest.json"), "w", newline="\n"
     ) as json_file:
-        json.dump(
+        json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
-            quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True
-        )
 
     for lang in languages:
         out_dict = {}
@@ -101,11 +91,7 @@ def quest_mst(qdb, old_path, languages):
 
         with io.open(langfile, "w", newline="\n") as lang_file:
             json.dump(
-                out_dict,
+                out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
-                lang_file,
-                ensure_ascii=False,
-                indent="\t",
-                sort_keys=True,
             )
 
     return quests

divatool.py

@@ -15,6 +15,16 @@ LANGUAGES = ["jpn", "eng", "rus"]
 def main():
     parser = argparse.ArgumentParser(description="Unpack and process XDU data")
 
+    global_flag_parser = argparse.ArgumentParser(add_help=False)
+    global_flag_parser.add_argument(
+        "-g",
+        "--global",
+        help="Whether we are running on Global XDU.",
+        action="store_true",
+        dest="is_global",
+        default=False,
+    )
+
     subparsers = parser.add_subparsers(
         metavar="<command>", title="subcommands", dest="command"
     )
@@ -81,7 +91,9 @@ def main():
     parser_utage_crypt.add_argument("TARGET", help="Input", type=str)
     # utage translate
     parser_utage_translate = parser_utage_subparsers.add_parser(
-        "translate", help="Generate keyed tsv and json files"
+        "translate",
+        help="Generate keyed tsv and json files",
+        parents=[global_flag_parser],
     )
     parser_utage_translate.add_argument("INPUT", help="Input", type=str)
     parser_utage_translate.add_argument(
@@ -92,7 +104,7 @@ def main():
     )
     # utage names
     parser_utage_names = parser_utage_subparsers.add_parser(
-        "names", help="Generate and update name files"
+        "names", help="Generate and update name files", parents=[global_flag_parser]
     )
     parser_utage_names.add_argument("INPUT", help="Input", type=str)
     parser_utage_names.add_argument(
@@ -152,23 +164,35 @@ def main():
                 raise FileNotFoundError("Could not find {}".format(args.TARGET))
         elif args.subcommand == "translate":
             if os.path.isfile(args.INPUT):
-                utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR)
+                utage.translate.translate_file(
+                    args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global
+                )
             elif os.path.isdir(args.INPUT):
-                utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR)
+                utage.translate.translate_dir(
+                    args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global
+                )
             else:
                 raise FileNotFoundError("Could not find {}".format(args.INPUT))
         elif args.subcommand == "names":
             if os.path.isdir(args.INPUT):
-                utage.names.update_names(args.INPUT, args.OLD, LANGUAGES)
+                utage.names.update_names(
+                    args.INPUT, args.OLD, LANGUAGES, args.is_global
+                )
             else:
                 raise FileNotFoundError(args.INPUT)
     elif args.command == "diva":
         if args.subcommand == "quest":
             diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES)
     elif args.command == "xdudata":
+        script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools"))
         if args.subcommand == "update":
             xdudata.update_all(
-                args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES
+                args.CACHE,
+                args.XDUDATA,
+                args.TRANS,
+                args.EXTRACT,
+                script_dir,
+                LANGUAGES,
             )
 
     return 0

tools/readme.txt

@@ -0,0 +1,34 @@
+Various files need to be in this directory.
+If you don't have them, ask someone who does.
+
+The required files are:
+
+Binaries:
+- acb_extract.exe
+- usm_extract.exe
+- clHCA
+
+Z shell scripts:
+- update_bgm.zsh
+- update_movie.zsh
+- update_se.zsh
+- update_voice.zsh
+
+Python scripts:
+- adx_parse.py
+- bingo_fuck.py
+- chara_room_mission.py
+- custom_mission.py
+- event_bonus.py
+- orphan_quest.py
+- pad_2_csv.py
+- pad_2_json.py
+- pad_2_pad.py
+- pad_2_pad_louisstyle.py
+- pad_stat.py
+- pad_update.py
+- store_parser.py
+- tsv_2_pad.py
+- tsv_parse.py
+- txt_2_pad.py
+- utage_decrypt.py

utage/names.py

@@ -9,14 +9,25 @@ from glob import glob
 from multiprocessing import Pool
 
 
-def update_names(dir_in, old_path, languages):
+def update_names(dir_in, old_path, languages, is_global):
-    names = extract_names(dir_in)
+    names = extract_names(dir_in, is_global)
+
+    global_languages = {"enm": 1, "zh": 2, "ko": 3}
+    if is_global:
+        languages = global_languages.keys()
 
     # we have to update japanese like the rest because CustomData exists
     # and POKELABO deleted the wedding gear event stuff from the game files
     for l in languages:
         if l == "jpn":
             out_dict = dict(zip(names, names))
+        elif l in global_languages.keys():
+            out_dict = {}
+            for name in names:
+                spl = name.split("\t")
+                idx = global_languages[l]
+                val = spl[idx] if spl[idx] != "None" else ""
+                out_dict[spl[0]] = val
         else:
             out_dict = dict.fromkeys(names, "")
 
@@ -30,21 +41,17 @@ def update_names(dir_in, old_path, languages):
             langfile, "w", newline="\n"
         ) as lang_file:  # you're using git right
             json.dump(
-                out_dict,
+                out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
-                lang_file,
-                ensure_ascii=False,
-                indent="\t",
-                sort_keys=True,
             )
 
 
-def extract_names(dir_in):
+def extract_names(dir_in, is_global):
     char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv")
 
     if not os.path.isfile(char_tsv_path):
         raise FileNotFoundError("Character.tsv not found")
 
-    names, sets = read_char_tsv(char_tsv_path)
+    names, sets = read_char_tsv(char_tsv_path, is_global)
 
     files = [
         f
@@ -54,18 +61,20 @@ def extract_names(dir_in):
     if len(files) == 0:
         raise FileNotFoundError("No valid files found in directory: " + dir_in)
 
-    # tfw gil
+    # TODO unsupported on global for now
-    # it's still faster on my machine so i'm keeping it
+    if not is_global:
-    read_partial = partial(read_mission, char_names=names, char_sets=sets)
+        # tfw gil
-    with Pool() as p:
+        # it's still faster on my machine so i'm keeping it
-        new_names = p.map(read_partial, files)
+        read_partial = partial(read_mission, char_names=names, char_sets=sets)
-    for x in new_names:
+        with Pool() as p:
-        names.update(x)
+            new_names = p.map(read_partial, files)
+        for x in new_names:
+            names.update(x)
 
     return names
 
 
-def read_char_tsv(char_tsv_path):
+def read_char_tsv(char_tsv_path, is_global):
     char_names = set()
     char_sets = set()
 
@@ -81,7 +90,12 @@ def read_char_tsv(char_tsv_path):
                 and row["CharacterName"].strip()
                 and row["NameText"].strip()
             ):
-                char_names.add(row["NameText"])
+                if is_global:
+                    char_names.add(
+                        f"{row['NameText']}\t{row['CharaEnglish']}\t{row['CharaChinese']}\t{row['CharaKorean']}"
+                    )
+                else:
+                    char_names.add(row["NameText"])
                 char_sets.add(row["CharacterName"])
 
     return char_names, char_sets
@@ -101,8 +115,6 @@ def read_mission(tsv_path, char_names, char_sets):
             if row["Command"] and row["Command"].startswith("//"):
                 continue
             if row["Text"] and row["Arg1"]:
-                if (row["Arg1"] not in char_names) and (
+                if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets):
-                    row["Arg1"] not in char_sets
-                ):
                     new_names.add(row["Arg1"])
     return new_names

utage/translate.py

@@ -4,18 +4,17 @@ import io
 import json
 import os
 import traceback
+from contextlib import suppress
 
 from functools import partial
 from glob import glob
 from multiprocessing.pool import Pool
 
 
-def translate_dir(dir_in, tsv_out_dir, json_out_dir):
+def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global):
     files = [
         f
-        for f in glob(
+        for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True)
-            os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True
-        )
         if "{os.path.sep}Diva{os.path.sep}" not in f
     ]
 
@@ -23,13 +22,16 @@ def translate_dir(dir_in, tsv_out_dir, json_out_dir):
         raise FileNotFoundError("No valid files found in directory: " + dir_in)
 
     ptrans = partial(
-        translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir
+        translate_file,
+        tsv_out_dir=tsv_out_dir,
+        json_out_dir=json_out_dir,
+        is_global=is_global,
     )
     with Pool() as p:
         p.map(ptrans, files)
 
 
-def translate_file(file_in, tsv_out_dir, json_out_dir):
+def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
     try:
         if not file_in.endswith(".tsv"):
             raise ValueError("Invalid File Type for {}".format(file_in))
@@ -48,11 +50,17 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
         tsv_output_path = os.path.join(
             tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num)
         )
-        json_output_path = os.path.join(
+        languages = ["jpn"]
-            json_out_dir,
+        if is_global:
-            event_folder,
+            languages += ["enm", "zh", "ko"]
-            "{}_translations_jpn.json".format(id_num),
+        json_output_paths = {
-        )
+            lang: os.path.join(
+                json_out_dir,
+                event_folder,
+                "{}_translations_{}.json".format(id_num, lang),
+            )
+            for lang in languages
+        }
 
         # need to create output paths and avoid races when threading
         if not os.path.exists(os.path.dirname(tsv_output_path)):
@@ -62,16 +70,21 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
                 if e.errno != errno.EEXIST:
                     raise
 
-        if not os.path.exists(os.path.dirname(json_output_path)):
+        if not os.path.exists(os.path.dirname(json_output_paths["jpn"])):
             try:
-                os.makedirs(os.path.dirname(json_output_path))
+                os.makedirs(os.path.dirname(json_output_paths["jpn"]))
             except OSError as e:
                 if e.errno != errno.EEXIST:
                     raise
 
         with open(file_in, "r") as tsv_file:
-            tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
+            tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE)
-            tsv_keyed, json_str = process_tsv(tsv, id_num)
+
+            t_fieldnames = tsv.fieldnames
+            if is_global:
+                t_fieldnames = tsv.fieldnames[: tsv.fieldnames.index("English") + 1]
+
+            tsv_keyed, json_str = process_tsv(tsv, id_num, is_global)
 
             # csv handles newlines, don't set it in io.open
             with io.open(tsv_output_path, "w", newline="") as tsv_out:
@@ -80,32 +93,39 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
                     delimiter="\t",
                     quotechar='"',
                     lineterminator="\n",
-                    fieldnames=tsv.fieldnames,
+                    fieldnames=t_fieldnames,
                     extrasaction="ignore",
                 )
                 writer.writeheader()
                 for row in tsv_keyed:
                     writer.writerow(row)
 
-        with io.open(json_output_path, "w", newline="\n") as json_out:
+        for lang in languages:
-            # we don't want to sort these because they're _1, ..., _10, etc
+            if not json_str[lang]:
-            json.dump(
+                continue
-                json_str,
+            with io.open(json_output_paths[lang], "w", newline="\n") as json_out:
-                json_out,
+                # we don't want to sort these because they're _1, ..., _10, etc
-                ensure_ascii=False,
+                json.dump(
-                indent="\t",
+                    json_str[lang],
-                sort_keys=False,
+                    json_out,
-            )
+                    ensure_ascii=False,
+                    indent="\t",
+                    sort_keys=False,
+                )
 
     except Exception:
         traceback.print_exc()
         print("Error processing file: " + file_in)
 
 
-def process_tsv(tsv, id_num):
+def process_tsv(tsv, id_num, is_global):
     i = 0
     tsv_keyed = []
-    key_dict = {}
+    key_dict = {"jpn": {}}
+    if is_global:
+        key_dict["enm"] = {}
+        key_dict["zh"] = {}
+        key_dict["ko"] = {}
 
     for row in tsv:
         try:
@@ -115,8 +135,14 @@ def process_tsv(tsv, id_num):
             if row["Text"] and row["Text"].strip():
                 key = "{}_{}".format(id_num, i)
                 i += 1
+                key_dict["jpn"][key] = row["Text"]
+                if is_global:
+                    with suppress(KeyError):
+                        if row["English"]:
+                            key_dict["enm"][key] = row["English"]
+                        key_dict["zh"][key] = row["Chinese"]
+                        key_dict["ko"][key] = row["Korean"]
                 row["English"] = key
-                key_dict[key] = row["Text"]
             tsv_keyed.append(row)
         except Exception:
             traceback.print_exc()

xdudata/update.py

@@ -1,4 +1,7 @@
+from contextlib import suppress
+
 import diva
+import shutil
 import os
 import subprocess
 import tempfile
@@ -8,35 +11,52 @@ from distutils.dir_util import copy_tree, remove_tree
 from glob import glob
 
 
-def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages):
+def update_all(
+    cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages,
+):
+    print("Cleaning cache")
+    cleanup_cache(cache_dir)
+
     print("Copying images")
     copy_images(cache_dir, update_dir)
 
     print("Processing TSV files")
-    update_tsv(cache_dir, update_dir, translation_dir, languages)
+    update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages)
 
     print("Processing Quest Json files")
-    diva.quest.update_missions(cache_dir, translation_dir, languages)
+    diva.quest.update_missions(extract_dir, translation_dir, languages)
 
-    # this code isn't going to run on windows because i need to rewrite it
+    # this code isn't going to run on windows (it is in WSL though) because i need to rewrite it
-    # it just calls my zsh scripts and generates it that way, need to replace them
+    # it just calls my zsh tools and generates it that way, need to replace them
     # with real python or something
+
+    # first copy over the necessary avtools to the extract dir
+    # TODO make it so that you don't need to copy them over (for example call them with python)
+    print("Copying necessary tools into extracting directory")
+    shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir)
+    shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir)
+    shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir)
+
     print("Processing Sound Effects")
     copy_tree(
         os.path.join(cache_dir, "Android/Asset/Sound/Se"),
         os.path.join(extract_dir, "Se"),
     )
-    subprocess.run("./update_se.zsh", shell=True, cwd=extract_dir)
+    subprocess.run(
-    copy_tree(os.path.join(extract_dir, "se"), os.path.join(update_dir, "Se"))
+        os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir
+    )
+    copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se"))
 
     print("Processing BGM - don't forget to copy that loop file")
     copy_tree(
         os.path.join(cache_dir, "Android/Asset/Sound/Bgm"),
         os.path.join(extract_dir, "Bgm"),
     )
-    subprocess.run("./update_bgm.zsh", shell=True, cwd=extract_dir)
+    subprocess.run(
+        os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir
+    )
     copy_tree(
-        os.path.join(extract_dir, "bgm"), os.path.join(update_dir, "Bgm")
+        os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm")
     )
 
     print("Processing voices")
@@ -44,9 +64,11 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages):
         os.path.join(cache_dir, "Android/Asset/Sound/Voice"),
         os.path.join(extract_dir, "Voice"),
     )
-    subprocess.run("./update_voice.zsh", shell=True, cwd=extract_dir)
+    subprocess.run(
+        os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir
+    )
     copy_tree(
-        os.path.join(extract_dir, "voice"), os.path.join(update_dir, "Voice")
+        os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice")
     )
 
     print("Processing movies - this might take forever")
@@ -54,36 +76,42 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages):
         os.path.join(cache_dir, "Common/Asset/Movie"),
         os.path.join(extract_dir, "Movie"),
     )
-    subprocess.run("./update_movie.zsh", shell=True, cwd=extract_dir)
+    subprocess.run(
+        os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir
+    )
     copy_tree(
-        os.path.join(extract_dir, "movie"),
+        os.path.join(extract_dir, "movie_extracted"),
         os.path.join(update_dir, "Asset/Movie"),
     )
 
 
-def update_tsv(cache_dir, update_dir, translation_dir, languages):
+def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages):
     utage_tmp = tempfile.mkdtemp()
     copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp)
     utage.crypt.crypt_dir(
         utage_tmp, bytearray("SampleSecretKey", "utf-8"), False, False
     )
-    for encrypted in glob(
+    for encrypted in glob(os.path.join(utage_tmp, "**/*.utage"), recursive=True):
-        os.path.join(utage_tmp, "**/*.utage"), recursive=True
-    ):
         try:
             os.remove(encrypted)
         except:
             pass
 
+    # copy over files for the quest updater
+    copy_tree(utage_tmp, os.path.join(extract_dir, "Common/Asset/Utage"))
+    shutil.copy2(os.path.join(cache_dir, "QuestMst.db"), extract_dir)
+    shutil.copy2(os.path.join(cache_dir, "QuestSceneMst.db"), extract_dir)
+    shutil.copy2(os.path.join(cache_dir, "QuestPartMst.db"), extract_dir)
+    shutil.copy2(os.path.join(cache_dir, "ResourceEntry.db"), extract_dir)
+
     utage.translate.translate_dir(
         utage_tmp,
         os.path.join(update_dir, "Utage"),
         os.path.join(translation_dir, "Missions"),
+        False,
     )
-    utage.names.update_names(utage_tmp, translation_dir, languages)
+    utage.names.update_names(utage_tmp, translation_dir, languages, False)
-    copy_tree(
+    copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva"))
-        os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")
-    )
     remove_tree(utage_tmp)
 
 
@@ -93,6 +121,15 @@ def copy_images(cache_dir, update_dir):
         os.path.join(update_dir, "Asset/Image"),
     )
     copy_tree(
-        os.path.join(cache_dir, "Common/Sample"),
+        os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"),
-        os.path.join(update_dir, "Sample"),
     )
+
+
+def cleanup_cache(cache_dir):
+    with suppress(FileNotFoundError):
+        shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32"))
+        os.remove(
+            os.path.join(
+                cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage"
+            )
+        )