Update README with a basic example

.gitignore

@@ -3,3 +3,6 @@ __pycache__/
 *$py.class
 *.swp
 virtualenv/
+.idea/
+tools/*
+!tools/.gitkeep

README.md

@@ -0,0 +1,36 @@
+# divatool
+
+A tool for that one game everyone hates but works on anyway.
+
+## Requirements
+
+Divatool is written in and uses [Python 3](https://www.python.org/downloads/).
+
+There are also several third party dependencies not provided as part of this repository,
+mostly used for the `xdudata` command. You need to have the binaries of the following tools
+either in your `$PATH` variable or in the [`tools`](./tools) folder. These tools are:
+
+* [clHCA](https://github.com/KinoMyu/FastHCADecoder)
+* acb_extract
+* usm_extract
+* [ffmpeg](https://www.ffmpeg.org/download.html)
+* [mono](https://www.mono-project.com/download/stable/) (if using Linux)
+
+Make sure these executables are called the same way on your machine.
+
+## Usage
+
+TODO: add detailed descriptions for all parameters.
+
+Using the tool to update XDUData for the player:
+
+```
+divatool.py xdudata update [path to the 'files' folder from the game] [path to the XDUData folder for the player] [path to the 'xdutranslations' repo folder] [path to the temp files folder]
+```
+
+Example:
+```
+divatool.py xdudata update /home/user/gameapp/files /home/user/player/XDUData /home/user/git/xdutranslations /var/tmp
+```
+
+Note that the 'files' folder from the app must contain the db files for this to work.

adx/adx_parser.py

@@ -74,7 +74,7 @@ class adx_file(object):
         self.encrypted = self.get_val(CRYPT_OFF, CRYPT_LEN)
         self.d_offset = self.get_val(DATA_OFF_OFF, DATA_OFF_LEN)
         if self.d_offset < DATA_OFF_MIN:
-            return None
+            return
         self.l_exists = 1
         # now load things for math
         self.sam_rate = self.get_val(SRATE_OFF, SRATE_LEN)
@@ -106,13 +106,11 @@ class adx_file(object):
             return None
         if self.l_start > self.sam_count or self.l_end > self.sam_count:
             raise ValueError("Invalid Loop Data")
-            return None
         if self.l_style not in LOOP_TYPE_SUP:
             raise NotImplementedError(
                 "Loop Style {} Not Implemented".format(str(self.l_style))
             )
-            return None
+        ret = dict()
-        ret = {}
         ret["duration"] = self.sam_count / self.sam_rate
         ret["loop_start"] = {}
         ret["loop_start"]["seconds"] = self.l_start / self.sam_rate
@@ -123,7 +121,5 @@ class adx_file(object):
         ret["loop_end"] = {}
         ret["loop_end"]["seconds"] = self.l_end / self.sam_rate
         ret["loop_end"]["samples_native"] = self.l_end
-        ret["loop_end"]["samples_48k"] = dumb_round(
+        ret["loop_end"]["samples_48k"] = dumb_round(self.l_end / self.sam_rate * 48000)
-            self.l_end / self.sam_rate * 48000
-        )
         return ret

db/base.py

@@ -23,7 +23,7 @@ class BaseDB(object):
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
-        self.quest_conn.close()
+        self.conn.close()
 
     def attach_dbs(self, databases, path):
         for d in databases:

db/quest.py

@@ -1,16 +1,24 @@
 from .base import BaseDB
 
-DATABASES = {
+
-    "QuestMst.db": "Quest",
+def get_dbs_by_lang(language=None):
-    "QuestSceneMst.db": "QuestScene",
+    databases = {
-    "QuestPartMst.db": "QuestPart",
+        "QuestMst": "Quest",
-    "ResourceEntry.db": "ResourceEntry",
+        "QuestSceneMst": "QuestScene",
-}
+        "QuestPartMst": "QuestPart",
+        "ResourceEntry": "ResourceEntry",
+    }
+    for key in list(databases):
+        if not language:
+            databases[key + ".db"] = databases.pop(key)
+        else:
+            databases[key + "_" + language + ".db"] = databases.pop(key)
+    return databases
 
 
 class QuestDB(BaseDB):
-    def __init__(self, path):
+    def __init__(self, path, language=None):
-        super().__init__(path, DATABASES)
+        super().__init__(path, get_dbs_by_lang(language))
 
     def get_quests(self):
         self.cursor.execute(
@@ -44,12 +52,14 @@ class QuestDB(BaseDB):
                 questSceneMstId,
                 name,
                 summaryText,
+                img,
                 group_concat(partIds) as parts
             FROM (
                     SELECT
                         qs.questSceneMstId,
                         qs.name,
                         qs.summaryText,
+                        qs.img,
                         CASE
                             WHEN qp.afterTalkName == '' AND qp.beforeTalkName == '' THEN NULL
                             WHEN qp.afterTalkName == '' AND qp.beforeTalkName != '' THEN qp.beforeTalkName
@@ -66,9 +76,11 @@ class QuestDB(BaseDB):
         )
         res = self.cursor.fetchall()
         scenes = {}
-        for scene_id, name, summary, parts in res:
+        for scene_id, name, summary, img, parts in res:
             scenes[str(scene_id)] = {}
             scenes[str(scene_id)]["Name"] = name
             scenes[str(scene_id)]["SummaryText"] = summary
+            if img:
+                scenes[str(scene_id)]["Image"] = img
             scenes[str(scene_id)]["Parts"] = parts.split(",")
         return scenes

diva/quest.py

@@ -7,16 +7,25 @@ import os
 from glob import glob
 
 
-def update_missions(dir_in, old_path, languages):
+def update_missions(dir_in, old_path, languages, is_global, utage_in=None):
+    if not utage_in:
+        utage_in = os.path.join(dir_in, "Common/Asset/Utage")
+    if is_global:
+        languages = ["enm", "zho", "kor"]
+        for lang in languages:
+            qdb = db.QuestDB(dir_in, lang)
+            quest_mst(qdb, old_path, [lang], is_global)
+            scene_mst(qdb, old_path, [lang], utage_in, is_global)
+    else:
         qdb = db.QuestDB(dir_in)
-    quest_mst(qdb, old_path, languages)
+        quest_mst(qdb, old_path, languages, is_global)
-    scene_mst(qdb, old_path, dir_in, languages)
+        scene_mst(qdb, old_path, languages, utage_in, is_global)
 
 
-def scene_mst(qdb, old_path, dir_in, languages):
+def scene_mst(qdb, old_path, languages, utage_in, is_global):
     scenes = qdb.get_scenes()
 
-    locations = load_locations(dir_in)
+    locations = load_locations(utage_in)
 
     for id, scene in scenes.items():
         i = 0
@@ -26,14 +35,15 @@ def scene_mst(qdb, old_path, dir_in, languages):
             except KeyError as exc:
                 i += 1
                 if i == len(scene["Parts"]):
-                    raise KeyError(
+                    if is_global:
-                        f"Could not find folder for scene {id}"
+                        break
-                    ) from exc
+                    raise KeyError(f"Could not find folder for scene {id}") from exc
                 continue
             break
 
+    if not is_global:
         with io.open(
-        os.path.join(old_path, "XduScene.json"), "w", newline="\n"
+            os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8"
         ) as json_file:
             json.dump(
                 scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True
@@ -41,44 +51,42 @@ def scene_mst(qdb, old_path, dir_in, languages):
 
     for lang in languages:
         out_dict = {}
-        if lang == "jpn":
         for key, value in scenes.items():
+            name = ""
+            summary = ""
+            credit = ""
+            enabled = False
+
+            if lang == "jpn" or is_global:
+                name = value["Name"]
+                summary = value["SummaryText"]
+                credit = "POKELABO"
+
             out_dict[key] = {
-                    "Name": value["Name"],
+                "Name": name,
-                    "SummaryText": value["SummaryText"],
+                "SummaryText": summary,
-                    "Credits": "POKELABO",
+                "Credits": credit,
-                    "Enabled": False,
+                "Enabled": enabled,
-                }
-        else:
-            for key in scenes.keys():
-                out_dict[key] = {
-                    "Name": "",
-                    "SummaryText": "",
-                    "Credits": "",
-                    "Enabled": False,
             }
 
         langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json")
         if os.path.isfile(langfile):
-            with open(langfile, "r") as lang_file:
+            with open(langfile, "r", encoding="utf-8") as lang_file:
                 lang_dict = json.load(lang_file)
                 out_dict.update(lang_dict)
 
-        with io.open(langfile, "w", newline="\n") as lang_file:
+        with io.open(langfile, "w", encoding="utf-8") as lang_file:
             json.dump(
-                out_dict,
+                out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
-                lang_file,
-                ensure_ascii=False,
-                indent="\t",
-                sort_keys=True,
             )
 
 
-def quest_mst(qdb, old_path, languages):
+def quest_mst(qdb, old_path, languages, is_global):
     quests = qdb.get_quests()
 
+    if not is_global:
         with io.open(
-        os.path.join(old_path, "XduQuest.json"), "w", newline="\n"
+            os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8"
         ) as json_file:
             json.dump(
                 quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True
@@ -86,7 +94,7 @@ def quest_mst(qdb, old_path, languages):
 
     for lang in languages:
         out_dict = {}
-        if lang == "jpn":
+        if lang == "jpn" or is_global:
             for key, value in quests.items():
                 out_dict[key] = {"Name": value["Name"], "Enabled": False}
         else:
@@ -95,41 +103,40 @@ def quest_mst(qdb, old_path, languages):
 
         langfile = os.path.join(old_path, f"XduQuestNames_{lang}.json")
         if os.path.isfile(langfile):
-            with open(langfile, "r") as lang_file:
+            with open(langfile, "r", encoding="utf-8") as lang_file:
                 lang_dict = json.load(lang_file)
                 out_dict.update(lang_dict)
 
-        with io.open(langfile, "w", newline="\n") as lang_file:
+        with io.open(langfile, "w", encoding="utf-8") as lang_file:
             json.dump(
-                out_dict,
+                out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
-                lang_file,
-                ensure_ascii=False,
-                indent="\t",
-                sort_keys=True,
             )
 
     return quests
 
 
-def load_locations(dir_in):
+def load_locations(utage_in):
     locations = {}
-    settings = glob(
+    settings = glob("{}/**/Settings/Scenario.tsv".format(utage_in), recursive=True,)
-        os.path.join(dir_in, "Common/Asset/Utage/**/Settings/Scenario.tsv"),
-        recursive=True,
-    )
 
     if len(settings) == 0:
         raise FileNotFoundError("Ya didn't decrypt the files dummy")
 
     for scenario in settings:
-        with open(scenario, "r") as tsv_file:
+        with open(scenario, "r", encoding="utf-8") as tsv_file:
             tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
             for row in tsv:
                 tokens = row["FileName"].split("/")
                 if len(tokens) < 3:
                     continue
-                if tokens[2] in locations:
+                folder, _scenario, part_id = tokens
-                    raise ValueError("Conflicting paths for file {tokens[2]}")
+                # side03 seems to only contain prototype scripts so we ignore it
-                locations[tokens[2]] = tokens[0]
+                if folder == "side03":
+                    continue
+                if part_id in locations:
+                    raise ValueError(
+                        f"Conflicting paths for scene part {part_id}, existing folder: {locations[part_id]}"
+                    )
+                locations[part_id] = folder
 
     return locations

divatool.py

@@ -9,12 +9,22 @@ import utage
 import db
 import xdudata
 
-LANGUAGES = ["jpn", "eng", "rus"]
+LANGUAGES = ["jpn", "eng", "rus", "cze"]
 
 
 def main():
     parser = argparse.ArgumentParser(description="Unpack and process XDU data")
 
+    global_flag_parser = argparse.ArgumentParser(add_help=False)
+    global_flag_parser.add_argument(
+        "-g",
+        "--global",
+        help="Whether we are running on Global XDU.",
+        action="store_true",
+        dest="is_global",
+        default=False,
+    )
+
     subparsers = parser.add_subparsers(
         metavar="<command>", title="subcommands", dest="command"
     )
@@ -81,7 +91,9 @@ def main():
     parser_utage_crypt.add_argument("TARGET", help="Input", type=str)
     # utage translate
     parser_utage_translate = parser_utage_subparsers.add_parser(
-        "translate", help="Generate keyed tsv and json files"
+        "translate",
+        help="Generate keyed tsv and json files",
+        parents=[global_flag_parser],
     )
     parser_utage_translate.add_argument("INPUT", help="Input", type=str)
     parser_utage_translate.add_argument(
@@ -92,7 +104,7 @@ def main():
     )
     # utage names
     parser_utage_names = parser_utage_subparsers.add_parser(
-        "names", help="Generate and update name files"
+        "names", help="Generate and update name files", parents=[global_flag_parser]
     )
     parser_utage_names.add_argument("INPUT", help="Input", type=str)
     parser_utage_names.add_argument(
@@ -105,13 +117,16 @@ def main():
         metavar="<command>", title="subcommand", dest="subcommand"
     )
     parser_diva_quest = parser_diva_subparsers.add_parser(
-        "quest", help="Generate quest JSON files"
+        "quest", help="Generate quest JSON files", parents=[global_flag_parser],
     )
     parser_diva_quest.add_argument(
         "INPUT",
         help="Directory of databases containing Quest*.db and ResourceEntry.db",
         type=str,
     )
+    parser_diva_quest.add_argument(
+        "--utage", help="Directory of Utage folders and files.", type=str, default=None,
+    )
     parser_diva_quest.add_argument(
         "OLD", help="Directory with old quest files", nargs="?", type=str, default="."
     )
@@ -152,23 +167,42 @@ def main():
                 raise FileNotFoundError("Could not find {}".format(args.TARGET))
         elif args.subcommand == "translate":
             if os.path.isfile(args.INPUT):
-                utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR)
+                utage.translate.translate_file(
+                    args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global
+                )
             elif os.path.isdir(args.INPUT):
-                utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR)
+                utage.translate.translate_dir(
+                    args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global
+                )
             else:
                 raise FileNotFoundError("Could not find {}".format(args.INPUT))
         elif args.subcommand == "names":
             if os.path.isdir(args.INPUT):
-                utage.names.update_names(args.INPUT, args.OLD, LANGUAGES)
+                utage.names.update_names(
+                    args.INPUT, args.OLD, LANGUAGES, args.is_global
+                )
             else:
                 raise FileNotFoundError(args.INPUT)
     elif args.command == "diva":
         if args.subcommand == "quest":
-            diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES)
+            if not args.utage:
+                diva.quest.update_missions(
+                    args.INPUT, args.OLD, LANGUAGES, args.is_global
+                )
+            else:
+                diva.quest.update_missions(
+                    args.INPUT, args.OLD, LANGUAGES, args.is_global, utage_in=args.utage
+                )
     elif args.command == "xdudata":
         if args.subcommand == "update":
+            # add the tools folder to PATH
+            os.environ["PATH"] += os.pathsep + os.path.join(
+                os.path.abspath(os.path.dirname(__file__)), "tools"
+            )
+
+            # update the data
             xdudata.update_all(
-                args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES
+                args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES,
             )
 
     return 0

utage/names.py

@@ -9,51 +9,61 @@ from glob import glob
 from multiprocessing import Pool
 
 
-def update_names(dir_in, old_path, languages):
+def update_names(dir_in, old_path, languages, is_global):
-    names = extract_names(dir_in)
+    names = extract_names(dir_in, is_global)
+
+    global_languages = {"enm": 1, "zho": 2, "kor": 3}
+    if is_global:
+        languages = global_languages.keys()
 
     # we have to update japanese like the rest because CustomData exists
     # and POKELABO deleted the wedding gear event stuff from the game files
     for l in languages:
         if l == "jpn":
             out_dict = dict(zip(names, names))
+        elif l in global_languages.keys():
+            out_dict = {}
+            for name in names:
+                spl = name.split("\t")
+                idx = global_languages[l]
+                val = spl[idx] if spl[idx] != "None" else ""
+                out_dict[spl[0]] = val
         else:
             out_dict = dict.fromkeys(names, "")
 
         langfile = os.path.join(old_path, "nametranslations_{}.json".format(l))
         if os.path.isfile(langfile):
-            with open(langfile, "r") as lang_file:
+            with open(langfile, "r", encoding="utf-8") as lang_file:
                 lang_dict = json.load(lang_file)
             out_dict.update(lang_dict)
 
         with io.open(
-            langfile, "w", newline="\n"
+            langfile, "w", encoding="utf-8"
         ) as lang_file:  # you're using git right
             json.dump(
-                out_dict,
+                out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
-                lang_file,
-                ensure_ascii=False,
-                indent="\t",
-                sort_keys=True,
             )
 
 
-def extract_names(dir_in):
+def extract_names(dir_in, is_global):
     char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv")
 
     if not os.path.isfile(char_tsv_path):
         raise FileNotFoundError("Character.tsv not found")
 
-    names, sets = read_char_tsv(char_tsv_path)
+    names, sets = read_char_tsv(char_tsv_path, is_global)
 
     files = [
         f
         for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
-        if re.search(r"/[0-9]{9}\.tsv$", f)
+        if re.search(r"[0-9]{9}\.tsv$", f)
+        and f"{os.path.sep}Diva{os.path.sep}" not in f
     ]
     if len(files) == 0:
         raise FileNotFoundError("No valid files found in directory: " + dir_in)
 
+    # TODO unsupported on global for now
+    if not is_global:
         # tfw gil
         # it's still faster on my machine so i'm keeping it
         read_partial = partial(read_mission, char_names=names, char_sets=sets)
@@ -65,11 +75,11 @@ def extract_names(dir_in):
     return names
 
 
-def read_char_tsv(char_tsv_path):
+def read_char_tsv(char_tsv_path, is_global):
     char_names = set()
     char_sets = set()
 
-    with open(char_tsv_path, "r") as char_tsv_file:
+    with open(char_tsv_path, "r", encoding="utf-8") as char_tsv_file:
         char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"')
 
         for row in char_tsv:
@@ -81,6 +91,11 @@ def read_char_tsv(char_tsv_path):
                 and row["CharacterName"].strip()
                 and row["NameText"].strip()
             ):
+                if is_global:
+                    char_names.add(
+                        f"{row['NameText']}\t{row['CharaEnglish']}\t{row['CharaChinese']}\t{row['CharaKorean']}"
+                    )
+                else:
                     char_names.add(row["NameText"])
                 char_sets.add(row["CharacterName"])
 
@@ -89,7 +104,7 @@ def read_char_tsv(char_tsv_path):
 
 def read_mission(tsv_path, char_names, char_sets):
     new_names = set()
-    with open(tsv_path, "r") as tsv_file:
+    with open(tsv_path, "r", encoding="utf-8") as tsv_file:
         tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
         if (
             ("Arg1" not in tsv.fieldnames)
@@ -101,8 +116,6 @@ def read_mission(tsv_path, char_names, char_sets):
             if row["Command"] and row["Command"].startswith("//"):
                 continue
             if row["Text"] and row["Arg1"]:
-                if (row["Arg1"] not in char_names) and (
+                if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets):
-                    row["Arg1"] not in char_sets
-                ):
                     new_names.add(row["Arg1"])
     return new_names

utage/translate.py

@@ -4,32 +4,34 @@ import io
 import json
 import os
 import traceback
+from contextlib import suppress
 
 from functools import partial
 from glob import glob
 from multiprocessing.pool import Pool
 
 
-def translate_dir(dir_in, tsv_out_dir, json_out_dir):
+def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global):
     files = [
         f
-        for f in glob(
+        for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True)
-            os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True
+        if f"{os.path.sep}Diva{os.path.sep}" not in f
-        )
-        if "{os.path.sep}Diva{os.path.sep}" not in f
     ]
 
     if len(files) == 0:
         raise FileNotFoundError("No valid files found in directory: " + dir_in)
 
     ptrans = partial(
-        translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir
+        translate_file,
+        tsv_out_dir=tsv_out_dir,
+        json_out_dir=json_out_dir,
+        is_global=is_global,
     )
     with Pool() as p:
         p.map(ptrans, files)
 
 
-def translate_file(file_in, tsv_out_dir, json_out_dir):
+def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
     try:
         if not file_in.endswith(".tsv"):
             raise ValueError("Invalid File Type for {}".format(file_in))
@@ -48,11 +50,17 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
         tsv_output_path = os.path.join(
             tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num)
         )
-        json_output_path = os.path.join(
+        languages = ["jpn"]
+        if is_global:
+            languages = ["enm", "zho", "kor"]
+        json_output_paths = {
+            lang: os.path.join(
                 json_out_dir,
                 event_folder,
-            "{}_translations_jpn.json".format(id_num),
+                "{}_translations_{}.json".format(id_num, lang),
             )
+            for lang in languages
+        }
 
         # need to create output paths and avoid races when threading
         if not os.path.exists(os.path.dirname(tsv_output_path)):
@@ -62,35 +70,42 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
                 if e.errno != errno.EEXIST:
                     raise
 
-        if not os.path.exists(os.path.dirname(json_output_path)):
+        if not os.path.exists(os.path.dirname(json_output_paths[languages[0]])):
             try:
-                os.makedirs(os.path.dirname(json_output_path))
+                os.makedirs(os.path.dirname(json_output_paths[languages[0]]))
             except OSError as e:
                 if e.errno != errno.EEXIST:
                     raise
 
-        with open(file_in, "r") as tsv_file:
+        with open(file_in, "r", encoding="utf-8") as tsv_file:
-            tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
+            tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE)
-            tsv_keyed, json_str = process_tsv(tsv, id_num)
+
+            t_fieldnames = tsv.fieldnames
+            if is_global:
+                t_fieldnames = tsv.fieldnames[: tsv.fieldnames.index("English") + 1]
+
+            tsv_keyed, json_str = process_tsv(tsv, id_num, is_global)
 
             # csv handles newlines, don't set it in io.open
-            with io.open(tsv_output_path, "w", newline="") as tsv_out:
+            with io.open(tsv_output_path, "w", newline="", encoding="utf-8") as tsv_out:
                 writer = csv.DictWriter(
                     tsv_out,
                     delimiter="\t",
                     quotechar='"',
-                    lineterminator="\n",
+                    fieldnames=t_fieldnames,
-                    fieldnames=tsv.fieldnames,
                     extrasaction="ignore",
                 )
                 writer.writeheader()
                 for row in tsv_keyed:
                     writer.writerow(row)
 
-        with io.open(json_output_path, "w", newline="\n") as json_out:
+        for lang in languages:
+            if not json_str[lang]:
+                continue
+            with io.open(json_output_paths[lang], "w", encoding="utf-8") as json_out:
                 # we don't want to sort these because they're _1, ..., _10, etc
                 json.dump(
-                json_str,
+                    json_str[lang],
                     json_out,
                     ensure_ascii=False,
                     indent="\t",
@@ -102,10 +117,14 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
         print("Error processing file: " + file_in)
 
 
-def process_tsv(tsv, id_num):
+def process_tsv(tsv, id_num, is_global):
     i = 0
     tsv_keyed = []
-    key_dict = {}
+    key_dict = {"jpn": {}}
+    if is_global:
+        key_dict["enm"] = {}
+        key_dict["zho"] = {}
+        key_dict["kor"] = {}
 
     for row in tsv:
         try:
@@ -115,8 +134,14 @@ def process_tsv(tsv, id_num):
             if row["Text"] and row["Text"].strip():
                 key = "{}_{}".format(id_num, i)
                 i += 1
+                key_dict["jpn"][key] = row["Text"]
+                if is_global:
+                    with suppress(KeyError):
+                        if row["English"]:
+                            key_dict["enm"][key] = row["English"]
+                        key_dict["zho"][key] = row["Chinese"]
+                        key_dict["kor"][key] = row["Korean"]
                 row["English"] = key
-                key_dict[key] = row["Text"]
             tsv_keyed.append(row)
         except Exception:
             traceback.print_exc()

xdudata/asset_extract.py

@@ -0,0 +1,303 @@
+import functools
+import multiprocessing
+import os
+import shutil
+import subprocess
+from contextlib import contextmanager
+from distutils.dir_util import copy_tree
+from glob import glob
+
+from adx import extract_loop_data_from_dir
+
+OPERATING_SYSTEM = os.name
+CHUNK_SIZE = 100
+
+
+def process_se(cache_dir, extract_dir, update_dir):
+    copy_tree(
+        os.path.join(cache_dir, "Android/Asset/Sound/Se"),
+        os.path.join(extract_dir, "Se"),
+    )
+
+    from_dir = "Se"
+    to_dir = "Se_Extracted"
+    acb_prefix = "_vgmt_acb_ext"
+
+    pool = get_pool()
+
+    with change_cwd(extract_dir):
+        os.makedirs(to_dir, exist_ok=True)
+
+        to_del = glob("{}/**/*.wav".format(from_dir), recursive=True)
+        to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True))
+        for file in to_del:
+            os.remove(file)
+
+        acb_files = [file for file in glob("{}/*.acb".format(from_dir))]
+        acb_cmd = get_dotnet_cmd("acb_extract.exe")
+        run_program(acb_files, cmd=acb_cmd)
+
+        hca_files = [
+            file for file in glob("{}/**/*.hca".format(from_dir), recursive=True)
+        ]
+        for hca_sublist in split_list(hca_files, CHUNK_SIZE):
+            run_program(hca_sublist, cmd="clHCA")
+
+        basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files]
+        char_secs = [basename.split("_", maxsplit=1) for basename in basenames]
+        for cs in char_secs:
+            char = cs[0]
+            sec = str(cs[1]) if len(cs) == 2 else ""
+            os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True)
+            wav_files = glob(
+                "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)),
+                recursive=True,
+            )
+            partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg")
+            wav_cmds = [
+                [
+                    "-i",
+                    os.path.abspath(file),
+                    "-acodec",
+                    "libopus",
+                    "-ab",
+                    "192k",
+                    "-af",
+                    "aresample=48000",
+                    os.path.abspath(
+                        "{}/{}/{}/{}.opus".format(
+                            to_dir,
+                            char,
+                            sec,
+                            os.path.splitext(os.path.basename(file))[0],
+                        )
+                    ),
+                    "-n",
+                ]
+                for file in wav_files
+            ]
+            pool.map(partial_ffmpeg_cmd, wav_cmds)
+
+    copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Se"))
+
+
+def process_bgm(cache_dir, extract_dir, update_dir):
+    copy_tree(
+        os.path.join(cache_dir, "Android/Asset/Sound/Bgm"),
+        os.path.join(extract_dir, "Bgm"),
+    )
+
+    from_dir = "Bgm"
+    to_dir = "Bgm_Extracted"
+    loop_file = "loop.json"
+
+    pool = get_pool()
+
+    with change_cwd(extract_dir):
+        os.makedirs(to_dir, exist_ok=True)
+
+        to_del = glob("{}/**/*.adx".format(from_dir), recursive=True)
+        for file in to_del:
+            os.remove(file)
+
+        acb_files = [file for file in glob("{}/*.acb".format(from_dir))]
+        acb_cmd = get_dotnet_cmd("acb_extract.exe")
+        run_program(acb_files, cmd=acb_cmd)
+
+        adx_files = [
+            file for file in glob("{}/**/*.adx".format(from_dir), recursive=True)
+        ]
+        partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg")
+        wav_cmds = [
+            [
+                "-i",
+                os.path.abspath(file),
+                "-acodec",
+                "libopus",
+                "-ab",
+                "192k",
+                "-af",
+                "aresample=48000",
+                os.path.abspath(
+                    "{}/{}.opus".format(
+                        to_dir, os.path.splitext(os.path.basename(file))[0],
+                    )
+                ),
+                "-n",
+            ]
+            for file in adx_files
+        ]
+        pool.map(partial_ffmpeg_cmd, wav_cmds)
+
+        extract_loop_data_from_dir(from_dir, loop_file)
+        shutil.copy2(loop_file, os.path.join(to_dir, "BgmLoop.json"))
+
+    copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Bgm"))
+
+
+def process_voice(cache_dir, extract_dir, update_dir):
+    copy_tree(
+        os.path.join(cache_dir, "Android/Asset/Sound/Voice"),
+        os.path.join(extract_dir, "Voice"),
+    )
+
+    from_dir = "Voice"
+    to_dir = "Voice_Extracted"
+    acb_prefix = "_vgmt_acb_ext"
+
+    pool = get_pool()
+
+    with change_cwd(extract_dir):
+        os.makedirs(to_dir, exist_ok=True)
+
+        to_del = glob("{}/**/*.wav".format(from_dir), recursive=True)
+        to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True))
+        for file in to_del:
+            os.remove(file)
+
+        acb_files = [file for file in glob("{}/*.acb".format(from_dir))]
+        acb_cmd = get_dotnet_cmd("acb_extract.exe")
+        run_program(acb_files, cmd=acb_cmd)
+
+        hca_files = [
+            file for file in glob("{}/**/*.hca".format(from_dir), recursive=True)
+        ]
+        for hca_sublist in split_list(hca_files, CHUNK_SIZE):
+            run_program(hca_sublist, cmd="clHCA")
+
+        basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files]
+        char_secs = [basename.split("_", maxsplit=1) for basename in basenames]
+        for cs in char_secs:
+            char = cs[0]
+            sec = str(cs[1]) if len(cs) == 2 else ""
+            os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True)
+            wav_files = glob(
+                "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)),
+                recursive=True,
+            )
+            partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg")
+
+            wav_cmds = [
+                [
+                    "-i",
+                    os.path.abspath(file),
+                    "-acodec",
+                    "libopus",
+                    "-ab",
+                    "192k",
+                    "-af",
+                    "aresample=48000",
+                    os.path.abspath(
+                        "{}/{}/{}/{}.opus".format(
+                            to_dir,
+                            char,
+                            sec,
+                            os.path.splitext(os.path.basename(file))[0],
+                        )
+                    ),
+                    "-n",
+                ]
+                for file in wav_files
+            ]
+            pool.map(partial_ffmpeg_cmd, wav_cmds)
+
+    copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Voice"))
+
+
+def process_movie(cache_dir, extract_dir, update_dir):
+    copy_tree(
+        os.path.join(cache_dir, "Common/Asset/Movie"),
+        os.path.join(extract_dir, "Movie"),
+    )
+
+    from_dir = "Movie"
+    to_dir = "Movie_Extracted"
+
+    pool = get_pool()
+
+    with change_cwd(extract_dir):
+        os.makedirs(to_dir, exist_ok=True)
+
+        to_del = glob("{}/**/*.adx".format(from_dir), recursive=True)
+        to_del.extend(glob("{}/**/*.m2v".format(from_dir), recursive=True))
+        for file in to_del:
+            os.remove(file)
+
+        movie_paths = [
+            os.path.abspath(os.path.splitext(file)[0])
+            for file in glob("{}/*.usm".format(from_dir))
+            if not os.path.basename(file).startswith("mini_radio")
+        ]
+
+        usm_files = [[path + ".usm"] for path in movie_paths]
+        usm_cmd = get_dotnet_cmd("usm_extract.exe")
+        partial_usm_cmd = functools.partial(run_program, cmd=usm_cmd)
+        pool.map(partial_usm_cmd, usm_files)
+
+        partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg")
+        mux_cmds = [
+            [
+                "-i",
+                path + ".adx",
+                "-i",
+                path + ".m2v",
+                "-c:a",
+                "libopus",
+                "-b:a",
+                "196k",
+                "-af",
+                "aresample=48000",
+                "-c:v",
+                "libvpx",
+                "-b:v",
+                "1M",
+                "-threads",
+                "1",
+                os.path.abspath("{}/{}.webm".format(to_dir, os.path.basename(path))),
+                "-n",
+            ]
+            for path in movie_paths
+        ]
+        pool.map(partial_ffmpeg_cmd, mux_cmds)
+
+    copy_tree(
+        os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Asset", "Movie")
+    )
+
+
+@contextmanager
+def change_cwd(new_path):
+    old_cwd = os.getcwd()
+    os.chdir(new_path)
+    try:
+        yield
+    finally:
+        os.chdir(old_cwd)
+
+
+def get_dotnet_cmd(name):
+    ext = os.path.splitext(name)[1]
+    if ext != ".exe":
+        raise RuntimeError("This doesn't seem like a .NET program.")
+    path = shutil.which(name)
+    if OPERATING_SYSTEM == "nt":
+        return [path]
+    return ["mono", path]
+
+
+def run_program(args, cmd=None):
+    if isinstance(cmd, str):
+        cmd = [cmd]
+    full_cmd = cmd + args
+    subprocess.run(full_cmd)
+
+
+def get_pool(size=multiprocessing.cpu_count()):
+    pool = multiprocessing.Pool(size)
+    return pool
+
+
+def split_list(input_list, chunk_size):
+    return [
+        input_list[i : i + chunk_size] for i in range(0, len(input_list), chunk_size)
+    ]

xdudata/update.py

@@ -1,90 +1,53 @@
+from contextlib import suppress
+
 import diva
+import shutil
 import os
-import subprocess
 import tempfile
 import utage
 
 from distutils.dir_util import copy_tree, remove_tree
 from glob import glob
 
+from xdudata.asset_extract import process_se, process_bgm, process_voice, process_movie
+
+
+def update_all(
+    cache_dir, update_dir, translation_dir, extract_dir, languages,
+):
+    print("Cleaning cache")
+    cleanup_cache(cache_dir)
 
-def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages):
     print("Copying images")
     copy_images(cache_dir, update_dir)
 
     print("Processing TSV files")
-    update_tsv(cache_dir, update_dir, translation_dir, languages)
+    update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages)
 
     print("Processing Quest Json files")
-    diva.quest.update_missions(cache_dir, translation_dir, languages)
+    diva.quest.update_missions(extract_dir, translation_dir, languages, False)
 
-    # this code isn't going to run on windows because i need to rewrite it
-    # it just calls my zsh scripts and generates it that way, need to replace them
-    # with real python or something
     print("Processing Sound Effects")
-    copy_tree(
+    process_se(cache_dir, extract_dir, update_dir)
-        os.path.join(cache_dir, "Android/Asset/Sound/Se"),
-        os.path.join(extract_dir, "Se"),
-    )
-    subprocess.run("./update_se.zsh", shell=True, cwd=extract_dir)
-    copy_tree(os.path.join(extract_dir, "se"), os.path.join(update_dir, "Se"))
 
-    print("Processing BGM - don't forget to copy that loop file")
+    print("Processing BGM")
-    copy_tree(
+    process_bgm(cache_dir, extract_dir, update_dir)
-        os.path.join(cache_dir, "Android/Asset/Sound/Bgm"),
-        os.path.join(extract_dir, "Bgm"),
-    )
-    subprocess.run("./update_bgm.zsh", shell=True, cwd=extract_dir)
-    copy_tree(
-        os.path.join(extract_dir, "bgm"), os.path.join(update_dir, "Bgm")
-    )
 
     print("Processing voices")
-    copy_tree(
+    process_voice(cache_dir, extract_dir, update_dir)
-        os.path.join(cache_dir, "Android/Asset/Sound/Voice"),
-        os.path.join(extract_dir, "Voice"),
-    )
-    subprocess.run("./update_voice.zsh", shell=True, cwd=extract_dir)
-    copy_tree(
-        os.path.join(extract_dir, "voice"), os.path.join(update_dir, "Voice")
-    )
 
     print("Processing movies - this might take forever")
-    copy_tree(
+    process_movie(cache_dir, extract_dir, update_dir)
-        os.path.join(cache_dir, "Common/Asset/Movie"),
-        os.path.join(extract_dir, "Movie"),
-    )
-    subprocess.run("./update_movie.zsh", shell=True, cwd=extract_dir)
-    copy_tree(
-        os.path.join(extract_dir, "movie"),
-        os.path.join(update_dir, "Asset/Movie"),
-    )
 
 
-def update_tsv(cache_dir, update_dir, translation_dir, languages):
+def cleanup_cache(cache_dir):
-    utage_tmp = tempfile.mkdtemp()
+    with suppress(FileNotFoundError):
-    copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp)
+        shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32"))
-    utage.crypt.crypt_dir(
+        os.remove(
-        utage_tmp, bytearray("SampleSecretKey", "utf-8"), False, False
+            os.path.join(
+                cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage"
             )
-    for encrypted in glob(
-        os.path.join(utage_tmp, "**/*.utage"), recursive=True
-    ):
-        try:
-            os.remove(encrypted)
-        except:
-            pass
-
-    utage.translate.translate_dir(
-        utage_tmp,
-        os.path.join(update_dir, "Utage"),
-        os.path.join(translation_dir, "Missions"),
         )
-    utage.names.update_names(utage_tmp, translation_dir, languages)
-    copy_tree(
-        os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")
-    )
-    remove_tree(utage_tmp)
 
 
 def copy_images(cache_dir, update_dir):
@@ -93,6 +56,35 @@ def copy_images(cache_dir, update_dir):
         os.path.join(update_dir, "Asset/Image"),
     )
     copy_tree(
-        os.path.join(cache_dir, "Common/Sample"),
+        os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"),
-        os.path.join(update_dir, "Sample"),
     )
+
+
+def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages):
+    utage_tmp = tempfile.mkdtemp()
+    copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp)
+    utage.crypt.crypt_dir(
+        utage_tmp, bytearray("SampleSecretKey", "utf-8"), False, False
+    )
+    for encrypted in glob(os.path.join(utage_tmp, "**/*.utage"), recursive=True):
+        try:
+            os.remove(encrypted)
+        except:
+            pass
+
+    # copy over files for the quest updater
+    copy_tree(utage_tmp, os.path.join(extract_dir, "Common/Asset/Utage"))
+    shutil.copy2(os.path.join(cache_dir, "QuestMst.db"), extract_dir)
+    shutil.copy2(os.path.join(cache_dir, "QuestSceneMst.db"), extract_dir)
+    shutil.copy2(os.path.join(cache_dir, "QuestPartMst.db"), extract_dir)
+    shutil.copy2(os.path.join(cache_dir, "ResourceEntry.db"), extract_dir)
+
+    utage.translate.translate_dir(
+        utage_tmp,
+        os.path.join(update_dir, "Utage"),
+        os.path.join(translation_dir, "Missions"),
+        False,
+    )
+    utage.names.update_names(utage_tmp, translation_dir, languages, False)
+    copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva"))
+    remove_tree(utage_tmp)