Update README with a basic example

.gitignore

@@ -3,6 +3,6 @@ __pycache__/
 *$py.class
 *.swp
 virtualenv/
-tools/*
-!tools/readme.txt
 .idea/
+tools/*
+!tools/.gitkeep

README.md

@@ -0,0 +1,36 @@
+# divatool
+
+A tool for that one game everyone hates but works on anyway.
+
+## Requirements
+
+Divatool is written in and uses [Python 3](https://www.python.org/downloads/).
+
+There are also several third party dependencies not provided as part of this repository,
+mostly used for the `xdudata` command. You need to have the binaries of the following tools
+either in your `$PATH` variable or in the [`tools`](./tools) folder. These tools are:
+
+* [clHCA](https://github.com/KinoMyu/FastHCADecoder)
+* acb_extract
+* usm_extract
+* [ffmpeg](https://www.ffmpeg.org/download.html)
+* [mono](https://www.mono-project.com/download/stable/) (if using Linux)
+
+Make sure these executables are called the same way on your machine.
+
+## Usage
+
+TODO: add detailed descriptions for all parameters.
+
+Using the tool to update XDUData for the player:
+
+```
+divatool.py xdudata update [path to the 'files' folder from the game] [path to the XDUData folder for the player] [path to the 'xdutranslations' repo folder] [path to the temp files folder]
+```
+
+Example:
+```
+divatool.py xdudata update /home/user/gameapp/files /home/user/player/XDUData /home/user/git/xdutranslations /var/tmp
+```
+
+Note that the 'files' folder from the app must contain the db files for this to work.

adx/adx_parser.py

@@ -74,7 +74,7 @@ class adx_file(object):
         self.encrypted = self.get_val(CRYPT_OFF, CRYPT_LEN)
         self.d_offset = self.get_val(DATA_OFF_OFF, DATA_OFF_LEN)
         if self.d_offset < DATA_OFF_MIN:
-            return None
+            return
         self.l_exists = 1
         # now load things for math
         self.sam_rate = self.get_val(SRATE_OFF, SRATE_LEN)
@@ -106,13 +106,11 @@ class adx_file(object):
             return None
         if self.l_start > self.sam_count or self.l_end > self.sam_count:
             raise ValueError("Invalid Loop Data")
-            return None
         if self.l_style not in LOOP_TYPE_SUP:
             raise NotImplementedError(
                 "Loop Style {} Not Implemented".format(str(self.l_style))
             )
-            return None
-        ret = {}
+        ret = dict()
         ret["duration"] = self.sam_count / self.sam_rate
         ret["loop_start"] = {}
         ret["loop_start"]["seconds"] = self.l_start / self.sam_rate
@@ -123,7 +121,5 @@ class adx_file(object):
         ret["loop_end"] = {}
         ret["loop_end"]["seconds"] = self.l_end / self.sam_rate
         ret["loop_end"]["samples_native"] = self.l_end
-        ret["loop_end"]["samples_48k"] = dumb_round(
-            self.l_end / self.sam_rate * 48000
-        )
+        ret["loop_end"]["samples_48k"] = dumb_round(self.l_end / self.sam_rate * 48000)
         return ret

db/base.py

@@ -23,7 +23,7 @@ class BaseDB(object):
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
-        self.quest_conn.close()
+        self.conn.close()
 
     def attach_dbs(self, databases, path):
         for d in databases:

db/quest.py

@@ -1,16 +1,24 @@
 from .base import BaseDB
 
-DATABASES = {
-    "QuestMst.db": "Quest",
-    "QuestSceneMst.db": "QuestScene",
-    "QuestPartMst.db": "QuestPart",
-    "ResourceEntry.db": "ResourceEntry",
-}
+
+def get_dbs_by_lang(language=None):
+    databases = {
+        "QuestMst": "Quest",
+        "QuestSceneMst": "QuestScene",
+        "QuestPartMst": "QuestPart",
+        "ResourceEntry": "ResourceEntry",
+    }
+    for key in list(databases):
+        if not language:
+            databases[key + ".db"] = databases.pop(key)
+        else:
+            databases[key + "_" + language + ".db"] = databases.pop(key)
+    return databases
 
 
 class QuestDB(BaseDB):
-    def __init__(self, path):
-        super().__init__(path, DATABASES)
+    def __init__(self, path, language=None):
+        super().__init__(path, get_dbs_by_lang(language))
 
     def get_quests(self):
         self.cursor.execute(
@@ -44,12 +52,14 @@ class QuestDB(BaseDB):
                 questSceneMstId,
                 name,
                 summaryText,
+                img,
                 group_concat(partIds) as parts
             FROM (
                     SELECT
                         qs.questSceneMstId,
                         qs.name,
                         qs.summaryText,
+                        qs.img,
                         CASE
                             WHEN qp.afterTalkName == '' AND qp.beforeTalkName == '' THEN NULL
                             WHEN qp.afterTalkName == '' AND qp.beforeTalkName != '' THEN qp.beforeTalkName
@@ -66,9 +76,11 @@ class QuestDB(BaseDB):
         )
         res = self.cursor.fetchall()
         scenes = {}
-        for scene_id, name, summary, parts in res:
+        for scene_id, name, summary, img, parts in res:
             scenes[str(scene_id)] = {}
             scenes[str(scene_id)]["Name"] = name
             scenes[str(scene_id)]["SummaryText"] = summary
+            if img:
+                scenes[str(scene_id)]["Image"] = img
             scenes[str(scene_id)]["Parts"] = parts.split(",")
         return scenes

diva/quest.py

@@ -7,16 +7,25 @@ import os
 from glob import glob
 
 
-def update_missions(dir_in, old_path, languages):
-    qdb = db.QuestDB(dir_in)
-    quest_mst(qdb, old_path, languages)
-    scene_mst(qdb, old_path, dir_in, languages)
+def update_missions(dir_in, old_path, languages, is_global, utage_in=None):
+    if not utage_in:
+        utage_in = os.path.join(dir_in, "Common/Asset/Utage")
+    if is_global:
+        languages = ["enm", "zho", "kor"]
+        for lang in languages:
+            qdb = db.QuestDB(dir_in, lang)
+            quest_mst(qdb, old_path, [lang], is_global)
+            scene_mst(qdb, old_path, [lang], utage_in, is_global)
+    else:
+        qdb = db.QuestDB(dir_in)
+        quest_mst(qdb, old_path, languages, is_global)
+        scene_mst(qdb, old_path, languages, utage_in, is_global)
 
 
-def scene_mst(qdb, old_path, dir_in, languages):
+def scene_mst(qdb, old_path, languages, utage_in, is_global):
     scenes = qdb.get_scenes()
 
-    locations = load_locations(dir_in)
+    locations = load_locations(utage_in)
 
     for id, scene in scenes.items():
         i = 0
@@ -26,57 +35,66 @@ def scene_mst(qdb, old_path, dir_in, languages):
             except KeyError as exc:
                 i += 1
                 if i == len(scene["Parts"]):
+                    if is_global:
+                        break
                     raise KeyError(f"Could not find folder for scene {id}") from exc
                 continue
             break
 
-    with io.open(
-        os.path.join(old_path, "XduScene.json"), "w", newline="\n"
-    ) as json_file:
-        json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
+    if not is_global:
+        with io.open(
+            os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8"
+        ) as json_file:
+            json.dump(
+                scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True
+            )
 
     for lang in languages:
         out_dict = {}
-        if lang == "jpn":
-            for key, value in scenes.items():
-                out_dict[key] = {
-                    "Name": value["Name"],
-                    "SummaryText": value["SummaryText"],
-                    "Credits": "POKELABO",
-                    "Enabled": False,
-                }
-        else:
-            for key in scenes.keys():
-                out_dict[key] = {
-                    "Name": "",
-                    "SummaryText": "",
-                    "Credits": "",
-                    "Enabled": False,
-                }
+        for key, value in scenes.items():
+            name = ""
+            summary = ""
+            credit = ""
+            enabled = False
+
+            if lang == "jpn" or is_global:
+                name = value["Name"]
+                summary = value["SummaryText"]
+                credit = "POKELABO"
+
+            out_dict[key] = {
+                "Name": name,
+                "SummaryText": summary,
+                "Credits": credit,
+                "Enabled": enabled,
+            }
 
         langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json")
         if os.path.isfile(langfile):
-            with open(langfile, "r") as lang_file:
+            with open(langfile, "r", encoding="utf-8") as lang_file:
                 lang_dict = json.load(lang_file)
                 out_dict.update(lang_dict)
 
-        with io.open(langfile, "w", newline="\n") as lang_file:
+        with io.open(langfile, "w", encoding="utf-8") as lang_file:
             json.dump(
                 out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
             )
 
 
-def quest_mst(qdb, old_path, languages):
+def quest_mst(qdb, old_path, languages, is_global):
     quests = qdb.get_quests()
 
-    with io.open(
-        os.path.join(old_path, "XduQuest.json"), "w", newline="\n"
-    ) as json_file:
-        json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
+    if not is_global:
+        with io.open(
+            os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8"
+        ) as json_file:
+            json.dump(
+                quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True
+            )
 
     for lang in languages:
         out_dict = {}
-        if lang == "jpn":
+        if lang == "jpn" or is_global:
             for key, value in quests.items():
                 out_dict[key] = {"Name": value["Name"], "Enabled": False}
         else:
@@ -85,11 +103,11 @@ def quest_mst(qdb, old_path, languages):
 
         langfile = os.path.join(old_path, f"XduQuestNames_{lang}.json")
         if os.path.isfile(langfile):
-            with open(langfile, "r") as lang_file:
+            with open(langfile, "r", encoding="utf-8") as lang_file:
                 lang_dict = json.load(lang_file)
                 out_dict.update(lang_dict)
 
-        with io.open(langfile, "w", newline="\n") as lang_file:
+        with io.open(langfile, "w", encoding="utf-8") as lang_file:
             json.dump(
                 out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
             )
@@ -97,25 +115,28 @@ def quest_mst(qdb, old_path, languages):
     return quests
 
 
-def load_locations(dir_in):
+def load_locations(utage_in):
     locations = {}
-    settings = glob(
-        os.path.join(dir_in, "Common/Asset/Utage/**/Settings/Scenario.tsv"),
-        recursive=True,
-    )
+    settings = glob("{}/**/Settings/Scenario.tsv".format(utage_in), recursive=True,)
 
     if len(settings) == 0:
         raise FileNotFoundError("Ya didn't decrypt the files dummy")
 
     for scenario in settings:
-        with open(scenario, "r") as tsv_file:
+        with open(scenario, "r", encoding="utf-8") as tsv_file:
             tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
             for row in tsv:
                 tokens = row["FileName"].split("/")
                 if len(tokens) < 3:
                     continue
-                if tokens[2] in locations:
-                    raise ValueError("Conflicting paths for file {tokens[2]}")
-                locations[tokens[2]] = tokens[0]
+                folder, _scenario, part_id = tokens
+                # side03 seems to only contain prototype scripts so we ignore it
+                if folder == "side03":
+                    continue
+                if part_id in locations:
+                    raise ValueError(
+                        f"Conflicting paths for scene part {part_id}, existing folder: {locations[part_id]}"
+                    )
+                locations[part_id] = folder
 
     return locations

divatool.py

@@ -9,7 +9,7 @@ import utage
 import db
 import xdudata
 
-LANGUAGES = ["jpn", "eng", "rus"]
+LANGUAGES = ["jpn", "eng", "rus", "cze"]
 
 
 def main():
@@ -117,13 +117,16 @@ def main():
         metavar="<command>", title="subcommand", dest="subcommand"
     )
     parser_diva_quest = parser_diva_subparsers.add_parser(
-        "quest", help="Generate quest JSON files"
+        "quest", help="Generate quest JSON files", parents=[global_flag_parser],
     )
     parser_diva_quest.add_argument(
         "INPUT",
         help="Directory of databases containing Quest*.db and ResourceEntry.db",
         type=str,
     )
+    parser_diva_quest.add_argument(
+        "--utage", help="Directory of Utage folders and files.", type=str, default=None,
+    )
     parser_diva_quest.add_argument(
         "OLD", help="Directory with old quest files", nargs="?", type=str, default="."
     )
@@ -182,17 +185,24 @@ def main():
                 raise FileNotFoundError(args.INPUT)
     elif args.command == "diva":
         if args.subcommand == "quest":
-            diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES)
+            if not args.utage:
+                diva.quest.update_missions(
+                    args.INPUT, args.OLD, LANGUAGES, args.is_global
+                )
+            else:
+                diva.quest.update_missions(
+                    args.INPUT, args.OLD, LANGUAGES, args.is_global, utage_in=args.utage
+                )
     elif args.command == "xdudata":
-        script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools"))
         if args.subcommand == "update":
+            # add the tools folder to PATH
+            os.environ["PATH"] += os.pathsep + os.path.join(
+                os.path.abspath(os.path.dirname(__file__)), "tools"
+            )
+
+            # update the data
             xdudata.update_all(
-                args.CACHE,
-                args.XDUDATA,
-                args.TRANS,
-                args.EXTRACT,
-                script_dir,
-                LANGUAGES,
+                args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES,
             )
 
     return 0

tools/readme.txt

@@ -1,34 +0,0 @@
-Various files need to be in this directory.
-If you don't have them, ask someone who does.
-
-The required files are:
-
-Binaries:
-- acb_extract.exe
-- usm_extract.exe
-- clHCA
-
-Z shell scripts:
-- update_bgm.zsh
-- update_movie.zsh
-- update_se.zsh
-- update_voice.zsh
-
-Python scripts:
-- adx_parse.py
-- bingo_fuck.py
-- chara_room_mission.py
-- custom_mission.py
-- event_bonus.py
-- orphan_quest.py
-- pad_2_csv.py
-- pad_2_json.py
-- pad_2_pad.py
-- pad_2_pad_louisstyle.py
-- pad_stat.py
-- pad_update.py
-- store_parser.py
-- tsv_2_pad.py
-- tsv_parse.py
-- txt_2_pad.py
-- utage_decrypt.py

utage/names.py

@@ -12,7 +12,7 @@ from multiprocessing import Pool
 def update_names(dir_in, old_path, languages, is_global):
     names = extract_names(dir_in, is_global)
 
-    global_languages = {"enm": 1, "zh": 2, "ko": 3}
+    global_languages = {"enm": 1, "zho": 2, "kor": 3}
     if is_global:
         languages = global_languages.keys()
 
@@ -33,12 +33,12 @@ def update_names(dir_in, old_path, languages, is_global):
 
         langfile = os.path.join(old_path, "nametranslations_{}.json".format(l))
         if os.path.isfile(langfile):
-            with open(langfile, "r") as lang_file:
+            with open(langfile, "r", encoding="utf-8") as lang_file:
                 lang_dict = json.load(lang_file)
             out_dict.update(lang_dict)
 
         with io.open(
-            langfile, "w", newline="\n"
+            langfile, "w", encoding="utf-8"
         ) as lang_file:  # you're using git right
             json.dump(
                 out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
@@ -56,7 +56,8 @@ def extract_names(dir_in, is_global):
     files = [
         f
         for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
-        if re.search(r"/[0-9]{9}\.tsv$", f)
+        if re.search(r"[0-9]{9}\.tsv$", f)
+        and f"{os.path.sep}Diva{os.path.sep}" not in f
     ]
     if len(files) == 0:
         raise FileNotFoundError("No valid files found in directory: " + dir_in)
@@ -78,7 +79,7 @@ def read_char_tsv(char_tsv_path, is_global):
     char_names = set()
     char_sets = set()
 
-    with open(char_tsv_path, "r") as char_tsv_file:
+    with open(char_tsv_path, "r", encoding="utf-8") as char_tsv_file:
         char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"')
 
         for row in char_tsv:
@@ -103,7 +104,7 @@ def read_char_tsv(char_tsv_path, is_global):
 
 def read_mission(tsv_path, char_names, char_sets):
     new_names = set()
-    with open(tsv_path, "r") as tsv_file:
+    with open(tsv_path, "r", encoding="utf-8") as tsv_file:
         tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
         if (
             ("Arg1" not in tsv.fieldnames)

utage/translate.py

@@ -15,7 +15,7 @@ def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global):
     files = [
         f
         for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True)
-        if "{os.path.sep}Diva{os.path.sep}" not in f
+        if f"{os.path.sep}Diva{os.path.sep}" not in f
     ]
 
     if len(files) == 0:
@@ -52,7 +52,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
         )
         languages = ["jpn"]
         if is_global:
-            languages += ["enm", "zh", "ko"]
+            languages = ["enm", "zho", "kor"]
         json_output_paths = {
             lang: os.path.join(
                 json_out_dir,
@@ -70,14 +70,14 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
                 if e.errno != errno.EEXIST:
                     raise
 
-        if not os.path.exists(os.path.dirname(json_output_paths["jpn"])):
+        if not os.path.exists(os.path.dirname(json_output_paths[languages[0]])):
             try:
-                os.makedirs(os.path.dirname(json_output_paths["jpn"]))
+                os.makedirs(os.path.dirname(json_output_paths[languages[0]]))
             except OSError as e:
                 if e.errno != errno.EEXIST:
                     raise
 
-        with open(file_in, "r") as tsv_file:
+        with open(file_in, "r", encoding="utf-8") as tsv_file:
             tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE)
 
             t_fieldnames = tsv.fieldnames
@@ -87,12 +87,11 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
             tsv_keyed, json_str = process_tsv(tsv, id_num, is_global)
 
             # csv handles newlines, don't set it in io.open
-            with io.open(tsv_output_path, "w", newline="") as tsv_out:
+            with io.open(tsv_output_path, "w", newline="", encoding="utf-8") as tsv_out:
                 writer = csv.DictWriter(
                     tsv_out,
                     delimiter="\t",
                     quotechar='"',
-                    lineterminator="\n",
                     fieldnames=t_fieldnames,
                     extrasaction="ignore",
                 )
@@ -103,7 +102,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
         for lang in languages:
             if not json_str[lang]:
                 continue
-            with io.open(json_output_paths[lang], "w", newline="\n") as json_out:
+            with io.open(json_output_paths[lang], "w", encoding="utf-8") as json_out:
                 # we don't want to sort these because they're _1, ..., _10, etc
                 json.dump(
                     json_str[lang],
@@ -124,8 +123,8 @@ def process_tsv(tsv, id_num, is_global):
     key_dict = {"jpn": {}}
     if is_global:
         key_dict["enm"] = {}
-        key_dict["zh"] = {}
-        key_dict["ko"] = {}
+        key_dict["zho"] = {}
+        key_dict["kor"] = {}
 
     for row in tsv:
         try:
@@ -140,8 +139,8 @@ def process_tsv(tsv, id_num, is_global):
                     with suppress(KeyError):
                         if row["English"]:
                             key_dict["enm"][key] = row["English"]
-                        key_dict["zh"][key] = row["Chinese"]
-                        key_dict["ko"][key] = row["Korean"]
+                        key_dict["zho"][key] = row["Chinese"]
+                        key_dict["kor"][key] = row["Korean"]
                 row["English"] = key
             tsv_keyed.append(row)
         except Exception:

xdudata/asset_extract.py

@@ -0,0 +1,303 @@
+import functools
+import multiprocessing
+import os
+import shutil
+import subprocess
+from contextlib import contextmanager
+from distutils.dir_util import copy_tree
+from glob import glob
+
+from adx import extract_loop_data_from_dir
+
+OPERATING_SYSTEM = os.name
+CHUNK_SIZE = 100
+
+
+def process_se(cache_dir, extract_dir, update_dir):
+    copy_tree(
+        os.path.join(cache_dir, "Android/Asset/Sound/Se"),
+        os.path.join(extract_dir, "Se"),
+    )
+
+    from_dir = "Se"
+    to_dir = "Se_Extracted"
+    acb_prefix = "_vgmt_acb_ext"
+
+    pool = get_pool()
+
+    with change_cwd(extract_dir):
+        os.makedirs(to_dir, exist_ok=True)
+
+        to_del = glob("{}/**/*.wav".format(from_dir), recursive=True)
+        to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True))
+        for file in to_del:
+            os.remove(file)
+
+        acb_files = [file for file in glob("{}/*.acb".format(from_dir))]
+        acb_cmd = get_dotnet_cmd("acb_extract.exe")
+        run_program(acb_files, cmd=acb_cmd)
+
+        hca_files = [
+            file for file in glob("{}/**/*.hca".format(from_dir), recursive=True)
+        ]
+        for hca_sublist in split_list(hca_files, CHUNK_SIZE):
+            run_program(hca_sublist, cmd="clHCA")
+
+        basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files]
+        char_secs = [basename.split("_", maxsplit=1) for basename in basenames]
+        for cs in char_secs:
+            char = cs[0]
+            sec = str(cs[1]) if len(cs) == 2 else ""
+            os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True)
+            wav_files = glob(
+                "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)),
+                recursive=True,
+            )
+            partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg")
+            wav_cmds = [
+                [
+                    "-i",
+                    os.path.abspath(file),
+                    "-acodec",
+                    "libopus",
+                    "-ab",
+                    "192k",
+                    "-af",
+                    "aresample=48000",
+                    os.path.abspath(
+                        "{}/{}/{}/{}.opus".format(
+                            to_dir,
+                            char,
+                            sec,
+                            os.path.splitext(os.path.basename(file))[0],
+                        )
+                    ),
+                    "-n",
+                ]
+                for file in wav_files
+            ]
+            pool.map(partial_ffmpeg_cmd, wav_cmds)
+
+    copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Se"))
+
+
+def process_bgm(cache_dir, extract_dir, update_dir):
+    copy_tree(
+        os.path.join(cache_dir, "Android/Asset/Sound/Bgm"),
+        os.path.join(extract_dir, "Bgm"),
+    )
+
+    from_dir = "Bgm"
+    to_dir = "Bgm_Extracted"
+    loop_file = "loop.json"
+
+    pool = get_pool()
+
+    with change_cwd(extract_dir):
+        os.makedirs(to_dir, exist_ok=True)
+
+        to_del = glob("{}/**/*.adx".format(from_dir), recursive=True)
+        for file in to_del:
+            os.remove(file)
+
+        acb_files = [file for file in glob("{}/*.acb".format(from_dir))]
+        acb_cmd = get_dotnet_cmd("acb_extract.exe")
+        run_program(acb_files, cmd=acb_cmd)
+
+        adx_files = [
+            file for file in glob("{}/**/*.adx".format(from_dir), recursive=True)
+        ]
+        partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg")
+        wav_cmds = [
+            [
+                "-i",
+                os.path.abspath(file),
+                "-acodec",
+                "libopus",
+                "-ab",
+                "192k",
+                "-af",
+                "aresample=48000",
+                os.path.abspath(
+                    "{}/{}.opus".format(
+                        to_dir, os.path.splitext(os.path.basename(file))[0],
+                    )
+                ),
+                "-n",
+            ]
+            for file in adx_files
+        ]
+        pool.map(partial_ffmpeg_cmd, wav_cmds)
+
+        extract_loop_data_from_dir(from_dir, loop_file)
+        shutil.copy2(loop_file, os.path.join(to_dir, "BgmLoop.json"))
+
+    copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Bgm"))
+
+
+def process_voice(cache_dir, extract_dir, update_dir):
+    copy_tree(
+        os.path.join(cache_dir, "Android/Asset/Sound/Voice"),
+        os.path.join(extract_dir, "Voice"),
+    )
+
+    from_dir = "Voice"
+    to_dir = "Voice_Extracted"
+    acb_prefix = "_vgmt_acb_ext"
+
+    pool = get_pool()
+
+    with change_cwd(extract_dir):
+        os.makedirs(to_dir, exist_ok=True)
+
+        to_del = glob("{}/**/*.wav".format(from_dir), recursive=True)
+        to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True))
+        for file in to_del:
+            os.remove(file)
+
+        acb_files = [file for file in glob("{}/*.acb".format(from_dir))]
+        acb_cmd = get_dotnet_cmd("acb_extract.exe")
+        run_program(acb_files, cmd=acb_cmd)
+
+        hca_files = [
+            file for file in glob("{}/**/*.hca".format(from_dir), recursive=True)
+        ]
+        for hca_sublist in split_list(hca_files, CHUNK_SIZE):
+            run_program(hca_sublist, cmd="clHCA")
+
+        basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files]
+        char_secs = [basename.split("_", maxsplit=1) for basename in basenames]
+        for cs in char_secs:
+            char = cs[0]
+            sec = str(cs[1]) if len(cs) == 2 else ""
+            os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True)
+            wav_files = glob(
+                "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)),
+                recursive=True,
+            )
+            partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg")
+
+            wav_cmds = [
+                [
+                    "-i",
+                    os.path.abspath(file),
+                    "-acodec",
+                    "libopus",
+                    "-ab",
+                    "192k",
+                    "-af",
+                    "aresample=48000",
+                    os.path.abspath(
+                        "{}/{}/{}/{}.opus".format(
+                            to_dir,
+                            char,
+                            sec,
+                            os.path.splitext(os.path.basename(file))[0],
+                        )
+                    ),
+                    "-n",
+                ]
+                for file in wav_files
+            ]
+            pool.map(partial_ffmpeg_cmd, wav_cmds)
+
+    copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Voice"))
+
+
+def process_movie(cache_dir, extract_dir, update_dir):
+    copy_tree(
+        os.path.join(cache_dir, "Common/Asset/Movie"),
+        os.path.join(extract_dir, "Movie"),
+    )
+
+    from_dir = "Movie"
+    to_dir = "Movie_Extracted"
+
+    pool = get_pool()
+
+    with change_cwd(extract_dir):
+        os.makedirs(to_dir, exist_ok=True)
+
+        to_del = glob("{}/**/*.adx".format(from_dir), recursive=True)
+        to_del.extend(glob("{}/**/*.m2v".format(from_dir), recursive=True))
+        for file in to_del:
+            os.remove(file)
+
+        movie_paths = [
+            os.path.abspath(os.path.splitext(file)[0])
+            for file in glob("{}/*.usm".format(from_dir))
+            if not os.path.basename(file).startswith("mini_radio")
+        ]
+
+        usm_files = [[path + ".usm"] for path in movie_paths]
+        usm_cmd = get_dotnet_cmd("usm_extract.exe")
+        partial_usm_cmd = functools.partial(run_program, cmd=usm_cmd)
+        pool.map(partial_usm_cmd, usm_files)
+
+        partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg")
+        mux_cmds = [
+            [
+                "-i",
+                path + ".adx",
+                "-i",
+                path + ".m2v",
+                "-c:a",
+                "libopus",
+                "-b:a",
+                "196k",
+                "-af",
+                "aresample=48000",
+                "-c:v",
+                "libvpx",
+                "-b:v",
+                "1M",
+                "-threads",
+                "1",
+                os.path.abspath("{}/{}.webm".format(to_dir, os.path.basename(path))),
+                "-n",
+            ]
+            for path in movie_paths
+        ]
+        pool.map(partial_ffmpeg_cmd, mux_cmds)
+
+    copy_tree(
+        os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Asset", "Movie")
+    )
+
+
+@contextmanager
+def change_cwd(new_path):
+    old_cwd = os.getcwd()
+    os.chdir(new_path)
+    try:
+        yield
+    finally:
+        os.chdir(old_cwd)
+
+
+def get_dotnet_cmd(name):
+    ext = os.path.splitext(name)[1]
+    if ext != ".exe":
+        raise RuntimeError("This doesn't seem like a .NET program.")
+    path = shutil.which(name)
+    if OPERATING_SYSTEM == "nt":
+        return [path]
+    return ["mono", path]
+
+
+def run_program(args, cmd=None):
+    if isinstance(cmd, str):
+        cmd = [cmd]
+    full_cmd = cmd + args
+    subprocess.run(full_cmd)
+
+
+def get_pool(size=multiprocessing.cpu_count()):
+    pool = multiprocessing.Pool(size)
+    return pool
+
+
+def split_list(input_list, chunk_size):
+    return [
+        input_list[i : i + chunk_size] for i in range(0, len(input_list), chunk_size)
+    ]

xdudata/update.py

@@ -3,16 +3,17 @@ from contextlib import suppress
 import diva
 import shutil
 import os
-import subprocess
 import tempfile
 import utage
 
 from distutils.dir_util import copy_tree, remove_tree
 from glob import glob
 
+from xdudata.asset_extract import process_se, process_bgm, process_voice, process_movie
+
 
 def update_all(
-    cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages,
+    cache_dir, update_dir, translation_dir, extract_dir, languages,
 ):
     print("Cleaning cache")
     cleanup_cache(cache_dir)
@@ -24,64 +25,38 @@ def update_all(
     update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages)
 
     print("Processing Quest Json files")
-    diva.quest.update_missions(extract_dir, translation_dir, languages)
-
-    # this code isn't going to run on windows (it is in WSL though) because i need to rewrite it
-    # it just calls my zsh tools and generates it that way, need to replace them
-    # with real python or something
-
-    # first copy over the necessary avtools to the extract dir
-    # TODO make it so that you don't need to copy them over (for example call them with python)
-    print("Copying necessary tools into extracting directory")
-    shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir)
-    shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir)
-    shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir)
+    diva.quest.update_missions(extract_dir, translation_dir, languages, False)
 
     print("Processing Sound Effects")
-    copy_tree(
-        os.path.join(cache_dir, "Android/Asset/Sound/Se"),
-        os.path.join(extract_dir, "Se"),
-    )
-    subprocess.run(
-        os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir
-    )
-    copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se"))
+    process_se(cache_dir, extract_dir, update_dir)
 
-    print("Processing BGM - don't forget to copy that loop file")
-    copy_tree(
-        os.path.join(cache_dir, "Android/Asset/Sound/Bgm"),
-        os.path.join(extract_dir, "Bgm"),
-    )
-    subprocess.run(
-        os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir
-    )
-    copy_tree(
-        os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm")
-    )
+    print("Processing BGM")
+    process_bgm(cache_dir, extract_dir, update_dir)
 
     print("Processing voices")
-    copy_tree(
-        os.path.join(cache_dir, "Android/Asset/Sound/Voice"),
-        os.path.join(extract_dir, "Voice"),
-    )
-    subprocess.run(
-        os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir
-    )
-    copy_tree(
-        os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice")
-    )
+    process_voice(cache_dir, extract_dir, update_dir)
 
     print("Processing movies - this might take forever")
+    process_movie(cache_dir, extract_dir, update_dir)
+
+
+def cleanup_cache(cache_dir):
+    with suppress(FileNotFoundError):
+        shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32"))
+        os.remove(
+            os.path.join(
+                cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage"
+            )
+        )
+
+
+def copy_images(cache_dir, update_dir):
     copy_tree(
-        os.path.join(cache_dir, "Common/Asset/Movie"),
-        os.path.join(extract_dir, "Movie"),
-    )
-    subprocess.run(
-        os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir
+        os.path.join(cache_dir, "Common/Asset/Image"),
+        os.path.join(update_dir, "Asset/Image"),
     )
     copy_tree(
-        os.path.join(extract_dir, "movie_extracted"),
-        os.path.join(update_dir, "Asset/Movie"),
+        os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"),
     )
 
 
@@ -113,23 +88,3 @@ def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages):
     utage.names.update_names(utage_tmp, translation_dir, languages, False)
     copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva"))
     remove_tree(utage_tmp)
-
-
-def copy_images(cache_dir, update_dir):
-    copy_tree(
-        os.path.join(cache_dir, "Common/Asset/Image"),
-        os.path.join(update_dir, "Asset/Image"),
-    )
-    copy_tree(
-        os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"),
-    )
-
-
-def cleanup_cache(cache_dir):
-    with suppress(FileNotFoundError):
-        shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32"))
-        os.remove(
-            os.path.join(
-                cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage"
-            )
-        )