Merge pull request 'Added --global flag for Global XDU' (#2) from feature/global_update into develop

.gitignore

@@ -3,6 +3,6 @@ __pycache__/
 *$py.class
 *.swp
 virtualenv/
-.idea/
 tools/*
-!tools/.gitkeep
+!tools/readme.txt
+.idea/

README.md

@@ -1,36 +0,0 @@
-# divatool
-
-A tool for that one game everyone hates but works on anyway.
-
-## Requirements
-
-Divatool is written in and uses [Python 3](https://www.python.org/downloads/).
-
-There are also several third party dependencies not provided as part of this repository,
-mostly used for the `xdudata` command. You need to have the binaries of the following tools
-either in your `$PATH` variable or in the [`tools`](./tools) folder. These tools are:
-
-* [clHCA](https://github.com/KinoMyu/FastHCADecoder)
-* acb_extract
-* usm_extract
-* [ffmpeg](https://www.ffmpeg.org/download.html)
-* [mono](https://www.mono-project.com/download/stable/) (if using Linux)
-
-Make sure these executables are called the same way on your machine.
-
-## Usage
-
-TODO: add detailed descriptions for all parameters.
-
-Using the tool to update XDUData for the player:
-
-```
-divatool.py xdudata update [path to the 'files' folder from the game] [path to the XDUData folder for the player] [path to the 'xdutranslations' repo folder] [path to the temp files folder]
-```
-
-Example:
-```
-divatool.py xdudata update /home/user/gameapp/files /home/user/player/XDUData /home/user/git/xdutranslations /var/tmp
-```
-
-Note that the 'files' folder from the app must contain the db files for this to work.

adx/adx_parser.py

@@ -74,7 +74,7 @@ class adx_file(object):
         self.encrypted = self.get_val(CRYPT_OFF, CRYPT_LEN)
         self.d_offset = self.get_val(DATA_OFF_OFF, DATA_OFF_LEN)
         if self.d_offset < DATA_OFF_MIN:
-            return
+            return None
         self.l_exists = 1
         # now load things for math
         self.sam_rate = self.get_val(SRATE_OFF, SRATE_LEN)
@@ -106,11 +106,13 @@ class adx_file(object):
             return None
         if self.l_start > self.sam_count or self.l_end > self.sam_count:
             raise ValueError("Invalid Loop Data")
+            return None
         if self.l_style not in LOOP_TYPE_SUP:
             raise NotImplementedError(
                 "Loop Style {} Not Implemented".format(str(self.l_style))
             )
-        ret = dict()
+            return None
+        ret = {}
         ret["duration"] = self.sam_count / self.sam_rate
         ret["loop_start"] = {}
         ret["loop_start"]["seconds"] = self.l_start / self.sam_rate
@@ -121,5 +123,7 @@ class adx_file(object):
         ret["loop_end"] = {}
         ret["loop_end"]["seconds"] = self.l_end / self.sam_rate
         ret["loop_end"]["samples_native"] = self.l_end
-        ret["loop_end"]["samples_48k"] = dumb_round(self.l_end / self.sam_rate * 48000)
+        ret["loop_end"]["samples_48k"] = dumb_round(
+            self.l_end / self.sam_rate * 48000
+        )
         return ret

db/base.py

@@ -23,7 +23,7 @@ class BaseDB(object):
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
-        self.conn.close()
+        self.quest_conn.close()
 
     def attach_dbs(self, databases, path):
         for d in databases:

db/quest.py

@@ -1,24 +1,16 @@
 from .base import BaseDB
 
-
+DATABASES = {
-def get_dbs_by_lang(language=None):
+    "QuestMst.db": "Quest",
-    databases = {
+    "QuestSceneMst.db": "QuestScene",
-        "QuestMst": "Quest",
+    "QuestPartMst.db": "QuestPart",
-        "QuestSceneMst": "QuestScene",
+    "ResourceEntry.db": "ResourceEntry",
-        "QuestPartMst": "QuestPart",
+}
-        "ResourceEntry": "ResourceEntry",
-    }
-    for key in list(databases):
-        if not language:
-            databases[key + ".db"] = databases.pop(key)
-        else:
-            databases[key + "_" + language + ".db"] = databases.pop(key)
-    return databases
 
 
 class QuestDB(BaseDB):
-    def __init__(self, path, language=None):
+    def __init__(self, path):
-        super().__init__(path, get_dbs_by_lang(language))
+        super().__init__(path, DATABASES)
 
     def get_quests(self):
         self.cursor.execute(
@@ -52,14 +44,12 @@ class QuestDB(BaseDB):
                 questSceneMstId,
                 name,
                 summaryText,
-                img,
                 group_concat(partIds) as parts
             FROM (
                     SELECT
                         qs.questSceneMstId,
                         qs.name,
                         qs.summaryText,
-                        qs.img,
                         CASE
                             WHEN qp.afterTalkName == '' AND qp.beforeTalkName == '' THEN NULL
                             WHEN qp.afterTalkName == '' AND qp.beforeTalkName != '' THEN qp.beforeTalkName
@@ -76,11 +66,9 @@ class QuestDB(BaseDB):
         )
         res = self.cursor.fetchall()
         scenes = {}
-        for scene_id, name, summary, img, parts in res:
+        for scene_id, name, summary, parts in res:
             scenes[str(scene_id)] = {}
             scenes[str(scene_id)]["Name"] = name
             scenes[str(scene_id)]["SummaryText"] = summary
-            if img:
-                scenes[str(scene_id)]["Image"] = img
             scenes[str(scene_id)]["Parts"] = parts.split(",")
         return scenes

diva/quest.py

@@ -7,25 +7,16 @@ import os
 from glob import glob
 
 
-def update_missions(dir_in, old_path, languages, is_global, utage_in=None):
+def update_missions(dir_in, old_path, languages):
-    if not utage_in:
+    qdb = db.QuestDB(dir_in)
-        utage_in = os.path.join(dir_in, "Common/Asset/Utage")
+    quest_mst(qdb, old_path, languages)
-    if is_global:
+    scene_mst(qdb, old_path, dir_in, languages)
-        languages = ["enm", "zho", "kor"]
-        for lang in languages:
-            qdb = db.QuestDB(dir_in, lang)
-            quest_mst(qdb, old_path, [lang], is_global)
-            scene_mst(qdb, old_path, [lang], utage_in, is_global)
-    else:
-        qdb = db.QuestDB(dir_in)
-        quest_mst(qdb, old_path, languages, is_global)
-        scene_mst(qdb, old_path, languages, utage_in, is_global)
 
 
-def scene_mst(qdb, old_path, languages, utage_in, is_global):
+def scene_mst(qdb, old_path, dir_in, languages):
     scenes = qdb.get_scenes()
 
-    locations = load_locations(utage_in)
+    locations = load_locations(dir_in)
 
     for id, scene in scenes.items():
         i = 0
@@ -35,66 +26,57 @@ def scene_mst(qdb, old_path, languages, utage_in, is_global):
             except KeyError as exc:
                 i += 1
                 if i == len(scene["Parts"]):
-                    if is_global:
-                        break
                     raise KeyError(f"Could not find folder for scene {id}") from exc
                 continue
             break
 
-    if not is_global:
+    with io.open(
-        with io.open(
+        os.path.join(old_path, "XduScene.json"), "w", newline="\n"
-            os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8"
+    ) as json_file:
-        ) as json_file:
+        json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
-            json.dump(
-                scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True
-            )
 
     for lang in languages:
         out_dict = {}
-        for key, value in scenes.items():
+        if lang == "jpn":
-            name = ""
+            for key, value in scenes.items():
-            summary = ""
+                out_dict[key] = {
-            credit = ""
+                    "Name": value["Name"],
-            enabled = False
+                    "SummaryText": value["SummaryText"],
-
+                    "Credits": "POKELABO",
-            if lang == "jpn" or is_global:
+                    "Enabled": False,
-                name = value["Name"]
+                }
-                summary = value["SummaryText"]
+        else:
-                credit = "POKELABO"
+            for key in scenes.keys():
-
+                out_dict[key] = {
-            out_dict[key] = {
+                    "Name": "",
-                "Name": name,
+                    "SummaryText": "",
-                "SummaryText": summary,
+                    "Credits": "",
-                "Credits": credit,
+                    "Enabled": False,
-                "Enabled": enabled,
+                }
-            }
 
         langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json")
         if os.path.isfile(langfile):
-            with open(langfile, "r", encoding="utf-8") as lang_file:
+            with open(langfile, "r") as lang_file:
                 lang_dict = json.load(lang_file)
                 out_dict.update(lang_dict)
 
-        with io.open(langfile, "w", encoding="utf-8") as lang_file:
+        with io.open(langfile, "w", newline="\n") as lang_file:
             json.dump(
                 out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
             )
 
 
-def quest_mst(qdb, old_path, languages, is_global):
+def quest_mst(qdb, old_path, languages):
     quests = qdb.get_quests()
 
-    if not is_global:
+    with io.open(
-        with io.open(
+        os.path.join(old_path, "XduQuest.json"), "w", newline="\n"
-            os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8"
+    ) as json_file:
-        ) as json_file:
+        json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
-            json.dump(
-                quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True
-            )
 
     for lang in languages:
         out_dict = {}
-        if lang == "jpn" or is_global:
+        if lang == "jpn":
             for key, value in quests.items():
                 out_dict[key] = {"Name": value["Name"], "Enabled": False}
         else:
@@ -103,11 +85,11 @@ def quest_mst(qdb, old_path, languages, is_global):
 
         langfile = os.path.join(old_path, f"XduQuestNames_{lang}.json")
         if os.path.isfile(langfile):
-            with open(langfile, "r", encoding="utf-8") as lang_file:
+            with open(langfile, "r") as lang_file:
                 lang_dict = json.load(lang_file)
                 out_dict.update(lang_dict)
 
-        with io.open(langfile, "w", encoding="utf-8") as lang_file:
+        with io.open(langfile, "w", newline="\n") as lang_file:
             json.dump(
                 out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
             )
@@ -115,28 +97,25 @@ def quest_mst(qdb, old_path, languages, is_global):
     return quests
 
 
-def load_locations(utage_in):
+def load_locations(dir_in):
     locations = {}
-    settings = glob("{}/**/Settings/Scenario.tsv".format(utage_in), recursive=True,)
+    settings = glob(
+        os.path.join(dir_in, "Common/Asset/Utage/**/Settings/Scenario.tsv"),
+        recursive=True,
+    )
 
     if len(settings) == 0:
         raise FileNotFoundError("Ya didn't decrypt the files dummy")
 
     for scenario in settings:
-        with open(scenario, "r", encoding="utf-8") as tsv_file:
+        with open(scenario, "r") as tsv_file:
             tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
             for row in tsv:
                 tokens = row["FileName"].split("/")
                 if len(tokens) < 3:
                     continue
-                folder, _scenario, part_id = tokens
+                if tokens[2] in locations:
-                # side03 seems to only contain prototype scripts so we ignore it
+                    raise ValueError("Conflicting paths for file {tokens[2]}")
-                if folder == "side03":
+                locations[tokens[2]] = tokens[0]
-                    continue
-                if part_id in locations:
-                    raise ValueError(
-                        f"Conflicting paths for scene part {part_id}, existing folder: {locations[part_id]}"
-                    )
-                locations[part_id] = folder
 
     return locations

divatool.py

@@ -9,7 +9,7 @@ import utage
 import db
 import xdudata
 
-LANGUAGES = ["jpn", "eng", "rus", "cze"]
+LANGUAGES = ["jpn", "eng", "rus"]
 
 
 def main():
@@ -117,16 +117,13 @@ def main():
         metavar="<command>", title="subcommand", dest="subcommand"
     )
     parser_diva_quest = parser_diva_subparsers.add_parser(
-        "quest", help="Generate quest JSON files", parents=[global_flag_parser],
+        "quest", help="Generate quest JSON files"
     )
     parser_diva_quest.add_argument(
         "INPUT",
         help="Directory of databases containing Quest*.db and ResourceEntry.db",
         type=str,
     )
-    parser_diva_quest.add_argument(
-        "--utage", help="Directory of Utage folders and files.", type=str, default=None,
-    )
     parser_diva_quest.add_argument(
         "OLD", help="Directory with old quest files", nargs="?", type=str, default="."
     )
@@ -185,24 +182,17 @@ def main():
                 raise FileNotFoundError(args.INPUT)
     elif args.command == "diva":
         if args.subcommand == "quest":
-            if not args.utage:
+            diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES)
-                diva.quest.update_missions(
-                    args.INPUT, args.OLD, LANGUAGES, args.is_global
-                )
-            else:
-                diva.quest.update_missions(
-                    args.INPUT, args.OLD, LANGUAGES, args.is_global, utage_in=args.utage
-                )
     elif args.command == "xdudata":
+        script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools"))
         if args.subcommand == "update":
-            # add the tools folder to PATH
-            os.environ["PATH"] += os.pathsep + os.path.join(
-                os.path.abspath(os.path.dirname(__file__)), "tools"
-            )
-
-            # update the data
             xdudata.update_all(
-                args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES,
+                args.CACHE,
+                args.XDUDATA,
+                args.TRANS,
+                args.EXTRACT,
+                script_dir,
+                LANGUAGES,
             )
 
     return 0

tools/readme.txt

@@ -0,0 +1,34 @@
+Various files need to be in this directory.
+If you don't have them, ask someone who does.
+
+The required files are:
+
+Binaries:
+- acb_extract.exe
+- usm_extract.exe
+- clHCA
+
+Z shell scripts:
+- update_bgm.zsh
+- update_movie.zsh
+- update_se.zsh
+- update_voice.zsh
+
+Python scripts:
+- adx_parse.py
+- bingo_fuck.py
+- chara_room_mission.py
+- custom_mission.py
+- event_bonus.py
+- orphan_quest.py
+- pad_2_csv.py
+- pad_2_json.py
+- pad_2_pad.py
+- pad_2_pad_louisstyle.py
+- pad_stat.py
+- pad_update.py
+- store_parser.py
+- tsv_2_pad.py
+- tsv_parse.py
+- txt_2_pad.py
+- utage_decrypt.py

utage/names.py

@@ -12,7 +12,7 @@ from multiprocessing import Pool
 def update_names(dir_in, old_path, languages, is_global):
     names = extract_names(dir_in, is_global)
 
-    global_languages = {"enm": 1, "zho": 2, "kor": 3}
+    global_languages = {"enm": 1, "zh": 2, "ko": 3}
     if is_global:
         languages = global_languages.keys()
 
@@ -33,12 +33,12 @@ def update_names(dir_in, old_path, languages, is_global):
 
         langfile = os.path.join(old_path, "nametranslations_{}.json".format(l))
         if os.path.isfile(langfile):
-            with open(langfile, "r", encoding="utf-8") as lang_file:
+            with open(langfile, "r") as lang_file:
                 lang_dict = json.load(lang_file)
             out_dict.update(lang_dict)
 
         with io.open(
-            langfile, "w", encoding="utf-8"
+            langfile, "w", newline="\n"
         ) as lang_file:  # you're using git right
             json.dump(
                 out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
@@ -56,8 +56,7 @@ def extract_names(dir_in, is_global):
     files = [
         f
         for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
-        if re.search(r"[0-9]{9}\.tsv$", f)
+        if re.search(r"/[0-9]{9}\.tsv$", f)
-        and f"{os.path.sep}Diva{os.path.sep}" not in f
     ]
     if len(files) == 0:
         raise FileNotFoundError("No valid files found in directory: " + dir_in)
@@ -79,7 +78,7 @@ def read_char_tsv(char_tsv_path, is_global):
     char_names = set()
     char_sets = set()
 
-    with open(char_tsv_path, "r", encoding="utf-8") as char_tsv_file:
+    with open(char_tsv_path, "r") as char_tsv_file:
         char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"')
 
         for row in char_tsv:
@@ -104,7 +103,7 @@ def read_char_tsv(char_tsv_path, is_global):
 
 def read_mission(tsv_path, char_names, char_sets):
     new_names = set()
-    with open(tsv_path, "r", encoding="utf-8") as tsv_file:
+    with open(tsv_path, "r") as tsv_file:
         tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
         if (
             ("Arg1" not in tsv.fieldnames)

utage/translate.py

@@ -15,7 +15,7 @@ def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global):
     files = [
         f
         for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True)
-        if f"{os.path.sep}Diva{os.path.sep}" not in f
+        if "{os.path.sep}Diva{os.path.sep}" not in f
     ]
 
     if len(files) == 0:
@@ -52,7 +52,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
         )
         languages = ["jpn"]
         if is_global:
-            languages = ["enm", "zho", "kor"]
+            languages += ["enm", "zh", "ko"]
         json_output_paths = {
             lang: os.path.join(
                 json_out_dir,
@@ -70,14 +70,14 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
                 if e.errno != errno.EEXIST:
                     raise
 
-        if not os.path.exists(os.path.dirname(json_output_paths[languages[0]])):
+        if not os.path.exists(os.path.dirname(json_output_paths["jpn"])):
             try:
-                os.makedirs(os.path.dirname(json_output_paths[languages[0]]))
+                os.makedirs(os.path.dirname(json_output_paths["jpn"]))
             except OSError as e:
                 if e.errno != errno.EEXIST:
                     raise
 
-        with open(file_in, "r", encoding="utf-8") as tsv_file:
+        with open(file_in, "r") as tsv_file:
             tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE)
 
             t_fieldnames = tsv.fieldnames
@@ -87,11 +87,12 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
             tsv_keyed, json_str = process_tsv(tsv, id_num, is_global)
 
             # csv handles newlines, don't set it in io.open
-            with io.open(tsv_output_path, "w", newline="", encoding="utf-8") as tsv_out:
+            with io.open(tsv_output_path, "w", newline="") as tsv_out:
                 writer = csv.DictWriter(
                     tsv_out,
                     delimiter="\t",
                     quotechar='"',
+                    lineterminator="\n",
                     fieldnames=t_fieldnames,
                     extrasaction="ignore",
                 )
@@ -102,7 +103,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
         for lang in languages:
             if not json_str[lang]:
                 continue
-            with io.open(json_output_paths[lang], "w", encoding="utf-8") as json_out:
+            with io.open(json_output_paths[lang], "w", newline="\n") as json_out:
                 # we don't want to sort these because they're _1, ..., _10, etc
                 json.dump(
                     json_str[lang],
@@ -123,8 +124,8 @@ def process_tsv(tsv, id_num, is_global):
     key_dict = {"jpn": {}}
     if is_global:
         key_dict["enm"] = {}
-        key_dict["zho"] = {}
+        key_dict["zh"] = {}
-        key_dict["kor"] = {}
+        key_dict["ko"] = {}
 
     for row in tsv:
         try:
@@ -139,8 +140,8 @@ def process_tsv(tsv, id_num, is_global):
                     with suppress(KeyError):
                         if row["English"]:
                             key_dict["enm"][key] = row["English"]
-                        key_dict["zho"][key] = row["Chinese"]
+                        key_dict["zh"][key] = row["Chinese"]
-                        key_dict["kor"][key] = row["Korean"]
+                        key_dict["ko"][key] = row["Korean"]
                 row["English"] = key
             tsv_keyed.append(row)
         except Exception:

xdudata/asset_extract.py

@@ -1,303 +0,0 @@
-import functools
-import multiprocessing
-import os
-import shutil
-import subprocess
-from contextlib import contextmanager
-from distutils.dir_util import copy_tree
-from glob import glob
-
-from adx import extract_loop_data_from_dir
-
-OPERATING_SYSTEM = os.name
-CHUNK_SIZE = 100
-
-
-def process_se(cache_dir, extract_dir, update_dir):
-    copy_tree(
-        os.path.join(cache_dir, "Android/Asset/Sound/Se"),
-        os.path.join(extract_dir, "Se"),
-    )
-
-    from_dir = "Se"
-    to_dir = "Se_Extracted"
-    acb_prefix = "_vgmt_acb_ext"
-
-    pool = get_pool()
-
-    with change_cwd(extract_dir):
-        os.makedirs(to_dir, exist_ok=True)
-
-        to_del = glob("{}/**/*.wav".format(from_dir), recursive=True)
-        to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True))
-        for file in to_del:
-            os.remove(file)
-
-        acb_files = [file for file in glob("{}/*.acb".format(from_dir))]
-        acb_cmd = get_dotnet_cmd("acb_extract.exe")
-        run_program(acb_files, cmd=acb_cmd)
-
-        hca_files = [
-            file for file in glob("{}/**/*.hca".format(from_dir), recursive=True)
-        ]
-        for hca_sublist in split_list(hca_files, CHUNK_SIZE):
-            run_program(hca_sublist, cmd="clHCA")
-
-        basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files]
-        char_secs = [basename.split("_", maxsplit=1) for basename in basenames]
-        for cs in char_secs:
-            char = cs[0]
-            sec = str(cs[1]) if len(cs) == 2 else ""
-            os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True)
-            wav_files = glob(
-                "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)),
-                recursive=True,
-            )
-            partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg")
-            wav_cmds = [
-                [
-                    "-i",
-                    os.path.abspath(file),
-                    "-acodec",
-                    "libopus",
-                    "-ab",
-                    "192k",
-                    "-af",
-                    "aresample=48000",
-                    os.path.abspath(
-                        "{}/{}/{}/{}.opus".format(
-                            to_dir,
-                            char,
-                            sec,
-                            os.path.splitext(os.path.basename(file))[0],
-                        )
-                    ),
-                    "-n",
-                ]
-                for file in wav_files
-            ]
-            pool.map(partial_ffmpeg_cmd, wav_cmds)
-
-    copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Se"))
-
-
-def process_bgm(cache_dir, extract_dir, update_dir):
-    copy_tree(
-        os.path.join(cache_dir, "Android/Asset/Sound/Bgm"),
-        os.path.join(extract_dir, "Bgm"),
-    )
-
-    from_dir = "Bgm"
-    to_dir = "Bgm_Extracted"
-    loop_file = "loop.json"
-
-    pool = get_pool()
-
-    with change_cwd(extract_dir):
-        os.makedirs(to_dir, exist_ok=True)
-
-        to_del = glob("{}/**/*.adx".format(from_dir), recursive=True)
-        for file in to_del:
-            os.remove(file)
-
-        acb_files = [file for file in glob("{}/*.acb".format(from_dir))]
-        acb_cmd = get_dotnet_cmd("acb_extract.exe")
-        run_program(acb_files, cmd=acb_cmd)
-
-        adx_files = [
-            file for file in glob("{}/**/*.adx".format(from_dir), recursive=True)
-        ]
-        partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg")
-        wav_cmds = [
-            [
-                "-i",
-                os.path.abspath(file),
-                "-acodec",
-                "libopus",
-                "-ab",
-                "192k",
-                "-af",
-                "aresample=48000",
-                os.path.abspath(
-                    "{}/{}.opus".format(
-                        to_dir, os.path.splitext(os.path.basename(file))[0],
-                    )
-                ),
-                "-n",
-            ]
-            for file in adx_files
-        ]
-        pool.map(partial_ffmpeg_cmd, wav_cmds)
-
-        extract_loop_data_from_dir(from_dir, loop_file)
-        shutil.copy2(loop_file, os.path.join(to_dir, "BgmLoop.json"))
-
-    copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Bgm"))
-
-
-def process_voice(cache_dir, extract_dir, update_dir):
-    copy_tree(
-        os.path.join(cache_dir, "Android/Asset/Sound/Voice"),
-        os.path.join(extract_dir, "Voice"),
-    )
-
-    from_dir = "Voice"
-    to_dir = "Voice_Extracted"
-    acb_prefix = "_vgmt_acb_ext"
-
-    pool = get_pool()
-
-    with change_cwd(extract_dir):
-        os.makedirs(to_dir, exist_ok=True)
-
-        to_del = glob("{}/**/*.wav".format(from_dir), recursive=True)
-        to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True))
-        for file in to_del:
-            os.remove(file)
-
-        acb_files = [file for file in glob("{}/*.acb".format(from_dir))]
-        acb_cmd = get_dotnet_cmd("acb_extract.exe")
-        run_program(acb_files, cmd=acb_cmd)
-
-        hca_files = [
-            file for file in glob("{}/**/*.hca".format(from_dir), recursive=True)
-        ]
-        for hca_sublist in split_list(hca_files, CHUNK_SIZE):
-            run_program(hca_sublist, cmd="clHCA")
-
-        basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files]
-        char_secs = [basename.split("_", maxsplit=1) for basename in basenames]
-        for cs in char_secs:
-            char = cs[0]
-            sec = str(cs[1]) if len(cs) == 2 else ""
-            os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True)
-            wav_files = glob(
-                "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)),
-                recursive=True,
-            )
-            partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg")
-
-            wav_cmds = [
-                [
-                    "-i",
-                    os.path.abspath(file),
-                    "-acodec",
-                    "libopus",
-                    "-ab",
-                    "192k",
-                    "-af",
-                    "aresample=48000",
-                    os.path.abspath(
-                        "{}/{}/{}/{}.opus".format(
-                            to_dir,
-                            char,
-                            sec,
-                            os.path.splitext(os.path.basename(file))[0],
-                        )
-                    ),
-                    "-n",
-                ]
-                for file in wav_files
-            ]
-            pool.map(partial_ffmpeg_cmd, wav_cmds)
-
-    copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Voice"))
-
-
-def process_movie(cache_dir, extract_dir, update_dir):
-    copy_tree(
-        os.path.join(cache_dir, "Common/Asset/Movie"),
-        os.path.join(extract_dir, "Movie"),
-    )
-
-    from_dir = "Movie"
-    to_dir = "Movie_Extracted"
-
-    pool = get_pool()
-
-    with change_cwd(extract_dir):
-        os.makedirs(to_dir, exist_ok=True)
-
-        to_del = glob("{}/**/*.adx".format(from_dir), recursive=True)
-        to_del.extend(glob("{}/**/*.m2v".format(from_dir), recursive=True))
-        for file in to_del:
-            os.remove(file)
-
-        movie_paths = [
-            os.path.abspath(os.path.splitext(file)[0])
-            for file in glob("{}/*.usm".format(from_dir))
-            if not os.path.basename(file).startswith("mini_radio")
-        ]
-
-        usm_files = [[path + ".usm"] for path in movie_paths]
-        usm_cmd = get_dotnet_cmd("usm_extract.exe")
-        partial_usm_cmd = functools.partial(run_program, cmd=usm_cmd)
-        pool.map(partial_usm_cmd, usm_files)
-
-        partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg")
-        mux_cmds = [
-            [
-                "-i",
-                path + ".adx",
-                "-i",
-                path + ".m2v",
-                "-c:a",
-                "libopus",
-                "-b:a",
-                "196k",
-                "-af",
-                "aresample=48000",
-                "-c:v",
-                "libvpx",
-                "-b:v",
-                "1M",
-                "-threads",
-                "1",
-                os.path.abspath("{}/{}.webm".format(to_dir, os.path.basename(path))),
-                "-n",
-            ]
-            for path in movie_paths
-        ]
-        pool.map(partial_ffmpeg_cmd, mux_cmds)
-
-    copy_tree(
-        os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Asset", "Movie")
-    )
-
-
-@contextmanager
-def change_cwd(new_path):
-    old_cwd = os.getcwd()
-    os.chdir(new_path)
-    try:
-        yield
-    finally:
-        os.chdir(old_cwd)
-
-
-def get_dotnet_cmd(name):
-    ext = os.path.splitext(name)[1]
-    if ext != ".exe":
-        raise RuntimeError("This doesn't seem like a .NET program.")
-    path = shutil.which(name)
-    if OPERATING_SYSTEM == "nt":
-        return [path]
-    return ["mono", path]
-
-
-def run_program(args, cmd=None):
-    if isinstance(cmd, str):
-        cmd = [cmd]
-    full_cmd = cmd + args
-    subprocess.run(full_cmd)
-
-
-def get_pool(size=multiprocessing.cpu_count()):
-    pool = multiprocessing.Pool(size)
-    return pool
-
-
-def split_list(input_list, chunk_size):
-    return [
-        input_list[i : i + chunk_size] for i in range(0, len(input_list), chunk_size)
-    ]

xdudata/update.py

@@ -3,17 +3,16 @@ from contextlib import suppress
 import diva
 import shutil
 import os
+import subprocess
 import tempfile
 import utage
 
 from distutils.dir_util import copy_tree, remove_tree
 from glob import glob
 
-from xdudata.asset_extract import process_se, process_bgm, process_voice, process_movie
-
 
 def update_all(
-    cache_dir, update_dir, translation_dir, extract_dir, languages,
+    cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages,
 ):
     print("Cleaning cache")
     cleanup_cache(cache_dir)
@@ -25,38 +24,64 @@ def update_all(
     update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages)
 
     print("Processing Quest Json files")
-    diva.quest.update_missions(extract_dir, translation_dir, languages, False)
+    diva.quest.update_missions(extract_dir, translation_dir, languages)
+
+    # this code isn't going to run on windows (it is in WSL though) because i need to rewrite it
+    # it just calls my zsh tools and generates it that way, need to replace them
+    # with real python or something
+
+    # first copy over the necessary avtools to the extract dir
+    # TODO make it so that you don't need to copy them over (for example call them with python)
+    print("Copying necessary tools into extracting directory")
+    shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir)
+    shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir)
+    shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir)
 
     print("Processing Sound Effects")
-    process_se(cache_dir, extract_dir, update_dir)
-
-    print("Processing BGM")
-    process_bgm(cache_dir, extract_dir, update_dir)
-
-    print("Processing voices")
-    process_voice(cache_dir, extract_dir, update_dir)
-
-    print("Processing movies - this might take forever")
-    process_movie(cache_dir, extract_dir, update_dir)
-
-
-def cleanup_cache(cache_dir):
-    with suppress(FileNotFoundError):
-        shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32"))
-        os.remove(
-            os.path.join(
-                cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage"
-            )
-        )
-
-
-def copy_images(cache_dir, update_dir):
     copy_tree(
-        os.path.join(cache_dir, "Common/Asset/Image"),
+        os.path.join(cache_dir, "Android/Asset/Sound/Se"),
-        os.path.join(update_dir, "Asset/Image"),
+        os.path.join(extract_dir, "Se"),
+    )
+    subprocess.run(
+        os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir
+    )
+    copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se"))
+
+    print("Processing BGM - don't forget to copy that loop file")
+    copy_tree(
+        os.path.join(cache_dir, "Android/Asset/Sound/Bgm"),
+        os.path.join(extract_dir, "Bgm"),
+    )
+    subprocess.run(
+        os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir
     )
     copy_tree(
-        os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"),
+        os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm")
+    )
+
+    print("Processing voices")
+    copy_tree(
+        os.path.join(cache_dir, "Android/Asset/Sound/Voice"),
+        os.path.join(extract_dir, "Voice"),
+    )
+    subprocess.run(
+        os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir
+    )
+    copy_tree(
+        os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice")
+    )
+
+    print("Processing movies - this might take forever")
+    copy_tree(
+        os.path.join(cache_dir, "Common/Asset/Movie"),
+        os.path.join(extract_dir, "Movie"),
+    )
+    subprocess.run(
+        os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir
+    )
+    copy_tree(
+        os.path.join(extract_dir, "movie_extracted"),
+        os.path.join(update_dir, "Asset/Movie"),
     )
 
 
@@ -88,3 +113,23 @@ def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages):
     utage.names.update_names(utage_tmp, translation_dir, languages, False)
     copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva"))
     remove_tree(utage_tmp)
+
+
+def copy_images(cache_dir, update_dir):
+    copy_tree(
+        os.path.join(cache_dir, "Common/Asset/Image"),
+        os.path.join(update_dir, "Asset/Image"),
+    )
+    copy_tree(
+        os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"),
+    )
+
+
+def cleanup_cache(cache_dir):
+    with suppress(FileNotFoundError):
+        shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32"))
+        os.remove(
+            os.path.join(
+                cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage"
+            )
+        )