diff --git a/.gitignore b/.gitignore index e691145..cd455e6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ __pycache__/ *$py.class *.swp virtualenv/ +.idea/ +tools/* +!tools/.gitkeep diff --git a/README.md b/README.md new file mode 100644 index 0000000..81ee494 --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# divatool + +A tool for that one game everyone hates but works on anyway. + +## Requirements + +Divatool is written in and uses [Python 3](https://www.python.org/downloads/). + +There are also several third party dependencies not provided as part of this repository, +mostly used for the `xdudata` command. You need to have the binaries of the following tools +either in your `$PATH` variable or in the [`tools`](./tools) folder. These tools are: + +* [clHCA](https://github.com/KinoMyu/FastHCADecoder) +* acb_extract +* usm_extract +* [ffmpeg](https://www.ffmpeg.org/download.html) +* [mono](https://www.mono-project.com/download/stable/) (if using Linux) + +Make sure these executables are called the same way on your machine. + +## Usage + +TODO: add detailed descriptions for all parameters. + +Using the tool to update XDUData for the player: + +``` +divatool.py xdudata update [path to the 'files' folder from the game] [path to the XDUData folder for the player] [path to the 'xdutranslations' repo folder] [path to the temp files folder] +``` + +Example: +``` +divatool.py xdudata update /home/user/gameapp/files /home/user/player/XDUData /home/user/git/xdutranslations /var/tmp +``` + +Note that the 'files' folder from the app must contain the db files for this to work. diff --git a/adx/adx_parser.py b/adx/adx_parser.py index 993fc3c..d6d4c5d 100644 --- a/adx/adx_parser.py +++ b/adx/adx_parser.py @@ -74,7 +74,7 @@ class adx_file(object): self.encrypted = self.get_val(CRYPT_OFF, CRYPT_LEN) self.d_offset = self.get_val(DATA_OFF_OFF, DATA_OFF_LEN) if self.d_offset < DATA_OFF_MIN: - return None + return self.l_exists = 1 # now load things for math self.sam_rate = self.get_val(SRATE_OFF, SRATE_LEN) @@ -106,13 +106,11 @@ class adx_file(object): return None if self.l_start > self.sam_count or self.l_end > self.sam_count: raise ValueError("Invalid Loop Data") - return None if self.l_style not in LOOP_TYPE_SUP: raise NotImplementedError( "Loop Style {} Not Implemented".format(str(self.l_style)) ) - return None - ret = {} + ret = dict() ret["duration"] = self.sam_count / self.sam_rate ret["loop_start"] = {} ret["loop_start"]["seconds"] = self.l_start / self.sam_rate @@ -123,7 +121,5 @@ class adx_file(object): ret["loop_end"] = {} ret["loop_end"]["seconds"] = self.l_end / self.sam_rate ret["loop_end"]["samples_native"] = self.l_end - ret["loop_end"]["samples_48k"] = dumb_round( - self.l_end / self.sam_rate * 48000 - ) + ret["loop_end"]["samples_48k"] = dumb_round(self.l_end / self.sam_rate * 48000) return ret diff --git a/db/base.py b/db/base.py index afa5817..80132be 100644 --- a/db/base.py +++ b/db/base.py @@ -23,7 +23,7 @@ class BaseDB(object): return self def __exit__(self, exc_type, exc_value, traceback): - self.quest_conn.close() + self.conn.close() def attach_dbs(self, databases, path): for d in databases: diff --git a/db/quest.py b/db/quest.py index c4b13f8..bedc636 100644 --- a/db/quest.py +++ b/db/quest.py @@ -1,16 +1,24 @@ from .base import BaseDB -DATABASES = { - "QuestMst.db": "Quest", - "QuestSceneMst.db": "QuestScene", - "QuestPartMst.db": "QuestPart", - "ResourceEntry.db": "ResourceEntry", -} + +def get_dbs_by_lang(language=None): + databases = { + "QuestMst": "Quest", + "QuestSceneMst": "QuestScene", + "QuestPartMst": "QuestPart", + "ResourceEntry": "ResourceEntry", + } + for key in list(databases): + if not language: + databases[key + ".db"] = databases.pop(key) + else: + databases[key + "_" + language + ".db"] = databases.pop(key) + return databases class QuestDB(BaseDB): - def __init__(self, path): - super().__init__(path, DATABASES) + def __init__(self, path, language=None): + super().__init__(path, get_dbs_by_lang(language)) def get_quests(self): self.cursor.execute( @@ -44,12 +52,14 @@ class QuestDB(BaseDB): questSceneMstId, name, summaryText, + img, group_concat(partIds) as parts FROM ( SELECT qs.questSceneMstId, qs.name, qs.summaryText, + qs.img, CASE WHEN qp.afterTalkName == '' AND qp.beforeTalkName == '' THEN NULL WHEN qp.afterTalkName == '' AND qp.beforeTalkName != '' THEN qp.beforeTalkName @@ -66,9 +76,11 @@ class QuestDB(BaseDB): ) res = self.cursor.fetchall() scenes = {} - for scene_id, name, summary, parts in res: + for scene_id, name, summary, img, parts in res: scenes[str(scene_id)] = {} scenes[str(scene_id)]["Name"] = name scenes[str(scene_id)]["SummaryText"] = summary + if img: + scenes[str(scene_id)]["Image"] = img scenes[str(scene_id)]["Parts"] = parts.split(",") return scenes diff --git a/diva/quest.py b/diva/quest.py index d84e3b2..a2306f0 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -7,16 +7,25 @@ import os from glob import glob -def update_missions(dir_in, old_path, languages): - qdb = db.QuestDB(dir_in) - quest_mst(qdb, old_path, languages) - scene_mst(qdb, old_path, dir_in, languages) +def update_missions(dir_in, old_path, languages, is_global, utage_in=None): + if not utage_in: + utage_in = os.path.join(dir_in, "Common/Asset/Utage") + if is_global: + languages = ["enm", "zho", "kor"] + for lang in languages: + qdb = db.QuestDB(dir_in, lang) + quest_mst(qdb, old_path, [lang], is_global) + scene_mst(qdb, old_path, [lang], utage_in, is_global) + else: + qdb = db.QuestDB(dir_in) + quest_mst(qdb, old_path, languages, is_global) + scene_mst(qdb, old_path, languages, utage_in, is_global) -def scene_mst(qdb, old_path, dir_in, languages): +def scene_mst(qdb, old_path, languages, utage_in, is_global): scenes = qdb.get_scenes() - locations = load_locations(dir_in) + locations = load_locations(utage_in) for id, scene in scenes.items(): i = 0 @@ -26,67 +35,66 @@ def scene_mst(qdb, old_path, dir_in, languages): except KeyError as exc: i += 1 if i == len(scene["Parts"]): - raise KeyError( - f"Could not find folder for scene {id}" - ) from exc + if is_global: + break + raise KeyError(f"Could not find folder for scene {id}") from exc continue break - with io.open( - os.path.join(old_path, "XduScene.json"), "w", newline="\n" - ) as json_file: - json.dump( - scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True - ) + if not is_global: + with io.open( + os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8" + ) as json_file: + json.dump( + scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True + ) for lang in languages: out_dict = {} - if lang == "jpn": - for key, value in scenes.items(): - out_dict[key] = { - "Name": value["Name"], - "SummaryText": value["SummaryText"], - "Credits": "POKELABO", - "Enabled": False, - } - else: - for key in scenes.keys(): - out_dict[key] = { - "Name": "", - "SummaryText": "", - "Credits": "", - "Enabled": False, - } + for key, value in scenes.items(): + name = "" + summary = "" + credit = "" + enabled = False + + if lang == "jpn" or is_global: + name = value["Name"] + summary = value["SummaryText"] + credit = "POKELABO" + + out_dict[key] = { + "Name": name, + "SummaryText": summary, + "Credits": credit, + "Enabled": enabled, + } langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json") if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) - with io.open(langfile, "w", newline="\n") as lang_file: + with io.open(langfile, "w", encoding="utf-8") as lang_file: json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) -def quest_mst(qdb, old_path, languages): +def quest_mst(qdb, old_path, languages, is_global): quests = qdb.get_quests() - with io.open( - os.path.join(old_path, "XduQuest.json"), "w", newline="\n" - ) as json_file: - json.dump( - quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True - ) + if not is_global: + with io.open( + os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8" + ) as json_file: + json.dump( + quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True + ) for lang in languages: out_dict = {} - if lang == "jpn": + if lang == "jpn" or is_global: for key, value in quests.items(): out_dict[key] = {"Name": value["Name"], "Enabled": False} else: @@ -95,41 +103,40 @@ def quest_mst(qdb, old_path, languages): langfile = os.path.join(old_path, f"XduQuestNames_{lang}.json") if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) - with io.open(langfile, "w", newline="\n") as lang_file: + with io.open(langfile, "w", encoding="utf-8") as lang_file: json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) return quests -def load_locations(dir_in): +def load_locations(utage_in): locations = {} - settings = glob( - os.path.join(dir_in, "Common/Asset/Utage/**/Settings/Scenario.tsv"), - recursive=True, - ) + settings = glob("{}/**/Settings/Scenario.tsv".format(utage_in), recursive=True,) if len(settings) == 0: raise FileNotFoundError("Ya didn't decrypt the files dummy") for scenario in settings: - with open(scenario, "r") as tsv_file: + with open(scenario, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') for row in tsv: tokens = row["FileName"].split("/") if len(tokens) < 3: continue - if tokens[2] in locations: - raise ValueError("Conflicting paths for file {tokens[2]}") - locations[tokens[2]] = tokens[0] + folder, _scenario, part_id = tokens + # side03 seems to only contain prototype scripts so we ignore it + if folder == "side03": + continue + if part_id in locations: + raise ValueError( + f"Conflicting paths for scene part {part_id}, existing folder: {locations[part_id]}" + ) + locations[part_id] = folder return locations diff --git a/divatool.py b/divatool.py index cd5f342..f43a927 100755 --- a/divatool.py +++ b/divatool.py @@ -9,12 +9,22 @@ import utage import db import xdudata -LANGUAGES = ["jpn", "eng", "rus"] +LANGUAGES = ["jpn", "eng", "rus", "cze"] def main(): parser = argparse.ArgumentParser(description="Unpack and process XDU data") + global_flag_parser = argparse.ArgumentParser(add_help=False) + global_flag_parser.add_argument( + "-g", + "--global", + help="Whether we are running on Global XDU.", + action="store_true", + dest="is_global", + default=False, + ) + subparsers = parser.add_subparsers( metavar="", title="subcommands", dest="command" ) @@ -81,7 +91,9 @@ def main(): parser_utage_crypt.add_argument("TARGET", help="Input", type=str) # utage translate parser_utage_translate = parser_utage_subparsers.add_parser( - "translate", help="Generate keyed tsv and json files" + "translate", + help="Generate keyed tsv and json files", + parents=[global_flag_parser], ) parser_utage_translate.add_argument("INPUT", help="Input", type=str) parser_utage_translate.add_argument( @@ -92,7 +104,7 @@ def main(): ) # utage names parser_utage_names = parser_utage_subparsers.add_parser( - "names", help="Generate and update name files" + "names", help="Generate and update name files", parents=[global_flag_parser] ) parser_utage_names.add_argument("INPUT", help="Input", type=str) parser_utage_names.add_argument( @@ -105,13 +117,16 @@ def main(): metavar="", title="subcommand", dest="subcommand" ) parser_diva_quest = parser_diva_subparsers.add_parser( - "quest", help="Generate quest JSON files" + "quest", help="Generate quest JSON files", parents=[global_flag_parser], ) parser_diva_quest.add_argument( "INPUT", help="Directory of databases containing Quest*.db and ResourceEntry.db", type=str, ) + parser_diva_quest.add_argument( + "--utage", help="Directory of Utage folders and files.", type=str, default=None, + ) parser_diva_quest.add_argument( "OLD", help="Directory with old quest files", nargs="?", type=str, default="." ) @@ -152,23 +167,42 @@ def main(): raise FileNotFoundError("Could not find {}".format(args.TARGET)) elif args.subcommand == "translate": if os.path.isfile(args.INPUT): - utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR) + utage.translate.translate_file( + args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global + ) elif os.path.isdir(args.INPUT): - utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR) + utage.translate.translate_dir( + args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global + ) else: raise FileNotFoundError("Could not find {}".format(args.INPUT)) elif args.subcommand == "names": if os.path.isdir(args.INPUT): - utage.names.update_names(args.INPUT, args.OLD, LANGUAGES) + utage.names.update_names( + args.INPUT, args.OLD, LANGUAGES, args.is_global + ) else: raise FileNotFoundError(args.INPUT) elif args.command == "diva": if args.subcommand == "quest": - diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) + if not args.utage: + diva.quest.update_missions( + args.INPUT, args.OLD, LANGUAGES, args.is_global + ) + else: + diva.quest.update_missions( + args.INPUT, args.OLD, LANGUAGES, args.is_global, utage_in=args.utage + ) elif args.command == "xdudata": if args.subcommand == "update": + # add the tools folder to PATH + os.environ["PATH"] += os.pathsep + os.path.join( + os.path.abspath(os.path.dirname(__file__)), "tools" + ) + + # update the data xdudata.update_all( - args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES + args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES, ) return 0 diff --git a/tools/.gitkeep b/tools/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/utage/names.py b/utage/names.py index 56ef85c..0ea03f8 100644 --- a/utage/names.py +++ b/utage/names.py @@ -9,67 +9,77 @@ from glob import glob from multiprocessing import Pool -def update_names(dir_in, old_path, languages): - names = extract_names(dir_in) +def update_names(dir_in, old_path, languages, is_global): + names = extract_names(dir_in, is_global) + + global_languages = {"enm": 1, "zho": 2, "kor": 3} + if is_global: + languages = global_languages.keys() # we have to update japanese like the rest because CustomData exists # and POKELABO deleted the wedding gear event stuff from the game files for l in languages: if l == "jpn": out_dict = dict(zip(names, names)) + elif l in global_languages.keys(): + out_dict = {} + for name in names: + spl = name.split("\t") + idx = global_languages[l] + val = spl[idx] if spl[idx] != "None" else "" + out_dict[spl[0]] = val else: out_dict = dict.fromkeys(names, "") langfile = os.path.join(old_path, "nametranslations_{}.json".format(l)) if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) with io.open( - langfile, "w", newline="\n" + langfile, "w", encoding="utf-8" ) as lang_file: # you're using git right json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) -def extract_names(dir_in): +def extract_names(dir_in, is_global): char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv") if not os.path.isfile(char_tsv_path): raise FileNotFoundError("Character.tsv not found") - names, sets = read_char_tsv(char_tsv_path) + names, sets = read_char_tsv(char_tsv_path, is_global) files = [ f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) - if re.search(r"/[0-9]{9}\.tsv$", f) + if re.search(r"[0-9]{9}\.tsv$", f) + and f"{os.path.sep}Diva{os.path.sep}" not in f ] if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) - # tfw gil - # it's still faster on my machine so i'm keeping it - read_partial = partial(read_mission, char_names=names, char_sets=sets) - with Pool() as p: - new_names = p.map(read_partial, files) - for x in new_names: - names.update(x) + # TODO unsupported on global for now + if not is_global: + # tfw gil + # it's still faster on my machine so i'm keeping it + read_partial = partial(read_mission, char_names=names, char_sets=sets) + with Pool() as p: + new_names = p.map(read_partial, files) + for x in new_names: + names.update(x) return names -def read_char_tsv(char_tsv_path): +def read_char_tsv(char_tsv_path, is_global): char_names = set() char_sets = set() - with open(char_tsv_path, "r") as char_tsv_file: + with open(char_tsv_path, "r", encoding="utf-8") as char_tsv_file: char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"') for row in char_tsv: @@ -81,7 +91,12 @@ def read_char_tsv(char_tsv_path): and row["CharacterName"].strip() and row["NameText"].strip() ): - char_names.add(row["NameText"]) + if is_global: + char_names.add( + f"{row['NameText']}\t{row['CharaEnglish']}\t{row['CharaChinese']}\t{row['CharaKorean']}" + ) + else: + char_names.add(row["NameText"]) char_sets.add(row["CharacterName"]) return char_names, char_sets @@ -89,7 +104,7 @@ def read_char_tsv(char_tsv_path): def read_mission(tsv_path, char_names, char_sets): new_names = set() - with open(tsv_path, "r") as tsv_file: + with open(tsv_path, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') if ( ("Arg1" not in tsv.fieldnames) @@ -101,8 +116,6 @@ def read_mission(tsv_path, char_names, char_sets): if row["Command"] and row["Command"].startswith("//"): continue if row["Text"] and row["Arg1"]: - if (row["Arg1"] not in char_names) and ( - row["Arg1"] not in char_sets - ): + if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets): new_names.add(row["Arg1"]) return new_names diff --git a/utage/translate.py b/utage/translate.py index 6c47289..cd3b875 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -4,32 +4,34 @@ import io import json import os import traceback +from contextlib import suppress from functools import partial from glob import glob from multiprocessing.pool import Pool -def translate_dir(dir_in, tsv_out_dir, json_out_dir): +def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global): files = [ f - for f in glob( - os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True - ) - if "{os.path.sep}Diva{os.path.sep}" not in f + for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True) + if f"{os.path.sep}Diva{os.path.sep}" not in f ] if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) ptrans = partial( - translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir + translate_file, + tsv_out_dir=tsv_out_dir, + json_out_dir=json_out_dir, + is_global=is_global, ) with Pool() as p: p.map(ptrans, files) -def translate_file(file_in, tsv_out_dir, json_out_dir): +def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): try: if not file_in.endswith(".tsv"): raise ValueError("Invalid File Type for {}".format(file_in)) @@ -48,11 +50,17 @@ def translate_file(file_in, tsv_out_dir, json_out_dir): tsv_output_path = os.path.join( tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num) ) - json_output_path = os.path.join( - json_out_dir, - event_folder, - "{}_translations_jpn.json".format(id_num), - ) + languages = ["jpn"] + if is_global: + languages = ["enm", "zho", "kor"] + json_output_paths = { + lang: os.path.join( + json_out_dir, + event_folder, + "{}_translations_{}.json".format(id_num, lang), + ) + for lang in languages + } # need to create output paths and avoid races when threading if not os.path.exists(os.path.dirname(tsv_output_path)): @@ -62,50 +70,61 @@ def translate_file(file_in, tsv_out_dir, json_out_dir): if e.errno != errno.EEXIST: raise - if not os.path.exists(os.path.dirname(json_output_path)): + if not os.path.exists(os.path.dirname(json_output_paths[languages[0]])): try: - os.makedirs(os.path.dirname(json_output_path)) + os.makedirs(os.path.dirname(json_output_paths[languages[0]])) except OSError as e: if e.errno != errno.EEXIST: raise - with open(file_in, "r") as tsv_file: - tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') - tsv_keyed, json_str = process_tsv(tsv, id_num) + with open(file_in, "r", encoding="utf-8") as tsv_file: + tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE) + + t_fieldnames = tsv.fieldnames + if is_global: + t_fieldnames = tsv.fieldnames[: tsv.fieldnames.index("English") + 1] + + tsv_keyed, json_str = process_tsv(tsv, id_num, is_global) # csv handles newlines, don't set it in io.open - with io.open(tsv_output_path, "w", newline="") as tsv_out: + with io.open(tsv_output_path, "w", newline="", encoding="utf-8") as tsv_out: writer = csv.DictWriter( tsv_out, delimiter="\t", quotechar='"', - lineterminator="\n", - fieldnames=tsv.fieldnames, + fieldnames=t_fieldnames, extrasaction="ignore", ) writer.writeheader() for row in tsv_keyed: writer.writerow(row) - with io.open(json_output_path, "w", newline="\n") as json_out: - # we don't want to sort these because they're _1, ..., _10, etc - json.dump( - json_str, - json_out, - ensure_ascii=False, - indent="\t", - sort_keys=False, - ) + for lang in languages: + if not json_str[lang]: + continue + with io.open(json_output_paths[lang], "w", encoding="utf-8") as json_out: + # we don't want to sort these because they're _1, ..., _10, etc + json.dump( + json_str[lang], + json_out, + ensure_ascii=False, + indent="\t", + sort_keys=False, + ) except Exception: traceback.print_exc() print("Error processing file: " + file_in) -def process_tsv(tsv, id_num): +def process_tsv(tsv, id_num, is_global): i = 0 tsv_keyed = [] - key_dict = {} + key_dict = {"jpn": {}} + if is_global: + key_dict["enm"] = {} + key_dict["zho"] = {} + key_dict["kor"] = {} for row in tsv: try: @@ -115,8 +134,14 @@ def process_tsv(tsv, id_num): if row["Text"] and row["Text"].strip(): key = "{}_{}".format(id_num, i) i += 1 + key_dict["jpn"][key] = row["Text"] + if is_global: + with suppress(KeyError): + if row["English"]: + key_dict["enm"][key] = row["English"] + key_dict["zho"][key] = row["Chinese"] + key_dict["kor"][key] = row["Korean"] row["English"] = key - key_dict[key] = row["Text"] tsv_keyed.append(row) except Exception: traceback.print_exc() diff --git a/xdudata/asset_extract.py b/xdudata/asset_extract.py new file mode 100644 index 0000000..04e088c --- /dev/null +++ b/xdudata/asset_extract.py @@ -0,0 +1,303 @@ +import functools +import multiprocessing +import os +import shutil +import subprocess +from contextlib import contextmanager +from distutils.dir_util import copy_tree +from glob import glob + +from adx import extract_loop_data_from_dir + +OPERATING_SYSTEM = os.name +CHUNK_SIZE = 100 + + +def process_se(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Se"), + os.path.join(extract_dir, "Se"), + ) + + from_dir = "Se" + to_dir = "Se_Extracted" + acb_prefix = "_vgmt_acb_ext" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.wav".format(from_dir), recursive=True) + to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True)) + for file in to_del: + os.remove(file) + + acb_files = [file for file in glob("{}/*.acb".format(from_dir))] + acb_cmd = get_dotnet_cmd("acb_extract.exe") + run_program(acb_files, cmd=acb_cmd) + + hca_files = [ + file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) + ] + for hca_sublist in split_list(hca_files, CHUNK_SIZE): + run_program(hca_sublist, cmd="clHCA") + + basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] + char_secs = [basename.split("_", maxsplit=1) for basename in basenames] + for cs in char_secs: + char = cs[0] + sec = str(cs[1]) if len(cs) == 2 else "" + os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True) + wav_files = glob( + "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)), + recursive=True, + ) + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + wav_cmds = [ + [ + "-i", + os.path.abspath(file), + "-acodec", + "libopus", + "-ab", + "192k", + "-af", + "aresample=48000", + os.path.abspath( + "{}/{}/{}/{}.opus".format( + to_dir, + char, + sec, + os.path.splitext(os.path.basename(file))[0], + ) + ), + "-n", + ] + for file in wav_files + ] + pool.map(partial_ffmpeg_cmd, wav_cmds) + + copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Se")) + + +def process_bgm(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), + os.path.join(extract_dir, "Bgm"), + ) + + from_dir = "Bgm" + to_dir = "Bgm_Extracted" + loop_file = "loop.json" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.adx".format(from_dir), recursive=True) + for file in to_del: + os.remove(file) + + acb_files = [file for file in glob("{}/*.acb".format(from_dir))] + acb_cmd = get_dotnet_cmd("acb_extract.exe") + run_program(acb_files, cmd=acb_cmd) + + adx_files = [ + file for file in glob("{}/**/*.adx".format(from_dir), recursive=True) + ] + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + wav_cmds = [ + [ + "-i", + os.path.abspath(file), + "-acodec", + "libopus", + "-ab", + "192k", + "-af", + "aresample=48000", + os.path.abspath( + "{}/{}.opus".format( + to_dir, os.path.splitext(os.path.basename(file))[0], + ) + ), + "-n", + ] + for file in adx_files + ] + pool.map(partial_ffmpeg_cmd, wav_cmds) + + extract_loop_data_from_dir(from_dir, loop_file) + shutil.copy2(loop_file, os.path.join(to_dir, "BgmLoop.json")) + + copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Bgm")) + + +def process_voice(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Voice"), + os.path.join(extract_dir, "Voice"), + ) + + from_dir = "Voice" + to_dir = "Voice_Extracted" + acb_prefix = "_vgmt_acb_ext" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.wav".format(from_dir), recursive=True) + to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True)) + for file in to_del: + os.remove(file) + + acb_files = [file for file in glob("{}/*.acb".format(from_dir))] + acb_cmd = get_dotnet_cmd("acb_extract.exe") + run_program(acb_files, cmd=acb_cmd) + + hca_files = [ + file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) + ] + for hca_sublist in split_list(hca_files, CHUNK_SIZE): + run_program(hca_sublist, cmd="clHCA") + + basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] + char_secs = [basename.split("_", maxsplit=1) for basename in basenames] + for cs in char_secs: + char = cs[0] + sec = str(cs[1]) if len(cs) == 2 else "" + os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True) + wav_files = glob( + "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)), + recursive=True, + ) + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + + wav_cmds = [ + [ + "-i", + os.path.abspath(file), + "-acodec", + "libopus", + "-ab", + "192k", + "-af", + "aresample=48000", + os.path.abspath( + "{}/{}/{}/{}.opus".format( + to_dir, + char, + sec, + os.path.splitext(os.path.basename(file))[0], + ) + ), + "-n", + ] + for file in wav_files + ] + pool.map(partial_ffmpeg_cmd, wav_cmds) + + copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Voice")) + + +def process_movie(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Common/Asset/Movie"), + os.path.join(extract_dir, "Movie"), + ) + + from_dir = "Movie" + to_dir = "Movie_Extracted" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.adx".format(from_dir), recursive=True) + to_del.extend(glob("{}/**/*.m2v".format(from_dir), recursive=True)) + for file in to_del: + os.remove(file) + + movie_paths = [ + os.path.abspath(os.path.splitext(file)[0]) + for file in glob("{}/*.usm".format(from_dir)) + if not os.path.basename(file).startswith("mini_radio") + ] + + usm_files = [[path + ".usm"] for path in movie_paths] + usm_cmd = get_dotnet_cmd("usm_extract.exe") + partial_usm_cmd = functools.partial(run_program, cmd=usm_cmd) + pool.map(partial_usm_cmd, usm_files) + + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + mux_cmds = [ + [ + "-i", + path + ".adx", + "-i", + path + ".m2v", + "-c:a", + "libopus", + "-b:a", + "196k", + "-af", + "aresample=48000", + "-c:v", + "libvpx", + "-b:v", + "1M", + "-threads", + "1", + os.path.abspath("{}/{}.webm".format(to_dir, os.path.basename(path))), + "-n", + ] + for path in movie_paths + ] + pool.map(partial_ffmpeg_cmd, mux_cmds) + + copy_tree( + os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Asset", "Movie") + ) + + +@contextmanager +def change_cwd(new_path): + old_cwd = os.getcwd() + os.chdir(new_path) + try: + yield + finally: + os.chdir(old_cwd) + + +def get_dotnet_cmd(name): + ext = os.path.splitext(name)[1] + if ext != ".exe": + raise RuntimeError("This doesn't seem like a .NET program.") + path = shutil.which(name) + if OPERATING_SYSTEM == "nt": + return [path] + return ["mono", path] + + +def run_program(args, cmd=None): + if isinstance(cmd, str): + cmd = [cmd] + full_cmd = cmd + args + subprocess.run(full_cmd) + + +def get_pool(size=multiprocessing.cpu_count()): + pool = multiprocessing.Pool(size) + return pool + + +def split_list(input_list, chunk_size): + return [ + input_list[i : i + chunk_size] for i in range(0, len(input_list), chunk_size) + ] diff --git a/xdudata/update.py b/xdudata/update.py index fd747f9..8f61b6f 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -1,90 +1,53 @@ +from contextlib import suppress + import diva +import shutil import os -import subprocess import tempfile import utage from distutils.dir_util import copy_tree, remove_tree from glob import glob +from xdudata.asset_extract import process_se, process_bgm, process_voice, process_movie + + +def update_all( + cache_dir, update_dir, translation_dir, extract_dir, languages, +): + print("Cleaning cache") + cleanup_cache(cache_dir) -def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): print("Copying images") copy_images(cache_dir, update_dir) print("Processing TSV files") - update_tsv(cache_dir, update_dir, translation_dir, languages) + update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) print("Processing Quest Json files") - diva.quest.update_missions(cache_dir, translation_dir, languages) + diva.quest.update_missions(extract_dir, translation_dir, languages, False) - # this code isn't going to run on windows because i need to rewrite it - # it just calls my zsh scripts and generates it that way, need to replace them - # with real python or something print("Processing Sound Effects") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Se"), - os.path.join(extract_dir, "Se"), - ) - subprocess.run("./update_se.zsh", shell=True, cwd=extract_dir) - copy_tree(os.path.join(extract_dir, "se"), os.path.join(update_dir, "Se")) + process_se(cache_dir, extract_dir, update_dir) - print("Processing BGM - don't forget to copy that loop file") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), - os.path.join(extract_dir, "Bgm"), - ) - subprocess.run("./update_bgm.zsh", shell=True, cwd=extract_dir) - copy_tree( - os.path.join(extract_dir, "bgm"), os.path.join(update_dir, "Bgm") - ) + print("Processing BGM") + process_bgm(cache_dir, extract_dir, update_dir) print("Processing voices") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Voice"), - os.path.join(extract_dir, "Voice"), - ) - subprocess.run("./update_voice.zsh", shell=True, cwd=extract_dir) - copy_tree( - os.path.join(extract_dir, "voice"), os.path.join(update_dir, "Voice") - ) + process_voice(cache_dir, extract_dir, update_dir) print("Processing movies - this might take forever") - copy_tree( - os.path.join(cache_dir, "Common/Asset/Movie"), - os.path.join(extract_dir, "Movie"), - ) - subprocess.run("./update_movie.zsh", shell=True, cwd=extract_dir) - copy_tree( - os.path.join(extract_dir, "movie"), - os.path.join(update_dir, "Asset/Movie"), - ) + process_movie(cache_dir, extract_dir, update_dir) -def update_tsv(cache_dir, update_dir, translation_dir, languages): - utage_tmp = tempfile.mkdtemp() - copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp) - utage.crypt.crypt_dir( - utage_tmp, bytearray("SampleSecretKey", "utf-8"), False, False - ) - for encrypted in glob( - os.path.join(utage_tmp, "**/*.utage"), recursive=True - ): - try: - os.remove(encrypted) - except: - pass - - utage.translate.translate_dir( - utage_tmp, - os.path.join(update_dir, "Utage"), - os.path.join(translation_dir, "Missions"), - ) - utage.names.update_names(utage_tmp, translation_dir, languages) - copy_tree( - os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva") - ) - remove_tree(utage_tmp) +def cleanup_cache(cache_dir): + with suppress(FileNotFoundError): + shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) + os.remove( + os.path.join( + cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" + ) + ) def copy_images(cache_dir, update_dir): @@ -93,6 +56,35 @@ def copy_images(cache_dir, update_dir): os.path.join(update_dir, "Asset/Image"), ) copy_tree( - os.path.join(cache_dir, "Common/Sample"), - os.path.join(update_dir, "Sample"), + os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), ) + + +def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages): + utage_tmp = tempfile.mkdtemp() + copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp) + utage.crypt.crypt_dir( + utage_tmp, bytearray("SampleSecretKey", "utf-8"), False, False + ) + for encrypted in glob(os.path.join(utage_tmp, "**/*.utage"), recursive=True): + try: + os.remove(encrypted) + except: + pass + + # copy over files for the quest updater + copy_tree(utage_tmp, os.path.join(extract_dir, "Common/Asset/Utage")) + shutil.copy2(os.path.join(cache_dir, "QuestMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "QuestSceneMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "QuestPartMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "ResourceEntry.db"), extract_dir) + + utage.translate.translate_dir( + utage_tmp, + os.path.join(update_dir, "Utage"), + os.path.join(translation_dir, "Missions"), + False, + ) + utage.names.update_names(utage_tmp, translation_dir, languages, False) + copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) + remove_tree(utage_tmp)