diff --git a/.gitignore b/.gitignore index f8d0fa1..cd455e6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ __pycache__/ *$py.class *.swp virtualenv/ -tools/* -!tools/readme.txt .idea/ +tools/* +!tools/.gitkeep diff --git a/README.md b/README.md new file mode 100644 index 0000000..81ee494 --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# divatool + +A tool for that one game everyone hates but works on anyway. + +## Requirements + +Divatool is written in and uses [Python 3](https://www.python.org/downloads/). + +There are also several third party dependencies not provided as part of this repository, +mostly used for the `xdudata` command. You need to have the binaries of the following tools +either in your `$PATH` variable or in the [`tools`](./tools) folder. These tools are: + +* [clHCA](https://github.com/KinoMyu/FastHCADecoder) +* acb_extract +* usm_extract +* [ffmpeg](https://www.ffmpeg.org/download.html) +* [mono](https://www.mono-project.com/download/stable/) (if using Linux) + +Make sure these executables are called the same way on your machine. + +## Usage + +TODO: add detailed descriptions for all parameters. + +Using the tool to update XDUData for the player: + +``` +divatool.py xdudata update [path to the 'files' folder from the game] [path to the XDUData folder for the player] [path to the 'xdutranslations' repo folder] [path to the temp files folder] +``` + +Example: +``` +divatool.py xdudata update /home/user/gameapp/files /home/user/player/XDUData /home/user/git/xdutranslations /var/tmp +``` + +Note that the 'files' folder from the app must contain the db files for this to work. diff --git a/adx/adx_parser.py b/adx/adx_parser.py index 993fc3c..d6d4c5d 100644 --- a/adx/adx_parser.py +++ b/adx/adx_parser.py @@ -74,7 +74,7 @@ class adx_file(object): self.encrypted = self.get_val(CRYPT_OFF, CRYPT_LEN) self.d_offset = self.get_val(DATA_OFF_OFF, DATA_OFF_LEN) if self.d_offset < DATA_OFF_MIN: - return None + return self.l_exists = 1 # now load things for math self.sam_rate = self.get_val(SRATE_OFF, SRATE_LEN) @@ -106,13 +106,11 @@ class adx_file(object): return None if self.l_start > self.sam_count or self.l_end > self.sam_count: raise ValueError("Invalid Loop Data") - return None if self.l_style not in LOOP_TYPE_SUP: raise NotImplementedError( "Loop Style {} Not Implemented".format(str(self.l_style)) ) - return None - ret = {} + ret = dict() ret["duration"] = self.sam_count / self.sam_rate ret["loop_start"] = {} ret["loop_start"]["seconds"] = self.l_start / self.sam_rate @@ -123,7 +121,5 @@ class adx_file(object): ret["loop_end"] = {} ret["loop_end"]["seconds"] = self.l_end / self.sam_rate ret["loop_end"]["samples_native"] = self.l_end - ret["loop_end"]["samples_48k"] = dumb_round( - self.l_end / self.sam_rate * 48000 - ) + ret["loop_end"]["samples_48k"] = dumb_round(self.l_end / self.sam_rate * 48000) return ret diff --git a/db/base.py b/db/base.py index afa5817..80132be 100644 --- a/db/base.py +++ b/db/base.py @@ -23,7 +23,7 @@ class BaseDB(object): return self def __exit__(self, exc_type, exc_value, traceback): - self.quest_conn.close() + self.conn.close() def attach_dbs(self, databases, path): for d in databases: diff --git a/db/quest.py b/db/quest.py index c4b13f8..bedc636 100644 --- a/db/quest.py +++ b/db/quest.py @@ -1,16 +1,24 @@ from .base import BaseDB -DATABASES = { - "QuestMst.db": "Quest", - "QuestSceneMst.db": "QuestScene", - "QuestPartMst.db": "QuestPart", - "ResourceEntry.db": "ResourceEntry", -} + +def get_dbs_by_lang(language=None): + databases = { + "QuestMst": "Quest", + "QuestSceneMst": "QuestScene", + "QuestPartMst": "QuestPart", + "ResourceEntry": "ResourceEntry", + } + for key in list(databases): + if not language: + databases[key + ".db"] = databases.pop(key) + else: + databases[key + "_" + language + ".db"] = databases.pop(key) + return databases class QuestDB(BaseDB): - def __init__(self, path): - super().__init__(path, DATABASES) + def __init__(self, path, language=None): + super().__init__(path, get_dbs_by_lang(language)) def get_quests(self): self.cursor.execute( @@ -44,12 +52,14 @@ class QuestDB(BaseDB): questSceneMstId, name, summaryText, + img, group_concat(partIds) as parts FROM ( SELECT qs.questSceneMstId, qs.name, qs.summaryText, + qs.img, CASE WHEN qp.afterTalkName == '' AND qp.beforeTalkName == '' THEN NULL WHEN qp.afterTalkName == '' AND qp.beforeTalkName != '' THEN qp.beforeTalkName @@ -66,9 +76,11 @@ class QuestDB(BaseDB): ) res = self.cursor.fetchall() scenes = {} - for scene_id, name, summary, parts in res: + for scene_id, name, summary, img, parts in res: scenes[str(scene_id)] = {} scenes[str(scene_id)]["Name"] = name scenes[str(scene_id)]["SummaryText"] = summary + if img: + scenes[str(scene_id)]["Image"] = img scenes[str(scene_id)]["Parts"] = parts.split(",") return scenes diff --git a/diva/quest.py b/diva/quest.py index f2342a1..a2306f0 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -7,16 +7,25 @@ import os from glob import glob -def update_missions(dir_in, old_path, languages): - qdb = db.QuestDB(dir_in) - quest_mst(qdb, old_path, languages) - scene_mst(qdb, old_path, dir_in, languages) +def update_missions(dir_in, old_path, languages, is_global, utage_in=None): + if not utage_in: + utage_in = os.path.join(dir_in, "Common/Asset/Utage") + if is_global: + languages = ["enm", "zho", "kor"] + for lang in languages: + qdb = db.QuestDB(dir_in, lang) + quest_mst(qdb, old_path, [lang], is_global) + scene_mst(qdb, old_path, [lang], utage_in, is_global) + else: + qdb = db.QuestDB(dir_in) + quest_mst(qdb, old_path, languages, is_global) + scene_mst(qdb, old_path, languages, utage_in, is_global) -def scene_mst(qdb, old_path, dir_in, languages): +def scene_mst(qdb, old_path, languages, utage_in, is_global): scenes = qdb.get_scenes() - locations = load_locations(dir_in) + locations = load_locations(utage_in) for id, scene in scenes.items(): i = 0 @@ -26,57 +35,66 @@ def scene_mst(qdb, old_path, dir_in, languages): except KeyError as exc: i += 1 if i == len(scene["Parts"]): + if is_global: + break raise KeyError(f"Could not find folder for scene {id}") from exc continue break - with io.open( - os.path.join(old_path, "XduScene.json"), "w", newline="\n" - ) as json_file: - json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True) + if not is_global: + with io.open( + os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8" + ) as json_file: + json.dump( + scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True + ) for lang in languages: out_dict = {} - if lang == "jpn": - for key, value in scenes.items(): - out_dict[key] = { - "Name": value["Name"], - "SummaryText": value["SummaryText"], - "Credits": "POKELABO", - "Enabled": False, - } - else: - for key in scenes.keys(): - out_dict[key] = { - "Name": "", - "SummaryText": "", - "Credits": "", - "Enabled": False, - } + for key, value in scenes.items(): + name = "" + summary = "" + credit = "" + enabled = False + + if lang == "jpn" or is_global: + name = value["Name"] + summary = value["SummaryText"] + credit = "POKELABO" + + out_dict[key] = { + "Name": name, + "SummaryText": summary, + "Credits": credit, + "Enabled": enabled, + } langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json") if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) - with io.open(langfile, "w", newline="\n") as lang_file: + with io.open(langfile, "w", encoding="utf-8") as lang_file: json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) -def quest_mst(qdb, old_path, languages): +def quest_mst(qdb, old_path, languages, is_global): quests = qdb.get_quests() - with io.open( - os.path.join(old_path, "XduQuest.json"), "w", newline="\n" - ) as json_file: - json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True) + if not is_global: + with io.open( + os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8" + ) as json_file: + json.dump( + quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True + ) for lang in languages: out_dict = {} - if lang == "jpn": + if lang == "jpn" or is_global: for key, value in quests.items(): out_dict[key] = {"Name": value["Name"], "Enabled": False} else: @@ -85,11 +103,11 @@ def quest_mst(qdb, old_path, languages): langfile = os.path.join(old_path, f"XduQuestNames_{lang}.json") if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) - with io.open(langfile, "w", newline="\n") as lang_file: + with io.open(langfile, "w", encoding="utf-8") as lang_file: json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -97,25 +115,28 @@ def quest_mst(qdb, old_path, languages): return quests -def load_locations(dir_in): +def load_locations(utage_in): locations = {} - settings = glob( - os.path.join(dir_in, "Common/Asset/Utage/**/Settings/Scenario.tsv"), - recursive=True, - ) + settings = glob("{}/**/Settings/Scenario.tsv".format(utage_in), recursive=True,) if len(settings) == 0: raise FileNotFoundError("Ya didn't decrypt the files dummy") for scenario in settings: - with open(scenario, "r") as tsv_file: + with open(scenario, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') for row in tsv: tokens = row["FileName"].split("/") if len(tokens) < 3: continue - if tokens[2] in locations: - raise ValueError("Conflicting paths for file {tokens[2]}") - locations[tokens[2]] = tokens[0] + folder, _scenario, part_id = tokens + # side03 seems to only contain prototype scripts so we ignore it + if folder == "side03": + continue + if part_id in locations: + raise ValueError( + f"Conflicting paths for scene part {part_id}, existing folder: {locations[part_id]}" + ) + locations[part_id] = folder return locations diff --git a/divatool.py b/divatool.py index ec36fed..f43a927 100755 --- a/divatool.py +++ b/divatool.py @@ -9,7 +9,7 @@ import utage import db import xdudata -LANGUAGES = ["jpn", "eng", "rus"] +LANGUAGES = ["jpn", "eng", "rus", "cze"] def main(): @@ -117,13 +117,16 @@ def main(): metavar="", title="subcommand", dest="subcommand" ) parser_diva_quest = parser_diva_subparsers.add_parser( - "quest", help="Generate quest JSON files" + "quest", help="Generate quest JSON files", parents=[global_flag_parser], ) parser_diva_quest.add_argument( "INPUT", help="Directory of databases containing Quest*.db and ResourceEntry.db", type=str, ) + parser_diva_quest.add_argument( + "--utage", help="Directory of Utage folders and files.", type=str, default=None, + ) parser_diva_quest.add_argument( "OLD", help="Directory with old quest files", nargs="?", type=str, default="." ) @@ -182,17 +185,24 @@ def main(): raise FileNotFoundError(args.INPUT) elif args.command == "diva": if args.subcommand == "quest": - diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) + if not args.utage: + diva.quest.update_missions( + args.INPUT, args.OLD, LANGUAGES, args.is_global + ) + else: + diva.quest.update_missions( + args.INPUT, args.OLD, LANGUAGES, args.is_global, utage_in=args.utage + ) elif args.command == "xdudata": - script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools")) if args.subcommand == "update": + # add the tools folder to PATH + os.environ["PATH"] += os.pathsep + os.path.join( + os.path.abspath(os.path.dirname(__file__)), "tools" + ) + + # update the data xdudata.update_all( - args.CACHE, - args.XDUDATA, - args.TRANS, - args.EXTRACT, - script_dir, - LANGUAGES, + args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES, ) return 0 diff --git a/tools/.gitkeep b/tools/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tools/readme.txt b/tools/readme.txt deleted file mode 100644 index f10fdc0..0000000 --- a/tools/readme.txt +++ /dev/null @@ -1,34 +0,0 @@ -Various files need to be in this directory. -If you don't have them, ask someone who does. - -The required files are: - -Binaries: -- acb_extract.exe -- usm_extract.exe -- clHCA - -Z shell scripts: -- update_bgm.zsh -- update_movie.zsh -- update_se.zsh -- update_voice.zsh - -Python scripts: -- adx_parse.py -- bingo_fuck.py -- chara_room_mission.py -- custom_mission.py -- event_bonus.py -- orphan_quest.py -- pad_2_csv.py -- pad_2_json.py -- pad_2_pad.py -- pad_2_pad_louisstyle.py -- pad_stat.py -- pad_update.py -- store_parser.py -- tsv_2_pad.py -- tsv_parse.py -- txt_2_pad.py -- utage_decrypt.py diff --git a/utage/names.py b/utage/names.py index b6633f0..0ea03f8 100644 --- a/utage/names.py +++ b/utage/names.py @@ -12,7 +12,7 @@ from multiprocessing import Pool def update_names(dir_in, old_path, languages, is_global): names = extract_names(dir_in, is_global) - global_languages = {"enm": 1, "zh": 2, "ko": 3} + global_languages = {"enm": 1, "zho": 2, "kor": 3} if is_global: languages = global_languages.keys() @@ -33,12 +33,12 @@ def update_names(dir_in, old_path, languages, is_global): langfile = os.path.join(old_path, "nametranslations_{}.json".format(l)) if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) with io.open( - langfile, "w", newline="\n" + langfile, "w", encoding="utf-8" ) as lang_file: # you're using git right json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, @@ -56,7 +56,8 @@ def extract_names(dir_in, is_global): files = [ f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) - if re.search(r"/[0-9]{9}\.tsv$", f) + if re.search(r"[0-9]{9}\.tsv$", f) + and f"{os.path.sep}Diva{os.path.sep}" not in f ] if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) @@ -78,7 +79,7 @@ def read_char_tsv(char_tsv_path, is_global): char_names = set() char_sets = set() - with open(char_tsv_path, "r") as char_tsv_file: + with open(char_tsv_path, "r", encoding="utf-8") as char_tsv_file: char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"') for row in char_tsv: @@ -103,7 +104,7 @@ def read_char_tsv(char_tsv_path, is_global): def read_mission(tsv_path, char_names, char_sets): new_names = set() - with open(tsv_path, "r") as tsv_file: + with open(tsv_path, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') if ( ("Arg1" not in tsv.fieldnames) diff --git a/utage/translate.py b/utage/translate.py index 2344b0a..cd3b875 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -15,7 +15,7 @@ def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global): files = [ f for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True) - if "{os.path.sep}Diva{os.path.sep}" not in f + if f"{os.path.sep}Diva{os.path.sep}" not in f ] if len(files) == 0: @@ -52,7 +52,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): ) languages = ["jpn"] if is_global: - languages += ["enm", "zh", "ko"] + languages = ["enm", "zho", "kor"] json_output_paths = { lang: os.path.join( json_out_dir, @@ -70,14 +70,14 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): if e.errno != errno.EEXIST: raise - if not os.path.exists(os.path.dirname(json_output_paths["jpn"])): + if not os.path.exists(os.path.dirname(json_output_paths[languages[0]])): try: - os.makedirs(os.path.dirname(json_output_paths["jpn"])) + os.makedirs(os.path.dirname(json_output_paths[languages[0]])) except OSError as e: if e.errno != errno.EEXIST: raise - with open(file_in, "r") as tsv_file: + with open(file_in, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE) t_fieldnames = tsv.fieldnames @@ -87,12 +87,11 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): tsv_keyed, json_str = process_tsv(tsv, id_num, is_global) # csv handles newlines, don't set it in io.open - with io.open(tsv_output_path, "w", newline="") as tsv_out: + with io.open(tsv_output_path, "w", newline="", encoding="utf-8") as tsv_out: writer = csv.DictWriter( tsv_out, delimiter="\t", quotechar='"', - lineterminator="\n", fieldnames=t_fieldnames, extrasaction="ignore", ) @@ -103,7 +102,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): for lang in languages: if not json_str[lang]: continue - with io.open(json_output_paths[lang], "w", newline="\n") as json_out: + with io.open(json_output_paths[lang], "w", encoding="utf-8") as json_out: # we don't want to sort these because they're _1, ..., _10, etc json.dump( json_str[lang], @@ -124,8 +123,8 @@ def process_tsv(tsv, id_num, is_global): key_dict = {"jpn": {}} if is_global: key_dict["enm"] = {} - key_dict["zh"] = {} - key_dict["ko"] = {} + key_dict["zho"] = {} + key_dict["kor"] = {} for row in tsv: try: @@ -140,8 +139,8 @@ def process_tsv(tsv, id_num, is_global): with suppress(KeyError): if row["English"]: key_dict["enm"][key] = row["English"] - key_dict["zh"][key] = row["Chinese"] - key_dict["ko"][key] = row["Korean"] + key_dict["zho"][key] = row["Chinese"] + key_dict["kor"][key] = row["Korean"] row["English"] = key tsv_keyed.append(row) except Exception: diff --git a/xdudata/asset_extract.py b/xdudata/asset_extract.py new file mode 100644 index 0000000..04e088c --- /dev/null +++ b/xdudata/asset_extract.py @@ -0,0 +1,303 @@ +import functools +import multiprocessing +import os +import shutil +import subprocess +from contextlib import contextmanager +from distutils.dir_util import copy_tree +from glob import glob + +from adx import extract_loop_data_from_dir + +OPERATING_SYSTEM = os.name +CHUNK_SIZE = 100 + + +def process_se(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Se"), + os.path.join(extract_dir, "Se"), + ) + + from_dir = "Se" + to_dir = "Se_Extracted" + acb_prefix = "_vgmt_acb_ext" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.wav".format(from_dir), recursive=True) + to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True)) + for file in to_del: + os.remove(file) + + acb_files = [file for file in glob("{}/*.acb".format(from_dir))] + acb_cmd = get_dotnet_cmd("acb_extract.exe") + run_program(acb_files, cmd=acb_cmd) + + hca_files = [ + file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) + ] + for hca_sublist in split_list(hca_files, CHUNK_SIZE): + run_program(hca_sublist, cmd="clHCA") + + basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] + char_secs = [basename.split("_", maxsplit=1) for basename in basenames] + for cs in char_secs: + char = cs[0] + sec = str(cs[1]) if len(cs) == 2 else "" + os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True) + wav_files = glob( + "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)), + recursive=True, + ) + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + wav_cmds = [ + [ + "-i", + os.path.abspath(file), + "-acodec", + "libopus", + "-ab", + "192k", + "-af", + "aresample=48000", + os.path.abspath( + "{}/{}/{}/{}.opus".format( + to_dir, + char, + sec, + os.path.splitext(os.path.basename(file))[0], + ) + ), + "-n", + ] + for file in wav_files + ] + pool.map(partial_ffmpeg_cmd, wav_cmds) + + copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Se")) + + +def process_bgm(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), + os.path.join(extract_dir, "Bgm"), + ) + + from_dir = "Bgm" + to_dir = "Bgm_Extracted" + loop_file = "loop.json" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.adx".format(from_dir), recursive=True) + for file in to_del: + os.remove(file) + + acb_files = [file for file in glob("{}/*.acb".format(from_dir))] + acb_cmd = get_dotnet_cmd("acb_extract.exe") + run_program(acb_files, cmd=acb_cmd) + + adx_files = [ + file for file in glob("{}/**/*.adx".format(from_dir), recursive=True) + ] + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + wav_cmds = [ + [ + "-i", + os.path.abspath(file), + "-acodec", + "libopus", + "-ab", + "192k", + "-af", + "aresample=48000", + os.path.abspath( + "{}/{}.opus".format( + to_dir, os.path.splitext(os.path.basename(file))[0], + ) + ), + "-n", + ] + for file in adx_files + ] + pool.map(partial_ffmpeg_cmd, wav_cmds) + + extract_loop_data_from_dir(from_dir, loop_file) + shutil.copy2(loop_file, os.path.join(to_dir, "BgmLoop.json")) + + copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Bgm")) + + +def process_voice(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Voice"), + os.path.join(extract_dir, "Voice"), + ) + + from_dir = "Voice" + to_dir = "Voice_Extracted" + acb_prefix = "_vgmt_acb_ext" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.wav".format(from_dir), recursive=True) + to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True)) + for file in to_del: + os.remove(file) + + acb_files = [file for file in glob("{}/*.acb".format(from_dir))] + acb_cmd = get_dotnet_cmd("acb_extract.exe") + run_program(acb_files, cmd=acb_cmd) + + hca_files = [ + file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) + ] + for hca_sublist in split_list(hca_files, CHUNK_SIZE): + run_program(hca_sublist, cmd="clHCA") + + basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] + char_secs = [basename.split("_", maxsplit=1) for basename in basenames] + for cs in char_secs: + char = cs[0] + sec = str(cs[1]) if len(cs) == 2 else "" + os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True) + wav_files = glob( + "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)), + recursive=True, + ) + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + + wav_cmds = [ + [ + "-i", + os.path.abspath(file), + "-acodec", + "libopus", + "-ab", + "192k", + "-af", + "aresample=48000", + os.path.abspath( + "{}/{}/{}/{}.opus".format( + to_dir, + char, + sec, + os.path.splitext(os.path.basename(file))[0], + ) + ), + "-n", + ] + for file in wav_files + ] + pool.map(partial_ffmpeg_cmd, wav_cmds) + + copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Voice")) + + +def process_movie(cache_dir, extract_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Common/Asset/Movie"), + os.path.join(extract_dir, "Movie"), + ) + + from_dir = "Movie" + to_dir = "Movie_Extracted" + + pool = get_pool() + + with change_cwd(extract_dir): + os.makedirs(to_dir, exist_ok=True) + + to_del = glob("{}/**/*.adx".format(from_dir), recursive=True) + to_del.extend(glob("{}/**/*.m2v".format(from_dir), recursive=True)) + for file in to_del: + os.remove(file) + + movie_paths = [ + os.path.abspath(os.path.splitext(file)[0]) + for file in glob("{}/*.usm".format(from_dir)) + if not os.path.basename(file).startswith("mini_radio") + ] + + usm_files = [[path + ".usm"] for path in movie_paths] + usm_cmd = get_dotnet_cmd("usm_extract.exe") + partial_usm_cmd = functools.partial(run_program, cmd=usm_cmd) + pool.map(partial_usm_cmd, usm_files) + + partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") + mux_cmds = [ + [ + "-i", + path + ".adx", + "-i", + path + ".m2v", + "-c:a", + "libopus", + "-b:a", + "196k", + "-af", + "aresample=48000", + "-c:v", + "libvpx", + "-b:v", + "1M", + "-threads", + "1", + os.path.abspath("{}/{}.webm".format(to_dir, os.path.basename(path))), + "-n", + ] + for path in movie_paths + ] + pool.map(partial_ffmpeg_cmd, mux_cmds) + + copy_tree( + os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Asset", "Movie") + ) + + +@contextmanager +def change_cwd(new_path): + old_cwd = os.getcwd() + os.chdir(new_path) + try: + yield + finally: + os.chdir(old_cwd) + + +def get_dotnet_cmd(name): + ext = os.path.splitext(name)[1] + if ext != ".exe": + raise RuntimeError("This doesn't seem like a .NET program.") + path = shutil.which(name) + if OPERATING_SYSTEM == "nt": + return [path] + return ["mono", path] + + +def run_program(args, cmd=None): + if isinstance(cmd, str): + cmd = [cmd] + full_cmd = cmd + args + subprocess.run(full_cmd) + + +def get_pool(size=multiprocessing.cpu_count()): + pool = multiprocessing.Pool(size) + return pool + + +def split_list(input_list, chunk_size): + return [ + input_list[i : i + chunk_size] for i in range(0, len(input_list), chunk_size) + ] diff --git a/xdudata/update.py b/xdudata/update.py index b659c3f..8f61b6f 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -3,16 +3,17 @@ from contextlib import suppress import diva import shutil import os -import subprocess import tempfile import utage from distutils.dir_util import copy_tree, remove_tree from glob import glob +from xdudata.asset_extract import process_se, process_bgm, process_voice, process_movie + def update_all( - cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages, + cache_dir, update_dir, translation_dir, extract_dir, languages, ): print("Cleaning cache") cleanup_cache(cache_dir) @@ -24,64 +25,38 @@ def update_all( update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) print("Processing Quest Json files") - diva.quest.update_missions(extract_dir, translation_dir, languages) - - # this code isn't going to run on windows (it is in WSL though) because i need to rewrite it - # it just calls my zsh tools and generates it that way, need to replace them - # with real python or something - - # first copy over the necessary avtools to the extract dir - # TODO make it so that you don't need to copy them over (for example call them with python) - print("Copying necessary tools into extracting directory") - shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir) - shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir) - shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir) + diva.quest.update_missions(extract_dir, translation_dir, languages, False) print("Processing Sound Effects") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Se"), - os.path.join(extract_dir, "Se"), - ) - subprocess.run( - os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir - ) - copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se")) + process_se(cache_dir, extract_dir, update_dir) - print("Processing BGM - don't forget to copy that loop file") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), - os.path.join(extract_dir, "Bgm"), - ) - subprocess.run( - os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir - ) - copy_tree( - os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") - ) + print("Processing BGM") + process_bgm(cache_dir, extract_dir, update_dir) print("Processing voices") - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Voice"), - os.path.join(extract_dir, "Voice"), - ) - subprocess.run( - os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir - ) - copy_tree( - os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") - ) + process_voice(cache_dir, extract_dir, update_dir) print("Processing movies - this might take forever") + process_movie(cache_dir, extract_dir, update_dir) + + +def cleanup_cache(cache_dir): + with suppress(FileNotFoundError): + shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) + os.remove( + os.path.join( + cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" + ) + ) + + +def copy_images(cache_dir, update_dir): copy_tree( - os.path.join(cache_dir, "Common/Asset/Movie"), - os.path.join(extract_dir, "Movie"), - ) - subprocess.run( - os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir + os.path.join(cache_dir, "Common/Asset/Image"), + os.path.join(update_dir, "Asset/Image"), ) copy_tree( - os.path.join(extract_dir, "movie_extracted"), - os.path.join(update_dir, "Asset/Movie"), + os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), ) @@ -113,23 +88,3 @@ def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages): utage.names.update_names(utage_tmp, translation_dir, languages, False) copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) remove_tree(utage_tmp) - - -def copy_images(cache_dir, update_dir): - copy_tree( - os.path.join(cache_dir, "Common/Asset/Image"), - os.path.join(update_dir, "Asset/Image"), - ) - copy_tree( - os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), - ) - - -def cleanup_cache(cache_dir): - with suppress(FileNotFoundError): - shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) - os.remove( - os.path.join( - cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" - ) - )