diff --git a/.gitignore b/.gitignore index cd455e6..f8d0fa1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ __pycache__/ *$py.class *.swp virtualenv/ -.idea/ tools/* -!tools/.gitkeep +!tools/readme.txt +.idea/ diff --git a/README.md b/README.md deleted file mode 100644 index 81ee494..0000000 --- a/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# divatool - -A tool for that one game everyone hates but works on anyway. - -## Requirements - -Divatool is written in and uses [Python 3](https://www.python.org/downloads/). - -There are also several third party dependencies not provided as part of this repository, -mostly used for the `xdudata` command. You need to have the binaries of the following tools -either in your `$PATH` variable or in the [`tools`](./tools) folder. These tools are: - -* [clHCA](https://github.com/KinoMyu/FastHCADecoder) -* acb_extract -* usm_extract -* [ffmpeg](https://www.ffmpeg.org/download.html) -* [mono](https://www.mono-project.com/download/stable/) (if using Linux) - -Make sure these executables are called the same way on your machine. - -## Usage - -TODO: add detailed descriptions for all parameters. - -Using the tool to update XDUData for the player: - -``` -divatool.py xdudata update [path to the 'files' folder from the game] [path to the XDUData folder for the player] [path to the 'xdutranslations' repo folder] [path to the temp files folder] -``` - -Example: -``` -divatool.py xdudata update /home/user/gameapp/files /home/user/player/XDUData /home/user/git/xdutranslations /var/tmp -``` - -Note that the 'files' folder from the app must contain the db files for this to work. diff --git a/adx/adx_parser.py b/adx/adx_parser.py index d6d4c5d..993fc3c 100644 --- a/adx/adx_parser.py +++ b/adx/adx_parser.py @@ -74,7 +74,7 @@ class adx_file(object): self.encrypted = self.get_val(CRYPT_OFF, CRYPT_LEN) self.d_offset = self.get_val(DATA_OFF_OFF, DATA_OFF_LEN) if self.d_offset < DATA_OFF_MIN: - return + return None self.l_exists = 1 # now load things for math self.sam_rate = self.get_val(SRATE_OFF, SRATE_LEN) @@ -106,11 +106,13 @@ class adx_file(object): return None if self.l_start > self.sam_count or self.l_end > self.sam_count: raise ValueError("Invalid Loop Data") + return None if self.l_style not in LOOP_TYPE_SUP: raise NotImplementedError( "Loop Style {} Not Implemented".format(str(self.l_style)) ) - ret = dict() + return None + ret = {} ret["duration"] = self.sam_count / self.sam_rate ret["loop_start"] = {} ret["loop_start"]["seconds"] = self.l_start / self.sam_rate @@ -121,5 +123,7 @@ class adx_file(object): ret["loop_end"] = {} ret["loop_end"]["seconds"] = self.l_end / self.sam_rate ret["loop_end"]["samples_native"] = self.l_end - ret["loop_end"]["samples_48k"] = dumb_round(self.l_end / self.sam_rate * 48000) + ret["loop_end"]["samples_48k"] = dumb_round( + self.l_end / self.sam_rate * 48000 + ) return ret diff --git a/db/base.py b/db/base.py index 80132be..afa5817 100644 --- a/db/base.py +++ b/db/base.py @@ -23,7 +23,7 @@ class BaseDB(object): return self def __exit__(self, exc_type, exc_value, traceback): - self.conn.close() + self.quest_conn.close() def attach_dbs(self, databases, path): for d in databases: diff --git a/db/quest.py b/db/quest.py index bedc636..c4b13f8 100644 --- a/db/quest.py +++ b/db/quest.py @@ -1,24 +1,16 @@ from .base import BaseDB - -def get_dbs_by_lang(language=None): - databases = { - "QuestMst": "Quest", - "QuestSceneMst": "QuestScene", - "QuestPartMst": "QuestPart", - "ResourceEntry": "ResourceEntry", - } - for key in list(databases): - if not language: - databases[key + ".db"] = databases.pop(key) - else: - databases[key + "_" + language + ".db"] = databases.pop(key) - return databases +DATABASES = { + "QuestMst.db": "Quest", + "QuestSceneMst.db": "QuestScene", + "QuestPartMst.db": "QuestPart", + "ResourceEntry.db": "ResourceEntry", +} class QuestDB(BaseDB): - def __init__(self, path, language=None): - super().__init__(path, get_dbs_by_lang(language)) + def __init__(self, path): + super().__init__(path, DATABASES) def get_quests(self): self.cursor.execute( @@ -52,14 +44,12 @@ class QuestDB(BaseDB): questSceneMstId, name, summaryText, - img, group_concat(partIds) as parts FROM ( SELECT qs.questSceneMstId, qs.name, qs.summaryText, - qs.img, CASE WHEN qp.afterTalkName == '' AND qp.beforeTalkName == '' THEN NULL WHEN qp.afterTalkName == '' AND qp.beforeTalkName != '' THEN qp.beforeTalkName @@ -76,11 +66,9 @@ class QuestDB(BaseDB): ) res = self.cursor.fetchall() scenes = {} - for scene_id, name, summary, img, parts in res: + for scene_id, name, summary, parts in res: scenes[str(scene_id)] = {} scenes[str(scene_id)]["Name"] = name scenes[str(scene_id)]["SummaryText"] = summary - if img: - scenes[str(scene_id)]["Image"] = img scenes[str(scene_id)]["Parts"] = parts.split(",") return scenes diff --git a/diva/quest.py b/diva/quest.py index a2306f0..f2342a1 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -7,25 +7,16 @@ import os from glob import glob -def update_missions(dir_in, old_path, languages, is_global, utage_in=None): - if not utage_in: - utage_in = os.path.join(dir_in, "Common/Asset/Utage") - if is_global: - languages = ["enm", "zho", "kor"] - for lang in languages: - qdb = db.QuestDB(dir_in, lang) - quest_mst(qdb, old_path, [lang], is_global) - scene_mst(qdb, old_path, [lang], utage_in, is_global) - else: - qdb = db.QuestDB(dir_in) - quest_mst(qdb, old_path, languages, is_global) - scene_mst(qdb, old_path, languages, utage_in, is_global) +def update_missions(dir_in, old_path, languages): + qdb = db.QuestDB(dir_in) + quest_mst(qdb, old_path, languages) + scene_mst(qdb, old_path, dir_in, languages) -def scene_mst(qdb, old_path, languages, utage_in, is_global): +def scene_mst(qdb, old_path, dir_in, languages): scenes = qdb.get_scenes() - locations = load_locations(utage_in) + locations = load_locations(dir_in) for id, scene in scenes.items(): i = 0 @@ -35,66 +26,57 @@ def scene_mst(qdb, old_path, languages, utage_in, is_global): except KeyError as exc: i += 1 if i == len(scene["Parts"]): - if is_global: - break raise KeyError(f"Could not find folder for scene {id}") from exc continue break - if not is_global: - with io.open( - os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8" - ) as json_file: - json.dump( - scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True - ) + with io.open( + os.path.join(old_path, "XduScene.json"), "w", newline="\n" + ) as json_file: + json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True) for lang in languages: out_dict = {} - for key, value in scenes.items(): - name = "" - summary = "" - credit = "" - enabled = False - - if lang == "jpn" or is_global: - name = value["Name"] - summary = value["SummaryText"] - credit = "POKELABO" - - out_dict[key] = { - "Name": name, - "SummaryText": summary, - "Credits": credit, - "Enabled": enabled, - } + if lang == "jpn": + for key, value in scenes.items(): + out_dict[key] = { + "Name": value["Name"], + "SummaryText": value["SummaryText"], + "Credits": "POKELABO", + "Enabled": False, + } + else: + for key in scenes.keys(): + out_dict[key] = { + "Name": "", + "SummaryText": "", + "Credits": "", + "Enabled": False, + } langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json") if os.path.isfile(langfile): - with open(langfile, "r", encoding="utf-8") as lang_file: + with open(langfile, "r") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) - with io.open(langfile, "w", encoding="utf-8") as lang_file: + with io.open(langfile, "w", newline="\n") as lang_file: json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) -def quest_mst(qdb, old_path, languages, is_global): +def quest_mst(qdb, old_path, languages): quests = qdb.get_quests() - if not is_global: - with io.open( - os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8" - ) as json_file: - json.dump( - quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True - ) + with io.open( + os.path.join(old_path, "XduQuest.json"), "w", newline="\n" + ) as json_file: + json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True) for lang in languages: out_dict = {} - if lang == "jpn" or is_global: + if lang == "jpn": for key, value in quests.items(): out_dict[key] = {"Name": value["Name"], "Enabled": False} else: @@ -103,11 +85,11 @@ def quest_mst(qdb, old_path, languages, is_global): langfile = os.path.join(old_path, f"XduQuestNames_{lang}.json") if os.path.isfile(langfile): - with open(langfile, "r", encoding="utf-8") as lang_file: + with open(langfile, "r") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) - with io.open(langfile, "w", encoding="utf-8") as lang_file: + with io.open(langfile, "w", newline="\n") as lang_file: json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -115,28 +97,25 @@ def quest_mst(qdb, old_path, languages, is_global): return quests -def load_locations(utage_in): +def load_locations(dir_in): locations = {} - settings = glob("{}/**/Settings/Scenario.tsv".format(utage_in), recursive=True,) + settings = glob( + os.path.join(dir_in, "Common/Asset/Utage/**/Settings/Scenario.tsv"), + recursive=True, + ) if len(settings) == 0: raise FileNotFoundError("Ya didn't decrypt the files dummy") for scenario in settings: - with open(scenario, "r", encoding="utf-8") as tsv_file: + with open(scenario, "r") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') for row in tsv: tokens = row["FileName"].split("/") if len(tokens) < 3: continue - folder, _scenario, part_id = tokens - # side03 seems to only contain prototype scripts so we ignore it - if folder == "side03": - continue - if part_id in locations: - raise ValueError( - f"Conflicting paths for scene part {part_id}, existing folder: {locations[part_id]}" - ) - locations[part_id] = folder + if tokens[2] in locations: + raise ValueError("Conflicting paths for file {tokens[2]}") + locations[tokens[2]] = tokens[0] return locations diff --git a/divatool.py b/divatool.py index f43a927..ec36fed 100755 --- a/divatool.py +++ b/divatool.py @@ -9,7 +9,7 @@ import utage import db import xdudata -LANGUAGES = ["jpn", "eng", "rus", "cze"] +LANGUAGES = ["jpn", "eng", "rus"] def main(): @@ -117,16 +117,13 @@ def main(): metavar="", title="subcommand", dest="subcommand" ) parser_diva_quest = parser_diva_subparsers.add_parser( - "quest", help="Generate quest JSON files", parents=[global_flag_parser], + "quest", help="Generate quest JSON files" ) parser_diva_quest.add_argument( "INPUT", help="Directory of databases containing Quest*.db and ResourceEntry.db", type=str, ) - parser_diva_quest.add_argument( - "--utage", help="Directory of Utage folders and files.", type=str, default=None, - ) parser_diva_quest.add_argument( "OLD", help="Directory with old quest files", nargs="?", type=str, default="." ) @@ -185,24 +182,17 @@ def main(): raise FileNotFoundError(args.INPUT) elif args.command == "diva": if args.subcommand == "quest": - if not args.utage: - diva.quest.update_missions( - args.INPUT, args.OLD, LANGUAGES, args.is_global - ) - else: - diva.quest.update_missions( - args.INPUT, args.OLD, LANGUAGES, args.is_global, utage_in=args.utage - ) + diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) elif args.command == "xdudata": + script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools")) if args.subcommand == "update": - # add the tools folder to PATH - os.environ["PATH"] += os.pathsep + os.path.join( - os.path.abspath(os.path.dirname(__file__)), "tools" - ) - - # update the data xdudata.update_all( - args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES, + args.CACHE, + args.XDUDATA, + args.TRANS, + args.EXTRACT, + script_dir, + LANGUAGES, ) return 0 diff --git a/tools/.gitkeep b/tools/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tools/readme.txt b/tools/readme.txt new file mode 100644 index 0000000..f10fdc0 --- /dev/null +++ b/tools/readme.txt @@ -0,0 +1,34 @@ +Various files need to be in this directory. +If you don't have them, ask someone who does. + +The required files are: + +Binaries: +- acb_extract.exe +- usm_extract.exe +- clHCA + +Z shell scripts: +- update_bgm.zsh +- update_movie.zsh +- update_se.zsh +- update_voice.zsh + +Python scripts: +- adx_parse.py +- bingo_fuck.py +- chara_room_mission.py +- custom_mission.py +- event_bonus.py +- orphan_quest.py +- pad_2_csv.py +- pad_2_json.py +- pad_2_pad.py +- pad_2_pad_louisstyle.py +- pad_stat.py +- pad_update.py +- store_parser.py +- tsv_2_pad.py +- tsv_parse.py +- txt_2_pad.py +- utage_decrypt.py diff --git a/utage/names.py b/utage/names.py index 0ea03f8..b6633f0 100644 --- a/utage/names.py +++ b/utage/names.py @@ -12,7 +12,7 @@ from multiprocessing import Pool def update_names(dir_in, old_path, languages, is_global): names = extract_names(dir_in, is_global) - global_languages = {"enm": 1, "zho": 2, "kor": 3} + global_languages = {"enm": 1, "zh": 2, "ko": 3} if is_global: languages = global_languages.keys() @@ -33,12 +33,12 @@ def update_names(dir_in, old_path, languages, is_global): langfile = os.path.join(old_path, "nametranslations_{}.json".format(l)) if os.path.isfile(langfile): - with open(langfile, "r", encoding="utf-8") as lang_file: + with open(langfile, "r") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) with io.open( - langfile, "w", encoding="utf-8" + langfile, "w", newline="\n" ) as lang_file: # you're using git right json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, @@ -56,8 +56,7 @@ def extract_names(dir_in, is_global): files = [ f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) - if re.search(r"[0-9]{9}\.tsv$", f) - and f"{os.path.sep}Diva{os.path.sep}" not in f + if re.search(r"/[0-9]{9}\.tsv$", f) ] if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) @@ -79,7 +78,7 @@ def read_char_tsv(char_tsv_path, is_global): char_names = set() char_sets = set() - with open(char_tsv_path, "r", encoding="utf-8") as char_tsv_file: + with open(char_tsv_path, "r") as char_tsv_file: char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"') for row in char_tsv: @@ -104,7 +103,7 @@ def read_char_tsv(char_tsv_path, is_global): def read_mission(tsv_path, char_names, char_sets): new_names = set() - with open(tsv_path, "r", encoding="utf-8") as tsv_file: + with open(tsv_path, "r") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') if ( ("Arg1" not in tsv.fieldnames) diff --git a/utage/translate.py b/utage/translate.py index cd3b875..2344b0a 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -15,7 +15,7 @@ def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global): files = [ f for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True) - if f"{os.path.sep}Diva{os.path.sep}" not in f + if "{os.path.sep}Diva{os.path.sep}" not in f ] if len(files) == 0: @@ -52,7 +52,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): ) languages = ["jpn"] if is_global: - languages = ["enm", "zho", "kor"] + languages += ["enm", "zh", "ko"] json_output_paths = { lang: os.path.join( json_out_dir, @@ -70,14 +70,14 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): if e.errno != errno.EEXIST: raise - if not os.path.exists(os.path.dirname(json_output_paths[languages[0]])): + if not os.path.exists(os.path.dirname(json_output_paths["jpn"])): try: - os.makedirs(os.path.dirname(json_output_paths[languages[0]])) + os.makedirs(os.path.dirname(json_output_paths["jpn"])) except OSError as e: if e.errno != errno.EEXIST: raise - with open(file_in, "r", encoding="utf-8") as tsv_file: + with open(file_in, "r") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE) t_fieldnames = tsv.fieldnames @@ -87,11 +87,12 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): tsv_keyed, json_str = process_tsv(tsv, id_num, is_global) # csv handles newlines, don't set it in io.open - with io.open(tsv_output_path, "w", newline="", encoding="utf-8") as tsv_out: + with io.open(tsv_output_path, "w", newline="") as tsv_out: writer = csv.DictWriter( tsv_out, delimiter="\t", quotechar='"', + lineterminator="\n", fieldnames=t_fieldnames, extrasaction="ignore", ) @@ -102,7 +103,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): for lang in languages: if not json_str[lang]: continue - with io.open(json_output_paths[lang], "w", encoding="utf-8") as json_out: + with io.open(json_output_paths[lang], "w", newline="\n") as json_out: # we don't want to sort these because they're _1, ..., _10, etc json.dump( json_str[lang], @@ -123,8 +124,8 @@ def process_tsv(tsv, id_num, is_global): key_dict = {"jpn": {}} if is_global: key_dict["enm"] = {} - key_dict["zho"] = {} - key_dict["kor"] = {} + key_dict["zh"] = {} + key_dict["ko"] = {} for row in tsv: try: @@ -139,8 +140,8 @@ def process_tsv(tsv, id_num, is_global): with suppress(KeyError): if row["English"]: key_dict["enm"][key] = row["English"] - key_dict["zho"][key] = row["Chinese"] - key_dict["kor"][key] = row["Korean"] + key_dict["zh"][key] = row["Chinese"] + key_dict["ko"][key] = row["Korean"] row["English"] = key tsv_keyed.append(row) except Exception: diff --git a/xdudata/asset_extract.py b/xdudata/asset_extract.py deleted file mode 100644 index 04e088c..0000000 --- a/xdudata/asset_extract.py +++ /dev/null @@ -1,303 +0,0 @@ -import functools -import multiprocessing -import os -import shutil -import subprocess -from contextlib import contextmanager -from distutils.dir_util import copy_tree -from glob import glob - -from adx import extract_loop_data_from_dir - -OPERATING_SYSTEM = os.name -CHUNK_SIZE = 100 - - -def process_se(cache_dir, extract_dir, update_dir): - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Se"), - os.path.join(extract_dir, "Se"), - ) - - from_dir = "Se" - to_dir = "Se_Extracted" - acb_prefix = "_vgmt_acb_ext" - - pool = get_pool() - - with change_cwd(extract_dir): - os.makedirs(to_dir, exist_ok=True) - - to_del = glob("{}/**/*.wav".format(from_dir), recursive=True) - to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True)) - for file in to_del: - os.remove(file) - - acb_files = [file for file in glob("{}/*.acb".format(from_dir))] - acb_cmd = get_dotnet_cmd("acb_extract.exe") - run_program(acb_files, cmd=acb_cmd) - - hca_files = [ - file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) - ] - for hca_sublist in split_list(hca_files, CHUNK_SIZE): - run_program(hca_sublist, cmd="clHCA") - - basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] - char_secs = [basename.split("_", maxsplit=1) for basename in basenames] - for cs in char_secs: - char = cs[0] - sec = str(cs[1]) if len(cs) == 2 else "" - os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True) - wav_files = glob( - "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)), - recursive=True, - ) - partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") - wav_cmds = [ - [ - "-i", - os.path.abspath(file), - "-acodec", - "libopus", - "-ab", - "192k", - "-af", - "aresample=48000", - os.path.abspath( - "{}/{}/{}/{}.opus".format( - to_dir, - char, - sec, - os.path.splitext(os.path.basename(file))[0], - ) - ), - "-n", - ] - for file in wav_files - ] - pool.map(partial_ffmpeg_cmd, wav_cmds) - - copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Se")) - - -def process_bgm(cache_dir, extract_dir, update_dir): - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), - os.path.join(extract_dir, "Bgm"), - ) - - from_dir = "Bgm" - to_dir = "Bgm_Extracted" - loop_file = "loop.json" - - pool = get_pool() - - with change_cwd(extract_dir): - os.makedirs(to_dir, exist_ok=True) - - to_del = glob("{}/**/*.adx".format(from_dir), recursive=True) - for file in to_del: - os.remove(file) - - acb_files = [file for file in glob("{}/*.acb".format(from_dir))] - acb_cmd = get_dotnet_cmd("acb_extract.exe") - run_program(acb_files, cmd=acb_cmd) - - adx_files = [ - file for file in glob("{}/**/*.adx".format(from_dir), recursive=True) - ] - partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") - wav_cmds = [ - [ - "-i", - os.path.abspath(file), - "-acodec", - "libopus", - "-ab", - "192k", - "-af", - "aresample=48000", - os.path.abspath( - "{}/{}.opus".format( - to_dir, os.path.splitext(os.path.basename(file))[0], - ) - ), - "-n", - ] - for file in adx_files - ] - pool.map(partial_ffmpeg_cmd, wav_cmds) - - extract_loop_data_from_dir(from_dir, loop_file) - shutil.copy2(loop_file, os.path.join(to_dir, "BgmLoop.json")) - - copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Bgm")) - - -def process_voice(cache_dir, extract_dir, update_dir): - copy_tree( - os.path.join(cache_dir, "Android/Asset/Sound/Voice"), - os.path.join(extract_dir, "Voice"), - ) - - from_dir = "Voice" - to_dir = "Voice_Extracted" - acb_prefix = "_vgmt_acb_ext" - - pool = get_pool() - - with change_cwd(extract_dir): - os.makedirs(to_dir, exist_ok=True) - - to_del = glob("{}/**/*.wav".format(from_dir), recursive=True) - to_del.extend(glob("{}/**/*.hca".format(from_dir), recursive=True)) - for file in to_del: - os.remove(file) - - acb_files = [file for file in glob("{}/*.acb".format(from_dir))] - acb_cmd = get_dotnet_cmd("acb_extract.exe") - run_program(acb_files, cmd=acb_cmd) - - hca_files = [ - file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) - ] - for hca_sublist in split_list(hca_files, CHUNK_SIZE): - run_program(hca_sublist, cmd="clHCA") - - basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] - char_secs = [basename.split("_", maxsplit=1) for basename in basenames] - for cs in char_secs: - char = cs[0] - sec = str(cs[1]) if len(cs) == 2 else "" - os.makedirs(os.path.join(to_dir, char, sec), exist_ok=True) - wav_files = glob( - "{}/{}_{}/**/*.wav".format(from_dir, acb_prefix, "_".join(cs)), - recursive=True, - ) - partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") - - wav_cmds = [ - [ - "-i", - os.path.abspath(file), - "-acodec", - "libopus", - "-ab", - "192k", - "-af", - "aresample=48000", - os.path.abspath( - "{}/{}/{}/{}.opus".format( - to_dir, - char, - sec, - os.path.splitext(os.path.basename(file))[0], - ) - ), - "-n", - ] - for file in wav_files - ] - pool.map(partial_ffmpeg_cmd, wav_cmds) - - copy_tree(os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Voice")) - - -def process_movie(cache_dir, extract_dir, update_dir): - copy_tree( - os.path.join(cache_dir, "Common/Asset/Movie"), - os.path.join(extract_dir, "Movie"), - ) - - from_dir = "Movie" - to_dir = "Movie_Extracted" - - pool = get_pool() - - with change_cwd(extract_dir): - os.makedirs(to_dir, exist_ok=True) - - to_del = glob("{}/**/*.adx".format(from_dir), recursive=True) - to_del.extend(glob("{}/**/*.m2v".format(from_dir), recursive=True)) - for file in to_del: - os.remove(file) - - movie_paths = [ - os.path.abspath(os.path.splitext(file)[0]) - for file in glob("{}/*.usm".format(from_dir)) - if not os.path.basename(file).startswith("mini_radio") - ] - - usm_files = [[path + ".usm"] for path in movie_paths] - usm_cmd = get_dotnet_cmd("usm_extract.exe") - partial_usm_cmd = functools.partial(run_program, cmd=usm_cmd) - pool.map(partial_usm_cmd, usm_files) - - partial_ffmpeg_cmd = functools.partial(run_program, cmd="ffmpeg") - mux_cmds = [ - [ - "-i", - path + ".adx", - "-i", - path + ".m2v", - "-c:a", - "libopus", - "-b:a", - "196k", - "-af", - "aresample=48000", - "-c:v", - "libvpx", - "-b:v", - "1M", - "-threads", - "1", - os.path.abspath("{}/{}.webm".format(to_dir, os.path.basename(path))), - "-n", - ] - for path in movie_paths - ] - pool.map(partial_ffmpeg_cmd, mux_cmds) - - copy_tree( - os.path.join(extract_dir, to_dir), os.path.join(update_dir, "Asset", "Movie") - ) - - -@contextmanager -def change_cwd(new_path): - old_cwd = os.getcwd() - os.chdir(new_path) - try: - yield - finally: - os.chdir(old_cwd) - - -def get_dotnet_cmd(name): - ext = os.path.splitext(name)[1] - if ext != ".exe": - raise RuntimeError("This doesn't seem like a .NET program.") - path = shutil.which(name) - if OPERATING_SYSTEM == "nt": - return [path] - return ["mono", path] - - -def run_program(args, cmd=None): - if isinstance(cmd, str): - cmd = [cmd] - full_cmd = cmd + args - subprocess.run(full_cmd) - - -def get_pool(size=multiprocessing.cpu_count()): - pool = multiprocessing.Pool(size) - return pool - - -def split_list(input_list, chunk_size): - return [ - input_list[i : i + chunk_size] for i in range(0, len(input_list), chunk_size) - ] diff --git a/xdudata/update.py b/xdudata/update.py index 8f61b6f..b659c3f 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -3,17 +3,16 @@ from contextlib import suppress import diva import shutil import os +import subprocess import tempfile import utage from distutils.dir_util import copy_tree, remove_tree from glob import glob -from xdudata.asset_extract import process_se, process_bgm, process_voice, process_movie - def update_all( - cache_dir, update_dir, translation_dir, extract_dir, languages, + cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages, ): print("Cleaning cache") cleanup_cache(cache_dir) @@ -25,38 +24,64 @@ def update_all( update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) print("Processing Quest Json files") - diva.quest.update_missions(extract_dir, translation_dir, languages, False) + diva.quest.update_missions(extract_dir, translation_dir, languages) + + # this code isn't going to run on windows (it is in WSL though) because i need to rewrite it + # it just calls my zsh tools and generates it that way, need to replace them + # with real python or something + + # first copy over the necessary avtools to the extract dir + # TODO make it so that you don't need to copy them over (for example call them with python) + print("Copying necessary tools into extracting directory") + shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir) + shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir) + shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir) print("Processing Sound Effects") - process_se(cache_dir, extract_dir, update_dir) - - print("Processing BGM") - process_bgm(cache_dir, extract_dir, update_dir) - - print("Processing voices") - process_voice(cache_dir, extract_dir, update_dir) - - print("Processing movies - this might take forever") - process_movie(cache_dir, extract_dir, update_dir) - - -def cleanup_cache(cache_dir): - with suppress(FileNotFoundError): - shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) - os.remove( - os.path.join( - cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" - ) - ) - - -def copy_images(cache_dir, update_dir): copy_tree( - os.path.join(cache_dir, "Common/Asset/Image"), - os.path.join(update_dir, "Asset/Image"), + os.path.join(cache_dir, "Android/Asset/Sound/Se"), + os.path.join(extract_dir, "Se"), + ) + subprocess.run( + os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir + ) + copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se")) + + print("Processing BGM - don't forget to copy that loop file") + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), + os.path.join(extract_dir, "Bgm"), + ) + subprocess.run( + os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir ) copy_tree( - os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), + os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") + ) + + print("Processing voices") + copy_tree( + os.path.join(cache_dir, "Android/Asset/Sound/Voice"), + os.path.join(extract_dir, "Voice"), + ) + subprocess.run( + os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir + ) + copy_tree( + os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") + ) + + print("Processing movies - this might take forever") + copy_tree( + os.path.join(cache_dir, "Common/Asset/Movie"), + os.path.join(extract_dir, "Movie"), + ) + subprocess.run( + os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir + ) + copy_tree( + os.path.join(extract_dir, "movie_extracted"), + os.path.join(update_dir, "Asset/Movie"), ) @@ -88,3 +113,23 @@ def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages): utage.names.update_names(utage_tmp, translation_dir, languages, False) copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) remove_tree(utage_tmp) + + +def copy_images(cache_dir, update_dir): + copy_tree( + os.path.join(cache_dir, "Common/Asset/Image"), + os.path.join(update_dir, "Asset/Image"), + ) + copy_tree( + os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), + ) + + +def cleanup_cache(cache_dir): + with suppress(FileNotFoundError): + shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) + os.remove( + os.path.join( + cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" + ) + )