diff --git a/.gitignore b/.gitignore index e691145..f8d0fa1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ __pycache__/ *$py.class *.swp virtualenv/ +tools/* +!tools/readme.txt +.idea/ diff --git a/diva/quest.py b/diva/quest.py index d84e3b2..f2342a1 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -26,18 +26,14 @@ def scene_mst(qdb, old_path, dir_in, languages): except KeyError as exc: i += 1 if i == len(scene["Parts"]): - raise KeyError( - f"Could not find folder for scene {id}" - ) from exc + raise KeyError(f"Could not find folder for scene {id}") from exc continue break with io.open( os.path.join(old_path, "XduScene.json"), "w", newline="\n" ) as json_file: - json.dump( - scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True - ) + json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True) for lang in languages: out_dict = {} @@ -66,11 +62,7 @@ def scene_mst(qdb, old_path, dir_in, languages): with io.open(langfile, "w", newline="\n") as lang_file: json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -80,9 +72,7 @@ def quest_mst(qdb, old_path, languages): with io.open( os.path.join(old_path, "XduQuest.json"), "w", newline="\n" ) as json_file: - json.dump( - quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True - ) + json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True) for lang in languages: out_dict = {} @@ -101,11 +91,7 @@ def quest_mst(qdb, old_path, languages): with io.open(langfile, "w", newline="\n") as lang_file: json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) return quests diff --git a/divatool.py b/divatool.py index cd5f342..ec36fed 100755 --- a/divatool.py +++ b/divatool.py @@ -15,6 +15,16 @@ LANGUAGES = ["jpn", "eng", "rus"] def main(): parser = argparse.ArgumentParser(description="Unpack and process XDU data") + global_flag_parser = argparse.ArgumentParser(add_help=False) + global_flag_parser.add_argument( + "-g", + "--global", + help="Whether we are running on Global XDU.", + action="store_true", + dest="is_global", + default=False, + ) + subparsers = parser.add_subparsers( metavar="", title="subcommands", dest="command" ) @@ -81,7 +91,9 @@ def main(): parser_utage_crypt.add_argument("TARGET", help="Input", type=str) # utage translate parser_utage_translate = parser_utage_subparsers.add_parser( - "translate", help="Generate keyed tsv and json files" + "translate", + help="Generate keyed tsv and json files", + parents=[global_flag_parser], ) parser_utage_translate.add_argument("INPUT", help="Input", type=str) parser_utage_translate.add_argument( @@ -92,7 +104,7 @@ def main(): ) # utage names parser_utage_names = parser_utage_subparsers.add_parser( - "names", help="Generate and update name files" + "names", help="Generate and update name files", parents=[global_flag_parser] ) parser_utage_names.add_argument("INPUT", help="Input", type=str) parser_utage_names.add_argument( @@ -152,23 +164,35 @@ def main(): raise FileNotFoundError("Could not find {}".format(args.TARGET)) elif args.subcommand == "translate": if os.path.isfile(args.INPUT): - utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR) + utage.translate.translate_file( + args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global + ) elif os.path.isdir(args.INPUT): - utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR) + utage.translate.translate_dir( + args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global + ) else: raise FileNotFoundError("Could not find {}".format(args.INPUT)) elif args.subcommand == "names": if os.path.isdir(args.INPUT): - utage.names.update_names(args.INPUT, args.OLD, LANGUAGES) + utage.names.update_names( + args.INPUT, args.OLD, LANGUAGES, args.is_global + ) else: raise FileNotFoundError(args.INPUT) elif args.command == "diva": if args.subcommand == "quest": diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) elif args.command == "xdudata": + script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools")) if args.subcommand == "update": xdudata.update_all( - args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES + args.CACHE, + args.XDUDATA, + args.TRANS, + args.EXTRACT, + script_dir, + LANGUAGES, ) return 0 diff --git a/tools/readme.txt b/tools/readme.txt new file mode 100644 index 0000000..f10fdc0 --- /dev/null +++ b/tools/readme.txt @@ -0,0 +1,34 @@ +Various files need to be in this directory. +If you don't have them, ask someone who does. + +The required files are: + +Binaries: +- acb_extract.exe +- usm_extract.exe +- clHCA + +Z shell scripts: +- update_bgm.zsh +- update_movie.zsh +- update_se.zsh +- update_voice.zsh + +Python scripts: +- adx_parse.py +- bingo_fuck.py +- chara_room_mission.py +- custom_mission.py +- event_bonus.py +- orphan_quest.py +- pad_2_csv.py +- pad_2_json.py +- pad_2_pad.py +- pad_2_pad_louisstyle.py +- pad_stat.py +- pad_update.py +- store_parser.py +- tsv_2_pad.py +- tsv_parse.py +- txt_2_pad.py +- utage_decrypt.py diff --git a/utage/names.py b/utage/names.py index 56ef85c..b6633f0 100644 --- a/utage/names.py +++ b/utage/names.py @@ -9,14 +9,25 @@ from glob import glob from multiprocessing import Pool -def update_names(dir_in, old_path, languages): - names = extract_names(dir_in) +def update_names(dir_in, old_path, languages, is_global): + names = extract_names(dir_in, is_global) + + global_languages = {"enm": 1, "zh": 2, "ko": 3} + if is_global: + languages = global_languages.keys() # we have to update japanese like the rest because CustomData exists # and POKELABO deleted the wedding gear event stuff from the game files for l in languages: if l == "jpn": out_dict = dict(zip(names, names)) + elif l in global_languages.keys(): + out_dict = {} + for name in names: + spl = name.split("\t") + idx = global_languages[l] + val = spl[idx] if spl[idx] != "None" else "" + out_dict[spl[0]] = val else: out_dict = dict.fromkeys(names, "") @@ -30,21 +41,17 @@ def update_names(dir_in, old_path, languages): langfile, "w", newline="\n" ) as lang_file: # you're using git right json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) -def extract_names(dir_in): +def extract_names(dir_in, is_global): char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv") if not os.path.isfile(char_tsv_path): raise FileNotFoundError("Character.tsv not found") - names, sets = read_char_tsv(char_tsv_path) + names, sets = read_char_tsv(char_tsv_path, is_global) files = [ f @@ -54,18 +61,20 @@ def extract_names(dir_in): if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) - # tfw gil - # it's still faster on my machine so i'm keeping it - read_partial = partial(read_mission, char_names=names, char_sets=sets) - with Pool() as p: - new_names = p.map(read_partial, files) - for x in new_names: - names.update(x) + # TODO unsupported on global for now + if not is_global: + # tfw gil + # it's still faster on my machine so i'm keeping it + read_partial = partial(read_mission, char_names=names, char_sets=sets) + with Pool() as p: + new_names = p.map(read_partial, files) + for x in new_names: + names.update(x) return names -def read_char_tsv(char_tsv_path): +def read_char_tsv(char_tsv_path, is_global): char_names = set() char_sets = set() @@ -81,7 +90,12 @@ def read_char_tsv(char_tsv_path): and row["CharacterName"].strip() and row["NameText"].strip() ): - char_names.add(row["NameText"]) + if is_global: + char_names.add( + f"{row['NameText']}\t{row['CharaEnglish']}\t{row['CharaChinese']}\t{row['CharaKorean']}" + ) + else: + char_names.add(row["NameText"]) char_sets.add(row["CharacterName"]) return char_names, char_sets @@ -101,8 +115,6 @@ def read_mission(tsv_path, char_names, char_sets): if row["Command"] and row["Command"].startswith("//"): continue if row["Text"] and row["Arg1"]: - if (row["Arg1"] not in char_names) and ( - row["Arg1"] not in char_sets - ): + if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets): new_names.add(row["Arg1"]) return new_names diff --git a/utage/translate.py b/utage/translate.py index 6c47289..2344b0a 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -4,18 +4,17 @@ import io import json import os import traceback +from contextlib import suppress from functools import partial from glob import glob from multiprocessing.pool import Pool -def translate_dir(dir_in, tsv_out_dir, json_out_dir): +def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global): files = [ f - for f in glob( - os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True - ) + for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True) if "{os.path.sep}Diva{os.path.sep}" not in f ] @@ -23,13 +22,16 @@ def translate_dir(dir_in, tsv_out_dir, json_out_dir): raise FileNotFoundError("No valid files found in directory: " + dir_in) ptrans = partial( - translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir + translate_file, + tsv_out_dir=tsv_out_dir, + json_out_dir=json_out_dir, + is_global=is_global, ) with Pool() as p: p.map(ptrans, files) -def translate_file(file_in, tsv_out_dir, json_out_dir): +def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): try: if not file_in.endswith(".tsv"): raise ValueError("Invalid File Type for {}".format(file_in)) @@ -48,11 +50,17 @@ def translate_file(file_in, tsv_out_dir, json_out_dir): tsv_output_path = os.path.join( tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num) ) - json_output_path = os.path.join( - json_out_dir, - event_folder, - "{}_translations_jpn.json".format(id_num), - ) + languages = ["jpn"] + if is_global: + languages += ["enm", "zh", "ko"] + json_output_paths = { + lang: os.path.join( + json_out_dir, + event_folder, + "{}_translations_{}.json".format(id_num, lang), + ) + for lang in languages + } # need to create output paths and avoid races when threading if not os.path.exists(os.path.dirname(tsv_output_path)): @@ -62,16 +70,21 @@ def translate_file(file_in, tsv_out_dir, json_out_dir): if e.errno != errno.EEXIST: raise - if not os.path.exists(os.path.dirname(json_output_path)): + if not os.path.exists(os.path.dirname(json_output_paths["jpn"])): try: - os.makedirs(os.path.dirname(json_output_path)) + os.makedirs(os.path.dirname(json_output_paths["jpn"])) except OSError as e: if e.errno != errno.EEXIST: raise with open(file_in, "r") as tsv_file: - tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') - tsv_keyed, json_str = process_tsv(tsv, id_num) + tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE) + + t_fieldnames = tsv.fieldnames + if is_global: + t_fieldnames = tsv.fieldnames[: tsv.fieldnames.index("English") + 1] + + tsv_keyed, json_str = process_tsv(tsv, id_num, is_global) # csv handles newlines, don't set it in io.open with io.open(tsv_output_path, "w", newline="") as tsv_out: @@ -80,32 +93,39 @@ def translate_file(file_in, tsv_out_dir, json_out_dir): delimiter="\t", quotechar='"', lineterminator="\n", - fieldnames=tsv.fieldnames, + fieldnames=t_fieldnames, extrasaction="ignore", ) writer.writeheader() for row in tsv_keyed: writer.writerow(row) - with io.open(json_output_path, "w", newline="\n") as json_out: - # we don't want to sort these because they're _1, ..., _10, etc - json.dump( - json_str, - json_out, - ensure_ascii=False, - indent="\t", - sort_keys=False, - ) + for lang in languages: + if not json_str[lang]: + continue + with io.open(json_output_paths[lang], "w", newline="\n") as json_out: + # we don't want to sort these because they're _1, ..., _10, etc + json.dump( + json_str[lang], + json_out, + ensure_ascii=False, + indent="\t", + sort_keys=False, + ) except Exception: traceback.print_exc() print("Error processing file: " + file_in) -def process_tsv(tsv, id_num): +def process_tsv(tsv, id_num, is_global): i = 0 tsv_keyed = [] - key_dict = {} + key_dict = {"jpn": {}} + if is_global: + key_dict["enm"] = {} + key_dict["zh"] = {} + key_dict["ko"] = {} for row in tsv: try: @@ -115,8 +135,14 @@ def process_tsv(tsv, id_num): if row["Text"] and row["Text"].strip(): key = "{}_{}".format(id_num, i) i += 1 + key_dict["jpn"][key] = row["Text"] + if is_global: + with suppress(KeyError): + if row["English"]: + key_dict["enm"][key] = row["English"] + key_dict["zh"][key] = row["Chinese"] + key_dict["ko"][key] = row["Korean"] row["English"] = key - key_dict[key] = row["Text"] tsv_keyed.append(row) except Exception: traceback.print_exc() diff --git a/xdudata/update.py b/xdudata/update.py index fd747f9..b659c3f 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -1,4 +1,7 @@ +from contextlib import suppress + import diva +import shutil import os import subprocess import tempfile @@ -8,35 +11,52 @@ from distutils.dir_util import copy_tree, remove_tree from glob import glob -def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): +def update_all( + cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages, +): + print("Cleaning cache") + cleanup_cache(cache_dir) + print("Copying images") copy_images(cache_dir, update_dir) print("Processing TSV files") - update_tsv(cache_dir, update_dir, translation_dir, languages) + update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) print("Processing Quest Json files") - diva.quest.update_missions(cache_dir, translation_dir, languages) + diva.quest.update_missions(extract_dir, translation_dir, languages) - # this code isn't going to run on windows because i need to rewrite it - # it just calls my zsh scripts and generates it that way, need to replace them + # this code isn't going to run on windows (it is in WSL though) because i need to rewrite it + # it just calls my zsh tools and generates it that way, need to replace them # with real python or something + + # first copy over the necessary avtools to the extract dir + # TODO make it so that you don't need to copy them over (for example call them with python) + print("Copying necessary tools into extracting directory") + shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir) + shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir) + shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir) + print("Processing Sound Effects") copy_tree( os.path.join(cache_dir, "Android/Asset/Sound/Se"), os.path.join(extract_dir, "Se"), ) - subprocess.run("./update_se.zsh", shell=True, cwd=extract_dir) - copy_tree(os.path.join(extract_dir, "se"), os.path.join(update_dir, "Se")) + subprocess.run( + os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir + ) + copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se")) print("Processing BGM - don't forget to copy that loop file") copy_tree( os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), os.path.join(extract_dir, "Bgm"), ) - subprocess.run("./update_bgm.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "bgm"), os.path.join(update_dir, "Bgm") + os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") ) print("Processing voices") @@ -44,9 +64,11 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): os.path.join(cache_dir, "Android/Asset/Sound/Voice"), os.path.join(extract_dir, "Voice"), ) - subprocess.run("./update_voice.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "voice"), os.path.join(update_dir, "Voice") + os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") ) print("Processing movies - this might take forever") @@ -54,36 +76,42 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): os.path.join(cache_dir, "Common/Asset/Movie"), os.path.join(extract_dir, "Movie"), ) - subprocess.run("./update_movie.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "movie"), + os.path.join(extract_dir, "movie_extracted"), os.path.join(update_dir, "Asset/Movie"), ) -def update_tsv(cache_dir, update_dir, translation_dir, languages): +def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages): utage_tmp = tempfile.mkdtemp() copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp) utage.crypt.crypt_dir( utage_tmp, bytearray("SampleSecretKey", "utf-8"), False, False ) - for encrypted in glob( - os.path.join(utage_tmp, "**/*.utage"), recursive=True - ): + for encrypted in glob(os.path.join(utage_tmp, "**/*.utage"), recursive=True): try: os.remove(encrypted) except: pass + # copy over files for the quest updater + copy_tree(utage_tmp, os.path.join(extract_dir, "Common/Asset/Utage")) + shutil.copy2(os.path.join(cache_dir, "QuestMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "QuestSceneMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "QuestPartMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "ResourceEntry.db"), extract_dir) + utage.translate.translate_dir( utage_tmp, os.path.join(update_dir, "Utage"), os.path.join(translation_dir, "Missions"), + False, ) - utage.names.update_names(utage_tmp, translation_dir, languages) - copy_tree( - os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva") - ) + utage.names.update_names(utage_tmp, translation_dir, languages, False) + copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) remove_tree(utage_tmp) @@ -93,6 +121,15 @@ def copy_images(cache_dir, update_dir): os.path.join(update_dir, "Asset/Image"), ) copy_tree( - os.path.join(cache_dir, "Common/Sample"), - os.path.join(update_dir, "Sample"), + os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), ) + + +def cleanup_cache(cache_dir): + with suppress(FileNotFoundError): + shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) + os.remove( + os.path.join( + cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" + ) + )