From 0991108b11ab34aeae033132c369b523c9290eac Mon Sep 17 00:00:00 2001 From: argoneus Date: Sat, 15 Feb 2020 20:16:16 +0100 Subject: [PATCH 1/7] updated .gitignore and added readme to scripts folder --- .gitignore | 3 +++ scripts/readme.txt | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 scripts/readme.txt diff --git a/.gitignore b/.gitignore index e691145..e74fd7e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ __pycache__/ *$py.class *.swp virtualenv/ +scripts/* +!scripts/readme.txt +.idea/ diff --git a/scripts/readme.txt b/scripts/readme.txt new file mode 100644 index 0000000..a8fad1a --- /dev/null +++ b/scripts/readme.txt @@ -0,0 +1,29 @@ +Various files need to be in this directory. +If you don't have them, ask someone who does. + +The required files are: + +Z shell: +- update_bgm.zsh +- update_movie.zsh +- update_se.zsh +- update_voice.zsh + +Python scripts: +- adx_parse.py +- bingo_fuck.py +- chara_room_mission.py +- custom_mission.py +- event_bonus.py +- orphan_quest.py +- pad_2_csv.py +- pad_2_json.py +- pad_2_pad.py +- pad_2_pad_louisstyle.py +- pad_stat.py +- pad_update.py +- store_parser.py +- tsv_2_pad.py +- tsv_parse.py +- txt_2_pad.py +- utage_decrypt.py From c5736014db0b1546d1cf32378432e9db0b3b44ce Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 16 Feb 2020 00:15:29 +0100 Subject: [PATCH 2/7] added "is_global" flag --- divatool.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/divatool.py b/divatool.py index cd5f342..f759b4f 100755 --- a/divatool.py +++ b/divatool.py @@ -15,6 +15,16 @@ LANGUAGES = ["jpn", "eng", "rus"] def main(): parser = argparse.ArgumentParser(description="Unpack and process XDU data") + global_flag_parser = argparse.ArgumentParser(add_help=False) + global_flag_parser.add_argument( + "-g", + "--global", + help="Whether we are running on Global XDU.", + action="store_true", + dest="is_global", + default=False, + ) + subparsers = parser.add_subparsers( metavar="", title="subcommands", dest="command" ) @@ -81,7 +91,9 @@ def main(): parser_utage_crypt.add_argument("TARGET", help="Input", type=str) # utage translate parser_utage_translate = parser_utage_subparsers.add_parser( - "translate", help="Generate keyed tsv and json files" + "translate", + help="Generate keyed tsv and json files", + parents=[global_flag_parser], ) parser_utage_translate.add_argument("INPUT", help="Input", type=str) parser_utage_translate.add_argument( @@ -92,7 +104,7 @@ def main(): ) # utage names parser_utage_names = parser_utage_subparsers.add_parser( - "names", help="Generate and update name files" + "names", help="Generate and update name files", parents=[global_flag_parser] ) parser_utage_names.add_argument("INPUT", help="Input", type=str) parser_utage_names.add_argument( @@ -124,7 +136,7 @@ def main(): metavar="", title="subcommand", dest="subcommand" ) parser_xdudata_update = parser_xdudata_subparsers.add_parser( - "update", help="Update an XDUData directory" + "update", help="Update an XDUData directory", parents=[global_flag_parser] ) parser_xdudata_update.add_argument( "CACHE", help="files/ directory from XDU cache", type=str @@ -152,14 +164,20 @@ def main(): raise FileNotFoundError("Could not find {}".format(args.TARGET)) elif args.subcommand == "translate": if os.path.isfile(args.INPUT): - utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR) + utage.translate.translate_file( + args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global + ) elif os.path.isdir(args.INPUT): - utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR) + utage.translate.translate_dir( + args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global + ) else: raise FileNotFoundError("Could not find {}".format(args.INPUT)) elif args.subcommand == "names": if os.path.isdir(args.INPUT): - utage.names.update_names(args.INPUT, args.OLD, LANGUAGES) + utage.names.update_names( + args.INPUT, args.OLD, LANGUAGES, args.is_global + ) else: raise FileNotFoundError(args.INPUT) elif args.command == "diva": @@ -168,7 +186,12 @@ def main(): elif args.command == "xdudata": if args.subcommand == "update": xdudata.update_all( - args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES + args.CACHE, + args.XDUDATA, + args.TRANS, + args.EXTRACT, + LANGUAGES, + args.is_global, ) return 0 From ecde988fd948c78bfc7d27a184efb8ec56f5d5e2 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 16 Feb 2020 00:55:31 +0100 Subject: [PATCH 3/7] sorting out zsh script calling and folder names --- .gitignore | 4 +- divatool.py | 2 + {scripts => tools}/readme.txt | 7 ++- utage/names.py | 12 ++--- utage/translate.py | 22 +++------- xdudata/update.py | 83 +++++++++++++++++++++++++---------- 6 files changed, 79 insertions(+), 51 deletions(-) rename {scripts => tools}/readme.txt (87%) diff --git a/.gitignore b/.gitignore index e74fd7e..f8d0fa1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ __pycache__/ *$py.class *.swp virtualenv/ -scripts/* -!scripts/readme.txt +tools/* +!tools/readme.txt .idea/ diff --git a/divatool.py b/divatool.py index f759b4f..caf2bef 100755 --- a/divatool.py +++ b/divatool.py @@ -184,12 +184,14 @@ def main(): if args.subcommand == "quest": diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) elif args.command == "xdudata": + script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools")) if args.subcommand == "update": xdudata.update_all( args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, + script_dir, LANGUAGES, args.is_global, ) diff --git a/scripts/readme.txt b/tools/readme.txt similarity index 87% rename from scripts/readme.txt rename to tools/readme.txt index a8fad1a..f10fdc0 100644 --- a/scripts/readme.txt +++ b/tools/readme.txt @@ -3,7 +3,12 @@ If you don't have them, ask someone who does. The required files are: -Z shell: +Binaries: +- acb_extract.exe +- usm_extract.exe +- clHCA + +Z shell scripts: - update_bgm.zsh - update_movie.zsh - update_se.zsh diff --git a/utage/names.py b/utage/names.py index 56ef85c..5c5f2e2 100644 --- a/utage/names.py +++ b/utage/names.py @@ -9,7 +9,7 @@ from glob import glob from multiprocessing import Pool -def update_names(dir_in, old_path, languages): +def update_names(dir_in, old_path, languages, is_global): names = extract_names(dir_in) # we have to update japanese like the rest because CustomData exists @@ -30,11 +30,7 @@ def update_names(dir_in, old_path, languages): langfile, "w", newline="\n" ) as lang_file: # you're using git right json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -101,8 +97,6 @@ def read_mission(tsv_path, char_names, char_sets): if row["Command"] and row["Command"].startswith("//"): continue if row["Text"] and row["Arg1"]: - if (row["Arg1"] not in char_names) and ( - row["Arg1"] not in char_sets - ): + if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets): new_names.add(row["Arg1"]) return new_names diff --git a/utage/translate.py b/utage/translate.py index 6c47289..f70fc07 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -10,26 +10,22 @@ from glob import glob from multiprocessing.pool import Pool -def translate_dir(dir_in, tsv_out_dir, json_out_dir): +def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global): files = [ f - for f in glob( - os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True - ) + for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True) if "{os.path.sep}Diva{os.path.sep}" not in f ] if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) - ptrans = partial( - translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir - ) + ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir) with Pool() as p: p.map(ptrans, files) -def translate_file(file_in, tsv_out_dir, json_out_dir): +def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): try: if not file_in.endswith(".tsv"): raise ValueError("Invalid File Type for {}".format(file_in)) @@ -49,9 +45,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir): tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num) ) json_output_path = os.path.join( - json_out_dir, - event_folder, - "{}_translations_jpn.json".format(id_num), + json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num), ) # need to create output paths and avoid races when threading @@ -90,11 +84,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir): with io.open(json_output_path, "w", newline="\n") as json_out: # we don't want to sort these because they're _1, ..., _10, etc json.dump( - json_str, - json_out, - ensure_ascii=False, - indent="\t", - sort_keys=False, + json_str, json_out, ensure_ascii=False, indent="\t", sort_keys=False, ) except Exception: diff --git a/xdudata/update.py b/xdudata/update.py index fd747f9..6924b13 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -1,4 +1,7 @@ +from contextlib import suppress + import diva +import shutil import os import subprocess import tempfile @@ -8,35 +11,59 @@ from distutils.dir_util import copy_tree, remove_tree from glob import glob -def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): +def update_all( + cache_dir, + update_dir, + translation_dir, + extract_dir, + script_dir, + languages, + is_global, +): + print("Cleaning cache") + cleanup_cache(cache_dir) + print("Copying images") - copy_images(cache_dir, update_dir) + # copy_images(cache_dir, update_dir) print("Processing TSV files") - update_tsv(cache_dir, update_dir, translation_dir, languages) + # TODO + # update_tsv(cache_dir, update_dir, translation_dir, languages, is_global) print("Processing Quest Json files") - diva.quest.update_missions(cache_dir, translation_dir, languages) + # diva.quest.update_missions(cache_dir, translation_dir, languages) - # this code isn't going to run on windows because i need to rewrite it - # it just calls my zsh scripts and generates it that way, need to replace them + # this code isn't going to run on windows (it will in WSL though) because i need to rewrite it + # it just calls my zsh tools and generates it that way, need to replace them # with real python or something + + # first copy over the necessary avtools to the extract dir + # TODO make it so that you don't need to copy them over (for example call them with python) + print("Copying necessary tools into extracting directory") + shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir) + shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir) + shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir) + print("Processing Sound Effects") copy_tree( os.path.join(cache_dir, "Android/Asset/Sound/Se"), os.path.join(extract_dir, "Se"), ) - subprocess.run("./update_se.zsh", shell=True, cwd=extract_dir) - copy_tree(os.path.join(extract_dir, "se"), os.path.join(update_dir, "Se")) + subprocess.run( + os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir + ) + copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se")) print("Processing BGM - don't forget to copy that loop file") copy_tree( os.path.join(cache_dir, "Android/Asset/Sound/Bgm"), os.path.join(extract_dir, "Bgm"), ) - subprocess.run("./update_bgm.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "bgm"), os.path.join(update_dir, "Bgm") + os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm") ) print("Processing voices") @@ -44,9 +71,11 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): os.path.join(cache_dir, "Android/Asset/Sound/Voice"), os.path.join(extract_dir, "Voice"), ) - subprocess.run("./update_voice.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "voice"), os.path.join(update_dir, "Voice") + os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice") ) print("Processing movies - this might take forever") @@ -54,22 +83,22 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages): os.path.join(cache_dir, "Common/Asset/Movie"), os.path.join(extract_dir, "Movie"), ) - subprocess.run("./update_movie.zsh", shell=True, cwd=extract_dir) + subprocess.run( + os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir + ) copy_tree( - os.path.join(extract_dir, "movie"), + os.path.join(extract_dir, "movie_extracted"), os.path.join(update_dir, "Asset/Movie"), ) -def update_tsv(cache_dir, update_dir, translation_dir, languages): +def update_tsv(cache_dir, update_dir, translation_dir, languages, is_global): utage_tmp = tempfile.mkdtemp() copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp) utage.crypt.crypt_dir( utage_tmp, bytearray("SampleSecretKey", "utf-8"), False, False ) - for encrypted in glob( - os.path.join(utage_tmp, "**/*.utage"), recursive=True - ): + for encrypted in glob(os.path.join(utage_tmp, "**/*.utage"), recursive=True): try: os.remove(encrypted) except: @@ -79,11 +108,10 @@ def update_tsv(cache_dir, update_dir, translation_dir, languages): utage_tmp, os.path.join(update_dir, "Utage"), os.path.join(translation_dir, "Missions"), + is_global, ) utage.names.update_names(utage_tmp, translation_dir, languages) - copy_tree( - os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva") - ) + copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) remove_tree(utage_tmp) @@ -93,6 +121,15 @@ def copy_images(cache_dir, update_dir): os.path.join(update_dir, "Asset/Image"), ) copy_tree( - os.path.join(cache_dir, "Common/Sample"), - os.path.join(update_dir, "Sample"), + os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"), ) + + +def cleanup_cache(cache_dir): + with suppress(FileNotFoundError): + shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32")) + os.remove( + os.path.join( + cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage" + ) + ) From fe9741a20a9240de60ee238db7d16db81b7d9d5a Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 16 Feb 2020 02:24:03 +0100 Subject: [PATCH 4/7] fixed up updater --- diva/quest.py | 26 +++++++------------------- divatool.py | 3 +-- utage/translate.py | 7 ++++++- xdudata/update.py | 30 +++++++++++++++--------------- 4 files changed, 29 insertions(+), 37 deletions(-) diff --git a/diva/quest.py b/diva/quest.py index d84e3b2..5a6d27d 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -18,6 +18,8 @@ def scene_mst(qdb, old_path, dir_in, languages): locations = load_locations(dir_in) + print(locations) + for id, scene in scenes.items(): i = 0 while True: @@ -26,18 +28,14 @@ def scene_mst(qdb, old_path, dir_in, languages): except KeyError as exc: i += 1 if i == len(scene["Parts"]): - raise KeyError( - f"Could not find folder for scene {id}" - ) from exc + raise KeyError(f"Could not find folder for scene {id}") from exc continue break with io.open( os.path.join(old_path, "XduScene.json"), "w", newline="\n" ) as json_file: - json.dump( - scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True - ) + json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True) for lang in languages: out_dict = {} @@ -66,11 +64,7 @@ def scene_mst(qdb, old_path, dir_in, languages): with io.open(langfile, "w", newline="\n") as lang_file: json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -80,9 +74,7 @@ def quest_mst(qdb, old_path, languages): with io.open( os.path.join(old_path, "XduQuest.json"), "w", newline="\n" ) as json_file: - json.dump( - quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True - ) + json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True) for lang in languages: out_dict = {} @@ -101,11 +93,7 @@ def quest_mst(qdb, old_path, languages): with io.open(langfile, "w", newline="\n") as lang_file: json.dump( - out_dict, - lang_file, - ensure_ascii=False, - indent="\t", - sort_keys=True, + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) return quests diff --git a/divatool.py b/divatool.py index caf2bef..ec36fed 100755 --- a/divatool.py +++ b/divatool.py @@ -136,7 +136,7 @@ def main(): metavar="", title="subcommand", dest="subcommand" ) parser_xdudata_update = parser_xdudata_subparsers.add_parser( - "update", help="Update an XDUData directory", parents=[global_flag_parser] + "update", help="Update an XDUData directory" ) parser_xdudata_update.add_argument( "CACHE", help="files/ directory from XDU cache", type=str @@ -193,7 +193,6 @@ def main(): args.EXTRACT, script_dir, LANGUAGES, - args.is_global, ) return 0 diff --git a/utage/translate.py b/utage/translate.py index f70fc07..8b70398 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -20,7 +20,12 @@ def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global): if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) - ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir) + ptrans = partial( + translate_file, + tsv_out_dir=tsv_out_dir, + json_out_dir=json_out_dir, + is_global=is_global, + ) with Pool() as p: p.map(ptrans, files) diff --git a/xdudata/update.py b/xdudata/update.py index 6924b13..b659c3f 100644 --- a/xdudata/update.py +++ b/xdudata/update.py @@ -12,28 +12,21 @@ from glob import glob def update_all( - cache_dir, - update_dir, - translation_dir, - extract_dir, - script_dir, - languages, - is_global, + cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages, ): print("Cleaning cache") cleanup_cache(cache_dir) print("Copying images") - # copy_images(cache_dir, update_dir) + copy_images(cache_dir, update_dir) print("Processing TSV files") - # TODO - # update_tsv(cache_dir, update_dir, translation_dir, languages, is_global) + update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages) print("Processing Quest Json files") - # diva.quest.update_missions(cache_dir, translation_dir, languages) + diva.quest.update_missions(extract_dir, translation_dir, languages) - # this code isn't going to run on windows (it will in WSL though) because i need to rewrite it + # this code isn't going to run on windows (it is in WSL though) because i need to rewrite it # it just calls my zsh tools and generates it that way, need to replace them # with real python or something @@ -92,7 +85,7 @@ def update_all( ) -def update_tsv(cache_dir, update_dir, translation_dir, languages, is_global): +def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages): utage_tmp = tempfile.mkdtemp() copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp) utage.crypt.crypt_dir( @@ -104,13 +97,20 @@ def update_tsv(cache_dir, update_dir, translation_dir, languages, is_global): except: pass + # copy over files for the quest updater + copy_tree(utage_tmp, os.path.join(extract_dir, "Common/Asset/Utage")) + shutil.copy2(os.path.join(cache_dir, "QuestMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "QuestSceneMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "QuestPartMst.db"), extract_dir) + shutil.copy2(os.path.join(cache_dir, "ResourceEntry.db"), extract_dir) + utage.translate.translate_dir( utage_tmp, os.path.join(update_dir, "Utage"), os.path.join(translation_dir, "Missions"), - is_global, + False, ) - utage.names.update_names(utage_tmp, translation_dir, languages) + utage.names.update_names(utage_tmp, translation_dir, languages, False) copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")) remove_tree(utage_tmp) From 1ef9a26aeea11a790fc346a275694591c889f036 Mon Sep 17 00:00:00 2001 From: argoneus Date: Sun, 16 Feb 2020 03:05:24 +0100 Subject: [PATCH 5/7] added basic nametranslations for global languages --- utage/names.py | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/utage/names.py b/utage/names.py index 5c5f2e2..b6633f0 100644 --- a/utage/names.py +++ b/utage/names.py @@ -10,13 +10,24 @@ from multiprocessing import Pool def update_names(dir_in, old_path, languages, is_global): - names = extract_names(dir_in) + names = extract_names(dir_in, is_global) + + global_languages = {"enm": 1, "zh": 2, "ko": 3} + if is_global: + languages = global_languages.keys() # we have to update japanese like the rest because CustomData exists # and POKELABO deleted the wedding gear event stuff from the game files for l in languages: if l == "jpn": out_dict = dict(zip(names, names)) + elif l in global_languages.keys(): + out_dict = {} + for name in names: + spl = name.split("\t") + idx = global_languages[l] + val = spl[idx] if spl[idx] != "None" else "" + out_dict[spl[0]] = val else: out_dict = dict.fromkeys(names, "") @@ -34,13 +45,13 @@ def update_names(dir_in, old_path, languages, is_global): ) -def extract_names(dir_in): +def extract_names(dir_in, is_global): char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv") if not os.path.isfile(char_tsv_path): raise FileNotFoundError("Character.tsv not found") - names, sets = read_char_tsv(char_tsv_path) + names, sets = read_char_tsv(char_tsv_path, is_global) files = [ f @@ -50,18 +61,20 @@ def extract_names(dir_in): if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) - # tfw gil - # it's still faster on my machine so i'm keeping it - read_partial = partial(read_mission, char_names=names, char_sets=sets) - with Pool() as p: - new_names = p.map(read_partial, files) - for x in new_names: - names.update(x) + # TODO unsupported on global for now + if not is_global: + # tfw gil + # it's still faster on my machine so i'm keeping it + read_partial = partial(read_mission, char_names=names, char_sets=sets) + with Pool() as p: + new_names = p.map(read_partial, files) + for x in new_names: + names.update(x) return names -def read_char_tsv(char_tsv_path): +def read_char_tsv(char_tsv_path, is_global): char_names = set() char_sets = set() @@ -77,7 +90,12 @@ def read_char_tsv(char_tsv_path): and row["CharacterName"].strip() and row["NameText"].strip() ): - char_names.add(row["NameText"]) + if is_global: + char_names.add( + f"{row['NameText']}\t{row['CharaEnglish']}\t{row['CharaChinese']}\t{row['CharaKorean']}" + ) + else: + char_names.add(row["NameText"]) char_sets.add(row["CharacterName"]) return char_names, char_sets From 05a29b6ee655e963c42363bccf7d8ba44e59779f Mon Sep 17 00:00:00 2001 From: argoneus Date: Thu, 20 Feb 2020 23:57:09 +0100 Subject: [PATCH 6/7] translate command now works with global languages --- diva/quest.py | 2 -- utage/translate.py | 61 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/diva/quest.py b/diva/quest.py index 5a6d27d..f2342a1 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -18,8 +18,6 @@ def scene_mst(qdb, old_path, dir_in, languages): locations = load_locations(dir_in) - print(locations) - for id, scene in scenes.items(): i = 0 while True: diff --git a/utage/translate.py b/utage/translate.py index 8b70398..6ea96b4 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -4,6 +4,7 @@ import io import json import os import traceback +from contextlib import suppress from functools import partial from glob import glob @@ -49,9 +50,17 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): tsv_output_path = os.path.join( tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num) ) - json_output_path = os.path.join( - json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num), - ) + languages = ["jpn"] + if is_global: + languages += ["enm", "zh", "ko"] + json_output_paths = { + lang: os.path.join( + json_out_dir, + event_folder, + "{}_translations_{}.json".format(id_num, lang), + ) + for lang in languages + } # need to create output paths and avoid races when threading if not os.path.exists(os.path.dirname(tsv_output_path)): @@ -61,16 +70,21 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): if e.errno != errno.EEXIST: raise - if not os.path.exists(os.path.dirname(json_output_path)): + if not os.path.exists(os.path.dirname(json_output_paths["jpn"])): try: - os.makedirs(os.path.dirname(json_output_path)) + os.makedirs(os.path.dirname(json_output_paths["jpn"])) except OSError as e: if e.errno != errno.EEXIST: raise with open(file_in, "r") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') - tsv_keyed, json_str = process_tsv(tsv, id_num) + + t_fieldnames = tsv.fieldnames + if is_global: + t_fieldnames = tsv.fieldnames[: tsv.fieldnames.index("English") + 1] + + tsv_keyed, json_str = process_tsv(tsv, id_num, is_global) # csv handles newlines, don't set it in io.open with io.open(tsv_output_path, "w", newline="") as tsv_out: @@ -79,28 +93,39 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): delimiter="\t", quotechar='"', lineterminator="\n", - fieldnames=tsv.fieldnames, + fieldnames=t_fieldnames, extrasaction="ignore", ) writer.writeheader() for row in tsv_keyed: writer.writerow(row) - with io.open(json_output_path, "w", newline="\n") as json_out: - # we don't want to sort these because they're _1, ..., _10, etc - json.dump( - json_str, json_out, ensure_ascii=False, indent="\t", sort_keys=False, - ) + for lang in languages: + if not json_str[lang]: + continue + with io.open(json_output_paths[lang], "w", newline="\n") as json_out: + # we don't want to sort these because they're _1, ..., _10, etc + json.dump( + json_str[lang], + json_out, + ensure_ascii=False, + indent="\t", + sort_keys=False, + ) except Exception: traceback.print_exc() print("Error processing file: " + file_in) -def process_tsv(tsv, id_num): +def process_tsv(tsv, id_num, is_global): i = 0 tsv_keyed = [] - key_dict = {} + key_dict = {"jpn": {}} + if is_global: + key_dict["enm"] = {} + key_dict["zh"] = {} + key_dict["ko"] = {} for row in tsv: try: @@ -110,8 +135,14 @@ def process_tsv(tsv, id_num): if row["Text"] and row["Text"].strip(): key = "{}_{}".format(id_num, i) i += 1 + key_dict["jpn"][key] = row["Text"] + if is_global: + with suppress(KeyError): + if row["English"]: + key_dict["enm"][key] = row["English"] + key_dict["zh"][key] = row["Chinese"] + key_dict["ko"][key] = row["Korean"] row["English"] = key - key_dict[key] = row["Text"] tsv_keyed.append(row) except Exception: traceback.print_exc() From 01fb281481459e6d07808f241ffbc74b73ef3183 Mon Sep 17 00:00:00 2001 From: argoneus Date: Fri, 21 Feb 2020 00:06:16 +0100 Subject: [PATCH 7/7] fixed quotes thanks j --- utage/translate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utage/translate.py b/utage/translate.py index 6ea96b4..2344b0a 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -78,7 +78,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): raise with open(file_in, "r") as tsv_file: - tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') + tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE) t_fieldnames = tsv.fieldnames if is_global: