diff --git a/diva/quest.py b/diva/quest.py index f2342a1..ed4d092 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -31,7 +31,7 @@ def scene_mst(qdb, old_path, dir_in, languages): break with io.open( - os.path.join(old_path, "XduScene.json"), "w", newline="\n" + os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8" ) as json_file: json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True) @@ -56,11 +56,11 @@ def scene_mst(qdb, old_path, dir_in, languages): langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json") if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) - with io.open(langfile, "w", newline="\n") as lang_file: + with io.open(langfile, "w", encoding="utf-8") as lang_file: json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -70,7 +70,7 @@ def quest_mst(qdb, old_path, languages): quests = qdb.get_quests() with io.open( - os.path.join(old_path, "XduQuest.json"), "w", newline="\n" + os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8" ) as json_file: json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True) @@ -85,11 +85,11 @@ def quest_mst(qdb, old_path, languages): langfile = os.path.join(old_path, f"XduQuestNames_{lang}.json") if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) - with io.open(langfile, "w", newline="\n") as lang_file: + with io.open(langfile, "w", encoding="utf-8") as lang_file: json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, ) @@ -108,7 +108,7 @@ def load_locations(dir_in): raise FileNotFoundError("Ya didn't decrypt the files dummy") for scenario in settings: - with open(scenario, "r") as tsv_file: + with open(scenario, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') for row in tsv: tokens = row["FileName"].split("/") diff --git a/utage/names.py b/utage/names.py index b6633f0..b47f3b8 100644 --- a/utage/names.py +++ b/utage/names.py @@ -12,7 +12,7 @@ from multiprocessing import Pool def update_names(dir_in, old_path, languages, is_global): names = extract_names(dir_in, is_global) - global_languages = {"enm": 1, "zh": 2, "ko": 3} + global_languages = {"enm": 1, "zho": 2, "kor": 3} if is_global: languages = global_languages.keys() @@ -33,12 +33,12 @@ def update_names(dir_in, old_path, languages, is_global): langfile = os.path.join(old_path, "nametranslations_{}.json".format(l)) if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: + with open(langfile, "r", encoding="utf-8") as lang_file: lang_dict = json.load(lang_file) out_dict.update(lang_dict) with io.open( - langfile, "w", newline="\n" + langfile, "w", encoding="utf-8" ) as lang_file: # you're using git right json.dump( out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True, @@ -56,7 +56,7 @@ def extract_names(dir_in, is_global): files = [ f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) - if re.search(r"/[0-9]{9}\.tsv$", f) + if re.search(r"[0-9]{9}\.tsv$", f) ] if len(files) == 0: raise FileNotFoundError("No valid files found in directory: " + dir_in) @@ -78,7 +78,7 @@ def read_char_tsv(char_tsv_path, is_global): char_names = set() char_sets = set() - with open(char_tsv_path, "r") as char_tsv_file: + with open(char_tsv_path, "r", encoding="utf-8") as char_tsv_file: char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"') for row in char_tsv: @@ -103,7 +103,7 @@ def read_char_tsv(char_tsv_path, is_global): def read_mission(tsv_path, char_names, char_sets): new_names = set() - with open(tsv_path, "r") as tsv_file: + with open(tsv_path, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') if ( ("Arg1" not in tsv.fieldnames) diff --git a/utage/translate.py b/utage/translate.py index 2344b0a..29dcd39 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -52,7 +52,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): ) languages = ["jpn"] if is_global: - languages += ["enm", "zh", "ko"] + languages += ["enm", "zho", "kor"] json_output_paths = { lang: os.path.join( json_out_dir, @@ -77,7 +77,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): if e.errno != errno.EEXIST: raise - with open(file_in, "r") as tsv_file: + with open(file_in, "r", encoding="utf-8") as tsv_file: tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE) t_fieldnames = tsv.fieldnames @@ -87,12 +87,11 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): tsv_keyed, json_str = process_tsv(tsv, id_num, is_global) # csv handles newlines, don't set it in io.open - with io.open(tsv_output_path, "w", newline="") as tsv_out: + with io.open(tsv_output_path, "w", newline="", encoding="utf-8") as tsv_out: writer = csv.DictWriter( tsv_out, delimiter="\t", quotechar='"', - lineterminator="\n", fieldnames=t_fieldnames, extrasaction="ignore", ) @@ -103,7 +102,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global): for lang in languages: if not json_str[lang]: continue - with io.open(json_output_paths[lang], "w", newline="\n") as json_out: + with io.open(json_output_paths[lang], "w", encoding="utf-8") as json_out: # we don't want to sort these because they're _1, ..., _10, etc json.dump( json_str[lang], @@ -124,8 +123,8 @@ def process_tsv(tsv, id_num, is_global): key_dict = {"jpn": {}} if is_global: key_dict["enm"] = {} - key_dict["zh"] = {} - key_dict["ko"] = {} + key_dict["zho"] = {} + key_dict["kor"] = {} for row in tsv: try: @@ -140,8 +139,8 @@ def process_tsv(tsv, id_num, is_global): with suppress(KeyError): if row["English"]: key_dict["enm"][key] = row["English"] - key_dict["zh"][key] = row["Chinese"] - key_dict["ko"][key] = row["Korean"] + key_dict["zho"][key] = row["Chinese"] + key_dict["kor"][key] = row["Korean"] row["English"] = key tsv_keyed.append(row) except Exception: diff --git a/xdudata/asset_extract.py b/xdudata/asset_extract.py index 8237162..04e088c 100644 --- a/xdudata/asset_extract.py +++ b/xdudata/asset_extract.py @@ -10,6 +10,7 @@ from glob import glob from adx import extract_loop_data_from_dir OPERATING_SYSTEM = os.name +CHUNK_SIZE = 100 def process_se(cache_dir, extract_dir, update_dir): @@ -39,7 +40,8 @@ def process_se(cache_dir, extract_dir, update_dir): hca_files = [ file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) ] - run_program(hca_files, cmd="clHCA") + for hca_sublist in split_list(hca_files, CHUNK_SIZE): + run_program(hca_sublist, cmd="clHCA") basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] char_secs = [basename.split("_", maxsplit=1) for basename in basenames] @@ -160,7 +162,8 @@ def process_voice(cache_dir, extract_dir, update_dir): hca_files = [ file for file in glob("{}/**/*.hca".format(from_dir), recursive=True) ] - run_program(hca_files, cmd="clHCA") + for hca_sublist in split_list(hca_files, CHUNK_SIZE): + run_program(hca_sublist, cmd="clHCA") basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files] char_secs = [basename.split("_", maxsplit=1) for basename in basenames] @@ -292,3 +295,9 @@ def run_program(args, cmd=None): def get_pool(size=multiprocessing.cpu_count()): pool = multiprocessing.Pool(size) return pool + + +def split_list(input_list, chunk_size): + return [ + input_list[i : i + chunk_size] for i in range(0, len(input_list), chunk_size) + ]