xdu/divatool
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixed newlines and other windows stuff

Browse source
This commit is contained in:
argoneus 2020-05-15 23:13:38 +02:00
parent 1f26e8a9ce
commit c15343b4b8
4 changed files with 32 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

14
diva/quest.py
View file

@ -31,7 +31,7 @@ def scene_mst(qdb, old_path, dir_in, languages):
break
with io.open(
os.path.join(old_path, "XduScene.json"), "w", newline="\n"
os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8"
) as json_file:
json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
@ -56,11 +56,11 @@ def scene_mst(qdb, old_path, dir_in, languages):
langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json")
if os.path.isfile(langfile):
with open(langfile, "r") as lang_file:
with open(langfile, "r", encoding="utf-8") as lang_file:
lang_dict = json.load(lang_file)
out_dict.update(lang_dict)
with io.open(langfile, "w", newline="\n") as lang_file:
with io.open(langfile, "w", encoding="utf-8") as lang_file:
json.dump(
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
)
@ -70,7 +70,7 @@ def quest_mst(qdb, old_path, languages):
quests = qdb.get_quests()
with io.open(
os.path.join(old_path, "XduQuest.json"), "w", newline="\n"
os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8"
) as json_file:
json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
@ -85,11 +85,11 @@ def quest_mst(qdb, old_path, languages):
langfile = os.path.join(old_path, f"XduQuestNames_{lang}.json")
if os.path.isfile(langfile):
with open(langfile, "r") as lang_file:
with open(langfile, "r", encoding="utf-8") as lang_file:
lang_dict = json.load(lang_file)
out_dict.update(lang_dict)
with io.open(langfile, "w", newline="\n") as lang_file:
with io.open(langfile, "w", encoding="utf-8") as lang_file:
json.dump(
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
)
@ -108,7 +108,7 @@ def load_locations(dir_in):
raise FileNotFoundError("Ya didn't decrypt the files dummy")
for scenario in settings:
with open(scenario, "r") as tsv_file:
with open(scenario, "r", encoding="utf-8") as tsv_file:
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
for row in tsv:
tokens = row["FileName"].split("/")

12
utage/names.py
View file

@ -12,7 +12,7 @@ from multiprocessing import Pool
def update_names(dir_in, old_path, languages, is_global):
names = extract_names(dir_in, is_global)
global_languages = {"enm": 1, "zh": 2, "ko": 3}
global_languages = {"enm": 1, "zho": 2, "kor": 3}
if is_global:
languages = global_languages.keys()
@ -33,12 +33,12 @@ def update_names(dir_in, old_path, languages, is_global):
langfile = os.path.join(old_path, "nametranslations_{}.json".format(l))
if os.path.isfile(langfile):
with open(langfile, "r") as lang_file:
with open(langfile, "r", encoding="utf-8") as lang_file:
lang_dict = json.load(lang_file)
out_dict.update(lang_dict)
with io.open(
langfile, "w", newline="\n"
langfile, "w", encoding="utf-8"
) as lang_file: # you're using git right
json.dump(
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
@ -56,7 +56,7 @@ def extract_names(dir_in, is_global):
files = [
f
for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
if re.search(r"/[0-9]{9}\.tsv$", f)
if re.search(r"[0-9]{9}\.tsv$", f)
]
if len(files) == 0:
raise FileNotFoundError("No valid files found in directory: " + dir_in)
@ -78,7 +78,7 @@ def read_char_tsv(char_tsv_path, is_global):
char_names = set()
char_sets = set()
with open(char_tsv_path, "r") as char_tsv_file:
with open(char_tsv_path, "r", encoding="utf-8") as char_tsv_file:
char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"')
for row in char_tsv:
@ -103,7 +103,7 @@ def read_char_tsv(char_tsv_path, is_global):
def read_mission(tsv_path, char_names, char_sets):
new_names = set()
with open(tsv_path, "r") as tsv_file:
with open(tsv_path, "r", encoding="utf-8") as tsv_file:
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
if (
("Arg1" not in tsv.fieldnames)

17
utage/translate.py
View file

@ -52,7 +52,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
)
languages = ["jpn"]
if is_global:
languages += ["enm", "zh", "ko"]
languages += ["enm", "zho", "kor"]
json_output_paths = {
lang: os.path.join(
json_out_dir,
@ -77,7 +77,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
if e.errno != errno.EEXIST:
raise
with open(file_in, "r") as tsv_file:
with open(file_in, "r", encoding="utf-8") as tsv_file:
tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE)
t_fieldnames = tsv.fieldnames
@ -87,12 +87,11 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
tsv_keyed, json_str = process_tsv(tsv, id_num, is_global)
# csv handles newlines, don't set it in io.open
with io.open(tsv_output_path, "w", newline="") as tsv_out:
with io.open(tsv_output_path, "w", newline="", encoding="utf-8") as tsv_out:
writer = csv.DictWriter(
tsv_out,
delimiter="\t",
quotechar='"',
lineterminator="\n",
fieldnames=t_fieldnames,
extrasaction="ignore",
)
@ -103,7 +102,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
for lang in languages:
if not json_str[lang]:
continue
with io.open(json_output_paths[lang], "w", newline="\n") as json_out:
with io.open(json_output_paths[lang], "w", encoding="utf-8") as json_out:
# we don't want to sort these because they're _1, ..., _10, etc
json.dump(
json_str[lang],
@ -124,8 +123,8 @@ def process_tsv(tsv, id_num, is_global):
key_dict = {"jpn": {}}
if is_global:
key_dict["enm"] = {}
key_dict["zh"] = {}
key_dict["ko"] = {}
key_dict["zho"] = {}
key_dict["kor"] = {}
for row in tsv:
try:
@ -140,8 +139,8 @@ def process_tsv(tsv, id_num, is_global):
with suppress(KeyError):
if row["English"]:
key_dict["enm"][key] = row["English"]
key_dict["zh"][key] = row["Chinese"]
key_dict["ko"][key] = row["Korean"]
key_dict["zho"][key] = row["Chinese"]
key_dict["kor"][key] = row["Korean"]
row["English"] = key
tsv_keyed.append(row)
except Exception:

13
xdudata/asset_extract.py
View file

@ -10,6 +10,7 @@ from glob import glob
from adx import extract_loop_data_from_dir
OPERATING_SYSTEM = os.name
CHUNK_SIZE = 100
def process_se(cache_dir, extract_dir, update_dir):
@ -39,7 +40,8 @@ def process_se(cache_dir, extract_dir, update_dir):
hca_files = [
file for file in glob("{}/**/*.hca".format(from_dir), recursive=True)
]
run_program(hca_files, cmd="clHCA")
for hca_sublist in split_list(hca_files, CHUNK_SIZE):
run_program(hca_sublist, cmd="clHCA")
basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files]
char_secs = [basename.split("_", maxsplit=1) for basename in basenames]
@ -160,7 +162,8 @@ def process_voice(cache_dir, extract_dir, update_dir):
hca_files = [
file for file in glob("{}/**/*.hca".format(from_dir), recursive=True)
]
run_program(hca_files, cmd="clHCA")
for hca_sublist in split_list(hca_files, CHUNK_SIZE):
run_program(hca_sublist, cmd="clHCA")
basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files]
char_secs = [basename.split("_", maxsplit=1) for basename in basenames]
@ -292,3 +295,9 @@ def run_program(args, cmd=None):
def get_pool(size=multiprocessing.cpu_count()):
pool = multiprocessing.Pool(size)
return pool
def split_list(input_list, chunk_size):
return [
input_list[i : i + chunk_size] for i in range(0, len(input_list), chunk_size)
]