xdu/divatool
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixed newlines and other windows stuff

Browse source
This commit is contained in:
argoneus 2020-05-15 23:13:38 +02:00 • committed by Gitea
parent 9f73c50d56
commit 5a9859cb56
4 changed files with 32 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

17
utage/translate.py
View file

@ -52,7 +52,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
)
languages = ["jpn"]
if is_global:
languages += ["enm", "zh", "ko"]
languages += ["enm", "zho", "kor"]
json_output_paths = {
lang: os.path.join(
json_out_dir,
@ -77,7 +77,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
if e.errno != errno.EEXIST:
raise
with open(file_in, "r") as tsv_file:
with open(file_in, "r", encoding="utf-8") as tsv_file:
tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE)
t_fieldnames = tsv.fieldnames
@ -87,12 +87,11 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
tsv_keyed, json_str = process_tsv(tsv, id_num, is_global)
# csv handles newlines, don't set it in io.open
with io.open(tsv_output_path, "w", newline="") as tsv_out:
with io.open(tsv_output_path, "w", newline="", encoding="utf-8") as tsv_out:
writer = csv.DictWriter(
tsv_out,
delimiter="\t",
quotechar='"',
lineterminator="\n",
fieldnames=t_fieldnames,
extrasaction="ignore",
)
@ -103,7 +102,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
for lang in languages:
if not json_str[lang]:
continue
with io.open(json_output_paths[lang], "w", newline="\n") as json_out:
with io.open(json_output_paths[lang], "w", encoding="utf-8") as json_out:
# we don't want to sort these because they're _1, ..., _10, etc
json.dump(
json_str[lang],
@ -124,8 +123,8 @@ def process_tsv(tsv, id_num, is_global):
key_dict = {"jpn": {}}
if is_global:
key_dict["enm"] = {}
key_dict["zh"] = {}
key_dict["ko"] = {}
key_dict["zho"] = {}
key_dict["kor"] = {}
for row in tsv:
try:
@ -140,8 +139,8 @@ def process_tsv(tsv, id_num, is_global):
with suppress(KeyError):
if row["English"]:
key_dict["enm"][key] = row["English"]
key_dict["zh"][key] = row["Chinese"]
key_dict["ko"][key] = row["Korean"]
key_dict["zho"][key] = row["Chinese"]
key_dict["kor"][key] = row["Korean"]
row["English"] = key
tsv_keyed.append(row)
except Exception: