xdu/divatool
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixed newlines and other windows stuff

Browse source
This commit is contained in:
argoneus 2020-05-15 23:13:38 +02:00 • committed by Gitea
parent 9f73c50d56
commit 5a9859cb56
4 changed files with 32 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

12
utage/names.py
View file

@ -12,7 +12,7 @@ from multiprocessing import Pool
def update_names(dir_in, old_path, languages, is_global):
names = extract_names(dir_in, is_global)
global_languages = {"enm": 1, "zh": 2, "ko": 3}
global_languages = {"enm": 1, "zho": 2, "kor": 3}
if is_global:
languages = global_languages.keys()
@ -33,12 +33,12 @@ def update_names(dir_in, old_path, languages, is_global):
langfile = os.path.join(old_path, "nametranslations_{}.json".format(l))
if os.path.isfile(langfile):
with open(langfile, "r") as lang_file:
with open(langfile, "r", encoding="utf-8") as lang_file:
lang_dict = json.load(lang_file)
out_dict.update(lang_dict)
with io.open(
langfile, "w", newline="\n"
langfile, "w", encoding="utf-8"
) as lang_file: # you're using git right
json.dump(
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
@ -56,7 +56,7 @@ def extract_names(dir_in, is_global):
files = [
f
for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
if re.search(r"/[0-9]{9}\.tsv$", f)
if re.search(r"[0-9]{9}\.tsv$", f)
]
if len(files) == 0:
raise FileNotFoundError("No valid files found in directory: " + dir_in)
@ -78,7 +78,7 @@ def read_char_tsv(char_tsv_path, is_global):
char_names = set()
char_sets = set()
with open(char_tsv_path, "r") as char_tsv_file:
with open(char_tsv_path, "r", encoding="utf-8") as char_tsv_file:
char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"')
for row in char_tsv:
@ -103,7 +103,7 @@ def read_char_tsv(char_tsv_path, is_global):
def read_mission(tsv_path, char_names, char_sets):
new_names = set()
with open(tsv_path, "r") as tsv_file:
with open(tsv_path, "r", encoding="utf-8") as tsv_file:
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
if (
("Arg1" not in tsv.fieldnames)

17
utage/translate.py
View file

@ -52,7 +52,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
)
languages = ["jpn"]
if is_global:
languages += ["enm", "zh", "ko"]
languages += ["enm", "zho", "kor"]
json_output_paths = {
lang: os.path.join(
json_out_dir,
@ -77,7 +77,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
if e.errno != errno.EEXIST:
raise
with open(file_in, "r") as tsv_file:
with open(file_in, "r", encoding="utf-8") as tsv_file:
tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE)
t_fieldnames = tsv.fieldnames
@ -87,12 +87,11 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
tsv_keyed, json_str = process_tsv(tsv, id_num, is_global)
# csv handles newlines, don't set it in io.open
with io.open(tsv_output_path, "w", newline="") as tsv_out:
with io.open(tsv_output_path, "w", newline="", encoding="utf-8") as tsv_out:
writer = csv.DictWriter(
tsv_out,
delimiter="\t",
quotechar='"',
lineterminator="\n",
fieldnames=t_fieldnames,
extrasaction="ignore",
)
@ -103,7 +102,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
for lang in languages:
if not json_str[lang]:
continue
with io.open(json_output_paths[lang], "w", newline="\n") as json_out:
with io.open(json_output_paths[lang], "w", encoding="utf-8") as json_out:
# we don't want to sort these because they're _1, ..., _10, etc
json.dump(
json_str[lang],
@ -124,8 +123,8 @@ def process_tsv(tsv, id_num, is_global):
key_dict = {"jpn": {}}
if is_global:
key_dict["enm"] = {}
key_dict["zh"] = {}
key_dict["ko"] = {}
key_dict["zho"] = {}
key_dict["kor"] = {}
for row in tsv:
try:
@ -140,8 +139,8 @@ def process_tsv(tsv, id_num, is_global):
with suppress(KeyError):
if row["English"]:
key_dict["enm"][key] = row["English"]
key_dict["zh"][key] = row["Chinese"]
key_dict["ko"][key] = row["Korean"]
key_dict["zho"][key] = row["Chinese"]
key_dict["kor"][key] = row["Korean"]
row["English"] = key
tsv_keyed.append(row)
except Exception: