121 lines
3.8 KiB
Python
121 lines
3.8 KiB
Python
import csv
|
|
import errno
|
|
import io
|
|
import json
|
|
import os
|
|
import traceback
|
|
|
|
from functools import partial
|
|
from glob import glob
|
|
from multiprocessing.pool import Pool
|
|
|
|
|
|
def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global):
|
|
files = [
|
|
f
|
|
for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True)
|
|
if "{os.path.sep}Diva{os.path.sep}" not in f
|
|
]
|
|
|
|
if len(files) == 0:
|
|
raise FileNotFoundError("No valid files found in directory: " + dir_in)
|
|
|
|
ptrans = partial(
|
|
translate_file,
|
|
tsv_out_dir=tsv_out_dir,
|
|
json_out_dir=json_out_dir,
|
|
is_global=is_global,
|
|
)
|
|
with Pool() as p:
|
|
p.map(ptrans, files)
|
|
|
|
|
|
def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
|
|
try:
|
|
if not file_in.endswith(".tsv"):
|
|
raise ValueError("Invalid File Type for {}".format(file_in))
|
|
|
|
# we need to get the event folder
|
|
# ie, Utage/>>>main01<<</Scenario/whatever.tsv
|
|
try:
|
|
event_folder = os.path.normpath(file_in).split(os.sep)[-3]
|
|
except Exception:
|
|
traceback.print_exc()
|
|
print("Error processing tsv: Please reference from Utage root")
|
|
raise
|
|
|
|
id_num = os.path.splitext(os.path.basename(file_in))[0]
|
|
|
|
tsv_output_path = os.path.join(
|
|
tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num)
|
|
)
|
|
json_output_path = os.path.join(
|
|
json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num),
|
|
)
|
|
|
|
# need to create output paths and avoid races when threading
|
|
if not os.path.exists(os.path.dirname(tsv_output_path)):
|
|
try:
|
|
os.makedirs(os.path.dirname(tsv_output_path))
|
|
except OSError as e:
|
|
if e.errno != errno.EEXIST:
|
|
raise
|
|
|
|
if not os.path.exists(os.path.dirname(json_output_path)):
|
|
try:
|
|
os.makedirs(os.path.dirname(json_output_path))
|
|
except OSError as e:
|
|
if e.errno != errno.EEXIST:
|
|
raise
|
|
|
|
with open(file_in, "r") as tsv_file:
|
|
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
|
|
tsv_keyed, json_str = process_tsv(tsv, id_num)
|
|
|
|
# csv handles newlines, don't set it in io.open
|
|
with io.open(tsv_output_path, "w", newline="") as tsv_out:
|
|
writer = csv.DictWriter(
|
|
tsv_out,
|
|
delimiter="\t",
|
|
quotechar='"',
|
|
lineterminator="\n",
|
|
fieldnames=tsv.fieldnames,
|
|
extrasaction="ignore",
|
|
)
|
|
writer.writeheader()
|
|
for row in tsv_keyed:
|
|
writer.writerow(row)
|
|
|
|
with io.open(json_output_path, "w", newline="\n") as json_out:
|
|
# we don't want to sort these because they're _1, ..., _10, etc
|
|
json.dump(
|
|
json_str, json_out, ensure_ascii=False, indent="\t", sort_keys=False,
|
|
)
|
|
|
|
except Exception:
|
|
traceback.print_exc()
|
|
print("Error processing file: " + file_in)
|
|
|
|
|
|
def process_tsv(tsv, id_num):
|
|
i = 0
|
|
tsv_keyed = []
|
|
key_dict = {}
|
|
|
|
for row in tsv:
|
|
try:
|
|
if row["Command"].startswith("//"):
|
|
tsv_keyed.append(row)
|
|
continue
|
|
if row["Text"] and row["Text"].strip():
|
|
key = "{}_{}".format(id_num, i)
|
|
i += 1
|
|
row["English"] = key
|
|
key_dict[key] = row["Text"]
|
|
tsv_keyed.append(row)
|
|
except Exception:
|
|
traceback.print_exc()
|
|
print("Error processing tsv: " + id_num)
|
|
raise
|
|
|
|
return tsv_keyed, key_dict
|