xdu/divatool
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

BLACKED

Browse source
This commit is contained in:
louis 2019-11-12 10:16:29 -05:00
parent 03e51318b0
commit b6c3c6a454
10 changed files with 819 additions and 654 deletions
Download patch file Download diff file Expand all files Collapse all files

156
utage/translate.py
View file

@ -10,89 +10,109 @@ from functools import partial
from glob import glob
from multiprocessing.pool import Pool
def translate_dir(dir_in, tsv_out_dir, json_out_dir):
# we only want to key files that are mission ids
files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)]
# we only want to key files that are mission ids
files = [
f
for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
if re.search(r"/[0-9]{9}\.tsv$", f)
]
if len(files) == 0:
raise FileNotFoundError("No valid files found in directory: " + dir_in)
if len(files) == 0:
raise FileNotFoundError("No valid files found in directory: " + dir_in)
ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir)
with Pool() as p:
p.map(ptrans, files)
ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir)
with Pool() as p:
p.map(ptrans, files)
def translate_file(file_in, tsv_out_dir, json_out_dir):
try:
if not file_in.endswith(".tsv"):
raise ValueError("Invalid File Type for {}".format(file_in))
try:
if not file_in.endswith(".tsv"):
raise ValueError("Invalid File Type for {}".format(file_in))
# we need to get the event folder
# ie, Utage/>>>main01<<</Scenario/whatever.tsv
try:
event_folder = os.path.normpath(file_in).split(os.sep)[-3]
except Exception as e:
traceback.print_exc()
print("Error processing tsv: Please reference from Utage root")
raise
# we need to get the event folder
# ie, Utage/>>>main01<<</Scenario/whatever.tsv
try:
event_folder = os.path.normpath(file_in).split(os.sep)[-3]
except Exception as e:
traceback.print_exc()
print("Error processing tsv: Please reference from Utage root")
raise
id_num = os.path.splitext(os.path.basename(file_in))[0]
id_num = os.path.splitext(os.path.basename(file_in))[0]
tsv_output_path = os.path.join(tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num))
json_output_path = os.path.join(json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num))
tsv_output_path = os.path.join(
tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num)
)
json_output_path = os.path.join(
json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num)
)
# need to create output paths and avoid races when threading
if not os.path.exists(os.path.dirname(tsv_output_path)):
try:
os.makedirs(os.path.dirname(tsv_output_path))
except OSError as e:
if e.errno != errno.EEXIST:
raise
# need to create output paths and avoid races when threading
if not os.path.exists(os.path.dirname(tsv_output_path)):
try:
os.makedirs(os.path.dirname(tsv_output_path))
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.exists(os.path.dirname(json_output_path)):
try:
os.makedirs(os.path.dirname(json_output_path))
except OSError as e:
if e.errno != errno.EEXIST:
raise
if not os.path.exists(os.path.dirname(json_output_path)):
try:
os.makedirs(os.path.dirname(json_output_path))
except OSError as e:
if e.errno != errno.EEXIST:
raise
with open(file_in, "r") as tsv_file:
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"")
tsv_keyed, json_str = process_tsv(tsv, id_num)
with open(file_in, "r") as tsv_file:
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
tsv_keyed, json_str = process_tsv(tsv, id_num)
# csv handles newlines, don't set it in io.open
with io.open(tsv_output_path, "w", newline='') as tsv_out:
writer = csv.DictWriter(tsv_out, delimiter="\t", quotechar="\"", lineterminator="\n", fieldnames=tsv.fieldnames, extrasaction='ignore')
writer.writeheader()
for row in tsv_keyed:
writer.writerow(row)
# csv handles newlines, don't set it in io.open
with io.open(tsv_output_path, "w", newline="") as tsv_out:
writer = csv.DictWriter(
tsv_out,
delimiter="\t",
quotechar='"',
lineterminator="\n",
fieldnames=tsv.fieldnames,
extrasaction="ignore",
)
writer.writeheader()
for row in tsv_keyed:
writer.writerow(row)
with io.open(json_output_path, "w", newline='\n') as json_out:
# we don't want to sort these because they're _1, ..., _10, etc
json.dump(json_str, json_out, ensure_ascii=False, indent='\t', sort_keys=False)
with io.open(json_output_path, "w", newline="\n") as json_out:
# we don't want to sort these because they're _1, ..., _10, etc
json.dump(
json_str, json_out, ensure_ascii=False, indent="\t", sort_keys=False
)
except Exception as e:
traceback.print_exc()
print("Error processing file: " + file_in)
except Exception as e:
traceback.print_exc()
print("Error processing file: " + file_in)
def process_tsv(tsv, id_num):
i = 0
tsv_keyed = []
key_dict = {}
i = 0
tsv_keyed = []
key_dict = {}
for row in tsv:
try:
if row['Command'].startswith("//"):
tsv_keyed.append(row)
continue
if row['Text'] and row['Text'].strip():
key = "{}_{}".format(id_num, i)
i += 1
row['English'] = key
key_dict[key] = row['Text']
tsv_keyed.append(row)
except Exception as e:
traceback.print_exc()
print("Error processing tsv: " + id_num)
raise
for row in tsv:
try:
if row["Command"].startswith("//"):
tsv_keyed.append(row)
continue
if row["Text"] and row["Text"].strip():
key = "{}_{}".format(id_num, i)
i += 1
row["English"] = key
key_dict[key] = row["Text"]
tsv_keyed.append(row)
except Exception as e:
traceback.print_exc()
print("Error processing tsv: " + id_num)
raise
return tsv_keyed, key_dict
return tsv_keyed, key_dict