98 lines
3 KiB
Python
98 lines
3 KiB
Python
import csv
|
|
import errno
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import traceback
|
|
|
|
from functools import partial
|
|
from glob import glob
|
|
from multiprocessing.pool import Pool
|
|
|
|
def translate_dir(dir_in, tsv_out_dir, json_out_dir):
|
|
# we only want to key files that are mission ids
|
|
files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)]
|
|
|
|
if len(files) == 0:
|
|
raise FileNotFoundError("No valid files found in directory: " + dir_in)
|
|
|
|
ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir)
|
|
with Pool() as p:
|
|
p.map(ptrans, files)
|
|
|
|
def translate_file(file_in, tsv_out_dir, json_out_dir):
|
|
try:
|
|
if not file_in.endswith(".tsv"):
|
|
raise ValueError("Invalid File Type for {}".format(file_in))
|
|
|
|
# we need to get the event folder
|
|
# ie, Utage/>>>main01<<</Scenario/whatever.tsv
|
|
try:
|
|
event_folder = os.path.normpath(file_in).split(os.sep)[-3]
|
|
except Exception as e:
|
|
traceback.print_exc()
|
|
print("Error processing tsv: Please reference from Utage root")
|
|
raise
|
|
|
|
id_num = os.path.splitext(os.path.basename(file_in))[0]
|
|
|
|
tsv_output_path = os.path.join(tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num))
|
|
json_output_path = os.path.join(json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num))
|
|
|
|
# need to create output paths and avoid races when threading
|
|
if not os.path.exists(os.path.dirname(tsv_output_path)):
|
|
try:
|
|
os.makedirs(os.path.dirname(tsv_output_path))
|
|
except OSError as e:
|
|
if e.errno != errno.EEXIST:
|
|
raise
|
|
|
|
if not os.path.exists(os.path.dirname(json_output_path)):
|
|
try:
|
|
os.makedirs(os.path.dirname(json_output_path))
|
|
except OSError as e:
|
|
if e.errno != errno.EEXIST:
|
|
raise
|
|
|
|
with open(file_in, "r") as tsv_file:
|
|
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"")
|
|
tsv_keyed, json_str = process_tsv(tsv, id_num)
|
|
|
|
# csv handles newlines, don't set it in io.open
|
|
with io.open(tsv_output_path, "w", newline='') as tsv_out:
|
|
writer = csv.DictWriter(tsv_out, delimiter="\t", quotechar="\"", lineterminator="\n", fieldnames=tsv.fieldnames, extrasaction='ignore')
|
|
writer.writeheader()
|
|
for row in tsv_keyed:
|
|
writer.writerow(row)
|
|
|
|
with io.open(json_output_path, "w", newline='\n') as json_out:
|
|
# we don't want to sort these because they're _1, ..., _10, etc
|
|
json.dump(json_str, json_out, ensure_ascii=False, indent='\t', sort_keys=False)
|
|
|
|
except Exception as e:
|
|
traceback.print_exc()
|
|
print("Error processing file: " + file_in)
|
|
|
|
def process_tsv(tsv, id_num):
|
|
i = 0
|
|
tsv_keyed = []
|
|
key_dict = {}
|
|
|
|
for row in tsv:
|
|
try:
|
|
if row['Command'].startswith("//"):
|
|
tsv_keyed.append(row)
|
|
continue
|
|
if row['Text'] and row['Text'].strip():
|
|
key = "{}_{}".format(id_num, i)
|
|
i += 1
|
|
row['English'] = key
|
|
key_dict[key] = row['Text']
|
|
tsv_keyed.append(row)
|
|
except Exception as e:
|
|
traceback.print_exc()
|
|
print("Error processing tsv: " + id_num)
|
|
raise
|
|
|
|
return tsv_keyed, key_dict
|