utage: mission key generation and name finding

should be compatible with bingo's xdusuitecsharp

utage/translate.py

@@ -0,0 +1,98 @@
+import csv
+import errno
+import io
+import json
+import os
+import re
+import traceback
+
+from functools import partial
+from glob import glob
+from multiprocessing.pool import Pool
+
+def translate_dir(dir_in, tsv_out_dir, json_out_dir):
+	# we only want to key files that are mission ids
+	files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)]
+
+	if len(files) == 0:
+		raise FileNotFoundError("No valid files found in directory: " + dir_in)
+
+	ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir)
+	with Pool() as p:
+		p.map(ptrans, files)
+
+def translate_file(file_in, tsv_out_dir, json_out_dir):
+	try:
+		if not file_in.endswith(".tsv"):
+			raise ValueError("Invalid File Type for {}".format(file_in))
+
+		# we need to get the event folder
+		# ie, Utage/>>>main01<<</Scenario/whatever.tsv
+		try:
+			event_folder = os.path.normpath(file_in).split(os.sep)[-3]
+		except Exception as e:
+			traceback.print_exc()
+			print("Error processing tsv: Please reference from Utage root")
+			raise
+
+		id_num = os.path.splitext(os.path.basename(file_in))[0]
+
+		tsv_output_path = os.path.join(tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num))
+		json_output_path = os.path.join(json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num))
+
+		# need to create output paths and avoid races when threading
+		if not os.path.exists(os.path.dirname(tsv_output_path)):
+			try:
+				os.makedirs(os.path.dirname(tsv_output_path))
+			except OSError as e:
+				if e.errno != errno.EEXIST:
+					raise
+
+		if not os.path.exists(os.path.dirname(json_output_path)):
+			try:
+				os.makedirs(os.path.dirname(json_output_path))
+			except OSError as e:
+				if e.errno != errno.EEXIST:
+					raise
+
+		with open(file_in, "r") as tsv_file:
+			tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"")
+			tsv_keyed, json_str = process_tsv(tsv, id_num)
+
+			# csv handles newlines, don't set it in io.open
+			with io.open(tsv_output_path, "w", newline='') as tsv_out:
+				writer = csv.DictWriter(tsv_out, delimiter="\t", quotechar="\"", lineterminator="\n", fieldnames=tsv.fieldnames, extrasaction='ignore')
+				writer.writeheader()
+				for row in tsv_keyed:
+					writer.writerow(row)
+
+		with io.open(json_output_path, "w", newline='\n') as json_out:
+			# we don't want to sort these because they're _1, ..., _10, etc
+			json.dump(json_str, json_out, ensure_ascii=False, indent='\t', sort_keys=False)
+
+	except Exception as e:
+		traceback.print_exc()
+		print("Error processing file: " + file_in)
+
+def process_tsv(tsv, id_num):
+	i = 0
+	tsv_keyed = []
+	key_dict = {}
+
+	for row in tsv:
+		try:
+			if row['Command'].startswith("//"):
+				tsv_keyed.append(row)
+				continue
+			if row['Text'] and row['Text'].strip():
+				key = "{}_{}".format(id_num, i)
+				i += 1
+				row['English'] = key
+				key_dict[key] = row['Text']
+			tsv_keyed.append(row)
+		except Exception as e:
+			traceback.print_exc()
+			print("Error processing tsv: " + id_num)
+			raise
+
+	return tsv_keyed, key_dict