utage: mission key generation and name finding

should be compatible with bingo's xdusuitecsharp

divatool.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 
 import argparse
+import os
 import sys
-from os import path
+import utage
 
 from adx import adx
-from utage import utage
+
+LANGUAGES = ["jpn", "eng", "rus"]
 
 def main():
 	parser = argparse.ArgumentParser(description="Unpack and process XDU data")
@@ -43,6 +45,20 @@ def main():
 	                                dest="hex", action="store_true", default=False)
 	parser_utage_crypt.add_argument("TARGET",
 	                                help="Input", type=str)
+	# utage translate
+	parser_utage_translate = parser_utage_subparsers.add_parser("translate", help="Generate keyed tsv and json file")
+	parser_utage_translate.add_argument("INPUT",
+	                                    help="Input", type=str)
+	parser_utage_translate.add_argument("TSVDIR", nargs='?',
+	                                    help="_t.tsv output directory", type=str, default=".")
+	parser_utage_translate.add_argument("JSONDIR", nargs='?',
+	                                    help="json output directory", type=str, default=".")
+	# utage names
+	parser_utage_names = parser_utage_subparsers.add_parser("names", help="Generate and update name files")
+	parser_utage_names.add_argument("INPUT",
+	                                help="Input", type=str)
+	parser_utage_names.add_argument("OLD", nargs='?',
+	                                help="Directory with old files", type=str, default="")
 
 	args = parser.parse_args()
 
@@ -55,12 +71,24 @@ def main():
 				key = bytearray(args.key, "utf-8")
 			else:
 				key = bytearray.fromhex(args.key)
-			if path.isfile(args.TARGET):
+			if os.path.isfile(args.TARGET):
-				utage.crypt_file(args.TARGET, key, args.encrypt, args.ncomp)
+				utage.crypt.crypt_file(args.TARGET, key, args.encrypt, args.ncomp)
-			elif path.isdir(args.TARGET):
+			elif os.path.isdir(args.TARGET):
-				utage.crypt_dir(args.TARGET, key, args.encrypt, args.ncomp)
+				utage.crypt.crypt_dir(args.TARGET, key, args.encrypt, args.ncomp)
 			else:
 				raise FileNotFoundError("Could not find {}".format(args.TARGET))
+		elif args.subcommand == "translate":
+			if os.path.isfile(args.INPUT):
+				utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR)
+			elif os.path.isdir(args.INPUT):
+				utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR)
+			else:
+				raise FileNotFoundError("Could not find {}".format(args.INPUT))
+		elif args.subcommand == "names":
+			if os.path.isdir(args.INPUT):
+				utage.names.update_names(args.INPUT, args.OLD, LANGUAGES)
+			else:
+				raise FileNotFoundError(args.INPUT)
 
 	return 0
 

utage/__init__.py

@@ -0,0 +1,3 @@
+from . import crypt
+from . import names
+from . import translate

utage/crypt.py

@@ -1,3 +1,51 @@
+import io
+import os
+import traceback
+
+from functools import partial
+from glob import glob
+from multiprocessing.pool import Pool
+
+from . import crypt
+from . import translate
+
+def crypt_dir(dir_in, key, encrypt=False, no_compress=False):
+	if not encrypt:
+		files = glob(os.path.join(dir_in, "**/*.utage"), recursive=True)
+	else:
+		files = glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
+
+	if len(files) == 0:
+		raise FileNotFoundError("No valid files found in directory: " + dir_in)
+
+	pcrypt = partial(crypt_file, key=key, encrypt=encrypt, no_compress=no_compress)
+	with Pool() as p:
+		p.map(pcrypt, files)
+
+def crypt_file(file_in, key, encrypt=False, no_compress=False):
+	with open(file_in, "rb") as inf:
+		in_data = inf.read()
+	try:
+		if encrypt:
+			if not file_in.endswith(".png") and not file_in.endswith(".jpg") and not file_in.endswith(".tsv"):
+				raise ValueError("Invalid File Type for {}".format(file_in))
+			if not file_in.endswith(".png") and not no_compress:
+				enc_data = crypt.compress(in_data)
+			enc_data = crypt.xor_crypt(enc_data, key)
+			with io.open(file_in + ".utage", "wb") as output:
+				output.write(enc_data)
+		else:
+			if not file_in.endswith(".utage"):
+				raise ValueError("Invalid File Type for {}".format(file_in))
+			dec_data = crypt.xor_crypt(in_data, key)
+			if not file_in.endswith(".png.utage") and not file_in.endswith(".jpg.utage") and not no_compress:
+				dec_data = crypt.decompress(dec_data)
+			with io.open(file_in.replace(".utage", ""), "wb") as output:
+				output.write(dec_data)
+	except Exception as e:
+		traceback.print_exc()
+		print("Error processing file: " + file_in)
+
 def decompress(data):
 	osize = 0
 	isize = len(data)

utage/names.py

@@ -0,0 +1,83 @@
+import csv
+import json
+import io
+import os
+import re
+
+from functools import partial
+from glob import glob
+from multiprocessing import Pool
+
+def update_names(dir_in, old_path, languages):
+	names = extract_names(dir_in)
+
+	# we have to update japanese like the rest because CustomData exists
+	# and POKELABO deleted the wedding gear event stuff from the game files
+	for x in languages:
+		langfile = os.path.join(old_path, "nametranslations_{}.json".format(x))
+		if os.path.isfile(langfile):
+			with open(langfile, "r") as lang_file:
+				lang_dict = json.load(lang_file)
+			if x == "jpn":
+				out_dict = dict(zip(names, names)).update(lang_dict)
+			else:
+				out_dict = dict.fromkeys(names, "").update(lang_dict)
+		elif x == "jpn": # bootstrap
+			out_dict = dict(zip(names, names))
+		else:
+			out_dict = dict.fromkeys(names, "")
+
+		with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right
+			json.dump(lang_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True)
+
+def extract_names(dir_in):
+	char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv")
+
+	if not os.path.isfile(char_tsv_path):
+		raise FileNotFoundError("Character.tsv not found")
+
+	names, sets = read_char_tsv(char_tsv_path)
+
+	files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)]
+	if len(files) == 0:
+		raise FileNotFoundError("No valid files found in directory: " + dir_in)
+
+	# tfw gil
+	# it's still faster on my machine so i'm keeping it
+	read_partial = partial(read_mission, char_names=names, char_sets=sets)
+	with Pool() as p:
+		new_names = p.map(read_partial, files)
+	for x in new_names:
+		names.update(x)
+
+	return names
+
+def read_char_tsv(char_tsv_path):
+	char_names = set()
+	char_sets = set()
+
+	with open(char_tsv_path, "r") as char_tsv_file:
+		char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar="\"")
+
+		for row in char_tsv:
+			if row['CharacterName'].startswith("//"):
+				continue
+			if row['CharacterName'] and row['NameText'] and row['CharacterName'].strip() and row['NameText'].strip():
+				char_names.add(row['NameText'])
+				char_sets.add(row['CharacterName'])
+
+	return char_names, char_sets
+
+def read_mission(tsv_path, char_names, char_sets):
+	new_names = set()
+	with open(tsv_path, "r") as tsv_file:
+		tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"")
+		if ('Arg1' not in tsv.fieldnames) or ('Text' not in tsv.fieldnames) or ('Command' not in tsv.fieldnames):
+			return new_names
+		for row in tsv:
+			if row['Command'] and row['Command'].startswith("//"):
+				continue
+			if row['Text'] and row['Arg1']:
+				if (row['Arg1'] not in char_names) and (row['Arg1'] not in char_sets):
+					new_names.add(row['Arg1'])
+	return new_names

utage/translate.py

@@ -0,0 +1,98 @@
+import csv
+import errno
+import io
+import json
+import os
+import re
+import traceback
+
+from functools import partial
+from glob import glob
+from multiprocessing.pool import Pool
+
+def translate_dir(dir_in, tsv_out_dir, json_out_dir):
+	# we only want to key files that are mission ids
+	files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)]
+
+	if len(files) == 0:
+		raise FileNotFoundError("No valid files found in directory: " + dir_in)
+
+	ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir)
+	with Pool() as p:
+		p.map(ptrans, files)
+
+def translate_file(file_in, tsv_out_dir, json_out_dir):
+	try:
+		if not file_in.endswith(".tsv"):
+			raise ValueError("Invalid File Type for {}".format(file_in))
+
+		# we need to get the event folder
+		# ie, Utage/>>>main01<<</Scenario/whatever.tsv
+		try:
+			event_folder = os.path.normpath(file_in).split(os.sep)[-3]
+		except Exception as e:
+			traceback.print_exc()
+			print("Error processing tsv: Please reference from Utage root")
+			raise
+
+		id_num = os.path.splitext(os.path.basename(file_in))[0]
+
+		tsv_output_path = os.path.join(tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num))
+		json_output_path = os.path.join(json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num))
+
+		# need to create output paths and avoid races when threading
+		if not os.path.exists(os.path.dirname(tsv_output_path)):
+			try:
+				os.makedirs(os.path.dirname(tsv_output_path))
+			except OSError as e:
+				if e.errno != errno.EEXIST:
+					raise
+
+		if not os.path.exists(os.path.dirname(json_output_path)):
+			try:
+				os.makedirs(os.path.dirname(json_output_path))
+			except OSError as e:
+				if e.errno != errno.EEXIST:
+					raise
+
+		with open(file_in, "r") as tsv_file:
+			tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"")
+			tsv_keyed, json_str = process_tsv(tsv, id_num)
+
+			# csv handles newlines, don't set it in io.open
+			with io.open(tsv_output_path, "w", newline='') as tsv_out:
+				writer = csv.DictWriter(tsv_out, delimiter="\t", quotechar="\"", lineterminator="\n", fieldnames=tsv.fieldnames, extrasaction='ignore')
+				writer.writeheader()
+				for row in tsv_keyed:
+					writer.writerow(row)
+
+		with io.open(json_output_path, "w", newline='\n') as json_out:
+			# we don't want to sort these because they're _1, ..., _10, etc
+			json.dump(json_str, json_out, ensure_ascii=False, indent='\t', sort_keys=False)
+
+	except Exception as e:
+		traceback.print_exc()
+		print("Error processing file: " + file_in)
+
+def process_tsv(tsv, id_num):
+	i = 0
+	tsv_keyed = []
+	key_dict = {}
+
+	for row in tsv:
+		try:
+			if row['Command'].startswith("//"):
+				tsv_keyed.append(row)
+				continue
+			if row['Text'] and row['Text'].strip():
+				key = "{}_{}".format(id_num, i)
+				i += 1
+				row['English'] = key
+				key_dict[key] = row['Text']
+			tsv_keyed.append(row)
+		except Exception as e:
+			traceback.print_exc()
+			print("Error processing tsv: " + id_num)
+			raise
+
+	return tsv_keyed, key_dict

utage/utage.py

@@ -1,46 +0,0 @@
-import glob
-import io
-import traceback
-
-from functools import partial
-from multiprocessing.pool import Pool
-from os import path
-
-from . import crypt
-
-def crypt_dir(dir_in, key, encrypt=False, no_compress=False):
-	if not encrypt:
-		files = glob.glob(path.join(dir_in, "**/*.utage"), recursive=True)
-	else:
-		files = glob.glob(path.join(dir_in, "**/*.tsv"), recursive=True)
-
-	if len(files) == 0:
-		raise FileNotFoundError("No valid files found in directory: " + dir_in)
-
-	pcrypt = partial(crypt_file, key=key, encrypt=encrypt, no_compress=no_compress)
-	with Pool() as p:
-		p.map(pcrypt, files)
-
-def crypt_file(file_in, key, encrypt=False, no_compress=False):
-	with open(file_in, "rb") as inf:
-		in_data = inf.read()
-	try:
-		if encrypt:
-			if not file_in.endswith(".png") and not file_in.endswith(".tsv"):
-				raise ValueError("Invalid File Type for {}".format(file_in))
-			if not file_in.endswith(".png") and not no_compress:
-				enc_data = crypt.compress(in_data)
-			enc_data = crypt.xor_crypt(enc_data, key)
-			with io.open(file_in + ".utage", "wb") as output:
-				output.write(enc_data)
-		else:
-			if not file_in.endswith(".utage"):
-				raise ValueError("Invalid File Type for {}".format(file_in))
-			dec_data = crypt.xor_crypt(in_data, key)
-			if not file_in.endswith(".png.utage") and not no_compress:
-				dec_data = crypt.decompress(dec_data)
-			with io.open(file_in.replace(".utage", ""), "wb") as output:
-				output.write(dec_data)
-	except Exception as e:
-		traceback.print_exc()
-		print("Error processing file: " + file_in)