From 6c455fe30b9d8dc08dca951754317f6f4723864a Mon Sep 17 00:00:00 2001 From: louis Date: Wed, 15 May 2019 05:17:37 -0400 Subject: [PATCH] utage: mission key generation and name finding should be compatible with bingo's xdusuitecsharp --- divatool.py | 40 ++++++++++++++++--- utage/__init__.py | 3 ++ utage/crypt.py | 48 +++++++++++++++++++++++ utage/names.py | 83 +++++++++++++++++++++++++++++++++++++++ utage/translate.py | 98 ++++++++++++++++++++++++++++++++++++++++++++++ utage/utage.py | 46 ---------------------- 6 files changed, 266 insertions(+), 52 deletions(-) create mode 100644 utage/__init__.py create mode 100644 utage/names.py create mode 100644 utage/translate.py delete mode 100644 utage/utage.py diff --git a/divatool.py b/divatool.py index 47f21f9..efbdf08 100755 --- a/divatool.py +++ b/divatool.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 import argparse +import os import sys -from os import path +import utage from adx import adx -from utage import utage + +LANGUAGES = ["jpn", "eng", "rus"] def main(): parser = argparse.ArgumentParser(description="Unpack and process XDU data") @@ -43,6 +45,20 @@ def main(): dest="hex", action="store_true", default=False) parser_utage_crypt.add_argument("TARGET", help="Input", type=str) + # utage translate + parser_utage_translate = parser_utage_subparsers.add_parser("translate", help="Generate keyed tsv and json file") + parser_utage_translate.add_argument("INPUT", + help="Input", type=str) + parser_utage_translate.add_argument("TSVDIR", nargs='?', + help="_t.tsv output directory", type=str, default=".") + parser_utage_translate.add_argument("JSONDIR", nargs='?', + help="json output directory", type=str, default=".") + # utage names + parser_utage_names = parser_utage_subparsers.add_parser("names", help="Generate and update name files") + parser_utage_names.add_argument("INPUT", + help="Input", type=str) + parser_utage_names.add_argument("OLD", nargs='?', + help="Directory with old files", type=str, default="") args = parser.parse_args() @@ -55,12 +71,24 @@ def main(): key = bytearray(args.key, "utf-8") else: key = bytearray.fromhex(args.key) - if path.isfile(args.TARGET): - utage.crypt_file(args.TARGET, key, args.encrypt, args.ncomp) - elif path.isdir(args.TARGET): - utage.crypt_dir(args.TARGET, key, args.encrypt, args.ncomp) + if os.path.isfile(args.TARGET): + utage.crypt.crypt_file(args.TARGET, key, args.encrypt, args.ncomp) + elif os.path.isdir(args.TARGET): + utage.crypt.crypt_dir(args.TARGET, key, args.encrypt, args.ncomp) else: raise FileNotFoundError("Could not find {}".format(args.TARGET)) + elif args.subcommand == "translate": + if os.path.isfile(args.INPUT): + utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR) + elif os.path.isdir(args.INPUT): + utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR) + else: + raise FileNotFoundError("Could not find {}".format(args.INPUT)) + elif args.subcommand == "names": + if os.path.isdir(args.INPUT): + utage.names.update_names(args.INPUT, args.OLD, LANGUAGES) + else: + raise FileNotFoundError(args.INPUT) return 0 diff --git a/utage/__init__.py b/utage/__init__.py new file mode 100644 index 0000000..f8c7953 --- /dev/null +++ b/utage/__init__.py @@ -0,0 +1,3 @@ +from . import crypt +from . import names +from . import translate diff --git a/utage/crypt.py b/utage/crypt.py index 719ad5f..79f084f 100644 --- a/utage/crypt.py +++ b/utage/crypt.py @@ -1,3 +1,51 @@ +import io +import os +import traceback + +from functools import partial +from glob import glob +from multiprocessing.pool import Pool + +from . import crypt +from . import translate + +def crypt_dir(dir_in, key, encrypt=False, no_compress=False): + if not encrypt: + files = glob(os.path.join(dir_in, "**/*.utage"), recursive=True) + else: + files = glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) + + if len(files) == 0: + raise FileNotFoundError("No valid files found in directory: " + dir_in) + + pcrypt = partial(crypt_file, key=key, encrypt=encrypt, no_compress=no_compress) + with Pool() as p: + p.map(pcrypt, files) + +def crypt_file(file_in, key, encrypt=False, no_compress=False): + with open(file_in, "rb") as inf: + in_data = inf.read() + try: + if encrypt: + if not file_in.endswith(".png") and not file_in.endswith(".jpg") and not file_in.endswith(".tsv"): + raise ValueError("Invalid File Type for {}".format(file_in)) + if not file_in.endswith(".png") and not no_compress: + enc_data = crypt.compress(in_data) + enc_data = crypt.xor_crypt(enc_data, key) + with io.open(file_in + ".utage", "wb") as output: + output.write(enc_data) + else: + if not file_in.endswith(".utage"): + raise ValueError("Invalid File Type for {}".format(file_in)) + dec_data = crypt.xor_crypt(in_data, key) + if not file_in.endswith(".png.utage") and not file_in.endswith(".jpg.utage") and not no_compress: + dec_data = crypt.decompress(dec_data) + with io.open(file_in.replace(".utage", ""), "wb") as output: + output.write(dec_data) + except Exception as e: + traceback.print_exc() + print("Error processing file: " + file_in) + def decompress(data): osize = 0 isize = len(data) diff --git a/utage/names.py b/utage/names.py new file mode 100644 index 0000000..0e95e88 --- /dev/null +++ b/utage/names.py @@ -0,0 +1,83 @@ +import csv +import json +import io +import os +import re + +from functools import partial +from glob import glob +from multiprocessing import Pool + +def update_names(dir_in, old_path, languages): + names = extract_names(dir_in) + + # we have to update japanese like the rest because CustomData exists + # and POKELABO deleted the wedding gear event stuff from the game files + for x in languages: + langfile = os.path.join(old_path, "nametranslations_{}.json".format(x)) + if os.path.isfile(langfile): + with open(langfile, "r") as lang_file: + lang_dict = json.load(lang_file) + if x == "jpn": + out_dict = dict(zip(names, names)).update(lang_dict) + else: + out_dict = dict.fromkeys(names, "").update(lang_dict) + elif x == "jpn": # bootstrap + out_dict = dict(zip(names, names)) + else: + out_dict = dict.fromkeys(names, "") + + with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right + json.dump(lang_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True) + +def extract_names(dir_in): + char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv") + + if not os.path.isfile(char_tsv_path): + raise FileNotFoundError("Character.tsv not found") + + names, sets = read_char_tsv(char_tsv_path) + + files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)] + if len(files) == 0: + raise FileNotFoundError("No valid files found in directory: " + dir_in) + + # tfw gil + # it's still faster on my machine so i'm keeping it + read_partial = partial(read_mission, char_names=names, char_sets=sets) + with Pool() as p: + new_names = p.map(read_partial, files) + for x in new_names: + names.update(x) + + return names + +def read_char_tsv(char_tsv_path): + char_names = set() + char_sets = set() + + with open(char_tsv_path, "r") as char_tsv_file: + char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar="\"") + + for row in char_tsv: + if row['CharacterName'].startswith("//"): + continue + if row['CharacterName'] and row['NameText'] and row['CharacterName'].strip() and row['NameText'].strip(): + char_names.add(row['NameText']) + char_sets.add(row['CharacterName']) + + return char_names, char_sets + +def read_mission(tsv_path, char_names, char_sets): + new_names = set() + with open(tsv_path, "r") as tsv_file: + tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"") + if ('Arg1' not in tsv.fieldnames) or ('Text' not in tsv.fieldnames) or ('Command' not in tsv.fieldnames): + return new_names + for row in tsv: + if row['Command'] and row['Command'].startswith("//"): + continue + if row['Text'] and row['Arg1']: + if (row['Arg1'] not in char_names) and (row['Arg1'] not in char_sets): + new_names.add(row['Arg1']) + return new_names diff --git a/utage/translate.py b/utage/translate.py new file mode 100644 index 0000000..6ad21a6 --- /dev/null +++ b/utage/translate.py @@ -0,0 +1,98 @@ +import csv +import errno +import io +import json +import os +import re +import traceback + +from functools import partial +from glob import glob +from multiprocessing.pool import Pool + +def translate_dir(dir_in, tsv_out_dir, json_out_dir): + # we only want to key files that are mission ids + files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)] + + if len(files) == 0: + raise FileNotFoundError("No valid files found in directory: " + dir_in) + + ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir) + with Pool() as p: + p.map(ptrans, files) + +def translate_file(file_in, tsv_out_dir, json_out_dir): + try: + if not file_in.endswith(".tsv"): + raise ValueError("Invalid File Type for {}".format(file_in)) + + # we need to get the event folder + # ie, Utage/>>>main01<<