utage: mission key generation and name finding
should be compatible with bingo's xdusuitecsharp
This commit is contained in:
louis
parent
71e8c9d257
commit
6c455fe30b
6 changed files with 266 additions and 52 deletions
40
divatool.py
40
divatool.py
|
|
@ -1,11 +1,13 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from os import path
|
||||
import utage
|
||||
|
||||
from adx import adx
|
||||
from utage import utage
|
||||
|
||||
LANGUAGES = ["jpn", "eng", "rus"]
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Unpack and process XDU data")
|
||||
|
|
@ -43,6 +45,20 @@ def main():
|
|||
dest="hex", action="store_true", default=False)
|
||||
parser_utage_crypt.add_argument("TARGET",
|
||||
help="Input", type=str)
|
||||
# utage translate
|
||||
parser_utage_translate = parser_utage_subparsers.add_parser("translate", help="Generate keyed tsv and json file")
|
||||
parser_utage_translate.add_argument("INPUT",
|
||||
help="Input", type=str)
|
||||
parser_utage_translate.add_argument("TSVDIR", nargs='?',
|
||||
help="_t.tsv output directory", type=str, default=".")
|
||||
parser_utage_translate.add_argument("JSONDIR", nargs='?',
|
||||
help="json output directory", type=str, default=".")
|
||||
# utage names
|
||||
parser_utage_names = parser_utage_subparsers.add_parser("names", help="Generate and update name files")
|
||||
parser_utage_names.add_argument("INPUT",
|
||||
help="Input", type=str)
|
||||
parser_utage_names.add_argument("OLD", nargs='?',
|
||||
help="Directory with old files", type=str, default="")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
|
@ -55,12 +71,24 @@ def main():
|
|||
key = bytearray(args.key, "utf-8")
|
||||
else:
|
||||
key = bytearray.fromhex(args.key)
|
||||
if path.isfile(args.TARGET):
|
||||
utage.crypt_file(args.TARGET, key, args.encrypt, args.ncomp)
|
||||
elif path.isdir(args.TARGET):
|
||||
utage.crypt_dir(args.TARGET, key, args.encrypt, args.ncomp)
|
||||
if os.path.isfile(args.TARGET):
|
||||
utage.crypt.crypt_file(args.TARGET, key, args.encrypt, args.ncomp)
|
||||
elif os.path.isdir(args.TARGET):
|
||||
utage.crypt.crypt_dir(args.TARGET, key, args.encrypt, args.ncomp)
|
||||
else:
|
||||
raise FileNotFoundError("Could not find {}".format(args.TARGET))
|
||||
elif args.subcommand == "translate":
|
||||
if os.path.isfile(args.INPUT):
|
||||
utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR)
|
||||
elif os.path.isdir(args.INPUT):
|
||||
utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR)
|
||||
else:
|
||||
raise FileNotFoundError("Could not find {}".format(args.INPUT))
|
||||
elif args.subcommand == "names":
|
||||
if os.path.isdir(args.INPUT):
|
||||
utage.names.update_names(args.INPUT, args.OLD, LANGUAGES)
|
||||
else:
|
||||
raise FileNotFoundError(args.INPUT)
|
||||
|
||||
return 0
|
||||
|
||||
|
|
|
|||
3
utage/__init__.py
Normal file
3
utage/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from . import crypt
|
||||
from . import names
|
||||
from . import translate
|
||||
|
|
@ -1,3 +1,51 @@
|
|||
import io
|
||||
import os
|
||||
import traceback
|
||||
|
||||
from functools import partial
|
||||
from glob import glob
|
||||
from multiprocessing.pool import Pool
|
||||
|
||||
from . import crypt
|
||||
from . import translate
|
||||
|
||||
def crypt_dir(dir_in, key, encrypt=False, no_compress=False):
|
||||
if not encrypt:
|
||||
files = glob(os.path.join(dir_in, "**/*.utage"), recursive=True)
|
||||
else:
|
||||
files = glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
|
||||
|
||||
if len(files) == 0:
|
||||
raise FileNotFoundError("No valid files found in directory: " + dir_in)
|
||||
|
||||
pcrypt = partial(crypt_file, key=key, encrypt=encrypt, no_compress=no_compress)
|
||||
with Pool() as p:
|
||||
p.map(pcrypt, files)
|
||||
|
||||
def crypt_file(file_in, key, encrypt=False, no_compress=False):
|
||||
with open(file_in, "rb") as inf:
|
||||
in_data = inf.read()
|
||||
try:
|
||||
if encrypt:
|
||||
if not file_in.endswith(".png") and not file_in.endswith(".jpg") and not file_in.endswith(".tsv"):
|
||||
raise ValueError("Invalid File Type for {}".format(file_in))
|
||||
if not file_in.endswith(".png") and not no_compress:
|
||||
enc_data = crypt.compress(in_data)
|
||||
enc_data = crypt.xor_crypt(enc_data, key)
|
||||
with io.open(file_in + ".utage", "wb") as output:
|
||||
output.write(enc_data)
|
||||
else:
|
||||
if not file_in.endswith(".utage"):
|
||||
raise ValueError("Invalid File Type for {}".format(file_in))
|
||||
dec_data = crypt.xor_crypt(in_data, key)
|
||||
if not file_in.endswith(".png.utage") and not file_in.endswith(".jpg.utage") and not no_compress:
|
||||
dec_data = crypt.decompress(dec_data)
|
||||
with io.open(file_in.replace(".utage", ""), "wb") as output:
|
||||
output.write(dec_data)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print("Error processing file: " + file_in)
|
||||
|
||||
def decompress(data):
|
||||
osize = 0
|
||||
isize = len(data)
|
||||
|
|
|
|||
83
utage/names.py
Normal file
83
utage/names.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
import csv
|
||||
import json
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
|
||||
from functools import partial
|
||||
from glob import glob
|
||||
from multiprocessing import Pool
|
||||
|
||||
def update_names(dir_in, old_path, languages):
|
||||
names = extract_names(dir_in)
|
||||
|
||||
# we have to update japanese like the rest because CustomData exists
|
||||
# and POKELABO deleted the wedding gear event stuff from the game files
|
||||
for x in languages:
|
||||
langfile = os.path.join(old_path, "nametranslations_{}.json".format(x))
|
||||
if os.path.isfile(langfile):
|
||||
with open(langfile, "r") as lang_file:
|
||||
lang_dict = json.load(lang_file)
|
||||
if x == "jpn":
|
||||
out_dict = dict(zip(names, names)).update(lang_dict)
|
||||
else:
|
||||
out_dict = dict.fromkeys(names, "").update(lang_dict)
|
||||
elif x == "jpn": # bootstrap
|
||||
out_dict = dict(zip(names, names))
|
||||
else:
|
||||
out_dict = dict.fromkeys(names, "")
|
||||
|
||||
with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right
|
||||
json.dump(lang_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True)
|
||||
|
||||
def extract_names(dir_in):
|
||||
char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv")
|
||||
|
||||
if not os.path.isfile(char_tsv_path):
|
||||
raise FileNotFoundError("Character.tsv not found")
|
||||
|
||||
names, sets = read_char_tsv(char_tsv_path)
|
||||
|
||||
files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)]
|
||||
if len(files) == 0:
|
||||
raise FileNotFoundError("No valid files found in directory: " + dir_in)
|
||||
|
||||
# tfw gil
|
||||
# it's still faster on my machine so i'm keeping it
|
||||
read_partial = partial(read_mission, char_names=names, char_sets=sets)
|
||||
with Pool() as p:
|
||||
new_names = p.map(read_partial, files)
|
||||
for x in new_names:
|
||||
names.update(x)
|
||||
|
||||
return names
|
||||
|
||||
def read_char_tsv(char_tsv_path):
|
||||
char_names = set()
|
||||
char_sets = set()
|
||||
|
||||
with open(char_tsv_path, "r") as char_tsv_file:
|
||||
char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar="\"")
|
||||
|
||||
for row in char_tsv:
|
||||
if row['CharacterName'].startswith("//"):
|
||||
continue
|
||||
if row['CharacterName'] and row['NameText'] and row['CharacterName'].strip() and row['NameText'].strip():
|
||||
char_names.add(row['NameText'])
|
||||
char_sets.add(row['CharacterName'])
|
||||
|
||||
return char_names, char_sets
|
||||
|
||||
def read_mission(tsv_path, char_names, char_sets):
|
||||
new_names = set()
|
||||
with open(tsv_path, "r") as tsv_file:
|
||||
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"")
|
||||
if ('Arg1' not in tsv.fieldnames) or ('Text' not in tsv.fieldnames) or ('Command' not in tsv.fieldnames):
|
||||
return new_names
|
||||
for row in tsv:
|
||||
if row['Command'] and row['Command'].startswith("//"):
|
||||
continue
|
||||
if row['Text'] and row['Arg1']:
|
||||
if (row['Arg1'] not in char_names) and (row['Arg1'] not in char_sets):
|
||||
new_names.add(row['Arg1'])
|
||||
return new_names
|
||||
98
utage/translate.py
Normal file
98
utage/translate.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
import csv
|
||||
import errno
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import traceback
|
||||
|
||||
from functools import partial
|
||||
from glob import glob
|
||||
from multiprocessing.pool import Pool
|
||||
|
||||
def translate_dir(dir_in, tsv_out_dir, json_out_dir):
|
||||
# we only want to key files that are mission ids
|
||||
files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)]
|
||||
|
||||
if len(files) == 0:
|
||||
raise FileNotFoundError("No valid files found in directory: " + dir_in)
|
||||
|
||||
ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir)
|
||||
with Pool() as p:
|
||||
p.map(ptrans, files)
|
||||
|
||||
def translate_file(file_in, tsv_out_dir, json_out_dir):
|
||||
try:
|
||||
if not file_in.endswith(".tsv"):
|
||||
raise ValueError("Invalid File Type for {}".format(file_in))
|
||||
|
||||
# we need to get the event folder
|
||||
# ie, Utage/>>>main01<<</Scenario/whatever.tsv
|
||||
try:
|
||||
event_folder = os.path.normpath(file_in).split(os.sep)[-3]
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print("Error processing tsv: Please reference from Utage root")
|
||||
raise
|
||||
|
||||
id_num = os.path.splitext(os.path.basename(file_in))[0]
|
||||
|
||||
tsv_output_path = os.path.join(tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num))
|
||||
json_output_path = os.path.join(json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num))
|
||||
|
||||
# need to create output paths and avoid races when threading
|
||||
if not os.path.exists(os.path.dirname(tsv_output_path)):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(tsv_output_path))
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
if not os.path.exists(os.path.dirname(json_output_path)):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(json_output_path))
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
with open(file_in, "r") as tsv_file:
|
||||
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"")
|
||||
tsv_keyed, json_str = process_tsv(tsv, id_num)
|
||||
|
||||
# csv handles newlines, don't set it in io.open
|
||||
with io.open(tsv_output_path, "w", newline='') as tsv_out:
|
||||
writer = csv.DictWriter(tsv_out, delimiter="\t", quotechar="\"", lineterminator="\n", fieldnames=tsv.fieldnames, extrasaction='ignore')
|
||||
writer.writeheader()
|
||||
for row in tsv_keyed:
|
||||
writer.writerow(row)
|
||||
|
||||
with io.open(json_output_path, "w", newline='\n') as json_out:
|
||||
# we don't want to sort these because they're _1, ..., _10, etc
|
||||
json.dump(json_str, json_out, ensure_ascii=False, indent='\t', sort_keys=False)
|
||||
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print("Error processing file: " + file_in)
|
||||
|
||||
def process_tsv(tsv, id_num):
|
||||
i = 0
|
||||
tsv_keyed = []
|
||||
key_dict = {}
|
||||
|
||||
for row in tsv:
|
||||
try:
|
||||
if row['Command'].startswith("//"):
|
||||
tsv_keyed.append(row)
|
||||
continue
|
||||
if row['Text'] and row['Text'].strip():
|
||||
key = "{}_{}".format(id_num, i)
|
||||
i += 1
|
||||
row['English'] = key
|
||||
key_dict[key] = row['Text']
|
||||
tsv_keyed.append(row)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print("Error processing tsv: " + id_num)
|
||||
raise
|
||||
|
||||
return tsv_keyed, key_dict
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
import glob
|
||||
import io
|
||||
import traceback
|
||||
|
||||
from functools import partial
|
||||
from multiprocessing.pool import Pool
|
||||
from os import path
|
||||
|
||||
from . import crypt
|
||||
|
||||
def crypt_dir(dir_in, key, encrypt=False, no_compress=False):
|
||||
if not encrypt:
|
||||
files = glob.glob(path.join(dir_in, "**/*.utage"), recursive=True)
|
||||
else:
|
||||
files = glob.glob(path.join(dir_in, "**/*.tsv"), recursive=True)
|
||||
|
||||
if len(files) == 0:
|
||||
raise FileNotFoundError("No valid files found in directory: " + dir_in)
|
||||
|
||||
pcrypt = partial(crypt_file, key=key, encrypt=encrypt, no_compress=no_compress)
|
||||
with Pool() as p:
|
||||
p.map(pcrypt, files)
|
||||
|
||||
def crypt_file(file_in, key, encrypt=False, no_compress=False):
|
||||
with open(file_in, "rb") as inf:
|
||||
in_data = inf.read()
|
||||
try:
|
||||
if encrypt:
|
||||
if not file_in.endswith(".png") and not file_in.endswith(".tsv"):
|
||||
raise ValueError("Invalid File Type for {}".format(file_in))
|
||||
if not file_in.endswith(".png") and not no_compress:
|
||||
enc_data = crypt.compress(in_data)
|
||||
enc_data = crypt.xor_crypt(enc_data, key)
|
||||
with io.open(file_in + ".utage", "wb") as output:
|
||||
output.write(enc_data)
|
||||
else:
|
||||
if not file_in.endswith(".utage"):
|
||||
raise ValueError("Invalid File Type for {}".format(file_in))
|
||||
dec_data = crypt.xor_crypt(in_data, key)
|
||||
if not file_in.endswith(".png.utage") and not no_compress:
|
||||
dec_data = crypt.decompress(dec_data)
|
||||
with io.open(file_in.replace(".utage", ""), "wb") as output:
|
||||
output.write(dec_data)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print("Error processing file: " + file_in)
|
||||
Loading…
Add table
Add a link
Reference in a new issue