xdu/divatool
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

BLACKED

Browse source
This commit is contained in:
louis 2019-11-12 10:16:29 -05:00
parent 03e51318b0
commit b6c3c6a454
10 changed files with 819 additions and 654 deletions
Download patch file Download diff file Expand all files Collapse all files

9
adx/adx.py
View file

@ -5,6 +5,7 @@ import traceback
from .adx_parser import adx_file from .adx_parser import adx_file
def extract_loop_data_from_dir(dir_in, fout): def extract_loop_data_from_dir(dir_in, fout):
if fout is None: if fout is None:
fout = "BgmLoop.json" fout = "BgmLoop.json"
@ -13,6 +14,7 @@ def extract_loop_data_from_dir(dir_in, fout):
raise FileNotFoundError("No valid files found in directory: " + dir_in) raise FileNotFoundError("No valid files found in directory: " + dir_in)
extract_loop_data_from_files(files, fout) extract_loop_data_from_files(files, fout)
def extract_loop_data_from_files(files_in, fout): def extract_loop_data_from_files(files_in, fout):
if not files_in: if not files_in:
raise FileNotFoundError("No input files found") raise FileNotFoundError("No input files found")
@ -21,15 +23,16 @@ def extract_loop_data_from_files(files_in, fout):
res = extract_loop_data_from_file(x) res = extract_loop_data_from_file(x)
if res is not None: if res is not None:
collect[os.path.splitext(os.path.basename(x))[0]] = res collect[os.path.splitext(os.path.basename(x))[0]] = res
with open(fout, 'w') as out: with open(fout, "w") as out:
json.dump(collect, out, ensure_ascii=False, indent="\t", sort_keys=True) json.dump(collect, out, ensure_ascii=False, indent="\t", sort_keys=True)
def extract_loop_data_from_file(file_in, fout=None): def extract_loop_data_from_file(file_in, fout=None):
if not file_in.endswith(".adx"): if not file_in.endswith(".adx"):
raise ValueError("Input file not .adx") raise ValueError("Input file not .adx")
try: try:
with open(file_in, 'rb') as x: with open(file_in, "rb") as x:
data = x.read(64) data = x.read(64)
ADX = adx_file(data) ADX = adx_file(data)
if ADX.l_exists and ADX.is_valid: if ADX.l_exists and ADX.is_valid:
@ -42,6 +45,6 @@ def extract_loop_data_from_file(file_in, fout=None):
return None return None
if fout is not None and res is not None: if fout is not None and res is not None:
with open(fout, 'w') as out: with open(fout, "w") as out:
json.dump(res, out, ensure_ascii=False, indent="\t", sort_keys=True) json.dump(res, out, ensure_ascii=False, indent="\t", sort_keys=True)
return res return res

34
adx/adx_parser.py
View file

@ -22,7 +22,7 @@ LOOP_TYPE_SUP = [4] # supported loop styles
SRATE_OFF = 0x08 SRATE_OFF = 0x08
SRATE_LEN = 4 SRATE_LEN = 4
SCOUNT_OFF = 0x0c SCOUNT_OFF = 0x0C
SCOUNT_LEN = 4 SCOUNT_LEN = 4
CRYPT_OFF = 0x13 CRYPT_OFF = 0x13
@ -40,13 +40,15 @@ LOOP4_END_LEN = 4
# class independent functions # class independent functions
def dumb_round(number): def dumb_round(number):
dec = modf(number)[0] dec = modf(number)[0]
if (dec <= 0.5): if dec <= 0.5:
return int(number) return int(number)
else: else:
return int(number + 1) return int(number + 1)
class adx_file(object): class adx_file(object):
magic = 0 magic = 0
form = 0 form = 0
@ -80,7 +82,9 @@ class adx_file(object):
self.validate() self.validate()
def get_val(self, offset, length): def get_val(self, offset, length):
return int.from_bytes(self.data[offset:offset+length], byteorder='big', signed=False) return int.from_bytes(
self.data[offset : offset + length], byteorder="big", signed=False
)
def validate(self): def validate(self):
if self.magic != MAGIC: if self.magic != MAGIC:
@ -104,16 +108,20 @@ class adx_file(object):
raise ValueError("Invalid Loop Data") raise ValueError("Invalid Loop Data")
return None return None
if not self.l_style in LOOP_TYPE_SUP: if not self.l_style in LOOP_TYPE_SUP:
raise NotImplementedError("Loop Style {} Not Implemented".format(str(self.l_style))) raise NotImplementedError(
"Loop Style {} Not Implemented".format(str(self.l_style))
)
return None return None
ret = {} ret = {}
ret['duration'] = self.sam_count / self.sam_rate ret["duration"] = self.sam_count / self.sam_rate
ret['loop_start'] = {} ret["loop_start"] = {}
ret['loop_start']['seconds'] = self.l_start / self.sam_rate ret["loop_start"]["seconds"] = self.l_start / self.sam_rate
ret['loop_start']['samples_native'] = self.l_start ret["loop_start"]["samples_native"] = self.l_start
ret['loop_start']['samples_48k'] = dumb_round(self.l_start / self.sam_rate * 48000) ret["loop_start"]["samples_48k"] = dumb_round(
ret['loop_end'] = {} self.l_start / self.sam_rate * 48000
ret['loop_end']['seconds'] = self.l_end / self.sam_rate )
ret['loop_end']['samples_native'] = self.l_end ret["loop_end"] = {}
ret['loop_end']['samples_48k'] = dumb_round(self.l_end / self.sam_rate * 48000) ret["loop_end"]["seconds"] = self.l_end / self.sam_rate
ret["loop_end"]["samples_native"] = self.l_end
ret["loop_end"]["samples_48k"] = dumb_round(self.l_end / self.sam_rate * 48000)
return ret return ret

5
db/base.py
View file

@ -7,7 +7,7 @@ from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base() Base = declarative_base()
engine = create_engine('sqlite:///:memory:', echo=False) engine = create_engine("sqlite:///:memory:", echo=False)
session = sessionmaker(bind=engine, autoflush=False, autocommit=False)() session = sessionmaker(bind=engine, autoflush=False, autocommit=False)()
session.flush = lambda: None session.flush = lambda: None
@ -15,9 +15,10 @@ DATABASES = {
"QuestMst.db": "Quest", "QuestMst.db": "Quest",
"QuestSceneMst.db": "QuestScene", "QuestSceneMst.db": "QuestScene",
"QuestPartMst.db": "QuestPart", "QuestPartMst.db": "QuestPart",
"ResourceEntry.db": "ResourceEntry" "ResourceEntry.db": "ResourceEntry",
} }
def init(path): def init(path):
for d in DATABASES: for d in DATABASES:
if not os.path.isfile(os.path.join(path, d)): if not os.path.isfile(os.path.join(path, d)):

27
db/quest.py
View file

@ -3,9 +3,10 @@ from sqlalchemy.orm import relationship
from .base import Base from .base import Base
class Quest(Base): class Quest(Base):
__tablename__ = 'QuestMstRecord' __tablename__ = "QuestMstRecord"
__table_args__ = {'schema': 'Quest'} __table_args__ = {"schema": "Quest"}
questMstId = Column(Integer, primary_key=True) questMstId = Column(Integer, primary_key=True)
questType = Column(Integer) questType = Column(Integer)
@ -22,9 +23,10 @@ class Quest(Base):
def __repr__(self): def __repr__(self):
return "<Quest(questMstId={}, name={})>".format(self.questMstId, self.name) return "<Quest(questMstId={}, name={})>".format(self.questMstId, self.name)
class QuestScene(Base): class QuestScene(Base):
__tablename__ = 'QuestSceneMstRecord' __tablename__ = "QuestSceneMstRecord"
__table_args__ = {'schema': 'QuestScene'} __table_args__ = {"schema": "QuestScene"}
questSceneMstId = Column(Integer, primary_key=True) questSceneMstId = Column(Integer, primary_key=True)
questMstId = Column(Integer, ForeignKey("Quest.QuestMstRecord.questMstId")) questMstId = Column(Integer, ForeignKey("Quest.QuestMstRecord.questMstId"))
@ -32,7 +34,9 @@ class QuestScene(Base):
sortNum = Column(Integer) sortNum = Column(Integer)
name = Column(String) name = Column(String)
_filter = Column("filter", Integer) _filter = Column("filter", Integer)
prevQuestSceneMstId = Column(Integer, ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId")) prevQuestSceneMstId = Column(
Integer, ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId")
)
releaseSerial = Column(Integer) releaseSerial = Column(Integer)
releaseEvolutionLevel = Column(Integer) releaseEvolutionLevel = Column(Integer)
summaryText = Column(String) summaryText = Column(String)
@ -48,11 +52,16 @@ class QuestScene(Base):
previous = relationship("QuestScene", remote_side=[questSceneMstId]) previous = relationship("QuestScene", remote_side=[questSceneMstId])
parts = relationship("QuestPart") parts = relationship("QuestPart")
class QuestPart(Base):
__tablename__ = 'QuestPartMstRecord'
__table_args__ = {'schema': 'QuestPart'}
questSceneMstId = Column(Integer, ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId"), primary_key=True) # now this is pod racing class QuestPart(Base):
__tablename__ = "QuestPartMstRecord"
__table_args__ = {"schema": "QuestPart"}
questSceneMstId = Column(
Integer,
ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId"),
primary_key=True,
) # now this is pod racing
partNum = Column(Integer, primary_key=True) partNum = Column(Integer, primary_key=True)
waveNum = Column(Integer) waveNum = Column(Integer)
stamina = Column(Integer) stamina = Column(Integer)

5
db/resource.py
View file

@ -2,9 +2,10 @@ from sqlalchemy import Column, BigInteger, String
from .base import Base from .base import Base
class ResourceEntry(Base): class ResourceEntry(Base):
__tablename__ = 'ResourceEntryRecord' __tablename__ = "ResourceEntryRecord"
__table_args__ = {'schema': 'ResourceEntry'} __table_args__ = {"schema": "ResourceEntry"}
path = Column(String, primary_key=True) path = Column(String, primary_key=True)
serverPath = Column(String) serverPath = Column(String)

99
diva/quest.py
View file

@ -5,25 +5,33 @@ import os
from sqlalchemy.orm import contains_eager from sqlalchemy.orm import contains_eager
def update_missions(dir_in, old_path, languages): def update_missions(dir_in, old_path, languages):
db.init(dir_in) db.init(dir_in)
quest_mst(old_path, languages) quest_mst(old_path, languages)
scene_mst(old_path, languages) scene_mst(old_path, languages)
def scene_mst(old_path, languages): def scene_mst(old_path, languages):
result = (db.session.query(db.QuestScene) result = (
db.session.query(db.QuestScene)
.join(db.QuestScene.parts) .join(db.QuestScene.parts)
.options(contains_eager(db.QuestScene.parts)) .options(contains_eager(db.QuestScene.parts))
.filter((db.QuestPart.beforeTalkName != "") | (db.QuestPart.afterTalkName != "")) .filter(
(db.QuestPart.beforeTalkName != "") | (db.QuestPart.afterTalkName != "")
)
.filter(db.QuestScene.parts.any()) .filter(db.QuestScene.parts.any())
.all()) .all()
)
scenes = {} scenes = {}
for x in result: for x in result:
scenes[str(x.questSceneMstId)] = { "Name": x.name, scenes[str(x.questSceneMstId)] = {
"Name": x.name,
"SummaryText": x.summaryText, "SummaryText": x.summaryText,
"Parts": [], "Parts": [],
"Folder": "" } "Folder": "",
}
id_num = "" id_num = ""
# this is slow as balls but i'm way too tired to figure out how to optimize it # this is slow as balls but i'm way too tired to figure out how to optimize it
@ -37,32 +45,51 @@ def scene_mst(old_path, languages):
# i checked every entry, moving it to do once for speed # i checked every entry, moving it to do once for speed
if not scenes[str(x.questSceneMstId)]["Folder"] and id_num: if not scenes[str(x.questSceneMstId)]["Folder"] and id_num:
path = (db.session.query(db.ResourceEntry) path = (
.filter(db.ResourceEntry.path.like("%/Scenario/{}.tsv.utage".format(id_num))) db.session.query(db.ResourceEntry)
.filter(db.ResourceEntry.path.notlike("%Utage/side03/%.tsv.utage")) # dead folder with dupes too lazy to parse settings tsv .filter(
.one()) db.ResourceEntry.path.like(
"%/Scenario/{}.tsv.utage".format(id_num)
)
)
.filter(
db.ResourceEntry.path.notlike("%Utage/side03/%.tsv.utage")
) # dead folder with dupes too lazy to parse settings tsv
.one()
)
folder = path.path.split("/")[2] folder = path.path.split("/")[2]
if scenes[str(x.questSceneMstId)]["Folder"] and scenes[str(x.questSceneMstId)]["Folder"] != folder: if (
raise KeyError("Internal path inconsitency, bailing out {}".format(id_num)) scenes[str(x.questSceneMstId)]["Folder"]
and scenes[str(x.questSceneMstId)]["Folder"] != folder
):
raise KeyError(
"Internal path inconsitency, bailing out {}".format(id_num)
)
scenes[str(x.questSceneMstId)]["Folder"] = folder scenes[str(x.questSceneMstId)]["Folder"] = folder
with io.open(os.path.join(old_path, "XduScene.json"), "w", newline='\n') as json_file: with io.open(
json.dump(scenes, json_file, ensure_ascii=False, indent='\t', sort_keys=True) os.path.join(old_path, "XduScene.json"), "w", newline="\n"
) as json_file:
json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
for l in languages: for l in languages:
out_dict = {} out_dict = {}
if l == "jpn": if l == "jpn":
for x in result: for x in result:
out_dict[str(x.questSceneMstId)] = { "Name": x.name, out_dict[str(x.questSceneMstId)] = {
"Name": x.name,
"SummaryText": x.summaryText, "SummaryText": x.summaryText,
"Credits": "POKELABO", "Credits": "POKELABO",
"Enabled": False } "Enabled": False,
}
else: else:
for x in result: for x in result:
out_dict[str(x.questSceneMstId)] = { "Name": "", out_dict[str(x.questSceneMstId)] = {
"Name": "",
"SummaryText": "", "SummaryText": "",
"Credits": "", "Credits": "",
"Enabled": False } "Enabled": False,
}
langfile = os.path.join(old_path, "XduSceneNames_{}.json".format(l)) langfile = os.path.join(old_path, "XduSceneNames_{}.json".format(l))
if os.path.isfile(langfile): if os.path.isfile(langfile):
@ -70,27 +97,38 @@ def scene_mst(old_path, languages):
lang_dict = json.load(lang_file) lang_dict = json.load(lang_file)
out_dict.update(lang_dict) out_dict.update(lang_dict)
with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right with io.open(
json.dump(out_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True) langfile, "w", newline="\n"
) as lang_file: # you're using git right
json.dump(
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True
)
def quest_mst(old_path, languages): def quest_mst(old_path, languages):
# i regret my life choices # i regret my life choices
result = (db.session.query(db.Quest) result = (
db.session.query(db.Quest)
.filter(db.Quest.baseQuestMstId == 0) .filter(db.Quest.baseQuestMstId == 0)
.join(db.Quest.scenes) .join(db.Quest.scenes)
.join(db.QuestScene.parts) .join(db.QuestScene.parts)
.options(contains_eager(db.Quest.scenes). .options(contains_eager(db.Quest.scenes).contains_eager(db.QuestScene.parts))
contains_eager(db.QuestScene.parts))
.filter(db.QuestPart.beforeTalkName != "" or db.QuestPart.afterTalkName != "") .filter(db.QuestPart.beforeTalkName != "" or db.QuestPart.afterTalkName != "")
.filter(db.QuestScene.parts.any()) .filter(db.QuestScene.parts.any())
.filter(db.Quest.scenes.any()) .filter(db.Quest.scenes.any())
.all()) .all()
)
quests = {} quests = {}
for x in result: for x in result:
quests[str(x.questMstId)] = { "Name": x.name, "Scenes": [d.questSceneMstId for d in x.scenes] } quests[str(x.questMstId)] = {
"Name": x.name,
"Scenes": [d.questSceneMstId for d in x.scenes],
}
with io.open(os.path.join(old_path, "XduQuest.json"), "w", newline='\n') as json_file: with io.open(
json.dump(quests, json_file, ensure_ascii=False, indent='\t', sort_keys=True) os.path.join(old_path, "XduQuest.json"), "w", newline="\n"
) as json_file:
json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
for l in languages: for l in languages:
out_dict = {} out_dict = {}
@ -107,8 +145,11 @@ def quest_mst(old_path, languages):
lang_dict = json.load(lang_file) lang_dict = json.load(lang_file)
out_dict.update(lang_dict) out_dict.update(lang_dict)
with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right with io.open(
json.dump(out_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True) langfile, "w", newline="\n"
) as lang_file: # you're using git right
json.dump(
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True
)
return quests return quests

122
divatool.py
View file

@ -10,65 +10,110 @@ import db
LANGUAGES = ["jpn", "eng", "rus"] LANGUAGES = ["jpn", "eng", "rus"]
def main(): def main():
parser = argparse.ArgumentParser(description="Unpack and process XDU data") parser = argparse.ArgumentParser(description="Unpack and process XDU data")
subparsers = parser.add_subparsers(metavar="<command>", title="subcommands", dest="command") subparsers = parser.add_subparsers(
metavar="<command>", title="subcommands", dest="command"
)
# adx subcommand # adx subcommand
parser_adx = subparsers.add_parser("adx", help="ADX stuff") parser_adx = subparsers.add_parser("adx", help="ADX stuff")
parser_adx_subparsers = parser_adx.add_subparsers(metavar="<command>", title="subcommands", dest="subcommand") parser_adx_subparsers = parser_adx.add_subparsers(
metavar="<command>", title="subcommands", dest="subcommand"
)
# adx loop # adx loop
parser_adx_loop = parser_adx_subparsers.add_parser("loop", help="Extract loop data") parser_adx_loop = parser_adx_subparsers.add_parser("loop", help="Extract loop data")
parser_adx_loop.add_argument("--output", "-o", parser_adx_loop.add_argument(
"--output",
"-o",
help="JSON output file for loop data", help="JSON output file for loop data",
type=str, dest="JSON_OUT") type=str,
parser_adx_loop.add_argument("ADX_DIR", dest="JSON_OUT",
help="Input directory containing ADX files", )
type=str) parser_adx_loop.add_argument(
"ADX_DIR", help="Input directory containing ADX files", type=str
)
# utage subcommand # utage subcommand
parser_utage = subparsers.add_parser("utage", help="UTAGE stuff") parser_utage = subparsers.add_parser("utage", help="UTAGE stuff")
parser_utage_subparsers = parser_utage.add_subparsers(metavar="<command>", title="subcommand", dest="subcommand") parser_utage_subparsers = parser_utage.add_subparsers(
metavar="<command>", title="subcommand", dest="subcommand"
)
# utage crypt # utage crypt
parser_utage_crypt = parser_utage_subparsers.add_parser("crypt", help="Encrypt/decrypt utage tsv files") parser_utage_crypt = parser_utage_subparsers.add_parser(
parser_utage_crypt.add_argument("--encrypt", "-e", "crypt", help="Encrypt/decrypt utage tsv files"
)
parser_utage_crypt.add_argument(
"--encrypt",
"-e",
help="Encrypt (default: Decrypt)", help="Encrypt (default: Decrypt)",
dest="encrypt", action="store_true", default=False) dest="encrypt",
parser_utage_crypt.add_argument("--no-compression", "-n", action="store_true",
default=False,
)
parser_utage_crypt.add_argument(
"--no-compression",
"-n",
help="Do not compress/decompress. Only applies to tsv, png will never be compressed", help="Do not compress/decompress. Only applies to tsv, png will never be compressed",
dest="ncomp", action="store_true", default=False) dest="ncomp",
parser_utage_crypt.add_argument("--key", "-k", action="store_true",
default=False,
)
parser_utage_crypt.add_argument(
"--key",
"-k",
help="Encryption key (Default: SampleSecretKey)", help="Encryption key (Default: SampleSecretKey)",
dest="key", type=str, default="SampleSecretKey") dest="key",
parser_utage_crypt.add_argument("--hex", "-x", type=str,
default="SampleSecretKey",
)
parser_utage_crypt.add_argument(
"--hex",
"-x",
help="KEY is hexadecimal (Default: False)", help="KEY is hexadecimal (Default: False)",
dest="hex", action="store_true", default=False) dest="hex",
parser_utage_crypt.add_argument("TARGET", action="store_true",
help="Input", type=str) default=False,
)
parser_utage_crypt.add_argument("TARGET", help="Input", type=str)
# utage translate # utage translate
parser_utage_translate = parser_utage_subparsers.add_parser("translate", help="Generate keyed tsv and json files") parser_utage_translate = parser_utage_subparsers.add_parser(
parser_utage_translate.add_argument("INPUT", "translate", help="Generate keyed tsv and json files"
help="Input", type=str) )
parser_utage_translate.add_argument("TSVDIR", nargs='?', parser_utage_translate.add_argument("INPUT", help="Input", type=str)
help="_t.tsv output directory", type=str, default=".") parser_utage_translate.add_argument(
parser_utage_translate.add_argument("JSONDIR", nargs='?', "TSVDIR", nargs="?", help="_t.tsv output directory", type=str, default="."
help="json output directory", type=str, default=".") )
parser_utage_translate.add_argument(
"JSONDIR", nargs="?", help="json output directory", type=str, default="."
)
# utage names # utage names
parser_utage_names = parser_utage_subparsers.add_parser("names", help="Generate and update name files") parser_utage_names = parser_utage_subparsers.add_parser(
parser_utage_names.add_argument("INPUT", "names", help="Generate and update name files"
help="Input", type=str) )
parser_utage_names.add_argument("OLD", nargs='?', parser_utage_names.add_argument("INPUT", help="Input", type=str)
help="Directory with old files", type=str, default=".") parser_utage_names.add_argument(
"OLD", nargs="?", help="Directory with old files", type=str, default="."
)
# diva subcommand # diva subcommand
parser_diva = subparsers.add_parser("diva", help="Diva stuff") parser_diva = subparsers.add_parser("diva", help="Diva stuff")
parser_diva_subparsers = parser_diva.add_subparsers(metavar="<command>", title="subcommand", dest="subcommand") parser_diva_subparsers = parser_diva.add_subparsers(
parser_diva_quest = parser_diva_subparsers.add_parser("quest", help="Generate quest JSON files") metavar="<command>", title="subcommand", dest="subcommand"
parser_diva_quest.add_argument("INPUT", help="Directory of databases containing Quest*.db and ResourceEntry.db", )
type=str) parser_diva_quest = parser_diva_subparsers.add_parser(
parser_diva_quest.add_argument("OLD", help="Directory with old quest files", "quest", help="Generate quest JSON files"
nargs='?', type=str, default=".") )
parser_diva_quest.add_argument(
"INPUT",
help="Directory of databases containing Quest*.db and ResourceEntry.db",
type=str,
)
parser_diva_quest.add_argument(
"OLD", help="Directory with old quest files", nargs="?", type=str, default="."
)
args = parser.parse_args() args = parser.parse_args()
@ -105,6 +150,7 @@ def main():
return 0 return 0
if __name__ == '__main__':
if __name__ == "__main__":
main() main()
sys.exit() sys.exit()

31
utage/crypt.py
View file

@ -9,6 +9,7 @@ from multiprocessing.pool import Pool
from . import crypt from . import crypt
from . import translate from . import translate
def crypt_dir(dir_in, key, encrypt=False, no_compress=False): def crypt_dir(dir_in, key, encrypt=False, no_compress=False):
if not encrypt: if not encrypt:
files = glob(os.path.join(dir_in, "**/*.utage"), recursive=True) files = glob(os.path.join(dir_in, "**/*.utage"), recursive=True)
@ -22,12 +23,17 @@ def crypt_dir(dir_in, key, encrypt=False, no_compress=False):
with Pool() as p: with Pool() as p:
p.map(pcrypt, files) p.map(pcrypt, files)
def crypt_file(file_in, key, encrypt=False, no_compress=False): def crypt_file(file_in, key, encrypt=False, no_compress=False):
with open(file_in, "rb") as inf: with open(file_in, "rb") as inf:
in_data = inf.read() in_data = inf.read()
try: try:
if encrypt: if encrypt:
if not file_in.endswith(".png") and not file_in.endswith(".jpg") and not file_in.endswith(".tsv"): if (
not file_in.endswith(".png")
and not file_in.endswith(".jpg")
and not file_in.endswith(".tsv")
):
raise ValueError("Invalid File Type for {}".format(file_in)) raise ValueError("Invalid File Type for {}".format(file_in))
if not file_in.endswith(".png") and not no_compress: if not file_in.endswith(".png") and not no_compress:
enc_data = crypt.compress(in_data) enc_data = crypt.compress(in_data)
@ -38,7 +44,11 @@ def crypt_file(file_in, key, encrypt=False, no_compress=False):
if not file_in.endswith(".utage"): if not file_in.endswith(".utage"):
raise ValueError("Invalid File Type for {}".format(file_in)) raise ValueError("Invalid File Type for {}".format(file_in))
dec_data = crypt.xor_crypt(in_data, key) dec_data = crypt.xor_crypt(in_data, key)
if not file_in.endswith(".png.utage") and not file_in.endswith(".jpg.utage") and not no_compress: if (
not file_in.endswith(".png.utage")
and not file_in.endswith(".jpg.utage")
and not no_compress
):
dec_data = crypt.decompress(dec_data) dec_data = crypt.decompress(dec_data)
with io.open(file_in.replace(".utage", ""), "wb") as output: with io.open(file_in.replace(".utage", ""), "wb") as output:
output.write(dec_data) output.write(dec_data)
@ -46,10 +56,11 @@ def crypt_file(file_in, key, encrypt=False, no_compress=False):
traceback.print_exc() traceback.print_exc()
print("Error processing file: " + file_in) print("Error processing file: " + file_in)
def decompress(data): def decompress(data):
osize = 0 osize = 0
isize = len(data) isize = len(data)
osize_expected = int.from_bytes(data[:4], byteorder='little', signed=False) osize_expected = int.from_bytes(data[:4], byteorder="little", signed=False)
odata = bytearray(osize_expected) odata = bytearray(osize_expected)
odata[:isize] = data odata[:isize] = data
i = 4 i = 4
@ -71,9 +82,10 @@ def decompress(data):
i += 1 i += 1
return odata return odata
def compress(data): def compress(data):
num = len(data) num = len(data)
bytes2 = num.to_bytes(4, byteorder='little') bytes2 = num.to_bytes(4, byteorder="little")
anum3 = num + num / 128 + 1 anum3 = num + num / 128 + 1
array = bytearray(int(anum3)) array = bytearray(int(anum3))
num2 = 0 num2 = 0
@ -106,7 +118,7 @@ def compress(data):
index.remove(data[num9], num9) index.remove(data[num9], num9)
index.add(data[num3 + j], num3 + j) index.add(data[num3 + j], num3 + j)
if num4 < num3: if num4 < num3:
array[num2] = (num3 - num4 - 1) array[num2] = num3 - num4 - 1
num2 = num2 + 1 num2 = num2 + 1
for j in range(num4, num3): for j in range(num4, num3):
array[num2] = data[j] array[num2] = data[j]
@ -116,7 +128,7 @@ def compress(data):
num12 = 0x80 | num10 num12 = 0x80 | num10
num12 |= (num11 & 0x700) >> 4 num12 |= (num11 & 0x700) >> 4
array[num2] = num12 array[num2] = num12
array[num2+1] = (num11&0xff) array[num2 + 1] = num11 & 0xFF
num2 = num2 + 2 num2 = num2 + 2
num3 = num3 + num5 num3 = num3 + num5
num4 = num3 num4 = num3
@ -145,11 +157,13 @@ def compress(data):
array2[4:] = array[:osize] array2[4:] = array[:osize]
return array2 return array2
class Node: class Node:
mNext = 0 mNext = 0
mPrev = 0 mPrev = 0
mPos = 0 mPos = 0
class Index: class Index:
mNodes = [] mNodes = []
mStack = [] mStack = []
@ -194,14 +208,15 @@ class Index:
def isEnd(self, idx): def isEnd(self, idx):
return idx >= 2048 return idx >= 2048
def xor_crypt(data, key): def xor_crypt(data, key):
odata = bytearray(len(data)) odata = bytearray(len(data))
odata[:] = data odata[:] = data
i = 0 i = 0
for x in data: for x in data:
if (x != 0): if x != 0:
m = key[i % len(key)] m = key[i % len(key)]
if (x != m): if x != m:
odata[i] = x ^ m odata[i] = x ^ m
i += 1 i += 1
return odata return odata

49
utage/names.py
View file

@ -8,6 +8,7 @@ from functools import partial
from glob import glob from glob import glob
from multiprocessing import Pool from multiprocessing import Pool
def update_names(dir_in, old_path, languages): def update_names(dir_in, old_path, languages):
names = extract_names(dir_in) names = extract_names(dir_in)
@ -25,8 +26,13 @@ def update_names(dir_in, old_path, languages):
lang_dict = json.load(lang_file) lang_dict = json.load(lang_file)
out_dict.update(lang_dict) out_dict.update(lang_dict)
with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right with io.open(
json.dump(out_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True) langfile, "w", newline="\n"
) as lang_file: # you're using git right
json.dump(
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True
)
def extract_names(dir_in): def extract_names(dir_in):
char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv") char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv")
@ -36,7 +42,11 @@ def extract_names(dir_in):
names, sets = read_char_tsv(char_tsv_path) names, sets = read_char_tsv(char_tsv_path)
files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)] files = [
f
for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
if re.search(r"/[0-9]{9}\.tsv$", f)
]
if len(files) == 0: if len(files) == 0:
raise FileNotFoundError("No valid files found in directory: " + dir_in) raise FileNotFoundError("No valid files found in directory: " + dir_in)
@ -50,32 +60,43 @@ def extract_names(dir_in):
return names return names
def read_char_tsv(char_tsv_path): def read_char_tsv(char_tsv_path):
char_names = set() char_names = set()
char_sets = set() char_sets = set()
with open(char_tsv_path, "r") as char_tsv_file: with open(char_tsv_path, "r") as char_tsv_file:
char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar="\"") char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"')
for row in char_tsv: for row in char_tsv:
if row['CharacterName'].startswith("//"): if row["CharacterName"].startswith("//"):
continue continue
if row['CharacterName'] and row['NameText'] and row['CharacterName'].strip() and row['NameText'].strip(): if (
char_names.add(row['NameText']) row["CharacterName"]
char_sets.add(row['CharacterName']) and row["NameText"]
and row["CharacterName"].strip()
and row["NameText"].strip()
):
char_names.add(row["NameText"])
char_sets.add(row["CharacterName"])
return char_names, char_sets return char_names, char_sets
def read_mission(tsv_path, char_names, char_sets): def read_mission(tsv_path, char_names, char_sets):
new_names = set() new_names = set()
with open(tsv_path, "r") as tsv_file: with open(tsv_path, "r") as tsv_file:
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"") tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
if ('Arg1' not in tsv.fieldnames) or ('Text' not in tsv.fieldnames) or ('Command' not in tsv.fieldnames): if (
("Arg1" not in tsv.fieldnames)
or ("Text" not in tsv.fieldnames)
or ("Command" not in tsv.fieldnames)
):
return new_names return new_names
for row in tsv: for row in tsv:
if row['Command'] and row['Command'].startswith("//"): if row["Command"] and row["Command"].startswith("//"):
continue continue
if row['Text'] and row['Arg1']: if row["Text"] and row["Arg1"]:
if (row['Arg1'] not in char_names) and (row['Arg1'] not in char_sets): if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets):
new_names.add(row['Arg1']) new_names.add(row["Arg1"])
return new_names return new_names

44
utage/translate.py
View file

@ -10,9 +10,14 @@ from functools import partial
from glob import glob from glob import glob
from multiprocessing.pool import Pool from multiprocessing.pool import Pool
def translate_dir(dir_in, tsv_out_dir, json_out_dir): def translate_dir(dir_in, tsv_out_dir, json_out_dir):
# we only want to key files that are mission ids # we only want to key files that are mission ids
files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)] files = [
f
for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
if re.search(r"/[0-9]{9}\.tsv$", f)
]
if len(files) == 0: if len(files) == 0:
raise FileNotFoundError("No valid files found in directory: " + dir_in) raise FileNotFoundError("No valid files found in directory: " + dir_in)
@ -21,6 +26,7 @@ def translate_dir(dir_in, tsv_out_dir, json_out_dir):
with Pool() as p: with Pool() as p:
p.map(ptrans, files) p.map(ptrans, files)
def translate_file(file_in, tsv_out_dir, json_out_dir): def translate_file(file_in, tsv_out_dir, json_out_dir):
try: try:
if not file_in.endswith(".tsv"): if not file_in.endswith(".tsv"):
@ -37,8 +43,12 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
id_num = os.path.splitext(os.path.basename(file_in))[0] id_num = os.path.splitext(os.path.basename(file_in))[0]
tsv_output_path = os.path.join(tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num)) tsv_output_path = os.path.join(
json_output_path = os.path.join(json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num)) tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num)
)
json_output_path = os.path.join(
json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num)
)
# need to create output paths and avoid races when threading # need to create output paths and avoid races when threading
if not os.path.exists(os.path.dirname(tsv_output_path)): if not os.path.exists(os.path.dirname(tsv_output_path)):
@ -56,24 +66,34 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
raise raise
with open(file_in, "r") as tsv_file: with open(file_in, "r") as tsv_file:
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"") tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
tsv_keyed, json_str = process_tsv(tsv, id_num) tsv_keyed, json_str = process_tsv(tsv, id_num)
# csv handles newlines, don't set it in io.open # csv handles newlines, don't set it in io.open
with io.open(tsv_output_path, "w", newline='') as tsv_out: with io.open(tsv_output_path, "w", newline="") as tsv_out:
writer = csv.DictWriter(tsv_out, delimiter="\t", quotechar="\"", lineterminator="\n", fieldnames=tsv.fieldnames, extrasaction='ignore') writer = csv.DictWriter(
tsv_out,
delimiter="\t",
quotechar='"',
lineterminator="\n",
fieldnames=tsv.fieldnames,
extrasaction="ignore",
)
writer.writeheader() writer.writeheader()
for row in tsv_keyed: for row in tsv_keyed:
writer.writerow(row) writer.writerow(row)
with io.open(json_output_path, "w", newline='\n') as json_out: with io.open(json_output_path, "w", newline="\n") as json_out:
# we don't want to sort these because they're _1, ..., _10, etc # we don't want to sort these because they're _1, ..., _10, etc
json.dump(json_str, json_out, ensure_ascii=False, indent='\t', sort_keys=False) json.dump(
json_str, json_out, ensure_ascii=False, indent="\t", sort_keys=False
)
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
print("Error processing file: " + file_in) print("Error processing file: " + file_in)
def process_tsv(tsv, id_num): def process_tsv(tsv, id_num):
i = 0 i = 0
tsv_keyed = [] tsv_keyed = []
@ -81,14 +101,14 @@ def process_tsv(tsv, id_num):
for row in tsv: for row in tsv:
try: try:
if row['Command'].startswith("//"): if row["Command"].startswith("//"):
tsv_keyed.append(row) tsv_keyed.append(row)
continue continue
if row['Text'] and row['Text'].strip(): if row["Text"] and row["Text"].strip():
key = "{}_{}".format(id_num, i) key = "{}_{}".format(id_num, i)
i += 1 i += 1
row['English'] = key row["English"] = key
key_dict[key] = row['Text'] key_dict[key] = row["Text"]
tsv_keyed.append(row) tsv_keyed.append(row)
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()