xdu/divatool
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

BLACKED

Browse source
This commit is contained in:
louis 2019-11-12 10:16:29 -05:00
parent 03e51318b0
commit b6c3c6a454
10 changed files with 819 additions and 654 deletions
Download patch file Download diff file Expand all files Collapse all files

69
adx/adx.py
View file

@ -5,43 +5,46 @@ import traceback
from .adx_parser import adx_file from .adx_parser import adx_file
def extract_loop_data_from_dir(dir_in, fout): def extract_loop_data_from_dir(dir_in, fout):
if fout is None: if fout is None:
fout = "BgmLoop.json" fout = "BgmLoop.json"
files = glob.glob(os.path.join(dir_in, "**/*.adx"), recursive=True) files = glob.glob(os.path.join(dir_in, "**/*.adx"), recursive=True)
if len(files) == 0: if len(files) == 0:
raise FileNotFoundError("No valid files found in directory: " + dir_in) raise FileNotFoundError("No valid files found in directory: " + dir_in)
extract_loop_data_from_files(files, fout) extract_loop_data_from_files(files, fout)
def extract_loop_data_from_files(files_in, fout): def extract_loop_data_from_files(files_in, fout):
if not files_in: if not files_in:
raise FileNotFoundError("No input files found") raise FileNotFoundError("No input files found")
collect = {} collect = {}
for x in files_in: for x in files_in:
res = extract_loop_data_from_file(x) res = extract_loop_data_from_file(x)
if res is not None: if res is not None:
collect[os.path.splitext(os.path.basename(x))[0]] = res collect[os.path.splitext(os.path.basename(x))[0]] = res
with open(fout, 'w') as out: with open(fout, "w") as out:
json.dump(collect, out, ensure_ascii=False, indent="\t", sort_keys=True) json.dump(collect, out, ensure_ascii=False, indent="\t", sort_keys=True)
def extract_loop_data_from_file(file_in, fout=None): def extract_loop_data_from_file(file_in, fout=None):
if not file_in.endswith(".adx"): if not file_in.endswith(".adx"):
raise ValueError("Input file not .adx") raise ValueError("Input file not .adx")
try: try:
with open(file_in, 'rb') as x: with open(file_in, "rb") as x:
data = x.read(64) data = x.read(64)
ADX = adx_file(data) ADX = adx_file(data)
if ADX.l_exists and ADX.is_valid: if ADX.l_exists and ADX.is_valid:
res = ADX.parse_loop_data() res = ADX.parse_loop_data()
else: else:
res = None res = None
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
print("Error processing file: " + file_in) print("Error processing file: " + file_in)
return None return None
if fout is not None and res is not None: if fout is not None and res is not None:
with open(fout, 'w') as out: with open(fout, "w") as out:
json.dump(res, out, ensure_ascii=False, indent="\t", sort_keys=True) json.dump(res, out, ensure_ascii=False, indent="\t", sort_keys=True)
return res return res

156
adx/adx_parser.py
View file

@ -1,4 +1,4 @@
from math import modf #fuck me rounding is hard from math import modf # fuck me rounding is hard
# header map # header map
# https://wiki.multimedia.cx/index.php/CRI_ADX_file # https://wiki.multimedia.cx/index.php/CRI_ADX_file
@ -13,16 +13,16 @@ DATA_OFF_MIN = 0x38
FORMAT_OFF = 0x04 FORMAT_OFF = 0x04
FORMAT_LEN = 1 FORMAT_LEN = 1
FORMAT = 3 # always 3 for adx apparently FORMAT = 3 # always 3 for adx apparently
LOOP_TYPE_OFF = 0x12 LOOP_TYPE_OFF = 0x12
LOOP_TYPE_LEN = 1 LOOP_TYPE_LEN = 1
LOOP_TYPE_SUP = [4] # supported loop styles LOOP_TYPE_SUP = [4] # supported loop styles
SRATE_OFF = 0x08 SRATE_OFF = 0x08
SRATE_LEN = 4 SRATE_LEN = 4
SCOUNT_OFF = 0x0c SCOUNT_OFF = 0x0C
SCOUNT_LEN = 4 SCOUNT_LEN = 4
CRYPT_OFF = 0x13 CRYPT_OFF = 0x13
@ -38,82 +38,90 @@ LOOP4_END_OFF = 0x30
LOOP4_END_LEN = 4 LOOP4_END_LEN = 4
#class independent functions # class independent functions
def dumb_round(number): def dumb_round(number):
dec = modf(number)[0] dec = modf(number)[0]
if (dec <= 0.5): if dec <= 0.5:
return int(number) return int(number)
else: else:
return int(number+1) return int(number + 1)
class adx_file(object): class adx_file(object):
magic = 0 magic = 0
form = 0 form = 0
l_style = 0 l_style = 0
encrypted = 0 encrypted = 0
sam_rate = 0 sam_rate = 0
sam_count = 0 sam_count = 0
l_flag = 0 l_flag = 0
l_start = 0 l_start = 0
l_end = 0 l_end = 0
l_exists = 0 l_exists = 0
d_offset = 0 d_offset = 0
is_valid = 0 is_valid = 0
data = None data = None
def __init__(self, data): def __init__(self, data):
# first, validate # first, validate
self.data = data self.data = data
self.magic = self.get_val(MAGIC_OFF, MAGIC_LEN) self.magic = self.get_val(MAGIC_OFF, MAGIC_LEN)
self.form = self.get_val(FORMAT_OFF, FORMAT_LEN) self.form = self.get_val(FORMAT_OFF, FORMAT_LEN)
self.l_style = self.get_val(LOOP_TYPE_OFF, LOOP_TYPE_LEN) self.l_style = self.get_val(LOOP_TYPE_OFF, LOOP_TYPE_LEN)
self.encrypted = self.get_val(CRYPT_OFF, CRYPT_LEN) self.encrypted = self.get_val(CRYPT_OFF, CRYPT_LEN)
self.d_offset = self.get_val(DATA_OFF_OFF, DATA_OFF_LEN) self.d_offset = self.get_val(DATA_OFF_OFF, DATA_OFF_LEN)
if self.d_offset < DATA_OFF_MIN: if self.d_offset < DATA_OFF_MIN:
return None return None
self.l_exists = 1 self.l_exists = 1
# now load things for math # now load things for math
self.sam_rate = self.get_val(SRATE_OFF, SRATE_LEN) self.sam_rate = self.get_val(SRATE_OFF, SRATE_LEN)
self.sam_count = self.get_val(SCOUNT_OFF, SCOUNT_LEN) self.sam_count = self.get_val(SCOUNT_OFF, SCOUNT_LEN)
self.validate() self.validate()
def get_val(self, offset, length): def get_val(self, offset, length):
return int.from_bytes(self.data[offset:offset+length], byteorder='big', signed=False) return int.from_bytes(
self.data[offset : offset + length], byteorder="big", signed=False
)
def validate(self): def validate(self):
if self.magic != MAGIC: if self.magic != MAGIC:
raise ValueError("Invalid ADX File") raise ValueError("Invalid ADX File")
if self.form != FORMAT: if self.form != FORMAT:
raise ValueError("Invalid ADX File") raise ValueError("Invalid ADX File")
if self.encrypted: if self.encrypted:
raise NotImplementedError("Encryption Not Supported") raise NotImplementedError("Encryption Not Supported")
self.is_valid = 1 self.is_valid = 1
def parse_loop_data(self): def parse_loop_data(self):
if self.l_style == 4: if self.l_style == 4:
self.l_flag = self.get_val(LOOP4_FLAG_OFF, LOOP4_FLAG_LEN) self.l_flag = self.get_val(LOOP4_FLAG_OFF, LOOP4_FLAG_LEN)
self.l_start = self.get_val(LOOP4_START_OFF, LOOP4_START_LEN) self.l_start = self.get_val(LOOP4_START_OFF, LOOP4_START_LEN)
self.l_end = self.get_val(LOOP4_END_OFF, LOOP4_END_LEN) self.l_end = self.get_val(LOOP4_END_OFF, LOOP4_END_LEN)
if not self.l_flag: if not self.l_flag:
return None return None
if self.l_start == 0 and self.l_end == self.sam_count: if self.l_start == 0 and self.l_end == self.sam_count:
return None return None
if self.l_start > self.sam_count or self.l_end > self.sam_count: if self.l_start > self.sam_count or self.l_end > self.sam_count:
raise ValueError("Invalid Loop Data") raise ValueError("Invalid Loop Data")
return None return None
if not self.l_style in LOOP_TYPE_SUP: if not self.l_style in LOOP_TYPE_SUP:
raise NotImplementedError("Loop Style {} Not Implemented".format(str(self.l_style))) raise NotImplementedError(
return None "Loop Style {} Not Implemented".format(str(self.l_style))
ret = {} )
ret['duration'] = self.sam_count / self.sam_rate return None
ret['loop_start'] = {} ret = {}
ret['loop_start']['seconds'] = self.l_start / self.sam_rate ret["duration"] = self.sam_count / self.sam_rate
ret['loop_start']['samples_native'] = self.l_start ret["loop_start"] = {}
ret['loop_start']['samples_48k'] = dumb_round(self.l_start / self.sam_rate * 48000) ret["loop_start"]["seconds"] = self.l_start / self.sam_rate
ret['loop_end'] = {} ret["loop_start"]["samples_native"] = self.l_start
ret['loop_end']['seconds'] = self.l_end / self.sam_rate ret["loop_start"]["samples_48k"] = dumb_round(
ret['loop_end']['samples_native'] = self.l_end self.l_start / self.sam_rate * 48000
ret['loop_end']['samples_48k'] = dumb_round(self.l_end / self.sam_rate * 48000) )
return ret ret["loop_end"] = {}
ret["loop_end"]["seconds"] = self.l_end / self.sam_rate
ret["loop_end"]["samples_native"] = self.l_end
ret["loop_end"]["samples_48k"] = dumb_round(self.l_end / self.sam_rate * 48000)
return ret

27
db/base.py
View file

@ -7,22 +7,23 @@ from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base() Base = declarative_base()
engine = create_engine('sqlite:///:memory:', echo=False) engine = create_engine("sqlite:///:memory:", echo=False)
session = sessionmaker(bind=engine, autoflush=False, autocommit=False)() session = sessionmaker(bind=engine, autoflush=False, autocommit=False)()
session.flush = lambda:None session.flush = lambda: None
DATABASES = { DATABASES = {
"QuestMst.db": "Quest", "QuestMst.db": "Quest",
"QuestSceneMst.db": "QuestScene", "QuestSceneMst.db": "QuestScene",
"QuestPartMst.db": "QuestPart", "QuestPartMst.db": "QuestPart",
"ResourceEntry.db": "ResourceEntry" "ResourceEntry.db": "ResourceEntry",
} }
def init(path):
for d in DATABASES:
if not os.path.isfile(os.path.join(path, d)):
raise FileNotFoundError("Database {} not found".format(d))
for d in DATABASES: def init(path):
t = text("attach database :path as :schema") for d in DATABASES:
engine.execute(t, path=os.path.join(path, d), schema=DATABASES[d]) if not os.path.isfile(os.path.join(path, d)):
raise FileNotFoundError("Database {} not found".format(d))
for d in DATABASES:
t = text("attach database :path as :schema")
engine.execute(t, path=os.path.join(path, d), schema=DATABASES[d])

111
db/quest.py
View file

@ -3,65 +3,74 @@ from sqlalchemy.orm import relationship
from .base import Base from .base import Base
class Quest(Base): class Quest(Base):
__tablename__ = 'QuestMstRecord' __tablename__ = "QuestMstRecord"
__table_args__ = {'schema': 'Quest'} __table_args__ = {"schema": "Quest"}
questMstId = Column(Integer, primary_key=True) questMstId = Column(Integer, primary_key=True)
questType = Column(Integer) questType = Column(Integer)
sortNum = Column(Integer) sortNum = Column(Integer)
name = Column(String) name = Column(String)
prevQuestMstId = Column(Integer) prevQuestMstId = Column(Integer)
eventItemMstIds = Column(Integer) eventItemMstIds = Column(Integer)
valid = Column(Integer) valid = Column(Integer)
baseQuestMstId = Column(Integer) baseQuestMstId = Column(Integer)
updatedTime = Column(BigInteger) updatedTime = Column(BigInteger)
scenes = relationship("QuestScene") scenes = relationship("QuestScene")
def __repr__(self):
return "<Quest(questMstId={}, name={})>".format(self.questMstId, self.name)
def __repr__(self):
return "<Quest(questMstId={}, name={})>".format(self.questMstId, self.name)
class QuestScene(Base): class QuestScene(Base):
__tablename__ = 'QuestSceneMstRecord' __tablename__ = "QuestSceneMstRecord"
__table_args__ = {'schema': 'QuestScene'} __table_args__ = {"schema": "QuestScene"}
questSceneMstId = Column(Integer, primary_key=True) questSceneMstId = Column(Integer, primary_key=True)
questMstId = Column(Integer, ForeignKey("Quest.QuestMstRecord.questMstId")) questMstId = Column(Integer, ForeignKey("Quest.QuestMstRecord.questMstId"))
_type = Column("type", Integer) _type = Column("type", Integer)
sortNum = Column(Integer) sortNum = Column(Integer)
name = Column(String) name = Column(String)
_filter = Column("filter", Integer) _filter = Column("filter", Integer)
prevQuestSceneMstId = Column(Integer, ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId")) prevQuestSceneMstId = Column(
releaseSerial = Column(Integer) Integer, ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId")
releaseEvolutionLevel = Column(Integer) )
summaryText = Column(String) releaseSerial = Column(Integer)
presentType = Column(String) releaseEvolutionLevel = Column(Integer)
objectId = Column(Integer) summaryText = Column(String)
num = Column(Integer) presentType = Column(String)
appearanceType = Column(Integer) objectId = Column(Integer)
isPrologue = Column(Integer) num = Column(Integer)
isEpilogue = Column(Integer) appearanceType = Column(Integer)
viewType = Column(Integer) isPrologue = Column(Integer)
updatedTime = Column(BigInteger) isEpilogue = Column(Integer)
viewType = Column(Integer)
updatedTime = Column(BigInteger)
previous = relationship("QuestScene", remote_side=[questSceneMstId])
parts = relationship("QuestPart")
previous = relationship("QuestScene", remote_side=[questSceneMstId])
parts = relationship("QuestPart")
class QuestPart(Base): class QuestPart(Base):
__tablename__ = 'QuestPartMstRecord' __tablename__ = "QuestPartMstRecord"
__table_args__ = {'schema': 'QuestPart'} __table_args__ = {"schema": "QuestPart"}
questSceneMstId = Column(Integer, ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId"), primary_key=True) # now this is pod racing questSceneMstId = Column(
partNum = Column(Integer, primary_key=True) Integer,
waveNum = Column(Integer) ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId"),
stamina = Column(Integer) primary_key=True,
exp = Column(Integer) ) # now this is pod racing
expertPoint = Column(Integer) partNum = Column(Integer, primary_key=True)
recommendLevel = Column(Integer) waveNum = Column(Integer)
beforeTalkName = Column(String) stamina = Column(Integer)
afterTalkName = Column(String) exp = Column(Integer)
battleBackgroundImg = Column(String) expertPoint = Column(Integer)
musicMstId = Column(Integer) recommendLevel = Column(Integer)
isFixedDeck = Column(Integer) beforeTalkName = Column(String)
updatedTime = Column(BigInteger) afterTalkName = Column(String)
battleBackgroundImg = Column(String)
musicMstId = Column(Integer)
isFixedDeck = Column(Integer)
updatedTime = Column(BigInteger)

17
db/resource.py
View file

@ -2,12 +2,13 @@ from sqlalchemy import Column, BigInteger, String
from .base import Base from .base import Base
class ResourceEntry(Base):
__tablename__ = 'ResourceEntryRecord'
__table_args__ = {'schema': 'ResourceEntry'}
path = Column(String, primary_key=True) class ResourceEntry(Base):
serverPath = Column(String) __tablename__ = "ResourceEntryRecord"
localPath = Column(String) __table_args__ = {"schema": "ResourceEntry"}
digest = Column(String)
fileSize = Column(BigInteger) path = Column(String, primary_key=True)
serverPath = Column(String)
localPath = Column(String)
digest = Column(String)
fileSize = Column(BigInteger)

221
diva/quest.py
View file

@ -5,110 +5,151 @@ import os
from sqlalchemy.orm import contains_eager from sqlalchemy.orm import contains_eager
def update_missions(dir_in, old_path, languages): def update_missions(dir_in, old_path, languages):
db.init(dir_in) db.init(dir_in)
quest_mst(old_path, languages) quest_mst(old_path, languages)
scene_mst(old_path, languages) scene_mst(old_path, languages)
def scene_mst(old_path, languages): def scene_mst(old_path, languages):
result = (db.session.query(db.QuestScene) result = (
.join(db.QuestScene.parts) db.session.query(db.QuestScene)
.options(contains_eager(db.QuestScene.parts)) .join(db.QuestScene.parts)
.filter((db.QuestPart.beforeTalkName != "") | (db.QuestPart.afterTalkName != "")) .options(contains_eager(db.QuestScene.parts))
.filter(db.QuestScene.parts.any()) .filter(
.all()) (db.QuestPart.beforeTalkName != "") | (db.QuestPart.afterTalkName != "")
)
.filter(db.QuestScene.parts.any())
.all()
)
scenes = {} scenes = {}
for x in result: for x in result:
scenes[str(x.questSceneMstId)] = { "Name": x.name, scenes[str(x.questSceneMstId)] = {
"SummaryText": x.summaryText, "Name": x.name,
"Parts": [], "SummaryText": x.summaryText,
"Folder": "" } "Parts": [],
"Folder": "",
}
id_num = "" id_num = ""
# this is slow as balls but i'm way too tired to figure out how to optimize it # this is slow as balls but i'm way too tired to figure out how to optimize it
for y in x.parts: for y in x.parts:
if y.beforeTalkName: if y.beforeTalkName:
id_num = y.beforeTalkName id_num = y.beforeTalkName
scenes[str(x.questSceneMstId)]["Parts"].append(y.beforeTalkName) scenes[str(x.questSceneMstId)]["Parts"].append(y.beforeTalkName)
if y.afterTalkName: if y.afterTalkName:
id_num = y.afterTalkName id_num = y.afterTalkName
scenes[str(x.questSceneMstId)]["Parts"].append(y.afterTalkName) scenes[str(x.questSceneMstId)]["Parts"].append(y.afterTalkName)
# i checked every entry, moving it to do once for speed # i checked every entry, moving it to do once for speed
if not scenes[str(x.questSceneMstId)]["Folder"] and id_num: if not scenes[str(x.questSceneMstId)]["Folder"] and id_num:
path = (db.session.query(db.ResourceEntry) path = (
.filter(db.ResourceEntry.path.like("%/Scenario/{}.tsv.utage".format(id_num))) db.session.query(db.ResourceEntry)
.filter(db.ResourceEntry.path.notlike("%Utage/side03/%.tsv.utage")) # dead folder with dupes too lazy to parse settings tsv .filter(
.one()) db.ResourceEntry.path.like(
folder = path.path.split("/")[2] "%/Scenario/{}.tsv.utage".format(id_num)
if scenes[str(x.questSceneMstId)]["Folder"] and scenes[str(x.questSceneMstId)]["Folder"] != folder: )
raise KeyError("Internal path inconsitency, bailing out {}".format(id_num)) )
scenes[str(x.questSceneMstId)]["Folder"] = folder .filter(
db.ResourceEntry.path.notlike("%Utage/side03/%.tsv.utage")
) # dead folder with dupes too lazy to parse settings tsv
.one()
)
folder = path.path.split("/")[2]
if (
scenes[str(x.questSceneMstId)]["Folder"]
and scenes[str(x.questSceneMstId)]["Folder"] != folder
):
raise KeyError(
"Internal path inconsitency, bailing out {}".format(id_num)
)
scenes[str(x.questSceneMstId)]["Folder"] = folder
with io.open(os.path.join(old_path, "XduScene.json"), "w", newline='\n') as json_file: with io.open(
json.dump(scenes, json_file, ensure_ascii=False, indent='\t', sort_keys=True) os.path.join(old_path, "XduScene.json"), "w", newline="\n"
) as json_file:
json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
for l in languages: for l in languages:
out_dict = {} out_dict = {}
if l == "jpn": if l == "jpn":
for x in result: for x in result:
out_dict[str(x.questSceneMstId)] = { "Name": x.name, out_dict[str(x.questSceneMstId)] = {
"SummaryText": x.summaryText, "Name": x.name,
"Credits": "POKELABO", "SummaryText": x.summaryText,
"Enabled": False } "Credits": "POKELABO",
else: "Enabled": False,
for x in result: }
out_dict[str(x.questSceneMstId)] = { "Name": "", else:
"SummaryText": "", for x in result:
"Credits": "", out_dict[str(x.questSceneMstId)] = {
"Enabled": False } "Name": "",
"SummaryText": "",
"Credits": "",
"Enabled": False,
}
langfile = os.path.join(old_path, "XduSceneNames_{}.json".format(l)) langfile = os.path.join(old_path, "XduSceneNames_{}.json".format(l))
if os.path.isfile(langfile): if os.path.isfile(langfile):
with open(langfile, "r") as lang_file: with open(langfile, "r") as lang_file:
lang_dict = json.load(lang_file) lang_dict = json.load(lang_file)
out_dict.update(lang_dict) out_dict.update(lang_dict)
with io.open(
langfile, "w", newline="\n"
) as lang_file: # you're using git right
json.dump(
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True
)
with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right
json.dump(out_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True)
def quest_mst(old_path, languages): def quest_mst(old_path, languages):
# i regret my life choices # i regret my life choices
result = (db.session.query(db.Quest) result = (
.filter(db.Quest.baseQuestMstId == 0) db.session.query(db.Quest)
.join(db.Quest.scenes) .filter(db.Quest.baseQuestMstId == 0)
.join(db.QuestScene.parts) .join(db.Quest.scenes)
.options(contains_eager(db.Quest.scenes). .join(db.QuestScene.parts)
contains_eager(db.QuestScene.parts)) .options(contains_eager(db.Quest.scenes).contains_eager(db.QuestScene.parts))
.filter(db.QuestPart.beforeTalkName != "" or db.QuestPart.afterTalkName != "") .filter(db.QuestPart.beforeTalkName != "" or db.QuestPart.afterTalkName != "")
.filter(db.QuestScene.parts.any()) .filter(db.QuestScene.parts.any())
.filter(db.Quest.scenes.any()) .filter(db.Quest.scenes.any())
.all()) .all()
quests = {} )
for x in result: quests = {}
quests[str(x.questMstId)] = { "Name": x.name, "Scenes": [d.questSceneMstId for d in x.scenes] } for x in result:
quests[str(x.questMstId)] = {
"Name": x.name,
"Scenes": [d.questSceneMstId for d in x.scenes],
}
with io.open(os.path.join(old_path, "XduQuest.json"), "w", newline='\n') as json_file: with io.open(
json.dump(quests, json_file, ensure_ascii=False, indent='\t', sort_keys=True) os.path.join(old_path, "XduQuest.json"), "w", newline="\n"
) as json_file:
json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
for l in languages: for l in languages:
out_dict = {} out_dict = {}
if l == "jpn": if l == "jpn":
for x in result: for x in result:
out_dict[str(x.questMstId)] = { "Name": x.name, "Enabled": False } out_dict[str(x.questMstId)] = {"Name": x.name, "Enabled": False}
else: else:
for x in result: for x in result:
out_dict[str(x.questMstId)] = { "Name": "", "Enabled": False } out_dict[str(x.questMstId)] = {"Name": "", "Enabled": False}
langfile = os.path.join(old_path, "XduQuestNames_{}.json".format(l)) langfile = os.path.join(old_path, "XduQuestNames_{}.json".format(l))
if os.path.isfile(langfile): if os.path.isfile(langfile):
with open(langfile, "r") as lang_file: with open(langfile, "r") as lang_file:
lang_dict = json.load(lang_file) lang_dict = json.load(lang_file)
out_dict.update(lang_dict) out_dict.update(lang_dict)
with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right with io.open(
json.dump(out_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True) langfile, "w", newline="\n"
) as lang_file: # you're using git right
json.dump(
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True
)
return quests
return quests

224
divatool.py
View file

@ -10,101 +10,147 @@ import db
LANGUAGES = ["jpn", "eng", "rus"] LANGUAGES = ["jpn", "eng", "rus"]
def main(): def main():
parser = argparse.ArgumentParser(description="Unpack and process XDU data") parser = argparse.ArgumentParser(description="Unpack and process XDU data")
subparsers = parser.add_subparsers(metavar="<command>", title="subcommands", dest="command") subparsers = parser.add_subparsers(
metavar="<command>", title="subcommands", dest="command"
)
# adx subcommand # adx subcommand
parser_adx = subparsers.add_parser("adx", help="ADX stuff") parser_adx = subparsers.add_parser("adx", help="ADX stuff")
parser_adx_subparsers = parser_adx.add_subparsers(metavar="<command>", title="subcommands", dest="subcommand") parser_adx_subparsers = parser_adx.add_subparsers(
# adx loop metavar="<command>", title="subcommands", dest="subcommand"
parser_adx_loop = parser_adx_subparsers.add_parser("loop", help="Extract loop data") )
parser_adx_loop.add_argument("--output", "-o", # adx loop
help="JSON output file for loop data", parser_adx_loop = parser_adx_subparsers.add_parser("loop", help="Extract loop data")
type=str, dest="JSON_OUT") parser_adx_loop.add_argument(
parser_adx_loop.add_argument("ADX_DIR", "--output",
help="Input directory containing ADX files", "-o",
type=str) help="JSON output file for loop data",
type=str,
dest="JSON_OUT",
)
parser_adx_loop.add_argument(
"ADX_DIR", help="Input directory containing ADX files", type=str
)
# utage subcommand # utage subcommand
parser_utage = subparsers.add_parser("utage", help="UTAGE stuff") parser_utage = subparsers.add_parser("utage", help="UTAGE stuff")
parser_utage_subparsers = parser_utage.add_subparsers(metavar="<command>", title="subcommand", dest="subcommand") parser_utage_subparsers = parser_utage.add_subparsers(
# utage crypt metavar="<command>", title="subcommand", dest="subcommand"
parser_utage_crypt = parser_utage_subparsers.add_parser("crypt", help="Encrypt/decrypt utage tsv files") )
parser_utage_crypt.add_argument("--encrypt", "-e", # utage crypt
help="Encrypt (default: Decrypt)", parser_utage_crypt = parser_utage_subparsers.add_parser(
dest="encrypt", action="store_true", default=False) "crypt", help="Encrypt/decrypt utage tsv files"
parser_utage_crypt.add_argument("--no-compression", "-n", )
help="Do not compress/decompress. Only applies to tsv, png will never be compressed", parser_utage_crypt.add_argument(
dest="ncomp", action="store_true", default=False) "--encrypt",
parser_utage_crypt.add_argument("--key", "-k", "-e",
help="Encryption key (Default: SampleSecretKey)", help="Encrypt (default: Decrypt)",
dest="key", type=str, default="SampleSecretKey") dest="encrypt",
parser_utage_crypt.add_argument("--hex", "-x", action="store_true",
help="KEY is hexadecimal (Default: False)", default=False,
dest="hex", action="store_true", default=False) )
parser_utage_crypt.add_argument("TARGET", parser_utage_crypt.add_argument(
help="Input", type=str) "--no-compression",
# utage translate "-n",
parser_utage_translate = parser_utage_subparsers.add_parser("translate", help="Generate keyed tsv and json files") help="Do not compress/decompress. Only applies to tsv, png will never be compressed",
parser_utage_translate.add_argument("INPUT", dest="ncomp",
help="Input", type=str) action="store_true",
parser_utage_translate.add_argument("TSVDIR", nargs='?', default=False,
help="_t.tsv output directory", type=str, default=".") )
parser_utage_translate.add_argument("JSONDIR", nargs='?', parser_utage_crypt.add_argument(
help="json output directory", type=str, default=".") "--key",
# utage names "-k",
parser_utage_names = parser_utage_subparsers.add_parser("names", help="Generate and update name files") help="Encryption key (Default: SampleSecretKey)",
parser_utage_names.add_argument("INPUT", dest="key",
help="Input", type=str) type=str,
parser_utage_names.add_argument("OLD", nargs='?', default="SampleSecretKey",
help="Directory with old files", type=str, default=".") )
parser_utage_crypt.add_argument(
"--hex",
"-x",
help="KEY is hexadecimal (Default: False)",
dest="hex",
action="store_true",
default=False,
)
parser_utage_crypt.add_argument("TARGET", help="Input", type=str)
# utage translate
parser_utage_translate = parser_utage_subparsers.add_parser(
"translate", help="Generate keyed tsv and json files"
)
parser_utage_translate.add_argument("INPUT", help="Input", type=str)
parser_utage_translate.add_argument(
"TSVDIR", nargs="?", help="_t.tsv output directory", type=str, default="."
)
parser_utage_translate.add_argument(
"JSONDIR", nargs="?", help="json output directory", type=str, default="."
)
# utage names
parser_utage_names = parser_utage_subparsers.add_parser(
"names", help="Generate and update name files"
)
parser_utage_names.add_argument("INPUT", help="Input", type=str)
parser_utage_names.add_argument(
"OLD", nargs="?", help="Directory with old files", type=str, default="."
)
# diva subcommand # diva subcommand
parser_diva = subparsers.add_parser("diva", help="Diva stuff") parser_diva = subparsers.add_parser("diva", help="Diva stuff")
parser_diva_subparsers = parser_diva.add_subparsers(metavar="<command>", title="subcommand", dest="subcommand") parser_diva_subparsers = parser_diva.add_subparsers(
parser_diva_quest = parser_diva_subparsers.add_parser("quest", help="Generate quest JSON files") metavar="<command>", title="subcommand", dest="subcommand"
parser_diva_quest.add_argument("INPUT", help="Directory of databases containing Quest*.db and ResourceEntry.db", )
type=str) parser_diva_quest = parser_diva_subparsers.add_parser(
parser_diva_quest.add_argument("OLD", help="Directory with old quest files", "quest", help="Generate quest JSON files"
nargs='?', type=str, default=".") )
parser_diva_quest.add_argument(
"INPUT",
help="Directory of databases containing Quest*.db and ResourceEntry.db",
type=str,
)
parser_diva_quest.add_argument(
"OLD", help="Directory with old quest files", nargs="?", type=str, default="."
)
args = parser.parse_args() args = parser.parse_args()
if args.command == "adx": if args.command == "adx":
if args.subcommand == "loop": if args.subcommand == "loop":
adx.extract_loop_data_from_dir(args.ADX_DIR, args.JSON_OUT) adx.extract_loop_data_from_dir(args.ADX_DIR, args.JSON_OUT)
elif args.command == "utage": elif args.command == "utage":
if args.subcommand == "crypt": if args.subcommand == "crypt":
if not args.hex: if not args.hex:
key = bytearray(args.key, "utf-8") key = bytearray(args.key, "utf-8")
else: else:
key = bytearray.fromhex(args.key) key = bytearray.fromhex(args.key)
if os.path.isfile(args.TARGET): if os.path.isfile(args.TARGET):
utage.crypt.crypt_file(args.TARGET, key, args.encrypt, args.ncomp) utage.crypt.crypt_file(args.TARGET, key, args.encrypt, args.ncomp)
elif os.path.isdir(args.TARGET): elif os.path.isdir(args.TARGET):
utage.crypt.crypt_dir(args.TARGET, key, args.encrypt, args.ncomp) utage.crypt.crypt_dir(args.TARGET, key, args.encrypt, args.ncomp)
else: else:
raise FileNotFoundError("Could not find {}".format(args.TARGET)) raise FileNotFoundError("Could not find {}".format(args.TARGET))
elif args.subcommand == "translate": elif args.subcommand == "translate":
if os.path.isfile(args.INPUT): if os.path.isfile(args.INPUT):
utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR) utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR)
elif os.path.isdir(args.INPUT): elif os.path.isdir(args.INPUT):
utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR) utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR)
else: else:
raise FileNotFoundError("Could not find {}".format(args.INPUT)) raise FileNotFoundError("Could not find {}".format(args.INPUT))
elif args.subcommand == "names": elif args.subcommand == "names":
if os.path.isdir(args.INPUT): if os.path.isdir(args.INPUT):
utage.names.update_names(args.INPUT, args.OLD, LANGUAGES) utage.names.update_names(args.INPUT, args.OLD, LANGUAGES)
else: else:
raise FileNotFoundError(args.INPUT) raise FileNotFoundError(args.INPUT)
elif args.command == "diva": elif args.command == "diva":
if args.subcommand == "quest": if args.subcommand == "quest":
diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES)
return 0 return 0
if __name__ == '__main__':
main() if __name__ == "__main__":
sys.exit() main()
sys.exit()

365
utage/crypt.py
View file

@ -9,199 +9,214 @@ from multiprocessing.pool import Pool
from . import crypt from . import crypt
from . import translate from . import translate
def crypt_dir(dir_in, key, encrypt=False, no_compress=False): def crypt_dir(dir_in, key, encrypt=False, no_compress=False):
if not encrypt: if not encrypt:
files = glob(os.path.join(dir_in, "**/*.utage"), recursive=True) files = glob(os.path.join(dir_in, "**/*.utage"), recursive=True)
else: else:
files = glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) files = glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
if len(files) == 0: if len(files) == 0:
raise FileNotFoundError("No valid files found in directory: " + dir_in) raise FileNotFoundError("No valid files found in directory: " + dir_in)
pcrypt = partial(crypt_file, key=key, encrypt=encrypt, no_compress=no_compress)
with Pool() as p:
p.map(pcrypt, files)
pcrypt = partial(crypt_file, key=key, encrypt=encrypt, no_compress=no_compress)
with Pool() as p:
p.map(pcrypt, files)
def crypt_file(file_in, key, encrypt=False, no_compress=False): def crypt_file(file_in, key, encrypt=False, no_compress=False):
with open(file_in, "rb") as inf: with open(file_in, "rb") as inf:
in_data = inf.read() in_data = inf.read()
try: try:
if encrypt: if encrypt:
if not file_in.endswith(".png") and not file_in.endswith(".jpg") and not file_in.endswith(".tsv"): if (
raise ValueError("Invalid File Type for {}".format(file_in)) not file_in.endswith(".png")
if not file_in.endswith(".png") and not no_compress: and not file_in.endswith(".jpg")
enc_data = crypt.compress(in_data) and not file_in.endswith(".tsv")
enc_data = crypt.xor_crypt(enc_data, key) ):
with io.open(file_in + ".utage", "wb") as output: raise ValueError("Invalid File Type for {}".format(file_in))
output.write(enc_data) if not file_in.endswith(".png") and not no_compress:
else: enc_data = crypt.compress(in_data)
if not file_in.endswith(".utage"): enc_data = crypt.xor_crypt(enc_data, key)
raise ValueError("Invalid File Type for {}".format(file_in)) with io.open(file_in + ".utage", "wb") as output:
dec_data = crypt.xor_crypt(in_data, key) output.write(enc_data)
if not file_in.endswith(".png.utage") and not file_in.endswith(".jpg.utage") and not no_compress: else:
dec_data = crypt.decompress(dec_data) if not file_in.endswith(".utage"):
with io.open(file_in.replace(".utage", ""), "wb") as output: raise ValueError("Invalid File Type for {}".format(file_in))
output.write(dec_data) dec_data = crypt.xor_crypt(in_data, key)
except Exception as e: if (
traceback.print_exc() not file_in.endswith(".png.utage")
print("Error processing file: " + file_in) and not file_in.endswith(".jpg.utage")
and not no_compress
):
dec_data = crypt.decompress(dec_data)
with io.open(file_in.replace(".utage", ""), "wb") as output:
output.write(dec_data)
except Exception as e:
traceback.print_exc()
print("Error processing file: " + file_in)
def decompress(data): def decompress(data):
osize = 0 osize = 0
isize = len(data) isize = len(data)
osize_expected = int.from_bytes(data[:4], byteorder='little', signed=False) osize_expected = int.from_bytes(data[:4], byteorder="little", signed=False)
odata = bytearray(osize_expected) odata = bytearray(osize_expected)
odata[:isize] = data odata[:isize] = data
i = 4 i = 4
while i < isize: while i < isize:
if (data[i] & 128) != 0: if (data[i] & 128) != 0:
num3 = data[i] & 15 num3 = data[i] & 15
num3 += 3 num3 += 3
num4 = (data[i] & 112) << 4 | data[i+1] num4 = (data[i] & 112) << 4 | data[i + 1]
num4 += 1 num4 += 1
for j in range(0, num3): for j in range(0, num3):
odata[osize + j] = odata[osize - num4 + j] odata[osize + j] = odata[osize - num4 + j]
i += 1 i += 1
else: else:
num3 = data[i] + 1 num3 = data[i] + 1
for j in range(0, num3): for j in range(0, num3):
odata[osize + j] = data[i + 1 + j] odata[osize + j] = data[i + 1 + j]
i += num3 i += num3
osize += num3 osize += num3
i += 1 i += 1
return odata return odata
def compress(data): def compress(data):
num = len(data) num = len(data)
bytes2 = num.to_bytes(4, byteorder='little') bytes2 = num.to_bytes(4, byteorder="little")
anum3 = num + num/128 + 1 anum3 = num + num / 128 + 1
array = bytearray(int(anum3)) array = bytearray(int(anum3))
num2 = 0 num2 = 0
num3 = 0 num3 = 0
num4 = 0 num4 = 0
index = Index() index = Index()
while num3 < num: while num3 < num:
num5 = 0 num5 = 0
num6 = 0 num6 = 0
num7 = min(18, num - num3) num7 = min(18, num - num3)
num8 = index.getFirst(data[num3]) num8 = index.getFirst(data[num3])
while not index.isEnd(num8): while not index.isEnd(num8):
node = index.getNode(num8) node = index.getNode(num8)
mPos = node.mPos mPos = node.mPos
i = 1 i = 1
while i < num7: while i < num7:
if data[mPos+i] != data[num3+i]: if data[mPos + i] != data[num3 + i]:
break break
i = i+1 i = i + 1
if num5 < i: if num5 < i:
num6 = mPos num6 = mPos
num5 = i num5 = i
if num5 == num7: if num5 == num7:
break break
num8 = node.mNext num8 = node.mNext
if num5 >= 3: if num5 >= 3:
for j in range(num5): for j in range(num5):
num9 = num3 + j - 2048 num9 = num3 + j - 2048
if num9 >= 0: if num9 >= 0:
index.remove(data[num9], num9) index.remove(data[num9], num9)
index.add(data[num3+j], num3+j) index.add(data[num3 + j], num3 + j)
if num4 < num3: if num4 < num3:
array[num2] = (num3 - num4 - 1) array[num2] = num3 - num4 - 1
num2 = num2+1 num2 = num2 + 1
for j in range(num4, num3): for j in range(num4, num3):
array[num2] = data[j] array[num2] = data[j]
num2 = num2+1 num2 = num2 + 1
num10 = num5 - 3 num10 = num5 - 3
num11 = num3 - num6 - 1 num11 = num3 - num6 - 1
num12 = 0x80 | num10 num12 = 0x80 | num10
num12 |= (num11 & 0x700) >> 4 num12 |= (num11 & 0x700) >> 4
array[num2] = num12 array[num2] = num12
array[num2+1] = (num11&0xff) array[num2 + 1] = num11 & 0xFF
num2 = num2+2 num2 = num2 + 2
num3 = num3 + num5 num3 = num3 + num5
num4 = num3 num4 = num3
else: else:
num9 = num3 - 2048 num9 = num3 - 2048
if num9 >= 0: if num9 >= 0:
index.remove(data[num9],num9) index.remove(data[num9], num9)
index.add(data[num3], num3) index.add(data[num3], num3)
num3 = num3+1 num3 = num3 + 1
if num3 - num4 == 128: if num3 - num4 == 128:
array[num2] = num3 - num4 - 1 array[num2] = num3 - num4 - 1
num2 = num2+1 num2 = num2 + 1
for j in range(num4, num3): for j in range(num4, num3):
array[num2] = data[j] array[num2] = data[j]
num2 = num2+1 num2 = num2 + 1
num4 = num3 num4 = num3
if num4 < num3: if num4 < num3:
array[num2] = num3 - num4 - 1 array[num2] = num3 - num4 - 1
num2 = num2+1 num2 = num2 + 1
for j in range(num4, num3): for j in range(num4, num3):
array[num2] = data[j] array[num2] = data[j]
num2 = num2+1 num2 = num2 + 1
osize = num2 osize = num2
array2 = bytearray(osize + 4) array2 = bytearray(osize + 4)
array2[:4] = bytes2[:4] array2[:4] = bytes2[:4]
array2[4:] = array[:osize] array2[4:] = array[:osize]
return array2 return array2
class Node: class Node:
mNext = 0 mNext = 0
mPrev = 0 mPrev = 0
mPos = 0 mPos = 0
class Index: class Index:
mNodes = [] mNodes = []
mStack = [] mStack = []
mStackPos = 0 mStackPos = 0
def __init__(self): def __init__(self):
for i in range(2304): for i in range(2304):
x = Node() x = Node()
self.mNodes.append(x) self.mNodes.append(x)
for i in range(2048, 2304): for i in range(2048, 2304):
self.mNodes[i].mNext = i self.mNodes[i].mNext = i
self.mNodes[i].mPrev = i self.mNodes[i].mPrev = i
for i in range(2048): for i in range(2048):
self.mStack.append(i) self.mStack.append(i)
self.mStackPos = 2048 self.mStackPos = 2048
def getFirst(self, c): def getFirst(self, c):
return self.mNodes[2048+c].mNext return self.mNodes[2048 + c].mNext
def getNode(self, i): def getNode(self, i):
return self.mNodes[i] return self.mNodes[i]
def add(self, c, pos): def add(self, c, pos):
self.mStackPos = self.mStackPos - 1 self.mStackPos = self.mStackPos - 1
num = self.mStack[self.mStackPos] num = self.mStack[self.mStackPos]
node = self.mNodes[num] node = self.mNodes[num]
node2 = self.mNodes[2048+c] node2 = self.mNodes[2048 + c]
node.mNext = node2.mNext node.mNext = node2.mNext
node.mPrev = 2048 + c node.mPrev = 2048 + c
node.mPos = pos node.mPos = pos
self.mNodes[node2.mNext].mPrev = num self.mNodes[node2.mNext].mPrev = num
node2.mNext = num node2.mNext = num
def remove(self, c, pos): def remove(self, c, pos):
mPrev = self.mNodes[2048+c].mPrev mPrev = self.mNodes[2048 + c].mPrev
node = self.mNodes[mPrev] node = self.mNodes[mPrev]
self.mStack[self.mStackPos] = self.mNodes[node.mPrev].mNext self.mStack[self.mStackPos] = self.mNodes[node.mPrev].mNext
self.mStackPos = self.mStackPos + 1 self.mStackPos = self.mStackPos + 1
self.mNodes[node.mPrev].mNext = node.mNext self.mNodes[node.mPrev].mNext = node.mNext
self.mNodes[node.mNext].mPrev = node.mPrev self.mNodes[node.mNext].mPrev = node.mPrev
def isEnd(self, idx):
return idx >= 2048
def isEnd(self, idx):
return idx >= 2048
def xor_crypt(data, key): def xor_crypt(data, key):
odata = bytearray(len(data)) odata = bytearray(len(data))
odata[:] = data odata[:] = data
i = 0 i = 0
for x in data: for x in data:
if (x != 0): if x != 0:
m = key[i % len(key)] m = key[i % len(key)]
if (x != m): if x != m:
odata[i] = x ^ m odata[i] = x ^ m
i += 1 i += 1
return odata return odata

127
utage/names.py
View file

@ -8,74 +8,95 @@ from functools import partial
from glob import glob from glob import glob
from multiprocessing import Pool from multiprocessing import Pool
def update_names(dir_in, old_path, languages): def update_names(dir_in, old_path, languages):
names = extract_names(dir_in) names = extract_names(dir_in)
# we have to update japanese like the rest because CustomData exists # we have to update japanese like the rest because CustomData exists
# and POKELABO deleted the wedding gear event stuff from the game files # and POKELABO deleted the wedding gear event stuff from the game files
for l in languages: for l in languages:
if l == "jpn": if l == "jpn":
out_dict = dict(zip(names, names)) out_dict = dict(zip(names, names))
else: else:
out_dict = dict.fromkeys(names, "") out_dict = dict.fromkeys(names, "")
langfile = os.path.join(old_path, "nametranslations_{}.json".format(l)) langfile = os.path.join(old_path, "nametranslations_{}.json".format(l))
if os.path.isfile(langfile): if os.path.isfile(langfile):
with open(langfile, "r") as lang_file: with open(langfile, "r") as lang_file:
lang_dict = json.load(lang_file) lang_dict = json.load(lang_file)
out_dict.update(lang_dict) out_dict.update(lang_dict)
with io.open(
langfile, "w", newline="\n"
) as lang_file: # you're using git right
json.dump(
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True
)
with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right
json.dump(out_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True)
def extract_names(dir_in): def extract_names(dir_in):
char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv") char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv")
if not os.path.isfile(char_tsv_path): if not os.path.isfile(char_tsv_path):
raise FileNotFoundError("Character.tsv not found") raise FileNotFoundError("Character.tsv not found")
names, sets = read_char_tsv(char_tsv_path) names, sets = read_char_tsv(char_tsv_path)
files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)] files = [
if len(files) == 0: f
raise FileNotFoundError("No valid files found in directory: " + dir_in) for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
if re.search(r"/[0-9]{9}\.tsv$", f)
]
if len(files) == 0:
raise FileNotFoundError("No valid files found in directory: " + dir_in)
# tfw gil # tfw gil
# it's still faster on my machine so i'm keeping it # it's still faster on my machine so i'm keeping it
read_partial = partial(read_mission, char_names=names, char_sets=sets) read_partial = partial(read_mission, char_names=names, char_sets=sets)
with Pool() as p: with Pool() as p:
new_names = p.map(read_partial, files) new_names = p.map(read_partial, files)
for x in new_names: for x in new_names:
names.update(x) names.update(x)
return names
return names
def read_char_tsv(char_tsv_path): def read_char_tsv(char_tsv_path):
char_names = set() char_names = set()
char_sets = set() char_sets = set()
with open(char_tsv_path, "r") as char_tsv_file: with open(char_tsv_path, "r") as char_tsv_file:
char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar="\"") char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"')
for row in char_tsv: for row in char_tsv:
if row['CharacterName'].startswith("//"): if row["CharacterName"].startswith("//"):
continue continue
if row['CharacterName'] and row['NameText'] and row['CharacterName'].strip() and row['NameText'].strip(): if (
char_names.add(row['NameText']) row["CharacterName"]
char_sets.add(row['CharacterName']) and row["NameText"]
and row["CharacterName"].strip()
and row["NameText"].strip()
):
char_names.add(row["NameText"])
char_sets.add(row["CharacterName"])
return char_names, char_sets
return char_names, char_sets
def read_mission(tsv_path, char_names, char_sets): def read_mission(tsv_path, char_names, char_sets):
new_names = set() new_names = set()
with open(tsv_path, "r") as tsv_file: with open(tsv_path, "r") as tsv_file:
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"") tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
if ('Arg1' not in tsv.fieldnames) or ('Text' not in tsv.fieldnames) or ('Command' not in tsv.fieldnames): if (
return new_names ("Arg1" not in tsv.fieldnames)
for row in tsv: or ("Text" not in tsv.fieldnames)
if row['Command'] and row['Command'].startswith("//"): or ("Command" not in tsv.fieldnames)
continue ):
if row['Text'] and row['Arg1']: return new_names
if (row['Arg1'] not in char_names) and (row['Arg1'] not in char_sets): for row in tsv:
new_names.add(row['Arg1']) if row["Command"] and row["Command"].startswith("//"):
return new_names continue
if row["Text"] and row["Arg1"]:
if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets):
new_names.add(row["Arg1"])
return new_names

156
utage/translate.py
View file

@ -10,89 +10,109 @@ from functools import partial
from glob import glob from glob import glob
from multiprocessing.pool import Pool from multiprocessing.pool import Pool
def translate_dir(dir_in, tsv_out_dir, json_out_dir): def translate_dir(dir_in, tsv_out_dir, json_out_dir):
# we only want to key files that are mission ids # we only want to key files that are mission ids
files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)] files = [
f
for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
if re.search(r"/[0-9]{9}\.tsv$", f)
]
if len(files) == 0: if len(files) == 0:
raise FileNotFoundError("No valid files found in directory: " + dir_in) raise FileNotFoundError("No valid files found in directory: " + dir_in)
ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir)
with Pool() as p:
p.map(ptrans, files)
ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir)
with Pool() as p:
p.map(ptrans, files)
def translate_file(file_in, tsv_out_dir, json_out_dir): def translate_file(file_in, tsv_out_dir, json_out_dir):
try: try:
if not file_in.endswith(".tsv"): if not file_in.endswith(".tsv"):
raise ValueError("Invalid File Type for {}".format(file_in)) raise ValueError("Invalid File Type for {}".format(file_in))
# we need to get the event folder # we need to get the event folder
# ie, Utage/>>>main01<<</Scenario/whatever.tsv # ie, Utage/>>>main01<<</Scenario/whatever.tsv
try: try:
event_folder = os.path.normpath(file_in).split(os.sep)[-3] event_folder = os.path.normpath(file_in).split(os.sep)[-3]
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
print("Error processing tsv: Please reference from Utage root") print("Error processing tsv: Please reference from Utage root")
raise raise
id_num = os.path.splitext(os.path.basename(file_in))[0] id_num = os.path.splitext(os.path.basename(file_in))[0]
tsv_output_path = os.path.join(tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num)) tsv_output_path = os.path.join(
json_output_path = os.path.join(json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num)) tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num)
)
json_output_path = os.path.join(
json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num)
)
# need to create output paths and avoid races when threading # need to create output paths and avoid races when threading
if not os.path.exists(os.path.dirname(tsv_output_path)): if not os.path.exists(os.path.dirname(tsv_output_path)):
try: try:
os.makedirs(os.path.dirname(tsv_output_path)) os.makedirs(os.path.dirname(tsv_output_path))
except OSError as e: except OSError as e:
if e.errno != errno.EEXIST: if e.errno != errno.EEXIST:
raise raise
if not os.path.exists(os.path.dirname(json_output_path)): if not os.path.exists(os.path.dirname(json_output_path)):
try: try:
os.makedirs(os.path.dirname(json_output_path)) os.makedirs(os.path.dirname(json_output_path))
except OSError as e: except OSError as e:
if e.errno != errno.EEXIST: if e.errno != errno.EEXIST:
raise raise
with open(file_in, "r") as tsv_file: with open(file_in, "r") as tsv_file:
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"") tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
tsv_keyed, json_str = process_tsv(tsv, id_num) tsv_keyed, json_str = process_tsv(tsv, id_num)
# csv handles newlines, don't set it in io.open # csv handles newlines, don't set it in io.open
with io.open(tsv_output_path, "w", newline='') as tsv_out: with io.open(tsv_output_path, "w", newline="") as tsv_out:
writer = csv.DictWriter(tsv_out, delimiter="\t", quotechar="\"", lineterminator="\n", fieldnames=tsv.fieldnames, extrasaction='ignore') writer = csv.DictWriter(
writer.writeheader() tsv_out,
for row in tsv_keyed: delimiter="\t",
writer.writerow(row) quotechar='"',
lineterminator="\n",
fieldnames=tsv.fieldnames,
extrasaction="ignore",
)
writer.writeheader()
for row in tsv_keyed:
writer.writerow(row)
with io.open(json_output_path, "w", newline='\n') as json_out: with io.open(json_output_path, "w", newline="\n") as json_out:
# we don't want to sort these because they're _1, ..., _10, etc # we don't want to sort these because they're _1, ..., _10, etc
json.dump(json_str, json_out, ensure_ascii=False, indent='\t', sort_keys=False) json.dump(
json_str, json_out, ensure_ascii=False, indent="\t", sort_keys=False
)
except Exception as e:
traceback.print_exc()
print("Error processing file: " + file_in)
except Exception as e:
traceback.print_exc()
print("Error processing file: " + file_in)
def process_tsv(tsv, id_num): def process_tsv(tsv, id_num):
i = 0 i = 0
tsv_keyed = [] tsv_keyed = []
key_dict = {} key_dict = {}
for row in tsv: for row in tsv:
try: try:
if row['Command'].startswith("//"): if row["Command"].startswith("//"):
tsv_keyed.append(row) tsv_keyed.append(row)
continue continue
if row['Text'] and row['Text'].strip(): if row["Text"] and row["Text"].strip():
key = "{}_{}".format(id_num, i) key = "{}_{}".format(id_num, i)
i += 1 i += 1
row['English'] = key row["English"] = key
key_dict[key] = row['Text'] key_dict[key] = row["Text"]
tsv_keyed.append(row) tsv_keyed.append(row)
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
print("Error processing tsv: " + id_num) print("Error processing tsv: " + id_num)
raise raise
return tsv_keyed, key_dict return tsv_keyed, key_dict