diff --git a/adx/adx.py b/adx/adx.py index 4ac4246..e334f58 100644 --- a/adx/adx.py +++ b/adx/adx.py @@ -5,43 +5,46 @@ import traceback from .adx_parser import adx_file + def extract_loop_data_from_dir(dir_in, fout): - if fout is None: - fout = "BgmLoop.json" - files = glob.glob(os.path.join(dir_in, "**/*.adx"), recursive=True) - if len(files) == 0: - raise FileNotFoundError("No valid files found in directory: " + dir_in) - extract_loop_data_from_files(files, fout) + if fout is None: + fout = "BgmLoop.json" + files = glob.glob(os.path.join(dir_in, "**/*.adx"), recursive=True) + if len(files) == 0: + raise FileNotFoundError("No valid files found in directory: " + dir_in) + extract_loop_data_from_files(files, fout) + def extract_loop_data_from_files(files_in, fout): - if not files_in: - raise FileNotFoundError("No input files found") - collect = {} - for x in files_in: - res = extract_loop_data_from_file(x) - if res is not None: - collect[os.path.splitext(os.path.basename(x))[0]] = res - with open(fout, 'w') as out: - json.dump(collect, out, ensure_ascii=False, indent="\t", sort_keys=True) + if not files_in: + raise FileNotFoundError("No input files found") + collect = {} + for x in files_in: + res = extract_loop_data_from_file(x) + if res is not None: + collect[os.path.splitext(os.path.basename(x))[0]] = res + with open(fout, "w") as out: + json.dump(collect, out, ensure_ascii=False, indent="\t", sort_keys=True) + def extract_loop_data_from_file(file_in, fout=None): - if not file_in.endswith(".adx"): - raise ValueError("Input file not .adx") + if not file_in.endswith(".adx"): + raise ValueError("Input file not .adx") - try: - with open(file_in, 'rb') as x: - data = x.read(64) - ADX = adx_file(data) - if ADX.l_exists and ADX.is_valid: - res = ADX.parse_loop_data() - else: - res = None - except Exception as e: - traceback.print_exc() - print("Error processing file: " + file_in) - return None + try: + with open(file_in, "rb") as x: + data = x.read(64) + ADX = adx_file(data) + if ADX.l_exists and ADX.is_valid: + res = ADX.parse_loop_data() + else: + res = None + except Exception as e: + traceback.print_exc() + print("Error processing file: " + file_in) + return None - if fout is not None and res is not None: - with open(fout, 'w') as out: - json.dump(res, out, ensure_ascii=False, indent="\t", sort_keys=True) - return res + if fout is not None and res is not None: + with open(fout, "w") as out: + json.dump(res, out, ensure_ascii=False, indent="\t", sort_keys=True) + return res diff --git a/adx/adx_parser.py b/adx/adx_parser.py index 9dea9a4..28974c2 100644 --- a/adx/adx_parser.py +++ b/adx/adx_parser.py @@ -1,4 +1,4 @@ -from math import modf #fuck me rounding is hard +from math import modf # fuck me rounding is hard # header map # https://wiki.multimedia.cx/index.php/CRI_ADX_file @@ -13,16 +13,16 @@ DATA_OFF_MIN = 0x38 FORMAT_OFF = 0x04 FORMAT_LEN = 1 -FORMAT = 3 # always 3 for adx apparently +FORMAT = 3 # always 3 for adx apparently LOOP_TYPE_OFF = 0x12 LOOP_TYPE_LEN = 1 -LOOP_TYPE_SUP = [4] # supported loop styles +LOOP_TYPE_SUP = [4] # supported loop styles SRATE_OFF = 0x08 SRATE_LEN = 4 -SCOUNT_OFF = 0x0c +SCOUNT_OFF = 0x0C SCOUNT_LEN = 4 CRYPT_OFF = 0x13 @@ -38,82 +38,90 @@ LOOP4_END_OFF = 0x30 LOOP4_END_LEN = 4 -#class independent functions +# class independent functions + def dumb_round(number): - dec = modf(number)[0] - if (dec <= 0.5): - return int(number) - else: - return int(number+1) + dec = modf(number)[0] + if dec <= 0.5: + return int(number) + else: + return int(number + 1) + class adx_file(object): - magic = 0 - form = 0 - l_style = 0 - encrypted = 0 - sam_rate = 0 - sam_count = 0 - l_flag = 0 - l_start = 0 - l_end = 0 - l_exists = 0 - d_offset = 0 - is_valid = 0 + magic = 0 + form = 0 + l_style = 0 + encrypted = 0 + sam_rate = 0 + sam_count = 0 + l_flag = 0 + l_start = 0 + l_end = 0 + l_exists = 0 + d_offset = 0 + is_valid = 0 - data = None + data = None - def __init__(self, data): - # first, validate - self.data = data - self.magic = self.get_val(MAGIC_OFF, MAGIC_LEN) - self.form = self.get_val(FORMAT_OFF, FORMAT_LEN) - self.l_style = self.get_val(LOOP_TYPE_OFF, LOOP_TYPE_LEN) - self.encrypted = self.get_val(CRYPT_OFF, CRYPT_LEN) - self.d_offset = self.get_val(DATA_OFF_OFF, DATA_OFF_LEN) - if self.d_offset < DATA_OFF_MIN: - return None - self.l_exists = 1 - # now load things for math - self.sam_rate = self.get_val(SRATE_OFF, SRATE_LEN) - self.sam_count = self.get_val(SCOUNT_OFF, SCOUNT_LEN) - self.validate() + def __init__(self, data): + # first, validate + self.data = data + self.magic = self.get_val(MAGIC_OFF, MAGIC_LEN) + self.form = self.get_val(FORMAT_OFF, FORMAT_LEN) + self.l_style = self.get_val(LOOP_TYPE_OFF, LOOP_TYPE_LEN) + self.encrypted = self.get_val(CRYPT_OFF, CRYPT_LEN) + self.d_offset = self.get_val(DATA_OFF_OFF, DATA_OFF_LEN) + if self.d_offset < DATA_OFF_MIN: + return None + self.l_exists = 1 + # now load things for math + self.sam_rate = self.get_val(SRATE_OFF, SRATE_LEN) + self.sam_count = self.get_val(SCOUNT_OFF, SCOUNT_LEN) + self.validate() - def get_val(self, offset, length): - return int.from_bytes(self.data[offset:offset+length], byteorder='big', signed=False) + def get_val(self, offset, length): + return int.from_bytes( + self.data[offset : offset + length], byteorder="big", signed=False + ) - def validate(self): - if self.magic != MAGIC: - raise ValueError("Invalid ADX File") - if self.form != FORMAT: - raise ValueError("Invalid ADX File") - if self.encrypted: - raise NotImplementedError("Encryption Not Supported") - self.is_valid = 1 + def validate(self): + if self.magic != MAGIC: + raise ValueError("Invalid ADX File") + if self.form != FORMAT: + raise ValueError("Invalid ADX File") + if self.encrypted: + raise NotImplementedError("Encryption Not Supported") + self.is_valid = 1 - def parse_loop_data(self): - if self.l_style == 4: - self.l_flag = self.get_val(LOOP4_FLAG_OFF, LOOP4_FLAG_LEN) - self.l_start = self.get_val(LOOP4_START_OFF, LOOP4_START_LEN) - self.l_end = self.get_val(LOOP4_END_OFF, LOOP4_END_LEN) - if not self.l_flag: - return None - if self.l_start == 0 and self.l_end == self.sam_count: - return None - if self.l_start > self.sam_count or self.l_end > self.sam_count: - raise ValueError("Invalid Loop Data") - return None - if not self.l_style in LOOP_TYPE_SUP: - raise NotImplementedError("Loop Style {} Not Implemented".format(str(self.l_style))) - return None - ret = {} - ret['duration'] = self.sam_count / self.sam_rate - ret['loop_start'] = {} - ret['loop_start']['seconds'] = self.l_start / self.sam_rate - ret['loop_start']['samples_native'] = self.l_start - ret['loop_start']['samples_48k'] = dumb_round(self.l_start / self.sam_rate * 48000) - ret['loop_end'] = {} - ret['loop_end']['seconds'] = self.l_end / self.sam_rate - ret['loop_end']['samples_native'] = self.l_end - ret['loop_end']['samples_48k'] = dumb_round(self.l_end / self.sam_rate * 48000) - return ret + def parse_loop_data(self): + if self.l_style == 4: + self.l_flag = self.get_val(LOOP4_FLAG_OFF, LOOP4_FLAG_LEN) + self.l_start = self.get_val(LOOP4_START_OFF, LOOP4_START_LEN) + self.l_end = self.get_val(LOOP4_END_OFF, LOOP4_END_LEN) + if not self.l_flag: + return None + if self.l_start == 0 and self.l_end == self.sam_count: + return None + if self.l_start > self.sam_count or self.l_end > self.sam_count: + raise ValueError("Invalid Loop Data") + return None + if not self.l_style in LOOP_TYPE_SUP: + raise NotImplementedError( + "Loop Style {} Not Implemented".format(str(self.l_style)) + ) + return None + ret = {} + ret["duration"] = self.sam_count / self.sam_rate + ret["loop_start"] = {} + ret["loop_start"]["seconds"] = self.l_start / self.sam_rate + ret["loop_start"]["samples_native"] = self.l_start + ret["loop_start"]["samples_48k"] = dumb_round( + self.l_start / self.sam_rate * 48000 + ) + ret["loop_end"] = {} + ret["loop_end"]["seconds"] = self.l_end / self.sam_rate + ret["loop_end"]["samples_native"] = self.l_end + ret["loop_end"]["samples_48k"] = dumb_round(self.l_end / self.sam_rate * 48000) + return ret diff --git a/db/base.py b/db/base.py index b23201f..097755c 100644 --- a/db/base.py +++ b/db/base.py @@ -7,22 +7,23 @@ from sqlalchemy.ext.declarative import declarative_base Base = declarative_base() -engine = create_engine('sqlite:///:memory:', echo=False) +engine = create_engine("sqlite:///:memory:", echo=False) session = sessionmaker(bind=engine, autoflush=False, autocommit=False)() -session.flush = lambda:None +session.flush = lambda: None DATABASES = { - "QuestMst.db": "Quest", - "QuestSceneMst.db": "QuestScene", - "QuestPartMst.db": "QuestPart", - "ResourceEntry.db": "ResourceEntry" + "QuestMst.db": "Quest", + "QuestSceneMst.db": "QuestScene", + "QuestPartMst.db": "QuestPart", + "ResourceEntry.db": "ResourceEntry", } -def init(path): - for d in DATABASES: - if not os.path.isfile(os.path.join(path, d)): - raise FileNotFoundError("Database {} not found".format(d)) - for d in DATABASES: - t = text("attach database :path as :schema") - engine.execute(t, path=os.path.join(path, d), schema=DATABASES[d]) +def init(path): + for d in DATABASES: + if not os.path.isfile(os.path.join(path, d)): + raise FileNotFoundError("Database {} not found".format(d)) + + for d in DATABASES: + t = text("attach database :path as :schema") + engine.execute(t, path=os.path.join(path, d), schema=DATABASES[d]) diff --git a/db/quest.py b/db/quest.py index 857bb8d..f4d2539 100644 --- a/db/quest.py +++ b/db/quest.py @@ -3,65 +3,74 @@ from sqlalchemy.orm import relationship from .base import Base + class Quest(Base): - __tablename__ = 'QuestMstRecord' - __table_args__ = {'schema': 'Quest'} + __tablename__ = "QuestMstRecord" + __table_args__ = {"schema": "Quest"} - questMstId = Column(Integer, primary_key=True) - questType = Column(Integer) - sortNum = Column(Integer) - name = Column(String) - prevQuestMstId = Column(Integer) - eventItemMstIds = Column(Integer) - valid = Column(Integer) - baseQuestMstId = Column(Integer) - updatedTime = Column(BigInteger) + questMstId = Column(Integer, primary_key=True) + questType = Column(Integer) + sortNum = Column(Integer) + name = Column(String) + prevQuestMstId = Column(Integer) + eventItemMstIds = Column(Integer) + valid = Column(Integer) + baseQuestMstId = Column(Integer) + updatedTime = Column(BigInteger) - scenes = relationship("QuestScene") + scenes = relationship("QuestScene") + + def __repr__(self): + return "".format(self.questMstId, self.name) - def __repr__(self): - return "".format(self.questMstId, self.name) class QuestScene(Base): - __tablename__ = 'QuestSceneMstRecord' - __table_args__ = {'schema': 'QuestScene'} + __tablename__ = "QuestSceneMstRecord" + __table_args__ = {"schema": "QuestScene"} - questSceneMstId = Column(Integer, primary_key=True) - questMstId = Column(Integer, ForeignKey("Quest.QuestMstRecord.questMstId")) - _type = Column("type", Integer) - sortNum = Column(Integer) - name = Column(String) - _filter = Column("filter", Integer) - prevQuestSceneMstId = Column(Integer, ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId")) - releaseSerial = Column(Integer) - releaseEvolutionLevel = Column(Integer) - summaryText = Column(String) - presentType = Column(String) - objectId = Column(Integer) - num = Column(Integer) - appearanceType = Column(Integer) - isPrologue = Column(Integer) - isEpilogue = Column(Integer) - viewType = Column(Integer) - updatedTime = Column(BigInteger) + questSceneMstId = Column(Integer, primary_key=True) + questMstId = Column(Integer, ForeignKey("Quest.QuestMstRecord.questMstId")) + _type = Column("type", Integer) + sortNum = Column(Integer) + name = Column(String) + _filter = Column("filter", Integer) + prevQuestSceneMstId = Column( + Integer, ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId") + ) + releaseSerial = Column(Integer) + releaseEvolutionLevel = Column(Integer) + summaryText = Column(String) + presentType = Column(String) + objectId = Column(Integer) + num = Column(Integer) + appearanceType = Column(Integer) + isPrologue = Column(Integer) + isEpilogue = Column(Integer) + viewType = Column(Integer) + updatedTime = Column(BigInteger) + + previous = relationship("QuestScene", remote_side=[questSceneMstId]) + parts = relationship("QuestPart") - previous = relationship("QuestScene", remote_side=[questSceneMstId]) - parts = relationship("QuestPart") class QuestPart(Base): - __tablename__ = 'QuestPartMstRecord' - __table_args__ = {'schema': 'QuestPart'} + __tablename__ = "QuestPartMstRecord" + __table_args__ = {"schema": "QuestPart"} - questSceneMstId = Column(Integer, ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId"), primary_key=True) # now this is pod racing - partNum = Column(Integer, primary_key=True) - waveNum = Column(Integer) - stamina = Column(Integer) - exp = Column(Integer) - expertPoint = Column(Integer) - recommendLevel = Column(Integer) - beforeTalkName = Column(String) - afterTalkName = Column(String) - battleBackgroundImg = Column(String) - musicMstId = Column(Integer) - isFixedDeck = Column(Integer) - updatedTime = Column(BigInteger) + questSceneMstId = Column( + Integer, + ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId"), + primary_key=True, + ) # now this is pod racing + partNum = Column(Integer, primary_key=True) + waveNum = Column(Integer) + stamina = Column(Integer) + exp = Column(Integer) + expertPoint = Column(Integer) + recommendLevel = Column(Integer) + beforeTalkName = Column(String) + afterTalkName = Column(String) + battleBackgroundImg = Column(String) + musicMstId = Column(Integer) + isFixedDeck = Column(Integer) + updatedTime = Column(BigInteger) diff --git a/db/resource.py b/db/resource.py index 835f101..b624ec0 100644 --- a/db/resource.py +++ b/db/resource.py @@ -2,12 +2,13 @@ from sqlalchemy import Column, BigInteger, String from .base import Base -class ResourceEntry(Base): - __tablename__ = 'ResourceEntryRecord' - __table_args__ = {'schema': 'ResourceEntry'} - path = Column(String, primary_key=True) - serverPath = Column(String) - localPath = Column(String) - digest = Column(String) - fileSize = Column(BigInteger) +class ResourceEntry(Base): + __tablename__ = "ResourceEntryRecord" + __table_args__ = {"schema": "ResourceEntry"} + + path = Column(String, primary_key=True) + serverPath = Column(String) + localPath = Column(String) + digest = Column(String) + fileSize = Column(BigInteger) diff --git a/diva/quest.py b/diva/quest.py index 46d73dc..6f166cf 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -5,110 +5,151 @@ import os from sqlalchemy.orm import contains_eager + def update_missions(dir_in, old_path, languages): - db.init(dir_in) - quest_mst(old_path, languages) - scene_mst(old_path, languages) + db.init(dir_in) + quest_mst(old_path, languages) + scene_mst(old_path, languages) + def scene_mst(old_path, languages): - result = (db.session.query(db.QuestScene) - .join(db.QuestScene.parts) - .options(contains_eager(db.QuestScene.parts)) - .filter((db.QuestPart.beforeTalkName != "") | (db.QuestPart.afterTalkName != "")) - .filter(db.QuestScene.parts.any()) - .all()) + result = ( + db.session.query(db.QuestScene) + .join(db.QuestScene.parts) + .options(contains_eager(db.QuestScene.parts)) + .filter( + (db.QuestPart.beforeTalkName != "") | (db.QuestPart.afterTalkName != "") + ) + .filter(db.QuestScene.parts.any()) + .all() + ) - scenes = {} - for x in result: - scenes[str(x.questSceneMstId)] = { "Name": x.name, - "SummaryText": x.summaryText, - "Parts": [], - "Folder": "" } + scenes = {} + for x in result: + scenes[str(x.questSceneMstId)] = { + "Name": x.name, + "SummaryText": x.summaryText, + "Parts": [], + "Folder": "", + } - id_num = "" - # this is slow as balls but i'm way too tired to figure out how to optimize it - for y in x.parts: - if y.beforeTalkName: - id_num = y.beforeTalkName - scenes[str(x.questSceneMstId)]["Parts"].append(y.beforeTalkName) - if y.afterTalkName: - id_num = y.afterTalkName - scenes[str(x.questSceneMstId)]["Parts"].append(y.afterTalkName) + id_num = "" + # this is slow as balls but i'm way too tired to figure out how to optimize it + for y in x.parts: + if y.beforeTalkName: + id_num = y.beforeTalkName + scenes[str(x.questSceneMstId)]["Parts"].append(y.beforeTalkName) + if y.afterTalkName: + id_num = y.afterTalkName + scenes[str(x.questSceneMstId)]["Parts"].append(y.afterTalkName) - # i checked every entry, moving it to do once for speed - if not scenes[str(x.questSceneMstId)]["Folder"] and id_num: - path = (db.session.query(db.ResourceEntry) - .filter(db.ResourceEntry.path.like("%/Scenario/{}.tsv.utage".format(id_num))) - .filter(db.ResourceEntry.path.notlike("%Utage/side03/%.tsv.utage")) # dead folder with dupes too lazy to parse settings tsv - .one()) - folder = path.path.split("/")[2] - if scenes[str(x.questSceneMstId)]["Folder"] and scenes[str(x.questSceneMstId)]["Folder"] != folder: - raise KeyError("Internal path inconsitency, bailing out {}".format(id_num)) - scenes[str(x.questSceneMstId)]["Folder"] = folder + # i checked every entry, moving it to do once for speed + if not scenes[str(x.questSceneMstId)]["Folder"] and id_num: + path = ( + db.session.query(db.ResourceEntry) + .filter( + db.ResourceEntry.path.like( + "%/Scenario/{}.tsv.utage".format(id_num) + ) + ) + .filter( + db.ResourceEntry.path.notlike("%Utage/side03/%.tsv.utage") + ) # dead folder with dupes too lazy to parse settings tsv + .one() + ) + folder = path.path.split("/")[2] + if ( + scenes[str(x.questSceneMstId)]["Folder"] + and scenes[str(x.questSceneMstId)]["Folder"] != folder + ): + raise KeyError( + "Internal path inconsitency, bailing out {}".format(id_num) + ) + scenes[str(x.questSceneMstId)]["Folder"] = folder - with io.open(os.path.join(old_path, "XduScene.json"), "w", newline='\n') as json_file: - json.dump(scenes, json_file, ensure_ascii=False, indent='\t', sort_keys=True) + with io.open( + os.path.join(old_path, "XduScene.json"), "w", newline="\n" + ) as json_file: + json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True) - for l in languages: - out_dict = {} - if l == "jpn": - for x in result: - out_dict[str(x.questSceneMstId)] = { "Name": x.name, - "SummaryText": x.summaryText, - "Credits": "POKELABO", - "Enabled": False } - else: - for x in result: - out_dict[str(x.questSceneMstId)] = { "Name": "", - "SummaryText": "", - "Credits": "", - "Enabled": False } + for l in languages: + out_dict = {} + if l == "jpn": + for x in result: + out_dict[str(x.questSceneMstId)] = { + "Name": x.name, + "SummaryText": x.summaryText, + "Credits": "POKELABO", + "Enabled": False, + } + else: + for x in result: + out_dict[str(x.questSceneMstId)] = { + "Name": "", + "SummaryText": "", + "Credits": "", + "Enabled": False, + } - langfile = os.path.join(old_path, "XduSceneNames_{}.json".format(l)) - if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: - lang_dict = json.load(lang_file) - out_dict.update(lang_dict) + langfile = os.path.join(old_path, "XduSceneNames_{}.json".format(l)) + if os.path.isfile(langfile): + with open(langfile, "r") as lang_file: + lang_dict = json.load(lang_file) + out_dict.update(lang_dict) + + with io.open( + langfile, "w", newline="\n" + ) as lang_file: # you're using git right + json.dump( + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True + ) - with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right - json.dump(out_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True) def quest_mst(old_path, languages): - # i regret my life choices - result = (db.session.query(db.Quest) - .filter(db.Quest.baseQuestMstId == 0) - .join(db.Quest.scenes) - .join(db.QuestScene.parts) - .options(contains_eager(db.Quest.scenes). - contains_eager(db.QuestScene.parts)) - .filter(db.QuestPart.beforeTalkName != "" or db.QuestPart.afterTalkName != "") - .filter(db.QuestScene.parts.any()) - .filter(db.Quest.scenes.any()) - .all()) - quests = {} - for x in result: - quests[str(x.questMstId)] = { "Name": x.name, "Scenes": [d.questSceneMstId for d in x.scenes] } + # i regret my life choices + result = ( + db.session.query(db.Quest) + .filter(db.Quest.baseQuestMstId == 0) + .join(db.Quest.scenes) + .join(db.QuestScene.parts) + .options(contains_eager(db.Quest.scenes).contains_eager(db.QuestScene.parts)) + .filter(db.QuestPart.beforeTalkName != "" or db.QuestPart.afterTalkName != "") + .filter(db.QuestScene.parts.any()) + .filter(db.Quest.scenes.any()) + .all() + ) + quests = {} + for x in result: + quests[str(x.questMstId)] = { + "Name": x.name, + "Scenes": [d.questSceneMstId for d in x.scenes], + } - with io.open(os.path.join(old_path, "XduQuest.json"), "w", newline='\n') as json_file: - json.dump(quests, json_file, ensure_ascii=False, indent='\t', sort_keys=True) + with io.open( + os.path.join(old_path, "XduQuest.json"), "w", newline="\n" + ) as json_file: + json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True) - for l in languages: - out_dict = {} - if l == "jpn": - for x in result: - out_dict[str(x.questMstId)] = { "Name": x.name, "Enabled": False } - else: - for x in result: - out_dict[str(x.questMstId)] = { "Name": "", "Enabled": False } + for l in languages: + out_dict = {} + if l == "jpn": + for x in result: + out_dict[str(x.questMstId)] = {"Name": x.name, "Enabled": False} + else: + for x in result: + out_dict[str(x.questMstId)] = {"Name": "", "Enabled": False} - langfile = os.path.join(old_path, "XduQuestNames_{}.json".format(l)) - if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: - lang_dict = json.load(lang_file) - out_dict.update(lang_dict) + langfile = os.path.join(old_path, "XduQuestNames_{}.json".format(l)) + if os.path.isfile(langfile): + with open(langfile, "r") as lang_file: + lang_dict = json.load(lang_file) + out_dict.update(lang_dict) - with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right - json.dump(out_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True) + with io.open( + langfile, "w", newline="\n" + ) as lang_file: # you're using git right + json.dump( + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True + ) - - return quests + return quests diff --git a/divatool.py b/divatool.py index cc0d227..74284fe 100755 --- a/divatool.py +++ b/divatool.py @@ -10,101 +10,147 @@ import db LANGUAGES = ["jpn", "eng", "rus"] + def main(): - parser = argparse.ArgumentParser(description="Unpack and process XDU data") + parser = argparse.ArgumentParser(description="Unpack and process XDU data") - subparsers = parser.add_subparsers(metavar="", title="subcommands", dest="command") + subparsers = parser.add_subparsers( + metavar="", title="subcommands", dest="command" + ) - # adx subcommand - parser_adx = subparsers.add_parser("adx", help="ADX stuff") - parser_adx_subparsers = parser_adx.add_subparsers(metavar="", title="subcommands", dest="subcommand") - # adx loop - parser_adx_loop = parser_adx_subparsers.add_parser("loop", help="Extract loop data") - parser_adx_loop.add_argument("--output", "-o", - help="JSON output file for loop data", - type=str, dest="JSON_OUT") - parser_adx_loop.add_argument("ADX_DIR", - help="Input directory containing ADX files", - type=str) + # adx subcommand + parser_adx = subparsers.add_parser("adx", help="ADX stuff") + parser_adx_subparsers = parser_adx.add_subparsers( + metavar="", title="subcommands", dest="subcommand" + ) + # adx loop + parser_adx_loop = parser_adx_subparsers.add_parser("loop", help="Extract loop data") + parser_adx_loop.add_argument( + "--output", + "-o", + help="JSON output file for loop data", + type=str, + dest="JSON_OUT", + ) + parser_adx_loop.add_argument( + "ADX_DIR", help="Input directory containing ADX files", type=str + ) - # utage subcommand - parser_utage = subparsers.add_parser("utage", help="UTAGE stuff") - parser_utage_subparsers = parser_utage.add_subparsers(metavar="", title="subcommand", dest="subcommand") - # utage crypt - parser_utage_crypt = parser_utage_subparsers.add_parser("crypt", help="Encrypt/decrypt utage tsv files") - parser_utage_crypt.add_argument("--encrypt", "-e", - help="Encrypt (default: Decrypt)", - dest="encrypt", action="store_true", default=False) - parser_utage_crypt.add_argument("--no-compression", "-n", - help="Do not compress/decompress. Only applies to tsv, png will never be compressed", - dest="ncomp", action="store_true", default=False) - parser_utage_crypt.add_argument("--key", "-k", - help="Encryption key (Default: SampleSecretKey)", - dest="key", type=str, default="SampleSecretKey") - parser_utage_crypt.add_argument("--hex", "-x", - help="KEY is hexadecimal (Default: False)", - dest="hex", action="store_true", default=False) - parser_utage_crypt.add_argument("TARGET", - help="Input", type=str) - # utage translate - parser_utage_translate = parser_utage_subparsers.add_parser("translate", help="Generate keyed tsv and json files") - parser_utage_translate.add_argument("INPUT", - help="Input", type=str) - parser_utage_translate.add_argument("TSVDIR", nargs='?', - help="_t.tsv output directory", type=str, default=".") - parser_utage_translate.add_argument("JSONDIR", nargs='?', - help="json output directory", type=str, default=".") - # utage names - parser_utage_names = parser_utage_subparsers.add_parser("names", help="Generate and update name files") - parser_utage_names.add_argument("INPUT", - help="Input", type=str) - parser_utage_names.add_argument("OLD", nargs='?', - help="Directory with old files", type=str, default=".") + # utage subcommand + parser_utage = subparsers.add_parser("utage", help="UTAGE stuff") + parser_utage_subparsers = parser_utage.add_subparsers( + metavar="", title="subcommand", dest="subcommand" + ) + # utage crypt + parser_utage_crypt = parser_utage_subparsers.add_parser( + "crypt", help="Encrypt/decrypt utage tsv files" + ) + parser_utage_crypt.add_argument( + "--encrypt", + "-e", + help="Encrypt (default: Decrypt)", + dest="encrypt", + action="store_true", + default=False, + ) + parser_utage_crypt.add_argument( + "--no-compression", + "-n", + help="Do not compress/decompress. Only applies to tsv, png will never be compressed", + dest="ncomp", + action="store_true", + default=False, + ) + parser_utage_crypt.add_argument( + "--key", + "-k", + help="Encryption key (Default: SampleSecretKey)", + dest="key", + type=str, + default="SampleSecretKey", + ) + parser_utage_crypt.add_argument( + "--hex", + "-x", + help="KEY is hexadecimal (Default: False)", + dest="hex", + action="store_true", + default=False, + ) + parser_utage_crypt.add_argument("TARGET", help="Input", type=str) + # utage translate + parser_utage_translate = parser_utage_subparsers.add_parser( + "translate", help="Generate keyed tsv and json files" + ) + parser_utage_translate.add_argument("INPUT", help="Input", type=str) + parser_utage_translate.add_argument( + "TSVDIR", nargs="?", help="_t.tsv output directory", type=str, default="." + ) + parser_utage_translate.add_argument( + "JSONDIR", nargs="?", help="json output directory", type=str, default="." + ) + # utage names + parser_utage_names = parser_utage_subparsers.add_parser( + "names", help="Generate and update name files" + ) + parser_utage_names.add_argument("INPUT", help="Input", type=str) + parser_utage_names.add_argument( + "OLD", nargs="?", help="Directory with old files", type=str, default="." + ) - # diva subcommand - parser_diva = subparsers.add_parser("diva", help="Diva stuff") - parser_diva_subparsers = parser_diva.add_subparsers(metavar="", title="subcommand", dest="subcommand") - parser_diva_quest = parser_diva_subparsers.add_parser("quest", help="Generate quest JSON files") - parser_diva_quest.add_argument("INPUT", help="Directory of databases containing Quest*.db and ResourceEntry.db", - type=str) - parser_diva_quest.add_argument("OLD", help="Directory with old quest files", - nargs='?', type=str, default=".") + # diva subcommand + parser_diva = subparsers.add_parser("diva", help="Diva stuff") + parser_diva_subparsers = parser_diva.add_subparsers( + metavar="", title="subcommand", dest="subcommand" + ) + parser_diva_quest = parser_diva_subparsers.add_parser( + "quest", help="Generate quest JSON files" + ) + parser_diva_quest.add_argument( + "INPUT", + help="Directory of databases containing Quest*.db and ResourceEntry.db", + type=str, + ) + parser_diva_quest.add_argument( + "OLD", help="Directory with old quest files", nargs="?", type=str, default="." + ) - args = parser.parse_args() + args = parser.parse_args() - if args.command == "adx": - if args.subcommand == "loop": - adx.extract_loop_data_from_dir(args.ADX_DIR, args.JSON_OUT) - elif args.command == "utage": - if args.subcommand == "crypt": - if not args.hex: - key = bytearray(args.key, "utf-8") - else: - key = bytearray.fromhex(args.key) - if os.path.isfile(args.TARGET): - utage.crypt.crypt_file(args.TARGET, key, args.encrypt, args.ncomp) - elif os.path.isdir(args.TARGET): - utage.crypt.crypt_dir(args.TARGET, key, args.encrypt, args.ncomp) - else: - raise FileNotFoundError("Could not find {}".format(args.TARGET)) - elif args.subcommand == "translate": - if os.path.isfile(args.INPUT): - utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR) - elif os.path.isdir(args.INPUT): - utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR) - else: - raise FileNotFoundError("Could not find {}".format(args.INPUT)) - elif args.subcommand == "names": - if os.path.isdir(args.INPUT): - utage.names.update_names(args.INPUT, args.OLD, LANGUAGES) - else: - raise FileNotFoundError(args.INPUT) - elif args.command == "diva": - if args.subcommand == "quest": - diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) + if args.command == "adx": + if args.subcommand == "loop": + adx.extract_loop_data_from_dir(args.ADX_DIR, args.JSON_OUT) + elif args.command == "utage": + if args.subcommand == "crypt": + if not args.hex: + key = bytearray(args.key, "utf-8") + else: + key = bytearray.fromhex(args.key) + if os.path.isfile(args.TARGET): + utage.crypt.crypt_file(args.TARGET, key, args.encrypt, args.ncomp) + elif os.path.isdir(args.TARGET): + utage.crypt.crypt_dir(args.TARGET, key, args.encrypt, args.ncomp) + else: + raise FileNotFoundError("Could not find {}".format(args.TARGET)) + elif args.subcommand == "translate": + if os.path.isfile(args.INPUT): + utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR) + elif os.path.isdir(args.INPUT): + utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR) + else: + raise FileNotFoundError("Could not find {}".format(args.INPUT)) + elif args.subcommand == "names": + if os.path.isdir(args.INPUT): + utage.names.update_names(args.INPUT, args.OLD, LANGUAGES) + else: + raise FileNotFoundError(args.INPUT) + elif args.command == "diva": + if args.subcommand == "quest": + diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES) - return 0 + return 0 -if __name__ == '__main__': - main() - sys.exit() + +if __name__ == "__main__": + main() + sys.exit() diff --git a/utage/crypt.py b/utage/crypt.py index 79f084f..c6459ca 100644 --- a/utage/crypt.py +++ b/utage/crypt.py @@ -9,199 +9,214 @@ from multiprocessing.pool import Pool from . import crypt from . import translate + def crypt_dir(dir_in, key, encrypt=False, no_compress=False): - if not encrypt: - files = glob(os.path.join(dir_in, "**/*.utage"), recursive=True) - else: - files = glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) + if not encrypt: + files = glob(os.path.join(dir_in, "**/*.utage"), recursive=True) + else: + files = glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) - if len(files) == 0: - raise FileNotFoundError("No valid files found in directory: " + dir_in) + if len(files) == 0: + raise FileNotFoundError("No valid files found in directory: " + dir_in) + + pcrypt = partial(crypt_file, key=key, encrypt=encrypt, no_compress=no_compress) + with Pool() as p: + p.map(pcrypt, files) - pcrypt = partial(crypt_file, key=key, encrypt=encrypt, no_compress=no_compress) - with Pool() as p: - p.map(pcrypt, files) def crypt_file(file_in, key, encrypt=False, no_compress=False): - with open(file_in, "rb") as inf: - in_data = inf.read() - try: - if encrypt: - if not file_in.endswith(".png") and not file_in.endswith(".jpg") and not file_in.endswith(".tsv"): - raise ValueError("Invalid File Type for {}".format(file_in)) - if not file_in.endswith(".png") and not no_compress: - enc_data = crypt.compress(in_data) - enc_data = crypt.xor_crypt(enc_data, key) - with io.open(file_in + ".utage", "wb") as output: - output.write(enc_data) - else: - if not file_in.endswith(".utage"): - raise ValueError("Invalid File Type for {}".format(file_in)) - dec_data = crypt.xor_crypt(in_data, key) - if not file_in.endswith(".png.utage") and not file_in.endswith(".jpg.utage") and not no_compress: - dec_data = crypt.decompress(dec_data) - with io.open(file_in.replace(".utage", ""), "wb") as output: - output.write(dec_data) - except Exception as e: - traceback.print_exc() - print("Error processing file: " + file_in) + with open(file_in, "rb") as inf: + in_data = inf.read() + try: + if encrypt: + if ( + not file_in.endswith(".png") + and not file_in.endswith(".jpg") + and not file_in.endswith(".tsv") + ): + raise ValueError("Invalid File Type for {}".format(file_in)) + if not file_in.endswith(".png") and not no_compress: + enc_data = crypt.compress(in_data) + enc_data = crypt.xor_crypt(enc_data, key) + with io.open(file_in + ".utage", "wb") as output: + output.write(enc_data) + else: + if not file_in.endswith(".utage"): + raise ValueError("Invalid File Type for {}".format(file_in)) + dec_data = crypt.xor_crypt(in_data, key) + if ( + not file_in.endswith(".png.utage") + and not file_in.endswith(".jpg.utage") + and not no_compress + ): + dec_data = crypt.decompress(dec_data) + with io.open(file_in.replace(".utage", ""), "wb") as output: + output.write(dec_data) + except Exception as e: + traceback.print_exc() + print("Error processing file: " + file_in) + def decompress(data): - osize = 0 - isize = len(data) - osize_expected = int.from_bytes(data[:4], byteorder='little', signed=False) - odata = bytearray(osize_expected) - odata[:isize] = data - i = 4 - while i < isize: - if (data[i] & 128) != 0: - num3 = data[i] & 15 - num3 += 3 - num4 = (data[i] & 112) << 4 | data[i+1] - num4 += 1 - for j in range(0, num3): - odata[osize + j] = odata[osize - num4 + j] - i += 1 - else: - num3 = data[i] + 1 - for j in range(0, num3): - odata[osize + j] = data[i + 1 + j] - i += num3 - osize += num3 - i += 1 - return odata + osize = 0 + isize = len(data) + osize_expected = int.from_bytes(data[:4], byteorder="little", signed=False) + odata = bytearray(osize_expected) + odata[:isize] = data + i = 4 + while i < isize: + if (data[i] & 128) != 0: + num3 = data[i] & 15 + num3 += 3 + num4 = (data[i] & 112) << 4 | data[i + 1] + num4 += 1 + for j in range(0, num3): + odata[osize + j] = odata[osize - num4 + j] + i += 1 + else: + num3 = data[i] + 1 + for j in range(0, num3): + odata[osize + j] = data[i + 1 + j] + i += num3 + osize += num3 + i += 1 + return odata + def compress(data): - num = len(data) - bytes2 = num.to_bytes(4, byteorder='little') - anum3 = num + num/128 + 1 - array = bytearray(int(anum3)) - num2 = 0 - num3 = 0 - num4 = 0 - index = Index() - while num3 < num: - num5 = 0 - num6 = 0 - num7 = min(18, num - num3) - num8 = index.getFirst(data[num3]) - while not index.isEnd(num8): - node = index.getNode(num8) - mPos = node.mPos - i = 1 - while i < num7: - if data[mPos+i] != data[num3+i]: - break - i = i+1 - if num5 < i: - num6 = mPos - num5 = i - if num5 == num7: - break - num8 = node.mNext - if num5 >= 3: - for j in range(num5): - num9 = num3 + j - 2048 - if num9 >= 0: - index.remove(data[num9], num9) - index.add(data[num3+j], num3+j) - if num4 < num3: - array[num2] = (num3 - num4 - 1) - num2 = num2+1 - for j in range(num4, num3): - array[num2] = data[j] - num2 = num2+1 - num10 = num5 - 3 - num11 = num3 - num6 - 1 - num12 = 0x80 | num10 - num12 |= (num11 & 0x700) >> 4 - array[num2] = num12 - array[num2+1] = (num11&0xff) - num2 = num2+2 - num3 = num3 + num5 - num4 = num3 - else: - num9 = num3 - 2048 - if num9 >= 0: - index.remove(data[num9],num9) - index.add(data[num3], num3) - num3 = num3+1 - if num3 - num4 == 128: - array[num2] = num3 - num4 - 1 - num2 = num2+1 - for j in range(num4, num3): - array[num2] = data[j] - num2 = num2+1 - num4 = num3 - if num4 < num3: - array[num2] = num3 - num4 - 1 - num2 = num2+1 - for j in range(num4, num3): - array[num2] = data[j] - num2 = num2+1 - osize = num2 - array2 = bytearray(osize + 4) - array2[:4] = bytes2[:4] - array2[4:] = array[:osize] - return array2 + num = len(data) + bytes2 = num.to_bytes(4, byteorder="little") + anum3 = num + num / 128 + 1 + array = bytearray(int(anum3)) + num2 = 0 + num3 = 0 + num4 = 0 + index = Index() + while num3 < num: + num5 = 0 + num6 = 0 + num7 = min(18, num - num3) + num8 = index.getFirst(data[num3]) + while not index.isEnd(num8): + node = index.getNode(num8) + mPos = node.mPos + i = 1 + while i < num7: + if data[mPos + i] != data[num3 + i]: + break + i = i + 1 + if num5 < i: + num6 = mPos + num5 = i + if num5 == num7: + break + num8 = node.mNext + if num5 >= 3: + for j in range(num5): + num9 = num3 + j - 2048 + if num9 >= 0: + index.remove(data[num9], num9) + index.add(data[num3 + j], num3 + j) + if num4 < num3: + array[num2] = num3 - num4 - 1 + num2 = num2 + 1 + for j in range(num4, num3): + array[num2] = data[j] + num2 = num2 + 1 + num10 = num5 - 3 + num11 = num3 - num6 - 1 + num12 = 0x80 | num10 + num12 |= (num11 & 0x700) >> 4 + array[num2] = num12 + array[num2 + 1] = num11 & 0xFF + num2 = num2 + 2 + num3 = num3 + num5 + num4 = num3 + else: + num9 = num3 - 2048 + if num9 >= 0: + index.remove(data[num9], num9) + index.add(data[num3], num3) + num3 = num3 + 1 + if num3 - num4 == 128: + array[num2] = num3 - num4 - 1 + num2 = num2 + 1 + for j in range(num4, num3): + array[num2] = data[j] + num2 = num2 + 1 + num4 = num3 + if num4 < num3: + array[num2] = num3 - num4 - 1 + num2 = num2 + 1 + for j in range(num4, num3): + array[num2] = data[j] + num2 = num2 + 1 + osize = num2 + array2 = bytearray(osize + 4) + array2[:4] = bytes2[:4] + array2[4:] = array[:osize] + return array2 + class Node: - mNext = 0 - mPrev = 0 - mPos = 0 + mNext = 0 + mPrev = 0 + mPos = 0 + class Index: - mNodes = [] - mStack = [] - mStackPos = 0 + mNodes = [] + mStack = [] + mStackPos = 0 - def __init__(self): - for i in range(2304): - x = Node() - self.mNodes.append(x) - for i in range(2048, 2304): - self.mNodes[i].mNext = i - self.mNodes[i].mPrev = i - for i in range(2048): - self.mStack.append(i) - self.mStackPos = 2048 + def __init__(self): + for i in range(2304): + x = Node() + self.mNodes.append(x) + for i in range(2048, 2304): + self.mNodes[i].mNext = i + self.mNodes[i].mPrev = i + for i in range(2048): + self.mStack.append(i) + self.mStackPos = 2048 - def getFirst(self, c): - return self.mNodes[2048+c].mNext + def getFirst(self, c): + return self.mNodes[2048 + c].mNext - def getNode(self, i): - return self.mNodes[i] + def getNode(self, i): + return self.mNodes[i] - def add(self, c, pos): - self.mStackPos = self.mStackPos - 1 - num = self.mStack[self.mStackPos] - node = self.mNodes[num] - node2 = self.mNodes[2048+c] - node.mNext = node2.mNext - node.mPrev = 2048 + c - node.mPos = pos - self.mNodes[node2.mNext].mPrev = num - node2.mNext = num + def add(self, c, pos): + self.mStackPos = self.mStackPos - 1 + num = self.mStack[self.mStackPos] + node = self.mNodes[num] + node2 = self.mNodes[2048 + c] + node.mNext = node2.mNext + node.mPrev = 2048 + c + node.mPos = pos + self.mNodes[node2.mNext].mPrev = num + node2.mNext = num - def remove(self, c, pos): - mPrev = self.mNodes[2048+c].mPrev - node = self.mNodes[mPrev] - self.mStack[self.mStackPos] = self.mNodes[node.mPrev].mNext - self.mStackPos = self.mStackPos + 1 - self.mNodes[node.mPrev].mNext = node.mNext - self.mNodes[node.mNext].mPrev = node.mPrev + def remove(self, c, pos): + mPrev = self.mNodes[2048 + c].mPrev + node = self.mNodes[mPrev] + self.mStack[self.mStackPos] = self.mNodes[node.mPrev].mNext + self.mStackPos = self.mStackPos + 1 + self.mNodes[node.mPrev].mNext = node.mNext + self.mNodes[node.mNext].mPrev = node.mPrev + + def isEnd(self, idx): + return idx >= 2048 - def isEnd(self, idx): - return idx >= 2048 def xor_crypt(data, key): - odata = bytearray(len(data)) - odata[:] = data - i = 0 - for x in data: - if (x != 0): - m = key[i % len(key)] - if (x != m): - odata[i] = x ^ m - i += 1 - return odata + odata = bytearray(len(data)) + odata[:] = data + i = 0 + for x in data: + if x != 0: + m = key[i % len(key)] + if x != m: + odata[i] = x ^ m + i += 1 + return odata diff --git a/utage/names.py b/utage/names.py index 2392b8c..7d4504a 100644 --- a/utage/names.py +++ b/utage/names.py @@ -8,74 +8,95 @@ from functools import partial from glob import glob from multiprocessing import Pool + def update_names(dir_in, old_path, languages): - names = extract_names(dir_in) + names = extract_names(dir_in) - # we have to update japanese like the rest because CustomData exists - # and POKELABO deleted the wedding gear event stuff from the game files - for l in languages: - if l == "jpn": - out_dict = dict(zip(names, names)) - else: - out_dict = dict.fromkeys(names, "") + # we have to update japanese like the rest because CustomData exists + # and POKELABO deleted the wedding gear event stuff from the game files + for l in languages: + if l == "jpn": + out_dict = dict(zip(names, names)) + else: + out_dict = dict.fromkeys(names, "") - langfile = os.path.join(old_path, "nametranslations_{}.json".format(l)) - if os.path.isfile(langfile): - with open(langfile, "r") as lang_file: - lang_dict = json.load(lang_file) - out_dict.update(lang_dict) + langfile = os.path.join(old_path, "nametranslations_{}.json".format(l)) + if os.path.isfile(langfile): + with open(langfile, "r") as lang_file: + lang_dict = json.load(lang_file) + out_dict.update(lang_dict) + + with io.open( + langfile, "w", newline="\n" + ) as lang_file: # you're using git right + json.dump( + out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True + ) - with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right - json.dump(out_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True) def extract_names(dir_in): - char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv") + char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv") - if not os.path.isfile(char_tsv_path): - raise FileNotFoundError("Character.tsv not found") + if not os.path.isfile(char_tsv_path): + raise FileNotFoundError("Character.tsv not found") - names, sets = read_char_tsv(char_tsv_path) + names, sets = read_char_tsv(char_tsv_path) - files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)] - if len(files) == 0: - raise FileNotFoundError("No valid files found in directory: " + dir_in) + files = [ + f + for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) + if re.search(r"/[0-9]{9}\.tsv$", f) + ] + if len(files) == 0: + raise FileNotFoundError("No valid files found in directory: " + dir_in) - # tfw gil - # it's still faster on my machine so i'm keeping it - read_partial = partial(read_mission, char_names=names, char_sets=sets) - with Pool() as p: - new_names = p.map(read_partial, files) - for x in new_names: - names.update(x) + # tfw gil + # it's still faster on my machine so i'm keeping it + read_partial = partial(read_mission, char_names=names, char_sets=sets) + with Pool() as p: + new_names = p.map(read_partial, files) + for x in new_names: + names.update(x) + + return names - return names def read_char_tsv(char_tsv_path): - char_names = set() - char_sets = set() + char_names = set() + char_sets = set() - with open(char_tsv_path, "r") as char_tsv_file: - char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar="\"") + with open(char_tsv_path, "r") as char_tsv_file: + char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"') - for row in char_tsv: - if row['CharacterName'].startswith("//"): - continue - if row['CharacterName'] and row['NameText'] and row['CharacterName'].strip() and row['NameText'].strip(): - char_names.add(row['NameText']) - char_sets.add(row['CharacterName']) + for row in char_tsv: + if row["CharacterName"].startswith("//"): + continue + if ( + row["CharacterName"] + and row["NameText"] + and row["CharacterName"].strip() + and row["NameText"].strip() + ): + char_names.add(row["NameText"]) + char_sets.add(row["CharacterName"]) + + return char_names, char_sets - return char_names, char_sets def read_mission(tsv_path, char_names, char_sets): - new_names = set() - with open(tsv_path, "r") as tsv_file: - tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"") - if ('Arg1' not in tsv.fieldnames) or ('Text' not in tsv.fieldnames) or ('Command' not in tsv.fieldnames): - return new_names - for row in tsv: - if row['Command'] and row['Command'].startswith("//"): - continue - if row['Text'] and row['Arg1']: - if (row['Arg1'] not in char_names) and (row['Arg1'] not in char_sets): - new_names.add(row['Arg1']) - return new_names + new_names = set() + with open(tsv_path, "r") as tsv_file: + tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"') + if ( + ("Arg1" not in tsv.fieldnames) + or ("Text" not in tsv.fieldnames) + or ("Command" not in tsv.fieldnames) + ): + return new_names + for row in tsv: + if row["Command"] and row["Command"].startswith("//"): + continue + if row["Text"] and row["Arg1"]: + if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets): + new_names.add(row["Arg1"]) + return new_names diff --git a/utage/translate.py b/utage/translate.py index 6ad21a6..3853b56 100644 --- a/utage/translate.py +++ b/utage/translate.py @@ -10,89 +10,109 @@ from functools import partial from glob import glob from multiprocessing.pool import Pool + def translate_dir(dir_in, tsv_out_dir, json_out_dir): - # we only want to key files that are mission ids - files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)] + # we only want to key files that are mission ids + files = [ + f + for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) + if re.search(r"/[0-9]{9}\.tsv$", f) + ] - if len(files) == 0: - raise FileNotFoundError("No valid files found in directory: " + dir_in) + if len(files) == 0: + raise FileNotFoundError("No valid files found in directory: " + dir_in) + + ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir) + with Pool() as p: + p.map(ptrans, files) - ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir) - with Pool() as p: - p.map(ptrans, files) def translate_file(file_in, tsv_out_dir, json_out_dir): - try: - if not file_in.endswith(".tsv"): - raise ValueError("Invalid File Type for {}".format(file_in)) + try: + if not file_in.endswith(".tsv"): + raise ValueError("Invalid File Type for {}".format(file_in)) - # we need to get the event folder - # ie, Utage/>>>main01<<>>main01<<