diff --git a/db/__init__.py b/db/__init__.py index eef08a0..533b8b4 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -1,4 +1 @@ -from .base import init -from .quest import Quest, QuestScene, QuestPart -from .resource import ResourceEntry -from .base import session +from .quest import QuestDB diff --git a/db/base.py b/db/base.py index 097755c..afa5817 100644 --- a/db/base.py +++ b/db/base.py @@ -1,15 +1,5 @@ import os - -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from sqlalchemy import text -from sqlalchemy.ext.declarative import declarative_base - -Base = declarative_base() - -engine = create_engine("sqlite:///:memory:", echo=False) -session = sessionmaker(bind=engine, autoflush=False, autocommit=False)() -session.flush = lambda: None +import sqlite3 DATABASES = { "QuestMst.db": "Quest", @@ -19,11 +9,28 @@ DATABASES = { } -def init(path): - for d in DATABASES: - if not os.path.isfile(os.path.join(path, d)): - raise FileNotFoundError("Database {} not found".format(d)) +class BaseDB(object): + conn = None + cursor = None - for d in DATABASES: - t = text("attach database :path as :schema") - engine.execute(t, path=os.path.join(path, d), schema=DATABASES[d]) + def __init__(self, path, databases): + self.conn = sqlite3.connect(":memory:") + self.conn.row_factory = sqlite3.Row + self.cursor = self.conn.cursor() + self.attach_dbs(databases, path) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.quest_conn.close() + + def attach_dbs(self, databases, path): + for d in databases: + if not os.path.isfile(os.path.join(path, d)): + raise FileNotFoundError(f"Database {d} not found") + + for d in databases.items(): + self.cursor.execute( + "ATTACH DATABASE ? AS ?", (os.path.join(path, d[0]), d[1]) + ) diff --git a/db/quest.py b/db/quest.py index f4d2539..5464324 100644 --- a/db/quest.py +++ b/db/quest.py @@ -1,76 +1,74 @@ -from sqlalchemy import Column, Integer, BigInteger, String, ForeignKey -from sqlalchemy.orm import relationship +from .base import BaseDB -from .base import Base +DATABASES = { + "QuestMst.db": "Quest", + "QuestSceneMst.db": "QuestScene", + "QuestPartMst.db": "QuestPart", + "ResourceEntry.db": "ResourceEntry", +} -class Quest(Base): - __tablename__ = "QuestMstRecord" - __table_args__ = {"schema": "Quest"} +class QuestDB(BaseDB): + def __init__(self, path): + super().__init__(path, DATABASES) - questMstId = Column(Integer, primary_key=True) - questType = Column(Integer) - sortNum = Column(Integer) - name = Column(String) - prevQuestMstId = Column(Integer) - eventItemMstIds = Column(Integer) - valid = Column(Integer) - baseQuestMstId = Column(Integer) - updatedTime = Column(BigInteger) + def get_quests(self): + self.cursor.execute( + """ + SELECT + q.questMstId, + q.name, + group_concat(DISTINCT qp.questSceneMstId) AS scenes + FROM Quest.QuestMstRecord q + INNER JOIN QuestScene.QuestSceneMstRecord qs + ON q.questMstId = qs.questMstId + INNER JOIN QuestPart.QuestPartMstRecord qp + ON qs.questSceneMstId = qp.questSceneMstId + WHERE q.baseQuestMstId = 0 + AND (qp.beforeTalkName != "" OR qp.afterTalkName != "") + GROUP BY q.questMstId + """ + ) + res = self.cursor.fetchall() + quests = {} + for quest_id, name, scenes in res: + quests[quest_id] = {} + quests[quest_id]["Name"] = name + quests[quest_id]["Scenes"] = scenes.split(",") + return quests - scenes = relationship("QuestScene") - - def __repr__(self): - return "".format(self.questMstId, self.name) - - -class QuestScene(Base): - __tablename__ = "QuestSceneMstRecord" - __table_args__ = {"schema": "QuestScene"} - - questSceneMstId = Column(Integer, primary_key=True) - questMstId = Column(Integer, ForeignKey("Quest.QuestMstRecord.questMstId")) - _type = Column("type", Integer) - sortNum = Column(Integer) - name = Column(String) - _filter = Column("filter", Integer) - prevQuestSceneMstId = Column( - Integer, ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId") - ) - releaseSerial = Column(Integer) - releaseEvolutionLevel = Column(Integer) - summaryText = Column(String) - presentType = Column(String) - objectId = Column(Integer) - num = Column(Integer) - appearanceType = Column(Integer) - isPrologue = Column(Integer) - isEpilogue = Column(Integer) - viewType = Column(Integer) - updatedTime = Column(BigInteger) - - previous = relationship("QuestScene", remote_side=[questSceneMstId]) - parts = relationship("QuestPart") - - -class QuestPart(Base): - __tablename__ = "QuestPartMstRecord" - __table_args__ = {"schema": "QuestPart"} - - questSceneMstId = Column( - Integer, - ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId"), - primary_key=True, - ) # now this is pod racing - partNum = Column(Integer, primary_key=True) - waveNum = Column(Integer) - stamina = Column(Integer) - exp = Column(Integer) - expertPoint = Column(Integer) - recommendLevel = Column(Integer) - beforeTalkName = Column(String) - afterTalkName = Column(String) - battleBackgroundImg = Column(String) - musicMstId = Column(Integer) - isFixedDeck = Column(Integer) - updatedTime = Column(BigInteger) + def get_scenes(self): + self.cursor.execute( + """ + SELECT + questSceneMstId, + name, + summaryText, + group_concat(partIds) as parts + FROM ( + SELECT + qs.questSceneMstId, + qs.name, + qs.summaryText, + CASE + WHEN qp.afterTalkName == '' AND qp.beforeTalkName == '' THEN NULL + WHEN qp.afterTalkName == '' AND qp.beforeTalkName != '' THEN qp.beforeTalkName + WHEN qp.afterTalkName != '' AND qp.beforeTalkName == '' THEN qp.afterTalkName + ELSE qp.beforeTalkName || ',' || qp.afterTalkName + END AS partIds + FROM QuestScene.QuestSceneMstRecord qs + INNER JOIN QuestPart.QuestPartMstRecord qp + ON qs.questSceneMstId = qp.questSceneMstId + WHERE (qp.beforeTalkName != "" OR qp.afterTalkName != "") + ) + GROUP BY questSceneMstId + """ + ) + res = self.cursor.fetchall() + scenes = {} + for scene_id, name, summary, parts in res: + scenes[scene_id] = {} + scenes[scene_id]["Name"] = name + scenes[scene_id]["SummaryText"] = summary + scenes[scene_id]["Parts"] = parts.split(",") + return scenes diff --git a/db/resource.py b/db/resource.py deleted file mode 100644 index b624ec0..0000000 --- a/db/resource.py +++ /dev/null @@ -1,14 +0,0 @@ -from sqlalchemy import Column, BigInteger, String - -from .base import Base - - -class ResourceEntry(Base): - __tablename__ = "ResourceEntryRecord" - __table_args__ = {"schema": "ResourceEntry"} - - path = Column(String, primary_key=True) - serverPath = Column(String) - localPath = Column(String) - digest = Column(String) - fileSize = Column(BigInteger) diff --git a/diva/quest.py b/diva/quest.py index 6f166cf..4a20477 100644 --- a/diva/quest.py +++ b/diva/quest.py @@ -3,153 +3,92 @@ import io import json import os -from sqlalchemy.orm import contains_eager - def update_missions(dir_in, old_path, languages): - db.init(dir_in) - quest_mst(old_path, languages) - scene_mst(old_path, languages) + qdb = db.QuestDB(dir_in) + quest_mst(qdb, old_path, languages) + scene_mst(qdb, old_path, languages) -def scene_mst(old_path, languages): - result = ( - db.session.query(db.QuestScene) - .join(db.QuestScene.parts) - .options(contains_eager(db.QuestScene.parts)) - .filter( - (db.QuestPart.beforeTalkName != "") | (db.QuestPart.afterTalkName != "") - ) - .filter(db.QuestScene.parts.any()) - .all() - ) - - scenes = {} - for x in result: - scenes[str(x.questSceneMstId)] = { - "Name": x.name, - "SummaryText": x.summaryText, - "Parts": [], - "Folder": "", - } - - id_num = "" - # this is slow as balls but i'm way too tired to figure out how to optimize it - for y in x.parts: - if y.beforeTalkName: - id_num = y.beforeTalkName - scenes[str(x.questSceneMstId)]["Parts"].append(y.beforeTalkName) - if y.afterTalkName: - id_num = y.afterTalkName - scenes[str(x.questSceneMstId)]["Parts"].append(y.afterTalkName) - - # i checked every entry, moving it to do once for speed - if not scenes[str(x.questSceneMstId)]["Folder"] and id_num: - path = ( - db.session.query(db.ResourceEntry) - .filter( - db.ResourceEntry.path.like( - "%/Scenario/{}.tsv.utage".format(id_num) - ) - ) - .filter( - db.ResourceEntry.path.notlike("%Utage/side03/%.tsv.utage") - ) # dead folder with dupes too lazy to parse settings tsv - .one() - ) - folder = path.path.split("/")[2] - if ( - scenes[str(x.questSceneMstId)]["Folder"] - and scenes[str(x.questSceneMstId)]["Folder"] != folder - ): - raise KeyError( - "Internal path inconsitency, bailing out {}".format(id_num) - ) - scenes[str(x.questSceneMstId)]["Folder"] = folder +def scene_mst(qdb, old_path, languages): + scenes = qdb.get_scenes() with io.open( os.path.join(old_path, "XduScene.json"), "w", newline="\n" ) as json_file: - json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True) + json.dump( + scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True + ) - for l in languages: + # TODO: reimplement folder finder + + for lang in languages: out_dict = {} - if l == "jpn": - for x in result: - out_dict[str(x.questSceneMstId)] = { - "Name": x.name, - "SummaryText": x.summaryText, + if lang == "jpn": + for key, value in scenes.items(): + out_dict[key] = { + "Name": value.name, + "SummaryText": value.summaryText, "Credits": "POKELABO", "Enabled": False, } else: - for x in result: - out_dict[str(x.questSceneMstId)] = { + for key in scenes.keys(): + out_dict[key] = { "Name": "", "SummaryText": "", "Credits": "", "Enabled": False, } - langfile = os.path.join(old_path, "XduSceneNames_{}.json".format(l)) + langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json") if os.path.isfile(langfile): with open(langfile, "r") as lang_file: lang_dict = json.load(lang_file) - out_dict.update(lang_dict) + out_dict.update(lang_dict) - with io.open( - langfile, "w", newline="\n" - ) as lang_file: # you're using git right + with io.open(langfile, "w", newline="\n") as lang_file: json.dump( - out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True + out_dict, + lang_file, + ensure_ascii=False, + indent="\t", + sort_keys=True, ) -def quest_mst(old_path, languages): - # i regret my life choices - result = ( - db.session.query(db.Quest) - .filter(db.Quest.baseQuestMstId == 0) - .join(db.Quest.scenes) - .join(db.QuestScene.parts) - .options(contains_eager(db.Quest.scenes).contains_eager(db.QuestScene.parts)) - .filter(db.QuestPart.beforeTalkName != "" or db.QuestPart.afterTalkName != "") - .filter(db.QuestScene.parts.any()) - .filter(db.Quest.scenes.any()) - .all() - ) - quests = {} - for x in result: - quests[str(x.questMstId)] = { - "Name": x.name, - "Scenes": [d.questSceneMstId for d in x.scenes], - } +def quest_mst(qdb, old_path, languages): + quests = qdb.get_quests() with io.open( os.path.join(old_path, "XduQuest.json"), "w", newline="\n" ) as json_file: - json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True) + json.dump( + quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True + ) - for l in languages: + for lang in languages: out_dict = {} - if l == "jpn": - for x in result: - out_dict[str(x.questMstId)] = {"Name": x.name, "Enabled": False} + if lang == "jpn": + for key, value in quests.items(): + out_dict[key] = {"Name": value.name, "Enabled": False} else: - for x in result: - out_dict[str(x.questMstId)] = {"Name": "", "Enabled": False} + for key, value in quests.items(): + out_dict[key] = {"Name": "", "Enabled": False} - langfile = os.path.join(old_path, "XduQuestNames_{}.json".format(l)) + langfile = os.path.join(old_path, f"XduQuestNames_{lang}.json") if os.path.isfile(langfile): with open(langfile, "r") as lang_file: lang_dict = json.load(lang_file) - out_dict.update(lang_dict) + out_dict.update(lang_dict) - with io.open( - langfile, "w", newline="\n" - ) as lang_file: # you're using git right + with io.open(langfile, "w", newline="\n") as lang_file: json.dump( - out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True + out_dict, + lang_file, + ensure_ascii=False, + indent="\t", + sort_keys=True, ) return quests