sqlalchemy ORM models for quest databases and json parsing scripts

also minor bugfixes
next is testing and implementation

adx/__init__.py

@@ -0,0 +1 @@
+from .adx import *

adx/adx.py

@@ -2,6 +2,7 @@ import glob
 import json
 import os
 import traceback
+
 from .adx_parser import adx_file
 
 def extract_loop_data_from_dir(dir_in, fout):

db/__init__.py

@@ -0,0 +1,4 @@
+from .base import init
+from .quest import Quest, QuestScene, QuestPart
+from .resource import ResourceEntry
+from .base import session

db/base.py

@@ -0,0 +1,28 @@
+import os
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import text
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
+
+engine = create_engine('sqlite:///:memory:', echo=False)
+session = sessionmaker(bind=engine, autoflush=False, autocommit=False)()
+session.flush = lambda:None
+
+DATABASES = {
+	"QuestMst.db": "Quest",
+	"QuestSceneMst.db": "QuestScene",
+	"QuestPartMst.db": "QuestPart",
+	"ResourceEntry.db": "ResourceEntry"
+}
+
+def init(path):
+	for d in DATABASES:
+		if not os.path.isfile(os.path.join(path, d)):
+			raise FileNotFoundError("Database {} not found".format(d))
+
+	for d in DATABASES:
+		t = text("attach database :path as :schema")
+		engine.execute(t, path=os.path.join(path, d), schema=DATABASES[d])

db/quest.py

@@ -0,0 +1,67 @@
+from sqlalchemy import Column, Integer, BigInteger, String, ForeignKey
+from sqlalchemy.orm import relationship
+
+from .base import Base
+
+class Quest(Base):
+	__tablename__ = 'QuestMstRecord'
+	__table_args__ = {'schema': 'Quest'}
+
+	questMstId = Column(Integer, primary_key=True)
+	questType = Column(Integer)
+	sortNum = Column(Integer)
+	name = Column(String)
+	prevQuestMstId = Column(Integer)
+	eventItemMstIds = Column(Integer)
+	valid = Column(Integer)
+	baseQuestMstId = Column(Integer)
+	updatedTime = Column(BigInteger)
+
+	scenes = relationship("QuestScene")
+
+	def __repr__(self):
+		return "<Quest(questMstId={}, name={})>".format(self.questMstId, self.name)
+
+class QuestScene(Base):
+	__tablename__ = 'QuestSceneMstRecord'
+	__table_args__ = {'schema': 'QuestScene'}
+
+	questSceneMstId = Column(Integer, primary_key=True)
+	questMstId = Column(Integer, ForeignKey("Quest.QuestMstRecord.questMstId"))
+	_type = Column("type", Integer)
+	sortNum = Column(Integer)
+	name = Column(String)
+	_filter = Column("filter", Integer)
+	prevQuestSceneMstId = Column(Integer, ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId"))
+	releaseSerial = Column(Integer)
+	releaseEvolutionLevel = Column(Integer)
+	summaryText = Column(String)
+	presentType = Column(String)
+	objectId = Column(Integer)
+	num = Column(Integer)
+	appearanceType = Column(Integer)
+	isPrologue = Column(Integer)
+	isEpilogue = Column(Integer)
+	viewType = Column(Integer)
+	updatedTime = Column(BigInteger)
+
+	previous = relationship("QuestScene", remote_side=[questSceneMstId])
+	parts = relationship("QuestPart")
+
+class QuestPart(Base):
+	__tablename__ = 'QuestPartMstRecord'
+	__table_args__ = {'schema': 'QuestPart'}
+
+	questSceneMstId = Column(Integer, ForeignKey("QuestScene.QuestSceneMstRecord.questSceneMstId"), primary_key=True) # now this is pod racing
+	partNum = Column(Integer, primary_key=True)
+	waveNum = Column(Integer)
+	stamina = Column(Integer)
+	exp = Column(Integer)
+	expertPoint = Column(Integer)
+	recommendLevel = Column(Integer)
+	beforeTalkName = Column(String)
+	afterTalkName = Column(String)
+	battleBackgroundImg = Column(String)
+	musicMstId = Column(Integer)
+	isFixedDeck = Column(Integer)
+	updatedTime = Column(BigInteger)

db/resource.py

@@ -0,0 +1,13 @@
+from sqlalchemy import Column, BigInteger, String
+
+from .base import Base
+
+class ResourceEntry(Base):
+	__tablename__ = 'ResourceEntryRecord'
+	__table_args__ = {'schema': 'ResourceEntry'}
+
+	path = Column(String, primary_key=True)
+	serverPath = Column(String)
+	localPath = Column(String)
+	digest = Column(String)
+	fileSize = Column(BigInteger)

diva/__init__.py

@@ -0,0 +1 @@
+from . import quest

diva/quest.py

@@ -0,0 +1,126 @@
+import db
+import io
+import json
+import os
+
+from sqlalchemy.orm import contains_eager
+
+def update_missions(dir_in, old_path, languages):
+	db.init(dir_in)
+	quest_mst(old_path, languages)
+	scene_mst(old_path, languages)
+
+def scene_mst(old_path, languages):
+	result = (db.session.query(db.QuestScene)
+	                    .join(db.QuestScene.parts)
+	                    .options(contains_eager(db.QuestScene.parts))
+	                    .filter(db.QuestPart.beforeTalkName != "" or db.QuestPart.afterTalkName != "")
+	                    .filter(db.QuestScene.parts.any())
+	                    .all())
+
+	scenes = {}
+	for x in result:
+		scenes[str(x.questSceneMstId)] = { "Name": x.name,
+		                                   "SummaryText": x.summaryText,
+		                                   "Parts": [],
+		                                   "Enabled": False,
+		                                   "Folder": "" }
+
+		id_num = ""
+		# this is slow as balls but i'm way too tired to figure out how to optimize it
+		for y in x.parts:
+			if y.beforeTalkName:
+				id_num = y.beforeTalkName
+				scenes[str(x.questSceneMstId)]["Parts"].append(y.beforeTalkName)
+			if y.afterTalkName:
+				id_num = y.afterTalkName
+				scenes[str(x.questSceneMstId)]["Parts"].append(y.afterTalkName)
+
+			# i checked every entry, moving it to do once for speed
+			if not scenes[str(x.questSceneMstId)]["Folder"] and id_num:
+				path = (db.session.query(db.ResourceEntry)
+				                  .filter(db.ResourceEntry.path.like("%/Scenario/{}.tsv.utage".format(id_num)))
+				                  .filter(db.ResourceEntry.path.notlike("%Utage/side03/%.tsv.utage")) # dead folder with dupes too lazy to parse settings tsv
+				                  .one())
+				folder = path.path.split("/")[2]
+				if scenes[str(x.questSceneMstId)]["Folder"] and scenes[str(x.questSceneMstId)]["Folder"] != folder:
+					raise KeyError("Internal path inconsitency, bailing out {}".format(y.afterTalkName))
+				scenes[str(x.questSceneMstId)]["Folder"] = folder
+
+	if os.path.isfile(os.path.join(old_path, "XduScene.json")):
+		with open(os.path.join(old_path, "XduScene.json"), "r") as old_json_file:
+			old_json = json.load(old_json_file)
+		for key in old_json:
+			if old_json[key]['Enabled'] == True and key in scenes:
+				scenes[key]['Enabled'] = True
+
+	with io.open(os.path.join(old_path, "XduScene.json"), "w", newline='\n') as json_file:
+		json.dump(scenes, json_file, ensure_ascii=False, indent='\t', sort_keys=True)
+
+	for l in languages:
+		out_dict = {}
+		if l == "jpn":
+			for x in result:
+				out_dict[str(x.questSceneMstId)] = { "Name": x.name,
+				                                     "SummaryText": x.summaryText }
+		else:
+			for x in result:
+				out_dict[str(x.questSceneMstId)] = { "Name": "",
+				                                     "SummaryText": "" }
+
+		langfile = os.path.join(old_path, "XduSceneNames_{}.json".format(l))
+		if os.path.isfile(langfile):
+			with open(langfile, "r") as lang_file:
+				lang_dict = json.load(lang_file)
+			out_dict.update(lang_dict)
+
+		with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right
+			json.dump(out_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True)
+
+def quest_mst(old_path, languages):
+	# i regret my life choices
+	result = (db.session.query(db.Quest)
+	                    .filter(db.Quest.baseQuestMstId == 0)
+	                    .join(db.Quest.scenes)
+	                    .join(db.QuestScene.parts)
+	                    .options(contains_eager(db.Quest.scenes).
+	                             contains_eager(db.QuestScene.parts))
+	                    .filter(db.QuestPart.beforeTalkName != "" or db.QuestPart.afterTalkName != "")
+	                    .filter(db.QuestScene.parts.any())
+	                    .filter(db.Quest.scenes.any())
+	                    .all())
+	quests = {}
+	for x in result:
+		quests[str(x.questMstId)] = { "Name": x.name, "Scenes": [d.questSceneMstId for d in x.scenes], "Enabled": False }
+
+	if os.path.isfile(os.path.join(old_path, "XduQuest.json")):
+		with open(os.path.join(old_path, "XduQuest.json"), "r") as old_json_file:
+			old_json = json.load(old_json_file)
+		for key in old_json:
+			if old_json[key]['Enabled'] == True and key in quests:
+				quests[key]['Enabled'] = True
+
+	with io.open(os.path.join(old_path, "XduQuest.json"), "w", newline='\n') as json_file:
+		json.dump(quests, json_file, ensure_ascii=False, indent='\t', sort_keys=True)
+
+
+	for l in languages:
+		out_dict = {}
+		if l == "jpn":
+			for x in result:
+				out_dict[str(x.questMstId)] = { "Name": x.name }
+		else:
+			for x in result:
+				out_dict[str(x.questMstId)] = { "Name": "" }
+
+		langfile = os.path.join(old_path, "XduQuestNames_{}.json".format(l))
+		if os.path.isfile(langfile):
+			with open(langfile, "r") as lang_file:
+				lang_dict = json.load(lang_file)
+			out_dict.update(lang_dict)
+
+		with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right
+			json.dump(out_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True)
+
+
+	return quests

divatool.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
 
+import adx
 import argparse
+import diva
 import os
 import sys
 import utage
-
+import db
-from adx import adx
 
 LANGUAGES = ["jpn", "eng", "rus"]
 
@@ -46,7 +47,7 @@ def main():
 	parser_utage_crypt.add_argument("TARGET",
 	                                help="Input", type=str)
 	# utage translate
-	parser_utage_translate = parser_utage_subparsers.add_parser("translate", help="Generate keyed tsv and json file")
+	parser_utage_translate = parser_utage_subparsers.add_parser("translate", help="Generate keyed tsv and json files")
 	parser_utage_translate.add_argument("INPUT",
 	                                    help="Input", type=str)
 	parser_utage_translate.add_argument("TSVDIR", nargs='?',
@@ -58,7 +59,16 @@ def main():
 	parser_utage_names.add_argument("INPUT",
 	                                help="Input", type=str)
 	parser_utage_names.add_argument("OLD", nargs='?',
-	                                help="Directory with old files", type=str, default="")
+	                                help="Directory with old files", type=str, default=".")
+
+	# diva subcommand
+	parser_diva = subparsers.add_parser("diva", help="Diva stuff")
+	parser_diva_subparsers = parser_diva.add_subparsers(metavar="<command>", title="subcommand", dest="subcommand")
+	parser_diva_quest = parser_diva_subparsers.add_parser("quest", help="Generate quest JSON files")
+	parser_diva_quest.add_argument("INPUT", help="Directory of databases containing Quest*.db and ResourceEntry.db",
+	                               type=str)
+	parser_diva_quest.add_argument("OLD", help="Directory with old quest files",
+	                               nargs='?', type=str, default=".")
 
 	args = parser.parse_args()
 
@@ -89,6 +99,9 @@ def main():
 				utage.names.update_names(args.INPUT, args.OLD, LANGUAGES)
 			else:
 				raise FileNotFoundError(args.INPUT)
+	elif args.command == "diva":
+		if args.subcommand == "quest":
+			diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES)
 
 	return 0
 

utage/names.py

@@ -13,22 +13,20 @@ def update_names(dir_in, old_path, languages):
 
 	# we have to update japanese like the rest because CustomData exists
 	# and POKELABO deleted the wedding gear event stuff from the game files
-	for x in languages:
+	for l in languages:
-		langfile = os.path.join(old_path, "nametranslations_{}.json".format(x))
+		if l == "jpn":
-		if os.path.isfile(langfile):
-			with open(langfile, "r") as lang_file:
-				lang_dict = json.load(lang_file)
-			if x == "jpn":
-				out_dict = dict(zip(names, names)).update(lang_dict)
-			else:
-				out_dict = dict.fromkeys(names, "").update(lang_dict)
-		elif x == "jpn": # bootstrap
 			out_dict = dict(zip(names, names))
 		else:
 			out_dict = dict.fromkeys(names, "")
 
+		langfile = os.path.join(old_path, "nametranslations_{}.json".format(l))
+		if os.path.isfile(langfile):
+			with open(langfile, "r") as lang_file:
+				lang_dict = json.load(lang_file)
+			out_dict.update(lang_dict)
+
 		with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right
-			json.dump(lang_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True)
+			json.dump(out_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True)
 
 def extract_names(dir_in):
 	char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv")