xdu/divatool
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request 'Added --global flag for Global XDU' (#2) from feature/global_update into develop

Browse source
This commit is contained in:
argo neus 2020-03-06 12:47:37 -05:00
commit 1a12512c7e
7 changed files with 219 additions and 97 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -3,3 +3,6 @@ __pycache__/
*$py.class
*.swp
virtualenv/
tools/*
!tools/readme.txt
.idea/

24
diva/quest.py
View file

@ -26,18 +26,14 @@ def scene_mst(qdb, old_path, dir_in, languages):
except KeyError as exc:
i += 1
if i == len(scene["Parts"]):
raise KeyError(
f"Could not find folder for scene {id}"
) from exc
raise KeyError(f"Could not find folder for scene {id}") from exc
continue
break
with io.open(
os.path.join(old_path, "XduScene.json"), "w", newline="\n"
) as json_file:
json.dump(
scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True
)
json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
for lang in languages:
out_dict = {}
@ -66,11 +62,7 @@ def scene_mst(qdb, old_path, dir_in, languages):
with io.open(langfile, "w", newline="\n") as lang_file:
json.dump(
out_dict,
lang_file,
ensure_ascii=False,
indent="\t",
sort_keys=True,
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
)
@ -80,9 +72,7 @@ def quest_mst(qdb, old_path, languages):
with io.open(
os.path.join(old_path, "XduQuest.json"), "w", newline="\n"
) as json_file:
json.dump(
quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True
)
json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
for lang in languages:
out_dict = {}
@ -101,11 +91,7 @@ def quest_mst(qdb, old_path, languages):
with io.open(langfile, "w", newline="\n") as lang_file:
json.dump(
out_dict,
lang_file,
ensure_ascii=False,
indent="\t",
sort_keys=True,
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
)
return quests

36
divatool.py
View file

@ -15,6 +15,16 @@ LANGUAGES = ["jpn", "eng", "rus"]
def main():
parser = argparse.ArgumentParser(description="Unpack and process XDU data")
global_flag_parser = argparse.ArgumentParser(add_help=False)
global_flag_parser.add_argument(
"-g",
"--global",
help="Whether we are running on Global XDU.",
action="store_true",
dest="is_global",
default=False,
)
subparsers = parser.add_subparsers(
metavar="<command>", title="subcommands", dest="command"
)
@ -81,7 +91,9 @@ def main():
parser_utage_crypt.add_argument("TARGET", help="Input", type=str)
# utage translate
parser_utage_translate = parser_utage_subparsers.add_parser(
"translate", help="Generate keyed tsv and json files"
"translate",
help="Generate keyed tsv and json files",
parents=[global_flag_parser],
)
parser_utage_translate.add_argument("INPUT", help="Input", type=str)
parser_utage_translate.add_argument(
@ -92,7 +104,7 @@ def main():
)
# utage names
parser_utage_names = parser_utage_subparsers.add_parser(
"names", help="Generate and update name files"
"names", help="Generate and update name files", parents=[global_flag_parser]
)
parser_utage_names.add_argument("INPUT", help="Input", type=str)
parser_utage_names.add_argument(
@ -152,23 +164,35 @@ def main():
raise FileNotFoundError("Could not find {}".format(args.TARGET))
elif args.subcommand == "translate":
if os.path.isfile(args.INPUT):
utage.translate.translate_file(args.INPUT, args.TSVDIR, args.JSONDIR)
utage.translate.translate_file(
args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global
)
elif os.path.isdir(args.INPUT):
utage.translate.translate_dir(args.INPUT, args.TSVDIR, args.JSONDIR)
utage.translate.translate_dir(
args.INPUT, args.TSVDIR, args.JSONDIR, args.is_global
)
else:
raise FileNotFoundError("Could not find {}".format(args.INPUT))
elif args.subcommand == "names":
if os.path.isdir(args.INPUT):
utage.names.update_names(args.INPUT, args.OLD, LANGUAGES)
utage.names.update_names(
args.INPUT, args.OLD, LANGUAGES, args.is_global
)
else:
raise FileNotFoundError(args.INPUT)
elif args.command == "diva":
if args.subcommand == "quest":
diva.quest.update_missions(args.INPUT, args.OLD, LANGUAGES)
elif args.command == "xdudata":
script_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tools"))
if args.subcommand == "update":
xdudata.update_all(
args.CACHE, args.XDUDATA, args.TRANS, args.EXTRACT, LANGUAGES
args.CACHE,
args.XDUDATA,
args.TRANS,
args.EXTRACT,
script_dir,
LANGUAGES,
)
return 0

34
tools/readme.txt Normal file
View file

@ -0,0 +1,34 @@
Various files need to be in this directory.
If you don't have them, ask someone who does.
The required files are:
Binaries:
- acb_extract.exe
- usm_extract.exe
- clHCA
Z shell scripts:
- update_bgm.zsh
- update_movie.zsh
- update_se.zsh
- update_voice.zsh
Python scripts:
- adx_parse.py
- bingo_fuck.py
- chara_room_mission.py
- custom_mission.py
- event_bonus.py
- orphan_quest.py
- pad_2_csv.py
- pad_2_json.py
- pad_2_pad.py
- pad_2_pad_louisstyle.py
- pad_stat.py
- pad_update.py
- store_parser.py
- tsv_2_pad.py
- tsv_parse.py
- txt_2_pad.py
- utage_decrypt.py

38
utage/names.py
View file

@ -9,14 +9,25 @@ from glob import glob
from multiprocessing import Pool
def update_names(dir_in, old_path, languages):
names = extract_names(dir_in)
def update_names(dir_in, old_path, languages, is_global):
names = extract_names(dir_in, is_global)
global_languages = {"enm": 1, "zh": 2, "ko": 3}
if is_global:
languages = global_languages.keys()
# we have to update japanese like the rest because CustomData exists
# and POKELABO deleted the wedding gear event stuff from the game files
for l in languages:
if l == "jpn":
out_dict = dict(zip(names, names))
elif l in global_languages.keys():
out_dict = {}
for name in names:
spl = name.split("\t")
idx = global_languages[l]
val = spl[idx] if spl[idx] != "None" else ""
out_dict[spl[0]] = val
else:
out_dict = dict.fromkeys(names, "")
@ -30,21 +41,17 @@ def update_names(dir_in, old_path, languages):
langfile, "w", newline="\n"
) as lang_file: # you're using git right
json.dump(
out_dict,
lang_file,
ensure_ascii=False,
indent="\t",
sort_keys=True,
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
)
def extract_names(dir_in):
def extract_names(dir_in, is_global):
char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv")
if not os.path.isfile(char_tsv_path):
raise FileNotFoundError("Character.tsv not found")
names, sets = read_char_tsv(char_tsv_path)
names, sets = read_char_tsv(char_tsv_path, is_global)
files = [
f
@ -54,6 +61,8 @@ def extract_names(dir_in):
if len(files) == 0:
raise FileNotFoundError("No valid files found in directory: " + dir_in)
# TODO unsupported on global for now
if not is_global:
# tfw gil
# it's still faster on my machine so i'm keeping it
read_partial = partial(read_mission, char_names=names, char_sets=sets)
@ -65,7 +74,7 @@ def extract_names(dir_in):
return names
def read_char_tsv(char_tsv_path):
def read_char_tsv(char_tsv_path, is_global):
char_names = set()
char_sets = set()
@ -81,6 +90,11 @@ def read_char_tsv(char_tsv_path):
and row["CharacterName"].strip()
and row["NameText"].strip()
):
if is_global:
char_names.add(
f"{row['NameText']}\t{row['CharaEnglish']}\t{row['CharaChinese']}\t{row['CharaKorean']}"
)
else:
char_names.add(row["NameText"])
char_sets.add(row["CharacterName"])
@ -101,8 +115,6 @@ def read_mission(tsv_path, char_names, char_sets):
if row["Command"] and row["Command"].startswith("//"):
continue
if row["Text"] and row["Arg1"]:
if (row["Arg1"] not in char_names) and (
row["Arg1"] not in char_sets
):
if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets):
new_names.add(row["Arg1"])
return new_names

62
utage/translate.py
View file

@ -4,18 +4,17 @@ import io
import json
import os
import traceback
from contextlib import suppress
from functools import partial
from glob import glob
from multiprocessing.pool import Pool
def translate_dir(dir_in, tsv_out_dir, json_out_dir):
def translate_dir(dir_in, tsv_out_dir, json_out_dir, is_global):
files = [
f
for f in glob(
os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True
)
for f in glob(os.path.join(dir_in, "**/Scenario/*.tsv"), recursive=True)
if "{os.path.sep}Diva{os.path.sep}" not in f
]
@ -23,13 +22,16 @@ def translate_dir(dir_in, tsv_out_dir, json_out_dir):
raise FileNotFoundError("No valid files found in directory: " + dir_in)
ptrans = partial(
translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir
translate_file,
tsv_out_dir=tsv_out_dir,
json_out_dir=json_out_dir,
is_global=is_global,
)
with Pool() as p:
p.map(ptrans, files)
def translate_file(file_in, tsv_out_dir, json_out_dir):
def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
try:
if not file_in.endswith(".tsv"):
raise ValueError("Invalid File Type for {}".format(file_in))
@ -48,11 +50,17 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
tsv_output_path = os.path.join(
tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num)
)
json_output_path = os.path.join(
languages = ["jpn"]
if is_global:
languages += ["enm", "zh", "ko"]
json_output_paths = {
lang: os.path.join(
json_out_dir,
event_folder,
"{}_translations_jpn.json".format(id_num),
"{}_translations_{}.json".format(id_num, lang),
)
for lang in languages
}
# need to create output paths and avoid races when threading
if not os.path.exists(os.path.dirname(tsv_output_path)):
@ -62,16 +70,21 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
if e.errno != errno.EEXIST:
raise
if not os.path.exists(os.path.dirname(json_output_path)):
if not os.path.exists(os.path.dirname(json_output_paths["jpn"])):
try:
os.makedirs(os.path.dirname(json_output_path))
os.makedirs(os.path.dirname(json_output_paths["jpn"]))
except OSError as e:
if e.errno != errno.EEXIST:
raise
with open(file_in, "r") as tsv_file:
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
tsv_keyed, json_str = process_tsv(tsv, id_num)
tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE)
t_fieldnames = tsv.fieldnames
if is_global:
t_fieldnames = tsv.fieldnames[: tsv.fieldnames.index("English") + 1]
tsv_keyed, json_str = process_tsv(tsv, id_num, is_global)
# csv handles newlines, don't set it in io.open
with io.open(tsv_output_path, "w", newline="") as tsv_out:
@ -80,17 +93,20 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
delimiter="\t",
quotechar='"',
lineterminator="\n",
fieldnames=tsv.fieldnames,
fieldnames=t_fieldnames,
extrasaction="ignore",
)
writer.writeheader()
for row in tsv_keyed:
writer.writerow(row)
with io.open(json_output_path, "w", newline="\n") as json_out:
for lang in languages:
if not json_str[lang]:
continue
with io.open(json_output_paths[lang], "w", newline="\n") as json_out:
# we don't want to sort these because they're _1, ..., _10, etc
json.dump(
json_str,
json_str[lang],
json_out,
ensure_ascii=False,
indent="\t",
@ -102,10 +118,14 @@ def translate_file(file_in, tsv_out_dir, json_out_dir):
print("Error processing file: " + file_in)
def process_tsv(tsv, id_num):
def process_tsv(tsv, id_num, is_global):
i = 0
tsv_keyed = []
key_dict = {}
key_dict = {"jpn": {}}
if is_global:
key_dict["enm"] = {}
key_dict["zh"] = {}
key_dict["ko"] = {}
for row in tsv:
try:
@ -115,8 +135,14 @@ def process_tsv(tsv, id_num):
if row["Text"] and row["Text"].strip():
key = "{}_{}".format(id_num, i)
i += 1
key_dict["jpn"][key] = row["Text"]
if is_global:
with suppress(KeyError):
if row["English"]:
key_dict["enm"][key] = row["English"]
key_dict["zh"][key] = row["Chinese"]
key_dict["ko"][key] = row["Korean"]
row["English"] = key
key_dict[key] = row["Text"]
tsv_keyed.append(row)
except Exception:
traceback.print_exc()

83
xdudata/update.py
View file

@ -1,4 +1,7 @@
from contextlib import suppress
import diva
import shutil
import os
import subprocess
import tempfile
@ -8,35 +11,52 @@ from distutils.dir_util import copy_tree, remove_tree
from glob import glob
def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages):
def update_all(
cache_dir, update_dir, translation_dir, extract_dir, script_dir, languages,
):
print("Cleaning cache")
cleanup_cache(cache_dir)
print("Copying images")
copy_images(cache_dir, update_dir)
print("Processing TSV files")
update_tsv(cache_dir, update_dir, translation_dir, languages)
update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages)
print("Processing Quest Json files")
diva.quest.update_missions(cache_dir, translation_dir, languages)
diva.quest.update_missions(extract_dir, translation_dir, languages)
# this code isn't going to run on windows because i need to rewrite it
# it just calls my zsh scripts and generates it that way, need to replace them
# this code isn't going to run on windows (it is in WSL though) because i need to rewrite it
# it just calls my zsh tools and generates it that way, need to replace them
# with real python or something
# first copy over the necessary avtools to the extract dir
# TODO make it so that you don't need to copy them over (for example call them with python)
print("Copying necessary tools into extracting directory")
shutil.copy2(os.path.join(script_dir, "acb_extract.exe"), extract_dir)
shutil.copy2(os.path.join(script_dir, "usm_extract.exe"), extract_dir)
shutil.copy2(os.path.join(script_dir, "clHCA"), extract_dir)
print("Processing Sound Effects")
copy_tree(
os.path.join(cache_dir, "Android/Asset/Sound/Se"),
os.path.join(extract_dir, "Se"),
)
subprocess.run("./update_se.zsh", shell=True, cwd=extract_dir)
copy_tree(os.path.join(extract_dir, "se"), os.path.join(update_dir, "Se"))
subprocess.run(
os.path.join(script_dir, "update_se.zsh"), shell=True, cwd=extract_dir
)
copy_tree(os.path.join(extract_dir, "se_extracted"), os.path.join(update_dir, "Se"))
print("Processing BGM - don't forget to copy that loop file")
copy_tree(
os.path.join(cache_dir, "Android/Asset/Sound/Bgm"),
os.path.join(extract_dir, "Bgm"),
)
subprocess.run("./update_bgm.zsh", shell=True, cwd=extract_dir)
subprocess.run(
os.path.join(script_dir, "update_bgm.zsh"), shell=True, cwd=extract_dir
)
copy_tree(
os.path.join(extract_dir, "bgm"), os.path.join(update_dir, "Bgm")
os.path.join(extract_dir, "bgm_extracted"), os.path.join(update_dir, "Bgm")
)
print("Processing voices")
@ -44,9 +64,11 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages):
os.path.join(cache_dir, "Android/Asset/Sound/Voice"),
os.path.join(extract_dir, "Voice"),
)
subprocess.run("./update_voice.zsh", shell=True, cwd=extract_dir)
subprocess.run(
os.path.join(script_dir, "update_voice.zsh"), shell=True, cwd=extract_dir
)
copy_tree(
os.path.join(extract_dir, "voice"), os.path.join(update_dir, "Voice")
os.path.join(extract_dir, "voice_extracted"), os.path.join(update_dir, "Voice")
)
print("Processing movies - this might take forever")
@ -54,36 +76,42 @@ def update_all(cache_dir, update_dir, translation_dir, extract_dir, languages):
os.path.join(cache_dir, "Common/Asset/Movie"),
os.path.join(extract_dir, "Movie"),
)
subprocess.run("./update_movie.zsh", shell=True, cwd=extract_dir)
subprocess.run(
os.path.join(script_dir, "update_movie.zsh"), shell=True, cwd=extract_dir
)
copy_tree(
os.path.join(extract_dir, "movie"),
os.path.join(extract_dir, "movie_extracted"),
os.path.join(update_dir, "Asset/Movie"),
)
def update_tsv(cache_dir, update_dir, translation_dir, languages):
def update_tsv(cache_dir, update_dir, extract_dir, translation_dir, languages):
utage_tmp = tempfile.mkdtemp()
copy_tree(os.path.join(cache_dir, "Common/Asset/Utage"), utage_tmp)
utage.crypt.crypt_dir(
utage_tmp, bytearray("SampleSecretKey", "utf-8"), False, False
)
for encrypted in glob(
os.path.join(utage_tmp, "**/*.utage"), recursive=True
):
for encrypted in glob(os.path.join(utage_tmp, "**/*.utage"), recursive=True):
try:
os.remove(encrypted)
except:
pass
# copy over files for the quest updater
copy_tree(utage_tmp, os.path.join(extract_dir, "Common/Asset/Utage"))
shutil.copy2(os.path.join(cache_dir, "QuestMst.db"), extract_dir)
shutil.copy2(os.path.join(cache_dir, "QuestSceneMst.db"), extract_dir)
shutil.copy2(os.path.join(cache_dir, "QuestPartMst.db"), extract_dir)
shutil.copy2(os.path.join(cache_dir, "ResourceEntry.db"), extract_dir)
utage.translate.translate_dir(
utage_tmp,
os.path.join(update_dir, "Utage"),
os.path.join(translation_dir, "Missions"),
False,
)
utage.names.update_names(utage_tmp, translation_dir, languages)
copy_tree(
os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva")
)
utage.names.update_names(utage_tmp, translation_dir, languages, False)
copy_tree(os.path.join(utage_tmp, "Diva"), os.path.join(update_dir, "Utage/Diva"))
remove_tree(utage_tmp)
@ -93,6 +121,15 @@ def copy_images(cache_dir, update_dir):
os.path.join(update_dir, "Asset/Image"),
)
copy_tree(
os.path.join(cache_dir, "Common/Sample"),
os.path.join(update_dir, "Sample"),
os.path.join(cache_dir, "Common/Sample"), os.path.join(update_dir, "Sample"),
)
def cleanup_cache(cache_dir):
with suppress(FileNotFoundError):
shutil.rmtree(os.path.join(cache_dir, "Common/Asset/Utage/event32"))
os.remove(
os.path.join(
cache_dir, "Common/Asset/Utage/side01/Scenario/Sheet2.tsv.utage"
)
)