BLACKED
This commit is contained in:
louis
parent
03e51318b0
commit
b6c3c6a454
10 changed files with 819 additions and 654 deletions
365
utage/crypt.py
365
utage/crypt.py
|
|
@ -9,199 +9,214 @@ from multiprocessing.pool import Pool
|
|||
from . import crypt
|
||||
from . import translate
|
||||
|
||||
|
||||
def crypt_dir(dir_in, key, encrypt=False, no_compress=False):
|
||||
if not encrypt:
|
||||
files = glob(os.path.join(dir_in, "**/*.utage"), recursive=True)
|
||||
else:
|
||||
files = glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
|
||||
if not encrypt:
|
||||
files = glob(os.path.join(dir_in, "**/*.utage"), recursive=True)
|
||||
else:
|
||||
files = glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
|
||||
|
||||
if len(files) == 0:
|
||||
raise FileNotFoundError("No valid files found in directory: " + dir_in)
|
||||
if len(files) == 0:
|
||||
raise FileNotFoundError("No valid files found in directory: " + dir_in)
|
||||
|
||||
pcrypt = partial(crypt_file, key=key, encrypt=encrypt, no_compress=no_compress)
|
||||
with Pool() as p:
|
||||
p.map(pcrypt, files)
|
||||
|
||||
pcrypt = partial(crypt_file, key=key, encrypt=encrypt, no_compress=no_compress)
|
||||
with Pool() as p:
|
||||
p.map(pcrypt, files)
|
||||
|
||||
def crypt_file(file_in, key, encrypt=False, no_compress=False):
|
||||
with open(file_in, "rb") as inf:
|
||||
in_data = inf.read()
|
||||
try:
|
||||
if encrypt:
|
||||
if not file_in.endswith(".png") and not file_in.endswith(".jpg") and not file_in.endswith(".tsv"):
|
||||
raise ValueError("Invalid File Type for {}".format(file_in))
|
||||
if not file_in.endswith(".png") and not no_compress:
|
||||
enc_data = crypt.compress(in_data)
|
||||
enc_data = crypt.xor_crypt(enc_data, key)
|
||||
with io.open(file_in + ".utage", "wb") as output:
|
||||
output.write(enc_data)
|
||||
else:
|
||||
if not file_in.endswith(".utage"):
|
||||
raise ValueError("Invalid File Type for {}".format(file_in))
|
||||
dec_data = crypt.xor_crypt(in_data, key)
|
||||
if not file_in.endswith(".png.utage") and not file_in.endswith(".jpg.utage") and not no_compress:
|
||||
dec_data = crypt.decompress(dec_data)
|
||||
with io.open(file_in.replace(".utage", ""), "wb") as output:
|
||||
output.write(dec_data)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print("Error processing file: " + file_in)
|
||||
with open(file_in, "rb") as inf:
|
||||
in_data = inf.read()
|
||||
try:
|
||||
if encrypt:
|
||||
if (
|
||||
not file_in.endswith(".png")
|
||||
and not file_in.endswith(".jpg")
|
||||
and not file_in.endswith(".tsv")
|
||||
):
|
||||
raise ValueError("Invalid File Type for {}".format(file_in))
|
||||
if not file_in.endswith(".png") and not no_compress:
|
||||
enc_data = crypt.compress(in_data)
|
||||
enc_data = crypt.xor_crypt(enc_data, key)
|
||||
with io.open(file_in + ".utage", "wb") as output:
|
||||
output.write(enc_data)
|
||||
else:
|
||||
if not file_in.endswith(".utage"):
|
||||
raise ValueError("Invalid File Type for {}".format(file_in))
|
||||
dec_data = crypt.xor_crypt(in_data, key)
|
||||
if (
|
||||
not file_in.endswith(".png.utage")
|
||||
and not file_in.endswith(".jpg.utage")
|
||||
and not no_compress
|
||||
):
|
||||
dec_data = crypt.decompress(dec_data)
|
||||
with io.open(file_in.replace(".utage", ""), "wb") as output:
|
||||
output.write(dec_data)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print("Error processing file: " + file_in)
|
||||
|
||||
|
||||
def decompress(data):
|
||||
osize = 0
|
||||
isize = len(data)
|
||||
osize_expected = int.from_bytes(data[:4], byteorder='little', signed=False)
|
||||
odata = bytearray(osize_expected)
|
||||
odata[:isize] = data
|
||||
i = 4
|
||||
while i < isize:
|
||||
if (data[i] & 128) != 0:
|
||||
num3 = data[i] & 15
|
||||
num3 += 3
|
||||
num4 = (data[i] & 112) << 4 | data[i+1]
|
||||
num4 += 1
|
||||
for j in range(0, num3):
|
||||
odata[osize + j] = odata[osize - num4 + j]
|
||||
i += 1
|
||||
else:
|
||||
num3 = data[i] + 1
|
||||
for j in range(0, num3):
|
||||
odata[osize + j] = data[i + 1 + j]
|
||||
i += num3
|
||||
osize += num3
|
||||
i += 1
|
||||
return odata
|
||||
osize = 0
|
||||
isize = len(data)
|
||||
osize_expected = int.from_bytes(data[:4], byteorder="little", signed=False)
|
||||
odata = bytearray(osize_expected)
|
||||
odata[:isize] = data
|
||||
i = 4
|
||||
while i < isize:
|
||||
if (data[i] & 128) != 0:
|
||||
num3 = data[i] & 15
|
||||
num3 += 3
|
||||
num4 = (data[i] & 112) << 4 | data[i + 1]
|
||||
num4 += 1
|
||||
for j in range(0, num3):
|
||||
odata[osize + j] = odata[osize - num4 + j]
|
||||
i += 1
|
||||
else:
|
||||
num3 = data[i] + 1
|
||||
for j in range(0, num3):
|
||||
odata[osize + j] = data[i + 1 + j]
|
||||
i += num3
|
||||
osize += num3
|
||||
i += 1
|
||||
return odata
|
||||
|
||||
|
||||
def compress(data):
|
||||
num = len(data)
|
||||
bytes2 = num.to_bytes(4, byteorder='little')
|
||||
anum3 = num + num/128 + 1
|
||||
array = bytearray(int(anum3))
|
||||
num2 = 0
|
||||
num3 = 0
|
||||
num4 = 0
|
||||
index = Index()
|
||||
while num3 < num:
|
||||
num5 = 0
|
||||
num6 = 0
|
||||
num7 = min(18, num - num3)
|
||||
num8 = index.getFirst(data[num3])
|
||||
while not index.isEnd(num8):
|
||||
node = index.getNode(num8)
|
||||
mPos = node.mPos
|
||||
i = 1
|
||||
while i < num7:
|
||||
if data[mPos+i] != data[num3+i]:
|
||||
break
|
||||
i = i+1
|
||||
if num5 < i:
|
||||
num6 = mPos
|
||||
num5 = i
|
||||
if num5 == num7:
|
||||
break
|
||||
num8 = node.mNext
|
||||
if num5 >= 3:
|
||||
for j in range(num5):
|
||||
num9 = num3 + j - 2048
|
||||
if num9 >= 0:
|
||||
index.remove(data[num9], num9)
|
||||
index.add(data[num3+j], num3+j)
|
||||
if num4 < num3:
|
||||
array[num2] = (num3 - num4 - 1)
|
||||
num2 = num2+1
|
||||
for j in range(num4, num3):
|
||||
array[num2] = data[j]
|
||||
num2 = num2+1
|
||||
num10 = num5 - 3
|
||||
num11 = num3 - num6 - 1
|
||||
num12 = 0x80 | num10
|
||||
num12 |= (num11 & 0x700) >> 4
|
||||
array[num2] = num12
|
||||
array[num2+1] = (num11&0xff)
|
||||
num2 = num2+2
|
||||
num3 = num3 + num5
|
||||
num4 = num3
|
||||
else:
|
||||
num9 = num3 - 2048
|
||||
if num9 >= 0:
|
||||
index.remove(data[num9],num9)
|
||||
index.add(data[num3], num3)
|
||||
num3 = num3+1
|
||||
if num3 - num4 == 128:
|
||||
array[num2] = num3 - num4 - 1
|
||||
num2 = num2+1
|
||||
for j in range(num4, num3):
|
||||
array[num2] = data[j]
|
||||
num2 = num2+1
|
||||
num4 = num3
|
||||
if num4 < num3:
|
||||
array[num2] = num3 - num4 - 1
|
||||
num2 = num2+1
|
||||
for j in range(num4, num3):
|
||||
array[num2] = data[j]
|
||||
num2 = num2+1
|
||||
osize = num2
|
||||
array2 = bytearray(osize + 4)
|
||||
array2[:4] = bytes2[:4]
|
||||
array2[4:] = array[:osize]
|
||||
return array2
|
||||
num = len(data)
|
||||
bytes2 = num.to_bytes(4, byteorder="little")
|
||||
anum3 = num + num / 128 + 1
|
||||
array = bytearray(int(anum3))
|
||||
num2 = 0
|
||||
num3 = 0
|
||||
num4 = 0
|
||||
index = Index()
|
||||
while num3 < num:
|
||||
num5 = 0
|
||||
num6 = 0
|
||||
num7 = min(18, num - num3)
|
||||
num8 = index.getFirst(data[num3])
|
||||
while not index.isEnd(num8):
|
||||
node = index.getNode(num8)
|
||||
mPos = node.mPos
|
||||
i = 1
|
||||
while i < num7:
|
||||
if data[mPos + i] != data[num3 + i]:
|
||||
break
|
||||
i = i + 1
|
||||
if num5 < i:
|
||||
num6 = mPos
|
||||
num5 = i
|
||||
if num5 == num7:
|
||||
break
|
||||
num8 = node.mNext
|
||||
if num5 >= 3:
|
||||
for j in range(num5):
|
||||
num9 = num3 + j - 2048
|
||||
if num9 >= 0:
|
||||
index.remove(data[num9], num9)
|
||||
index.add(data[num3 + j], num3 + j)
|
||||
if num4 < num3:
|
||||
array[num2] = num3 - num4 - 1
|
||||
num2 = num2 + 1
|
||||
for j in range(num4, num3):
|
||||
array[num2] = data[j]
|
||||
num2 = num2 + 1
|
||||
num10 = num5 - 3
|
||||
num11 = num3 - num6 - 1
|
||||
num12 = 0x80 | num10
|
||||
num12 |= (num11 & 0x700) >> 4
|
||||
array[num2] = num12
|
||||
array[num2 + 1] = num11 & 0xFF
|
||||
num2 = num2 + 2
|
||||
num3 = num3 + num5
|
||||
num4 = num3
|
||||
else:
|
||||
num9 = num3 - 2048
|
||||
if num9 >= 0:
|
||||
index.remove(data[num9], num9)
|
||||
index.add(data[num3], num3)
|
||||
num3 = num3 + 1
|
||||
if num3 - num4 == 128:
|
||||
array[num2] = num3 - num4 - 1
|
||||
num2 = num2 + 1
|
||||
for j in range(num4, num3):
|
||||
array[num2] = data[j]
|
||||
num2 = num2 + 1
|
||||
num4 = num3
|
||||
if num4 < num3:
|
||||
array[num2] = num3 - num4 - 1
|
||||
num2 = num2 + 1
|
||||
for j in range(num4, num3):
|
||||
array[num2] = data[j]
|
||||
num2 = num2 + 1
|
||||
osize = num2
|
||||
array2 = bytearray(osize + 4)
|
||||
array2[:4] = bytes2[:4]
|
||||
array2[4:] = array[:osize]
|
||||
return array2
|
||||
|
||||
|
||||
class Node:
|
||||
mNext = 0
|
||||
mPrev = 0
|
||||
mPos = 0
|
||||
mNext = 0
|
||||
mPrev = 0
|
||||
mPos = 0
|
||||
|
||||
|
||||
class Index:
|
||||
mNodes = []
|
||||
mStack = []
|
||||
mStackPos = 0
|
||||
mNodes = []
|
||||
mStack = []
|
||||
mStackPos = 0
|
||||
|
||||
def __init__(self):
|
||||
for i in range(2304):
|
||||
x = Node()
|
||||
self.mNodes.append(x)
|
||||
for i in range(2048, 2304):
|
||||
self.mNodes[i].mNext = i
|
||||
self.mNodes[i].mPrev = i
|
||||
for i in range(2048):
|
||||
self.mStack.append(i)
|
||||
self.mStackPos = 2048
|
||||
def __init__(self):
|
||||
for i in range(2304):
|
||||
x = Node()
|
||||
self.mNodes.append(x)
|
||||
for i in range(2048, 2304):
|
||||
self.mNodes[i].mNext = i
|
||||
self.mNodes[i].mPrev = i
|
||||
for i in range(2048):
|
||||
self.mStack.append(i)
|
||||
self.mStackPos = 2048
|
||||
|
||||
def getFirst(self, c):
|
||||
return self.mNodes[2048+c].mNext
|
||||
def getFirst(self, c):
|
||||
return self.mNodes[2048 + c].mNext
|
||||
|
||||
def getNode(self, i):
|
||||
return self.mNodes[i]
|
||||
def getNode(self, i):
|
||||
return self.mNodes[i]
|
||||
|
||||
def add(self, c, pos):
|
||||
self.mStackPos = self.mStackPos - 1
|
||||
num = self.mStack[self.mStackPos]
|
||||
node = self.mNodes[num]
|
||||
node2 = self.mNodes[2048+c]
|
||||
node.mNext = node2.mNext
|
||||
node.mPrev = 2048 + c
|
||||
node.mPos = pos
|
||||
self.mNodes[node2.mNext].mPrev = num
|
||||
node2.mNext = num
|
||||
def add(self, c, pos):
|
||||
self.mStackPos = self.mStackPos - 1
|
||||
num = self.mStack[self.mStackPos]
|
||||
node = self.mNodes[num]
|
||||
node2 = self.mNodes[2048 + c]
|
||||
node.mNext = node2.mNext
|
||||
node.mPrev = 2048 + c
|
||||
node.mPos = pos
|
||||
self.mNodes[node2.mNext].mPrev = num
|
||||
node2.mNext = num
|
||||
|
||||
def remove(self, c, pos):
|
||||
mPrev = self.mNodes[2048+c].mPrev
|
||||
node = self.mNodes[mPrev]
|
||||
self.mStack[self.mStackPos] = self.mNodes[node.mPrev].mNext
|
||||
self.mStackPos = self.mStackPos + 1
|
||||
self.mNodes[node.mPrev].mNext = node.mNext
|
||||
self.mNodes[node.mNext].mPrev = node.mPrev
|
||||
def remove(self, c, pos):
|
||||
mPrev = self.mNodes[2048 + c].mPrev
|
||||
node = self.mNodes[mPrev]
|
||||
self.mStack[self.mStackPos] = self.mNodes[node.mPrev].mNext
|
||||
self.mStackPos = self.mStackPos + 1
|
||||
self.mNodes[node.mPrev].mNext = node.mNext
|
||||
self.mNodes[node.mNext].mPrev = node.mPrev
|
||||
|
||||
def isEnd(self, idx):
|
||||
return idx >= 2048
|
||||
|
||||
def isEnd(self, idx):
|
||||
return idx >= 2048
|
||||
|
||||
def xor_crypt(data, key):
|
||||
odata = bytearray(len(data))
|
||||
odata[:] = data
|
||||
i = 0
|
||||
for x in data:
|
||||
if (x != 0):
|
||||
m = key[i % len(key)]
|
||||
if (x != m):
|
||||
odata[i] = x ^ m
|
||||
i += 1
|
||||
return odata
|
||||
odata = bytearray(len(data))
|
||||
odata[:] = data
|
||||
i = 0
|
||||
for x in data:
|
||||
if x != 0:
|
||||
m = key[i % len(key)]
|
||||
if x != m:
|
||||
odata[i] = x ^ m
|
||||
i += 1
|
||||
return odata
|
||||
|
|
|
|||
127
utage/names.py
127
utage/names.py
|
|
@ -8,74 +8,95 @@ from functools import partial
|
|||
from glob import glob
|
||||
from multiprocessing import Pool
|
||||
|
||||
|
||||
def update_names(dir_in, old_path, languages):
|
||||
names = extract_names(dir_in)
|
||||
names = extract_names(dir_in)
|
||||
|
||||
# we have to update japanese like the rest because CustomData exists
|
||||
# and POKELABO deleted the wedding gear event stuff from the game files
|
||||
for l in languages:
|
||||
if l == "jpn":
|
||||
out_dict = dict(zip(names, names))
|
||||
else:
|
||||
out_dict = dict.fromkeys(names, "")
|
||||
# we have to update japanese like the rest because CustomData exists
|
||||
# and POKELABO deleted the wedding gear event stuff from the game files
|
||||
for l in languages:
|
||||
if l == "jpn":
|
||||
out_dict = dict(zip(names, names))
|
||||
else:
|
||||
out_dict = dict.fromkeys(names, "")
|
||||
|
||||
langfile = os.path.join(old_path, "nametranslations_{}.json".format(l))
|
||||
if os.path.isfile(langfile):
|
||||
with open(langfile, "r") as lang_file:
|
||||
lang_dict = json.load(lang_file)
|
||||
out_dict.update(lang_dict)
|
||||
langfile = os.path.join(old_path, "nametranslations_{}.json".format(l))
|
||||
if os.path.isfile(langfile):
|
||||
with open(langfile, "r") as lang_file:
|
||||
lang_dict = json.load(lang_file)
|
||||
out_dict.update(lang_dict)
|
||||
|
||||
with io.open(
|
||||
langfile, "w", newline="\n"
|
||||
) as lang_file: # you're using git right
|
||||
json.dump(
|
||||
out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True
|
||||
)
|
||||
|
||||
with io.open(langfile, "w", newline='\n') as lang_file: # you're using git right
|
||||
json.dump(out_dict, lang_file, ensure_ascii=False, indent='\t', sort_keys=True)
|
||||
|
||||
def extract_names(dir_in):
|
||||
char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv")
|
||||
char_tsv_path = os.path.join(dir_in, "Diva", "Settings", "Character.tsv")
|
||||
|
||||
if not os.path.isfile(char_tsv_path):
|
||||
raise FileNotFoundError("Character.tsv not found")
|
||||
if not os.path.isfile(char_tsv_path):
|
||||
raise FileNotFoundError("Character.tsv not found")
|
||||
|
||||
names, sets = read_char_tsv(char_tsv_path)
|
||||
names, sets = read_char_tsv(char_tsv_path)
|
||||
|
||||
files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)]
|
||||
if len(files) == 0:
|
||||
raise FileNotFoundError("No valid files found in directory: " + dir_in)
|
||||
files = [
|
||||
f
|
||||
for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
|
||||
if re.search(r"/[0-9]{9}\.tsv$", f)
|
||||
]
|
||||
if len(files) == 0:
|
||||
raise FileNotFoundError("No valid files found in directory: " + dir_in)
|
||||
|
||||
# tfw gil
|
||||
# it's still faster on my machine so i'm keeping it
|
||||
read_partial = partial(read_mission, char_names=names, char_sets=sets)
|
||||
with Pool() as p:
|
||||
new_names = p.map(read_partial, files)
|
||||
for x in new_names:
|
||||
names.update(x)
|
||||
# tfw gil
|
||||
# it's still faster on my machine so i'm keeping it
|
||||
read_partial = partial(read_mission, char_names=names, char_sets=sets)
|
||||
with Pool() as p:
|
||||
new_names = p.map(read_partial, files)
|
||||
for x in new_names:
|
||||
names.update(x)
|
||||
|
||||
return names
|
||||
|
||||
return names
|
||||
|
||||
def read_char_tsv(char_tsv_path):
|
||||
char_names = set()
|
||||
char_sets = set()
|
||||
char_names = set()
|
||||
char_sets = set()
|
||||
|
||||
with open(char_tsv_path, "r") as char_tsv_file:
|
||||
char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar="\"")
|
||||
with open(char_tsv_path, "r") as char_tsv_file:
|
||||
char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"')
|
||||
|
||||
for row in char_tsv:
|
||||
if row['CharacterName'].startswith("//"):
|
||||
continue
|
||||
if row['CharacterName'] and row['NameText'] and row['CharacterName'].strip() and row['NameText'].strip():
|
||||
char_names.add(row['NameText'])
|
||||
char_sets.add(row['CharacterName'])
|
||||
for row in char_tsv:
|
||||
if row["CharacterName"].startswith("//"):
|
||||
continue
|
||||
if (
|
||||
row["CharacterName"]
|
||||
and row["NameText"]
|
||||
and row["CharacterName"].strip()
|
||||
and row["NameText"].strip()
|
||||
):
|
||||
char_names.add(row["NameText"])
|
||||
char_sets.add(row["CharacterName"])
|
||||
|
||||
return char_names, char_sets
|
||||
|
||||
return char_names, char_sets
|
||||
|
||||
def read_mission(tsv_path, char_names, char_sets):
|
||||
new_names = set()
|
||||
with open(tsv_path, "r") as tsv_file:
|
||||
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"")
|
||||
if ('Arg1' not in tsv.fieldnames) or ('Text' not in tsv.fieldnames) or ('Command' not in tsv.fieldnames):
|
||||
return new_names
|
||||
for row in tsv:
|
||||
if row['Command'] and row['Command'].startswith("//"):
|
||||
continue
|
||||
if row['Text'] and row['Arg1']:
|
||||
if (row['Arg1'] not in char_names) and (row['Arg1'] not in char_sets):
|
||||
new_names.add(row['Arg1'])
|
||||
return new_names
|
||||
new_names = set()
|
||||
with open(tsv_path, "r") as tsv_file:
|
||||
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
|
||||
if (
|
||||
("Arg1" not in tsv.fieldnames)
|
||||
or ("Text" not in tsv.fieldnames)
|
||||
or ("Command" not in tsv.fieldnames)
|
||||
):
|
||||
return new_names
|
||||
for row in tsv:
|
||||
if row["Command"] and row["Command"].startswith("//"):
|
||||
continue
|
||||
if row["Text"] and row["Arg1"]:
|
||||
if (row["Arg1"] not in char_names) and (row["Arg1"] not in char_sets):
|
||||
new_names.add(row["Arg1"])
|
||||
return new_names
|
||||
|
|
|
|||
|
|
@ -10,89 +10,109 @@ from functools import partial
|
|||
from glob import glob
|
||||
from multiprocessing.pool import Pool
|
||||
|
||||
|
||||
def translate_dir(dir_in, tsv_out_dir, json_out_dir):
|
||||
# we only want to key files that are mission ids
|
||||
files = [f for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True) if re.search(r'/[0-9]{9}\.tsv$', f)]
|
||||
# we only want to key files that are mission ids
|
||||
files = [
|
||||
f
|
||||
for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
|
||||
if re.search(r"/[0-9]{9}\.tsv$", f)
|
||||
]
|
||||
|
||||
if len(files) == 0:
|
||||
raise FileNotFoundError("No valid files found in directory: " + dir_in)
|
||||
if len(files) == 0:
|
||||
raise FileNotFoundError("No valid files found in directory: " + dir_in)
|
||||
|
||||
ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir)
|
||||
with Pool() as p:
|
||||
p.map(ptrans, files)
|
||||
|
||||
ptrans = partial(translate_file, tsv_out_dir=tsv_out_dir, json_out_dir=json_out_dir)
|
||||
with Pool() as p:
|
||||
p.map(ptrans, files)
|
||||
|
||||
def translate_file(file_in, tsv_out_dir, json_out_dir):
|
||||
try:
|
||||
if not file_in.endswith(".tsv"):
|
||||
raise ValueError("Invalid File Type for {}".format(file_in))
|
||||
try:
|
||||
if not file_in.endswith(".tsv"):
|
||||
raise ValueError("Invalid File Type for {}".format(file_in))
|
||||
|
||||
# we need to get the event folder
|
||||
# ie, Utage/>>>main01<<</Scenario/whatever.tsv
|
||||
try:
|
||||
event_folder = os.path.normpath(file_in).split(os.sep)[-3]
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print("Error processing tsv: Please reference from Utage root")
|
||||
raise
|
||||
# we need to get the event folder
|
||||
# ie, Utage/>>>main01<<</Scenario/whatever.tsv
|
||||
try:
|
||||
event_folder = os.path.normpath(file_in).split(os.sep)[-3]
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print("Error processing tsv: Please reference from Utage root")
|
||||
raise
|
||||
|
||||
id_num = os.path.splitext(os.path.basename(file_in))[0]
|
||||
id_num = os.path.splitext(os.path.basename(file_in))[0]
|
||||
|
||||
tsv_output_path = os.path.join(tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num))
|
||||
json_output_path = os.path.join(json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num))
|
||||
tsv_output_path = os.path.join(
|
||||
tsv_out_dir, event_folder, "Scenario", "{}_t.tsv".format(id_num)
|
||||
)
|
||||
json_output_path = os.path.join(
|
||||
json_out_dir, event_folder, "{}_translations_jpn.json".format(id_num)
|
||||
)
|
||||
|
||||
# need to create output paths and avoid races when threading
|
||||
if not os.path.exists(os.path.dirname(tsv_output_path)):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(tsv_output_path))
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
# need to create output paths and avoid races when threading
|
||||
if not os.path.exists(os.path.dirname(tsv_output_path)):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(tsv_output_path))
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
if not os.path.exists(os.path.dirname(json_output_path)):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(json_output_path))
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
if not os.path.exists(os.path.dirname(json_output_path)):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(json_output_path))
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
with open(file_in, "r") as tsv_file:
|
||||
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar="\"")
|
||||
tsv_keyed, json_str = process_tsv(tsv, id_num)
|
||||
with open(file_in, "r") as tsv_file:
|
||||
tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
|
||||
tsv_keyed, json_str = process_tsv(tsv, id_num)
|
||||
|
||||
# csv handles newlines, don't set it in io.open
|
||||
with io.open(tsv_output_path, "w", newline='') as tsv_out:
|
||||
writer = csv.DictWriter(tsv_out, delimiter="\t", quotechar="\"", lineterminator="\n", fieldnames=tsv.fieldnames, extrasaction='ignore')
|
||||
writer.writeheader()
|
||||
for row in tsv_keyed:
|
||||
writer.writerow(row)
|
||||
# csv handles newlines, don't set it in io.open
|
||||
with io.open(tsv_output_path, "w", newline="") as tsv_out:
|
||||
writer = csv.DictWriter(
|
||||
tsv_out,
|
||||
delimiter="\t",
|
||||
quotechar='"',
|
||||
lineterminator="\n",
|
||||
fieldnames=tsv.fieldnames,
|
||||
extrasaction="ignore",
|
||||
)
|
||||
writer.writeheader()
|
||||
for row in tsv_keyed:
|
||||
writer.writerow(row)
|
||||
|
||||
with io.open(json_output_path, "w", newline='\n') as json_out:
|
||||
# we don't want to sort these because they're _1, ..., _10, etc
|
||||
json.dump(json_str, json_out, ensure_ascii=False, indent='\t', sort_keys=False)
|
||||
with io.open(json_output_path, "w", newline="\n") as json_out:
|
||||
# we don't want to sort these because they're _1, ..., _10, etc
|
||||
json.dump(
|
||||
json_str, json_out, ensure_ascii=False, indent="\t", sort_keys=False
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print("Error processing file: " + file_in)
|
||||
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print("Error processing file: " + file_in)
|
||||
|
||||
def process_tsv(tsv, id_num):
|
||||
i = 0
|
||||
tsv_keyed = []
|
||||
key_dict = {}
|
||||
i = 0
|
||||
tsv_keyed = []
|
||||
key_dict = {}
|
||||
|
||||
for row in tsv:
|
||||
try:
|
||||
if row['Command'].startswith("//"):
|
||||
tsv_keyed.append(row)
|
||||
continue
|
||||
if row['Text'] and row['Text'].strip():
|
||||
key = "{}_{}".format(id_num, i)
|
||||
i += 1
|
||||
row['English'] = key
|
||||
key_dict[key] = row['Text']
|
||||
tsv_keyed.append(row)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print("Error processing tsv: " + id_num)
|
||||
raise
|
||||
for row in tsv:
|
||||
try:
|
||||
if row["Command"].startswith("//"):
|
||||
tsv_keyed.append(row)
|
||||
continue
|
||||
if row["Text"] and row["Text"].strip():
|
||||
key = "{}_{}".format(id_num, i)
|
||||
i += 1
|
||||
row["English"] = key
|
||||
key_dict[key] = row["Text"]
|
||||
tsv_keyed.append(row)
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print("Error processing tsv: " + id_num)
|
||||
raise
|
||||
|
||||
return tsv_keyed, key_dict
|
||||
return tsv_keyed, key_dict
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue