fixed newlines and other windows stuff

diva/quest.py

@@ -31,7 +31,7 @@ def scene_mst(qdb, old_path, dir_in, languages):
             break
 
     with io.open(
-        os.path.join(old_path, "XduScene.json"), "w", newline="\n"
+        os.path.join(old_path, "XduScene.json"), "w", encoding="utf-8"
     ) as json_file:
         json.dump(scenes, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
 
@@ -56,11 +56,11 @@ def scene_mst(qdb, old_path, dir_in, languages):
 
         langfile = os.path.join(old_path, f"XduSceneNames_{lang}.json")
         if os.path.isfile(langfile):
-            with open(langfile, "r") as lang_file:
+            with open(langfile, "r", encoding="utf-8") as lang_file:
                 lang_dict = json.load(lang_file)
                 out_dict.update(lang_dict)
 
-        with io.open(langfile, "w", newline="\n") as lang_file:
+        with io.open(langfile, "w", encoding="utf-8") as lang_file:
             json.dump(
                 out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
             )
@@ -70,7 +70,7 @@ def quest_mst(qdb, old_path, languages):
     quests = qdb.get_quests()
 
     with io.open(
-        os.path.join(old_path, "XduQuest.json"), "w", newline="\n"
+        os.path.join(old_path, "XduQuest.json"), "w", encoding="utf-8"
     ) as json_file:
         json.dump(quests, json_file, ensure_ascii=False, indent="\t", sort_keys=True)
 
@@ -85,11 +85,11 @@ def quest_mst(qdb, old_path, languages):
 
         langfile = os.path.join(old_path, f"XduQuestNames_{lang}.json")
         if os.path.isfile(langfile):
-            with open(langfile, "r") as lang_file:
+            with open(langfile, "r", encoding="utf-8") as lang_file:
                 lang_dict = json.load(lang_file)
                 out_dict.update(lang_dict)
 
-        with io.open(langfile, "w", newline="\n") as lang_file:
+        with io.open(langfile, "w", encoding="utf-8") as lang_file:
             json.dump(
                 out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
             )
@@ -108,7 +108,7 @@ def load_locations(dir_in):
         raise FileNotFoundError("Ya didn't decrypt the files dummy")
 
     for scenario in settings:
-        with open(scenario, "r") as tsv_file:
+        with open(scenario, "r", encoding="utf-8") as tsv_file:
             tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
             for row in tsv:
                 tokens = row["FileName"].split("/")

utage/names.py

@@ -12,7 +12,7 @@ from multiprocessing import Pool
 def update_names(dir_in, old_path, languages, is_global):
     names = extract_names(dir_in, is_global)
 
-    global_languages = {"enm": 1, "zh": 2, "ko": 3}
+    global_languages = {"enm": 1, "zho": 2, "kor": 3}
     if is_global:
         languages = global_languages.keys()
 
@@ -33,12 +33,12 @@ def update_names(dir_in, old_path, languages, is_global):
 
         langfile = os.path.join(old_path, "nametranslations_{}.json".format(l))
         if os.path.isfile(langfile):
-            with open(langfile, "r") as lang_file:
+            with open(langfile, "r", encoding="utf-8") as lang_file:
                 lang_dict = json.load(lang_file)
             out_dict.update(lang_dict)
 
         with io.open(
-            langfile, "w", newline="\n"
+            langfile, "w", encoding="utf-8"
         ) as lang_file:  # you're using git right
             json.dump(
                 out_dict, lang_file, ensure_ascii=False, indent="\t", sort_keys=True,
@@ -56,7 +56,7 @@ def extract_names(dir_in, is_global):
     files = [
         f
         for f in glob(os.path.join(dir_in, "**/*.tsv"), recursive=True)
-        if re.search(r"/[0-9]{9}\.tsv$", f)
+        if re.search(r"[0-9]{9}\.tsv$", f)
     ]
     if len(files) == 0:
         raise FileNotFoundError("No valid files found in directory: " + dir_in)
@@ -78,7 +78,7 @@ def read_char_tsv(char_tsv_path, is_global):
     char_names = set()
     char_sets = set()
 
-    with open(char_tsv_path, "r") as char_tsv_file:
+    with open(char_tsv_path, "r", encoding="utf-8") as char_tsv_file:
         char_tsv = csv.DictReader(char_tsv_file, delimiter="\t", quotechar='"')
 
         for row in char_tsv:
@@ -103,7 +103,7 @@ def read_char_tsv(char_tsv_path, is_global):
 
 def read_mission(tsv_path, char_names, char_sets):
     new_names = set()
-    with open(tsv_path, "r") as tsv_file:
+    with open(tsv_path, "r", encoding="utf-8") as tsv_file:
         tsv = csv.DictReader(tsv_file, delimiter="\t", quotechar='"')
         if (
             ("Arg1" not in tsv.fieldnames)

utage/translate.py

@@ -52,7 +52,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
         )
         languages = ["jpn"]
         if is_global:
-            languages += ["enm", "zh", "ko"]
+            languages += ["enm", "zho", "kor"]
         json_output_paths = {
             lang: os.path.join(
                 json_out_dir,
@@ -77,7 +77,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
                 if e.errno != errno.EEXIST:
                     raise
 
-        with open(file_in, "r") as tsv_file:
+        with open(file_in, "r", encoding="utf-8") as tsv_file:
             tsv = csv.DictReader(tsv_file, delimiter="\t", quoting=csv.QUOTE_NONE)
 
             t_fieldnames = tsv.fieldnames
@@ -87,12 +87,11 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
             tsv_keyed, json_str = process_tsv(tsv, id_num, is_global)
 
             # csv handles newlines, don't set it in io.open
-            with io.open(tsv_output_path, "w", newline="") as tsv_out:
+            with io.open(tsv_output_path, "w", newline="", encoding="utf-8") as tsv_out:
                 writer = csv.DictWriter(
                     tsv_out,
                     delimiter="\t",
                     quotechar='"',
-                    lineterminator="\n",
                     fieldnames=t_fieldnames,
                     extrasaction="ignore",
                 )
@@ -103,7 +102,7 @@ def translate_file(file_in, tsv_out_dir, json_out_dir, is_global):
         for lang in languages:
             if not json_str[lang]:
                 continue
-            with io.open(json_output_paths[lang], "w", newline="\n") as json_out:
+            with io.open(json_output_paths[lang], "w", encoding="utf-8") as json_out:
                 # we don't want to sort these because they're _1, ..., _10, etc
                 json.dump(
                     json_str[lang],
@@ -124,8 +123,8 @@ def process_tsv(tsv, id_num, is_global):
     key_dict = {"jpn": {}}
     if is_global:
         key_dict["enm"] = {}
-        key_dict["zh"] = {}
+        key_dict["zho"] = {}
-        key_dict["ko"] = {}
+        key_dict["kor"] = {}
 
     for row in tsv:
         try:
@@ -140,8 +139,8 @@ def process_tsv(tsv, id_num, is_global):
                     with suppress(KeyError):
                         if row["English"]:
                             key_dict["enm"][key] = row["English"]
-                        key_dict["zh"][key] = row["Chinese"]
+                        key_dict["zho"][key] = row["Chinese"]
-                        key_dict["ko"][key] = row["Korean"]
+                        key_dict["kor"][key] = row["Korean"]
                 row["English"] = key
             tsv_keyed.append(row)
         except Exception:

xdudata/asset_extract.py

@@ -10,6 +10,7 @@ from glob import glob
 from adx import extract_loop_data_from_dir
 
 OPERATING_SYSTEM = os.name
+CHUNK_SIZE = 100
 
 
 def process_se(cache_dir, extract_dir, update_dir):
@@ -39,7 +40,8 @@ def process_se(cache_dir, extract_dir, update_dir):
         hca_files = [
             file for file in glob("{}/**/*.hca".format(from_dir), recursive=True)
         ]
-        run_program(hca_files, cmd="clHCA")
+        for hca_sublist in split_list(hca_files, CHUNK_SIZE):
+            run_program(hca_sublist, cmd="clHCA")
 
         basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files]
         char_secs = [basename.split("_", maxsplit=1) for basename in basenames]
@@ -160,7 +162,8 @@ def process_voice(cache_dir, extract_dir, update_dir):
         hca_files = [
             file for file in glob("{}/**/*.hca".format(from_dir), recursive=True)
         ]
-        run_program(hca_files, cmd="clHCA")
+        for hca_sublist in split_list(hca_files, CHUNK_SIZE):
+            run_program(hca_sublist, cmd="clHCA")
 
         basenames = [os.path.splitext(os.path.basename(file))[0] for file in acb_files]
         char_secs = [basename.split("_", maxsplit=1) for basename in basenames]
@@ -292,3 +295,9 @@ def run_program(args, cmd=None):
 def get_pool(size=multiprocessing.cpu_count()):
     pool = multiprocessing.Pool(size)
     return pool
+
+
+def split_list(input_list, chunk_size):
+    return [
+        input_list[i : i + chunk_size] for i in range(0, len(input_list), chunk_size)
+    ]