From 8c0a5e3aa731e40cfcea02248b3e7e11424a5552 Mon Sep 17 00:00:00 2001 From: louis Date: Tue, 18 Feb 2020 18:44:58 -0500 Subject: [PATCH] initial commit of media shit --- acb_extract/acb_extract.csproj | 83 + acb_extract/acb_extract.sln | 23 + acb_extract/acb_extract.userprefs | 13 + acb_extract/src/ByteConversion.cs | 252 +++ .../src/CalculatingOffsetDescription.cs | 13 + acb_extract/src/ChecksumUtil.cs | 42 + acb_extract/src/Constants.cs | 453 ++++ acb_extract/src/CriAcbFile.cs | 684 ++++++ acb_extract/src/CriAfs2Archive.cs | 155 ++ acb_extract/src/CriUtfTable.cs | 864 ++++++++ acb_extract/src/FileUtil.cs | 667 ++++++ acb_extract/src/MathUtil.cs | 26 + acb_extract/src/OffsetDescription.cs | 9 + acb_extract/src/ParseFile.cs | 864 ++++++++ acb_extract/src/Program.cs | 17 + update_bgm.zsh | 11 + update_movie.zsh | 6 + update_se.zsh | 14 + update_voice.zsh | 18 + usm_extract/src/CriUsmStream.cs | 222 ++ usm_extract/src/Mpeg1Stream.cs | 39 + usm_extract/src/MpegStream.cs | 503 +++++ usm_extract/src/Program.cs | 21 + usm_extract/src/SofdecStream.cs | 58 + usm_extract/src/util/ByteConversion.cs | 266 +++ .../ByteSearchCalculatingOffsetDescription.cs | 16 + .../src/util/CalculatingOffsetDescription.cs | 13 + usm_extract/src/util/ChecksumUtil.cs | 196 ++ usm_extract/src/util/CompressionUtil.cs | 496 +++++ usm_extract/src/util/Constants.cs | 453 ++++ usm_extract/src/util/FileUtil.cs | 706 +++++++ usm_extract/src/util/IDeepCopy.cs | 7 + usm_extract/src/util/MathUtil.cs | 26 + usm_extract/src/util/OffsetDescription.cs | 9 + usm_extract/src/util/ParseFile.cs | 1826 +++++++++++++++++ .../util/RiffCalculatingOffsetDescription.cs | 15 + usm_extract/usm_extract.csproj | 86 + usm_extract/usm_extract.sln | 23 + 38 files changed, 9195 insertions(+) create mode 100644 acb_extract/acb_extract.csproj create mode 100644 acb_extract/acb_extract.sln create mode 100644 acb_extract/acb_extract.userprefs create mode 100644 acb_extract/src/ByteConversion.cs create mode 100644 acb_extract/src/CalculatingOffsetDescription.cs create mode 100644 acb_extract/src/ChecksumUtil.cs create mode 100644 acb_extract/src/Constants.cs create mode 100644 acb_extract/src/CriAcbFile.cs create mode 100644 acb_extract/src/CriAfs2Archive.cs create mode 100644 acb_extract/src/CriUtfTable.cs create mode 100644 acb_extract/src/FileUtil.cs create mode 100644 acb_extract/src/MathUtil.cs create mode 100644 acb_extract/src/OffsetDescription.cs create mode 100644 acb_extract/src/ParseFile.cs create mode 100644 acb_extract/src/Program.cs create mode 100755 update_bgm.zsh create mode 100755 update_movie.zsh create mode 100755 update_se.zsh create mode 100755 update_voice.zsh create mode 100644 usm_extract/src/CriUsmStream.cs create mode 100644 usm_extract/src/Mpeg1Stream.cs create mode 100644 usm_extract/src/MpegStream.cs create mode 100644 usm_extract/src/Program.cs create mode 100644 usm_extract/src/SofdecStream.cs create mode 100644 usm_extract/src/util/ByteConversion.cs create mode 100644 usm_extract/src/util/ByteSearchCalculatingOffsetDescription.cs create mode 100644 usm_extract/src/util/CalculatingOffsetDescription.cs create mode 100644 usm_extract/src/util/ChecksumUtil.cs create mode 100644 usm_extract/src/util/CompressionUtil.cs create mode 100644 usm_extract/src/util/Constants.cs create mode 100644 usm_extract/src/util/FileUtil.cs create mode 100644 usm_extract/src/util/IDeepCopy.cs create mode 100644 usm_extract/src/util/MathUtil.cs create mode 100644 usm_extract/src/util/OffsetDescription.cs create mode 100644 usm_extract/src/util/ParseFile.cs create mode 100644 usm_extract/src/util/RiffCalculatingOffsetDescription.cs create mode 100644 usm_extract/usm_extract.csproj create mode 100644 usm_extract/usm_extract.sln diff --git a/acb_extract/acb_extract.csproj b/acb_extract/acb_extract.csproj new file mode 100644 index 0000000..9a6d5ab --- /dev/null +++ b/acb_extract/acb_extract.csproj @@ -0,0 +1,83 @@ + + + + Debug + AnyCPU + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83} + Exe + Properties + acb_extract + acb_extract + v4.5 + 512 + + + + + 3.5 + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + bin\x86\Debug\ + DEBUG;TRACE + full + x86 + prompt + 4 + false + + + bin\x86\Release\ + TRACE + true + pdbonly + x86 + prompt + 4 + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/acb_extract/acb_extract.sln b/acb_extract/acb_extract.sln new file mode 100644 index 0000000..e585ef3 --- /dev/null +++ b/acb_extract/acb_extract.sln @@ -0,0 +1,23 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2012 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "acb_extract", "acb_extract.csproj", "{3951D6E5-61BE-4D7C-A464-F44D98E9CE83}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + Debug|x86 = Debug|x86 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Debug|x86.ActiveCfg = Debug|x86 + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Debug|x86.Build.0 = Debug|x86 + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Release|Any CPU.Build.0 = Release|Any CPU + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Release|x86.ActiveCfg = Release|x86 + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Release|x86.Build.0 = Release|x86 + EndGlobalSection +EndGlobal diff --git a/acb_extract/acb_extract.userprefs b/acb_extract/acb_extract.userprefs new file mode 100644 index 0000000..e60c8cd --- /dev/null +++ b/acb_extract/acb_extract.userprefs @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/acb_extract/src/ByteConversion.cs b/acb_extract/src/ByteConversion.cs new file mode 100644 index 0000000..e04fbad --- /dev/null +++ b/acb_extract/src/ByteConversion.cs @@ -0,0 +1,252 @@ +using System; +using System.Globalization; +using System.Collections.Generic; +using System.Text; + +namespace vgm_acb.extract +{ + /// + /// Class containing static text conversion functions. + /// + public sealed class ByteConversion + { + /// + /// Codepage value for Shift JIS (Jp) + /// + public const int CodePageJapan = 932; + /// + /// Codepage value for Cyrillic (US) + /// + public const int CodePageUnitedStates = 1251; + + /// + /// Codepage value for OEM DOS + /// + public const int CodePageOEM = 437; + + private ByteConversion() { } + + /// + /// Get string from bytes. + /// + /// Bytes to convert to a string. + /// Codepage to use in converting bytes. + /// String encoding using the input Codepage. + public static string GetEncodedText(byte[] value, int codePage) + { + //return Encoding.Unicode.GetString(value); + return System.Text.Encoding.GetEncoding(codePage).GetString(value); + } + + /// + /// Get text encoded in Shift JIS + /// + /// Bytes to decode. + /// String encoded using the Shift JIS codepage. + public static string GetJapaneseEncodedText(byte[] value) + { + return GetEncodedText(value, CodePageJapan); + } + /// + /// Get text encoded in Cyrillic + /// + /// Bytes to decode. + /// String encoded using the Cyrillic codepage. + public static string GetUnitedStatesEncodedText(byte[] value) + { + return GetEncodedText(value, CodePageUnitedStates); + } + + /// + /// Get text encoded in ASCII + /// + /// Bytes to decode. + /// String encoded using ASCII. + public static string GetAsciiText(byte[] value) + { + System.Text.Encoding ascii = System.Text.Encoding.ASCII; + return ascii.GetString(value); + } + + /// + /// Get text encoded in ASCII, stops at null. + /// + /// Bytes to decode. + /// Offet within byte array of string. + /// String encoded using ASCII. + public static string GetAsciiText(byte[] value, long offset) + { + StringBuilder sb = new StringBuilder(); + System.Text.Encoding ascii = System.Text.Encoding.ASCII; + + for (long i = offset; i < value.Length; i++) + { + if (value[i] == 0) + { + break; + } + else + { + sb.Append((char)value[i]); + } + } + + return sb.ToString(); + } + + public static string GetUtf16LeText(byte[] value) + { + System.Text.Encoding encoding = System.Text.Encoding.Unicode; + return encoding.GetString(value); + } + + + /// + /// Convert input string to a long. Works for Decimal and Hexidecimal (use 0x prefix). + /// + /// String containing a Decimal and Hexidecimal number. + /// Long representing the input string. + public static long GetLongValueFromString(string value) + { + long ret; + bool isNegative = false; + string parseValue; + + if (value.StartsWith("-")) + { + parseValue = value.Substring(1); + isNegative = true; + } + else + { + parseValue = value; + } + + if (parseValue.StartsWith("0x", StringComparison.CurrentCultureIgnoreCase)) + { + parseValue = parseValue.Substring(2); + ret = long.Parse(parseValue, System.Globalization.NumberStyles.HexNumber, null); + } + else + { + ret = long.Parse(parseValue, System.Globalization.NumberStyles.Integer, null); + } + + if (isNegative) + { + ret *= -1; + } + + return ret; + } + + /// + /// Get the UInt32 Value of the Incoming Byte Array, which is in Big Endian order. + /// + /// Bytes to convert. + /// The UInt32 Value of the Incoming Byte Array. + public static UInt32 GetUInt32BigEndian(byte[] value) + { + byte[] workingArray = new byte[value.Length]; + Array.Copy(value, 0, workingArray, 0, value.Length); + + if (BitConverter.IsLittleEndian) + { + Array.Reverse(workingArray); + } + return BitConverter.ToUInt32(workingArray, 0); + } + + /// + /// Get the UInt16 Value of the Incoming Byte Array, which is in Big Endian order. + /// + /// Bytes to convert. + /// The UInt16 Value of the Incoming Byte Array. + public static UInt16 GetUInt16BigEndian(byte[] value) + { + byte[] workingArray = new byte[value.Length]; + Array.Copy(value, 0, workingArray, 0, value.Length); + + if (BitConverter.IsLittleEndian) + { + Array.Reverse(workingArray); + } + return BitConverter.ToUInt16(workingArray, 0); + } + + /// + /// Predicts Code Page between Cyrillic and Shift-JIS based on whether high ASCII is included or not. + /// + /// Bytes containing the tags in an unknown language. + /// Integer representing the predicted code page. + public static int GetPredictedCodePageForTags(byte[] tagBytes) + { + int predictedCodePage = CodePageUnitedStates; + + foreach (byte b in tagBytes) + { + if ((int)b > 0x7F) + { + predictedCodePage = CodePageJapan; + break; + } + } + + return predictedCodePage; + } + + public static byte GetHighNibble(byte value) + { + return (byte)(((value) >> 4) & 0x0F); + } + + public static byte GetLowNibble(byte value) + { + return (byte)((value) & 0x0F); ; + } + + public static byte[] GetBytesFromHexString(string hexValue) + { + int j = 0; + byte[] bytes = new byte[hexValue.Length / 2]; + + // convert the string to bytes + for (int i = 0; i < hexValue.Length; i += 2) + { + bytes[j] = BitConverter.GetBytes(Int16.Parse(hexValue.Substring(i, 2), System.Globalization.NumberStyles.AllowHexSpecifier, CultureInfo.CurrentCulture))[0]; + j++; + } + + return bytes; + } + + public static byte[] GetBytesBigEndian(uint value) + { + byte[] ret = BitConverter.GetBytes(value); + Array.Reverse(ret); + return ret; + } + + public static DateTime GetDateTimeFromFAT32Date(int value) + { + short xDate = (short)(value >> 0x10); + short xTime = (short)(value & 0xFFFF); + + if (xDate == 0 && xTime == 0) + { + return DateTime.Now; + } + else + { + return new DateTime( + (((xDate & 0xFE00) >> 9) + 0x7BC), + ((xDate & 0x1E0) >> 5), + (xDate & 0x1F), + ((xTime & 0xF800) >> 0xB), + ((xTime & 0x7E0) >> 5), + ((xTime & 0x1F) * 2)); + } + } + + } +} diff --git a/acb_extract/src/CalculatingOffsetDescription.cs b/acb_extract/src/CalculatingOffsetDescription.cs new file mode 100644 index 0000000..e2179bb --- /dev/null +++ b/acb_extract/src/CalculatingOffsetDescription.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace vgm_acb.extract +{ + public class CalculatingOffsetDescription : OffsetDescription + { + public const string OFFSET_VARIABLE_STRING = "$V"; + + public string CalculationString { set; get; } + } +} diff --git a/acb_extract/src/ChecksumUtil.cs b/acb_extract/src/ChecksumUtil.cs new file mode 100644 index 0000000..8534334 --- /dev/null +++ b/acb_extract/src/ChecksumUtil.cs @@ -0,0 +1,42 @@ +// +// Copyright (c) 2008 All Rights Reserved +// +// +// +// +// Contains the ChecksumUtil class. +using System; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; + +namespace vgm_acb.extract +{ + /// + /// Class containing static functions related to checksum generation. + /// + public sealed class ChecksumUtil + { + /// + /// Prevents a default instance of the ChecksumUtil class from being created. + /// + private ChecksumUtil() + { + } + + /// + /// Get the MD5 checksum of the input stream. + /// + /// File Stream for which to generate the checksum. + /// String containing the hexidecimal representation of the MD5 of the input stream. + public static string GetMd5OfFullFile(FileStream stream) + { + MD5CryptoServiceProvider hashMd5 = new MD5CryptoServiceProvider(); + + stream.Seek(0, SeekOrigin.Begin); + hashMd5.ComputeHash(stream); + return ParseFile.ByteArrayToString(hashMd5.Hash); + } + + } +} diff --git a/acb_extract/src/Constants.cs b/acb_extract/src/Constants.cs new file mode 100644 index 0000000..72bbbd0 --- /dev/null +++ b/acb_extract/src/Constants.cs @@ -0,0 +1,453 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Windows.Forms; + +namespace vgm_acb.extract +{ + /// + /// Struct containing criteria used to find offsets. + /// + public struct FindOffsetStruct + { + private string searchString; + private string startingOffset; + private bool treatSearchStringAsHex; + private bool cutFile; + private string searchStringOffset; + private string cutSize; + private string cutSizeOffsetSize; + private bool isCutSizeAnOffset; + private string outputFileExtension; + private bool isLittleEndian; + private bool useTerminatorForCutSize; + private string terminatorString; + private bool treatTerminatorStringAsHex; + private bool includeTerminatorLength; + private string extraCutSizeBytes; + + public bool DoSearchStringModulo + { + set; + get; + } + + public string SearchStringModuloDivisor + { + set; + get; + } + + public string SearchStringModuloResult + { + set; + get; + } + + public bool DoTerminatorModulo + { + set; + get; + } + + public string TerminatorStringModuloDivisor + { + set; + get; + } + + public string TerminatorStringModuloResult + { + set; + get; + } + + public string MinimumSize + { + set; + get; + } + + public string SearchString + { + get { return searchString; } + set { searchString = value; } + } + + /// + /// Gets or sets offset to being searching at + /// + public string StartingOffset + { + set { startingOffset = value; } + get { return startingOffset; } + } + + /// + /// Gets or sets flag to indicate search string is a hex value. + /// + public bool TreatSearchStringAsHex + { + get { return treatSearchStringAsHex; } + set { treatSearchStringAsHex = value; } + } + + /// + /// Gets or sets flag to cut the file when the offset is found. + /// + public bool CutFile + { + get { return cutFile; } + set { cutFile = value; } + } + + /// + /// Gets or sets offset within destination file Search String would reside. + /// + public string SearchStringOffset + { + get { return searchStringOffset; } + set { searchStringOffset = value; } + } + + /// + /// Gets or sets size to cut from file + /// + public string CutSize + { + get { return cutSize; } + set { cutSize = value; } + } + + /// + /// Gets or sets size of offset holding cut size + /// + public string CutSizeOffsetSize + { + get { return cutSizeOffsetSize; } + set { cutSizeOffsetSize = value; } + } + + /// + /// Gets or sets flag indicating that cut size is an offset. + /// + public bool IsCutSizeAnOffset + { + get { return isCutSizeAnOffset; } + set { isCutSizeAnOffset = value; } + } + + /// + /// Gets or sets file extension to use for cut files. + /// + public string OutputFileExtension + { + get { return outputFileExtension; } + set { outputFileExtension = value; } + } + + /// + /// Gets or sets flag indicating that offset based cut size is stored in Little Endian byte order. + /// + public bool IsLittleEndian + { + get { return isLittleEndian; } + set { isLittleEndian = value; } + } + + /// + /// Gets or sets flag indicating that a terminator should be used to determine the cut size. + /// + + public bool UseLengthMultiplier { set; get; } + public string LengthMultiplier { set; get; } + + public bool UseTerminatorForCutSize + { + get { return useTerminatorForCutSize; } + set { useTerminatorForCutSize = value; } + } + + /// + /// Gets or sets terminator string to search for. + /// + public string TerminatorString + { + get { return terminatorString; } + set { terminatorString = value; } + } + + /// + /// Gets or sets flag indicating that Terminator String is hex. + /// + public bool TreatTerminatorStringAsHex + { + get { return treatTerminatorStringAsHex; } + set { treatTerminatorStringAsHex = value; } + } + + /// + /// Gets or sets flag indicating that the length of the terminator should be included in the cut size. + /// + public bool IncludeTerminatorLength + { + get { return includeTerminatorLength; } + set { includeTerminatorLength = value; } + } + + public bool CutToEofIfTerminatorNotFound { set; get; } + + /// + /// Gets or sets additional bytes to include in the cut size. + /// + public string ExtraCutSizeBytes + { + get { return extraCutSizeBytes; } + set { extraCutSizeBytes = value; } + } + + public string OutputFolder { get; set; } + } + + /// + /// Struct used to send messages conveying progress. + /// + public struct ProgressStruct + { + /// + /// File name to display in progress bar. + /// + private string fileName; + + /// + /// Error message to display in output window. + /// + private string errorMessage; + + /// + /// Generic message to display in output window. + /// + private string genericMessage; + + /// + /// New tree node to add to a TreeView. + /// + private TreeNode newNode; + + /// + /// Gets or sets fileName. + /// + public string FileName + { + get { return fileName; } + set { fileName = value; } + } + + /// + /// Gets or sets errorMessage. + /// + public string ErrorMessage + { + get { return errorMessage; } + set { errorMessage = value; } + } + + /// + /// Gets or sets genericMessage. + /// + public string GenericMessage + { + get { return genericMessage; } + set { genericMessage = value; } + } + + /// + /// Gets or sets newNode. + /// + public TreeNode NewNode + { + get { return newNode; } + set { newNode = value; } + } + + /// + /// Reset this node's values + /// + public void Clear() + { + fileName = String.Empty; + errorMessage = String.Empty; + genericMessage = String.Empty; + newNode = null; + } + } + + /// + /// Struct used to allow TreeView to select a specific form and modify the originating node upon completion of a task. + /// + public struct NodeTagStruct + { + /// + /// Class name of the Form this node will bring to focus. + /// + private string formClass; + + /// + /// Object type this node represents. + /// + private string objectType; + + /// + /// File path of the file this node represents. + /// + private string filePath; + + /// + /// Gets or sets formClass + /// + public string FormClass + { + get { return formClass; } + set { formClass = value; } + } + + /// + /// Gets or sets objectType + /// + public string ObjectType + { + get { return objectType; } + set { objectType = value; } + } + + /// + /// Gets or sets filePath + /// + public string FilePath + { + get { return filePath; } + set { filePath = value; } + } + } + + public enum VfsFileRecordRelativeOffsetLocationType + { + FileRecordStart, + FileRecordEnd + } + + public struct VfsExtractionStruct + { + // header size + public bool UseStaticHeaderSize { set; get; } + public string StaticHeaderSize { set; get; } + public bool UseHeaderSizeOffset { set; get; } + public OffsetDescription HeaderSizeOffsetDescription { set; get; } + public bool ReadHeaderToEof { set; get; } + + // file count + public bool UseStaticFileCount { set; get; } + public string StaticFileCount { set; get; } + public bool UseFileCountOffset { set; get; } + public OffsetDescription FileCountOffsetDescription { set; get; } + + // file record basic information + public string FileRecordsStartOffset { set; get; } + public string FileRecordSize { set; get; } + + // file offset + public bool UseFileOffsetOffset { set; get; } + public CalculatingOffsetDescription FileOffsetOffsetDescription { set; get; } + + public bool UsePreviousFilesSizeToDetermineOffset { set; get; } + public string BeginCuttingFilesAtOffset { set; get; } + public bool UseByteAlignmentValue { set; get; } + public string ByteAlignmentValue { set; get; } + + // file length/size + public bool UseFileLengthOffset { set; get; } + public CalculatingOffsetDescription FileLengthOffsetDescription { set; get; } + + public bool UseLocationOfNextFileToDetermineLength { set; get; } + + // file name + public bool FileNameIsPresent { set; get; } + + public bool UseStaticFileNameOffsetWithinRecord { set; get; } + public string StaticFileNameOffsetWithinRecord { set; get; } + + public bool UseAbsoluteFileNameOffset { set; get; } + public OffsetDescription AbsoluteFileNameOffsetDescription { set; get; } + + public bool UseRelativeFileNameOffset { set; get; } + public OffsetDescription RelativeFileNameOffsetDescription { set; get; } + public VfsFileRecordRelativeOffsetLocationType FileRecordNameRelativeOffsetLocation { set; get; } + + // name size + public bool UseStaticFileNameLength { set; get; } + public string StaticFileNameLength { set; get; } + + public bool UseFileNameTerminatorString { set; get; } + public string FileNameTerminatorString { set; get; } + } + + public struct SimpleFileExtractionStruct + { + public string FilePath { set; get; } + public long FileOffset { set; get; } + public long FileLength { set; get; } + public long FileNameLength { set; get; } + + public void Clear() + { + this.FilePath = String.Empty; + this.FileOffset = -1; + this.FileLength = -1; + this.FileNameLength = -1; + } + } + + /// + /// Class containing universal constants. + /// + public sealed class Constants + { + /// + /// Chunk size to use when reading from files. Used to grab maximum buffer + /// size without using the large object heap which has poor collection. + /// + public const int FileReadChunkSize = 71680; + + /// + /// Constant used to send an ignore the value message to the progress bar. + /// + public const int IgnoreProgress = -1; + + /// + /// Constant used to send a generic message to the progress bar. + /// + public const int ProgressMessageOnly = -2; + + /// + /// Text description to use when describing a Big Endian option + /// + public const string BigEndianByteOrder = "Big Endian"; + + /// + /// Text description to use when describing a Little Endian option + /// + public const string LittleEndianByteOrder = "Little Endian"; + + public static readonly byte[] RiffHeaderBytes = new byte[] { 0x52, 0x49, 0x46, 0x46 }; + public static readonly byte[] RiffDataBytes = new byte[] { 0x64, 0x61, 0x74, 0x61 }; + public static readonly byte[] RiffWaveBytes = new byte[] { 0x57, 0x41, 0x56, 0x45 }; + public static readonly byte[] RiffFmtBytes = new byte[] { 0x66, 0x6D, 0x74, 0x20 }; + + public static readonly byte[] NullByteArray = new byte[] { 0x00 }; + + public const string StringNullTerminator = "\0"; + + // empty constructor + private Constants() { } + } +} diff --git a/acb_extract/src/CriAcbFile.cs b/acb_extract/src/CriAcbFile.cs new file mode 100644 index 0000000..95bc9af --- /dev/null +++ b/acb_extract/src/CriAcbFile.cs @@ -0,0 +1,684 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +//using VGMToolbox.util; + +namespace vgm_acb.extract +{ + public class CriAcbCueRecord + { + public uint CueId { set; get; } + public byte ReferenceType { set; get; } + public ushort ReferenceIndex { set; get; } + + public ushort numWaveforms { set; get; } + public bool IsWaveformIdentified { set; get; } + public ushort WaveformIndex { set; get; } + public ushort WaveformId { set; get; } + public byte EncodeType { set; get; } + public bool IsStreaming { set; get; } + + public string CueName { set; get; } + } + + public class CriAcbSeqRecord + { + public ushort NumTracks { set; get; } + public ushort[] TrackIndex { set; get; } + public byte Type { set; get; } + } + public class CriAcbFile : CriUtfTable + { + public const string EXTRACTION_FOLDER_FORMAT = "_vgmt_acb_ext_{0}"; + public static readonly string ACB_AWB_EXTRACTION_FOLDER = "acb" + Path.DirectorySeparatorChar + "awb"; + public static readonly string EXT_AWB_EXTRACTION_FOLDER = "awb"; + + public const byte WAVEFORM_ENCODE_TYPE_ADX = 0; + public const byte WAVEFORM_ENCODE_TYPE_HCA = 2; + public const byte WAVEFORM_ENCODE_TYPE_VAG = 7; + public const byte WAVEFORM_ENCODE_TYPE_ATRAC3 = 8; + public const byte WAVEFORM_ENCODE_TYPE_BCWAV = 9; + public const byte WAVEFORM_ENCODE_TYPE_ATRAC9 = 11; + public const byte WAVEFORM_ENCODE_TYPE_NINTENDO_DSP = 13; + + public const string AWB_FORMAT1 = "{0}_streamfiles.awb"; + public const string AWB_FORMAT2 = "{0}.awb"; + public const string AWB_FORMAT3 = "{0}_STR.awb"; + + protected enum AwbToExtract { Internal, External }; + + public CriAcbFile(FileStream fs, long offset, bool includeCueIdInFileName) + { + // initialize UTF + this.Initialize(fs, offset); + + // initialize ACB specific items + this.InitializeCueList(fs); + this.InitializeCueNameToWaveformMap(fs, includeCueIdInFileName); + + // initialize internal AWB + if (this.InternalAwbFileSize > 0) + { + this.InternalAwb = new CriAfs2Archive(fs, (long)this.InternalAwbFileOffset); + } + + // initialize external AWB + if (this.StreamAwbAfs2HeaderSize > 0) + { + this.ExternalAwb = this.InitializeExternalAwbArchive(); + } + } + + public string Name + { + get + { + return (string)CriUtfTable.GetUtfFieldForRow(this, 0, "Name"); + } + } + public string VersionString + { + get + { + return (string)CriUtfTable.GetUtfFieldForRow(this, 0, "VersionString"); + } + } + + public ulong InternalAwbFileOffset + { + get + { + return (ulong)CriUtfTable.GetOffsetForUtfFieldForRow(this, 0, "AwbFile"); + } + + } + public ulong InternalAwbFileSize + { + get + { + return (ulong)CriUtfTable.GetSizeForUtfFieldForRow(this, 0, "AwbFile"); + } + + } + + public ulong CueTableOffset + { + get + { + return CriUtfTable.GetOffsetForUtfFieldForRow(this, 0, "CueTable"); + } + } + public ulong CueNameTableOffset + { + get + { + return CriUtfTable.GetOffsetForUtfFieldForRow(this, 0, "CueNameTable"); + } + } + public ulong WaveformTableOffset + { + get + { + return CriUtfTable.GetOffsetForUtfFieldForRow(this, 0, "WaveformTable"); + } + } + public ulong SynthTableOffset + { + get + { + return CriUtfTable.GetOffsetForUtfFieldForRow(this, 0, "SynthTable"); + } + } + + public ulong SeqTableOffset + { + get + { + return CriUtfTable.GetOffsetForUtfFieldForRow(this, 0, "SequenceTable"); + } + } + + public CriAcbCueRecord[] CueList { set; get; } + public CriAcbSeqRecord[] SeqList { set; get; } + + public Dictionary CueNamesToWaveforms { set; get; } + + public byte[] AcfMd5Hash + { + get + { + return (byte[])CriUtfTable.GetUtfFieldForRow(this, 0, "AcfMd5Hash"); + } + } + public ulong AwbFileOffset + { + get + { + return CriUtfTable.GetOffsetForUtfFieldForRow(this, 0, "AwbFile"); + } + } + public CriAfs2Archive InternalAwb { set; get; } + + public byte[] StreamAwbHash + { + get + { + return (byte[])CriUtfTable.GetUtfFieldForRow(this, 0, "StreamAwbHash"); + } + } + public ulong StreamAwbAfs2HeaderOffset + { + get + { + return (ulong)CriUtfTable.GetOffsetForUtfFieldForRow(this, 0, "StreamAwbAfs2Header"); + } + } + public ulong StreamAwbAfs2HeaderSize + { + get + { + return (ulong)CriUtfTable.GetSizeForUtfFieldForRow(this, 0, "StreamAwbAfs2Header"); + } + } + public CriAfs2Archive ExternalAwb { set; get; } + + public void ExtractAll() + { + string baseExtractionFolder = Path.Combine(Path.GetDirectoryName(this.SourceFile), + String.Format(EXTRACTION_FOLDER_FORMAT, Path.GetFileNameWithoutExtension(this.SourceFile))); + + this.ExtractAllUsingCueList(baseExtractionFolder); + } + + + protected void InitializeCueNameToWaveformMap(FileStream fs, bool includeCueIdInFileName) + { + ushort cueIndex; + string cueName; + + CriUtfTable cueNameTableUtf = new CriUtfTable(); + cueNameTableUtf.Initialize(fs, (long)this.CueNameTableOffset); + + for (int i = 0; i < cueNameTableUtf.NumberOfRows; i++) + { + cueIndex = (ushort)CriUtfTable.GetUtfFieldForRow(cueNameTableUtf, i, "CueIndex"); + + // skip cues with unidentified waveforms (see 'vc05_0140.acb, vc10_0372.acb' in Kidou_Senshi_Gundam_AGE_Universe_Accel (PSP)) + if (this.CueList[cueIndex].IsWaveformIdentified) + { + cueName = (string)CriUtfTable.GetUtfFieldForRow(cueNameTableUtf, i, "CueName"); + + this.CueList[cueIndex].CueName = cueName; + this.CueList[cueIndex].CueName += CriAcbFile.GetFileExtensionForEncodeType(this.CueList[cueIndex].EncodeType); + + if (includeCueIdInFileName) + { + this.CueList[cueIndex].CueName = String.Format("{0}_{1}", + this.CueList[cueIndex].CueId.ToString("D5"), this.CueList[cueIndex].CueName); + } + + this.CueNamesToWaveforms.Add(this.CueList[cueIndex].CueName, this.CueList[cueIndex].WaveformId); + } + } + } + + protected void InitializeCueList(FileStream fs) + { + this.CueNamesToWaveforms = new Dictionary(); + + ulong referenceItemsOffset = 0; + ulong referenceItemsSize = 0; + ulong referenceCorrection = 0; + + object dummy; + byte isStreaming = 0; + + CriUtfTable cueTableUtf = new CriUtfTable(); + cueTableUtf.Initialize(fs, (long)this.CueTableOffset); + + CriUtfTable waveformTableUtf = new CriUtfTable(); + waveformTableUtf.Initialize(fs, (long)this.WaveformTableOffset); + + CriUtfTable synthTableUtf = new CriUtfTable(); + synthTableUtf.Initialize(fs, (long)this.SynthTableOffset); + + CriUtfTable seqTableUtf = new CriUtfTable(); + seqTableUtf.Initialize(fs, (long)this.SeqTableOffset); + + + this.SeqList = new CriAcbSeqRecord[seqTableUtf.NumberOfRows]; + this.CueList = new CriAcbCueRecord[cueTableUtf.NumberOfRows]; + + for (int i = 0; i < seqTableUtf.NumberOfRows; i++) + { + this.SeqList[i] = new CriAcbSeqRecord(); + + this.SeqList[i].NumTracks = (ushort)CriUtfTable.GetUtfFieldForRow(seqTableUtf, i, "NumTracks"); + this.SeqList[i].Type = (byte)CriUtfTable.GetUtfFieldForRow(seqTableUtf, i, "Type"); + + var tindex = (byte[])CriUtfTable.GetUtfFieldForRow(seqTableUtf, i, "TrackIndex"); + this.SeqList[i].TrackIndex = new ushort[(int)Math.Ceiling(tindex.Length / 2.0)]; + + Buffer.BlockCopy(tindex, 0, this.SeqList[i].TrackIndex, 0, tindex.Length); + + for (int j = 0; j < this.SeqList[i].NumTracks; j++) { + // i'm a lazy bastard and don't want to look up a better way + // byte order is dumb + ushort a = (ushort) (this.SeqList[i].TrackIndex[j] & 0xff00u); + ushort b = (ushort) (this.SeqList[i].TrackIndex[j] & 0xffu); + this.SeqList[i].TrackIndex[j] = (ushort) ((b << 8) | (a >> 8)); + } + } + + for (int i = 0; i < cueTableUtf.NumberOfRows; i++) + { + this.CueList[i] = new CriAcbCueRecord(); + this.CueList[i].IsWaveformIdentified = false; + + this.CueList[i].CueId = (uint)CriUtfTable.GetUtfFieldForRow(cueTableUtf, i, "CueId"); + this.CueList[i].ReferenceType = (byte)CriUtfTable.GetUtfFieldForRow(cueTableUtf, i, "ReferenceType"); + this.CueList[i].ReferenceIndex = (ushort)CriUtfTable.GetUtfFieldForRow(cueTableUtf, i, "ReferenceIndex"); + + this.CueList[i].numWaveforms = this.SeqList[i].NumTracks; + + switch (this.CueList[i].ReferenceType) + { + case 2: + referenceItemsOffset = (ulong)CriUtfTable.GetOffsetForUtfFieldForRow(synthTableUtf, this.CueList[i].ReferenceIndex, "ReferenceItems"); + referenceItemsSize = CriUtfTable.GetSizeForUtfFieldForRow(synthTableUtf, this.CueList[i].ReferenceIndex, "ReferenceItems"); + referenceCorrection = referenceItemsSize + 2; + break; + case 3: + case 8: + if (i == 0) + { + referenceItemsOffset = (ulong)CriUtfTable.GetOffsetForUtfFieldForRow(synthTableUtf, 0, "ReferenceItems"); + referenceItemsSize = CriUtfTable.GetSizeForUtfFieldForRow(synthTableUtf, 0, "ReferenceItems"); + referenceCorrection = referenceItemsSize - 2; // samples found have only had a '01 00' record => Always Waveform[0]?. + } + else + { + //referenceCorrection += 4; // relative to previous offset, do not lookup + // @TODO: Should this do a referenceItemsSize - 2 for the ReferenceIndex? Need to find + // one where size > 4. + //referenceItemsOffset = (ulong)CriUtfTable.GetOffsetForUtfFieldForRow(synthTableUtf, this.CueList[i].ReferenceIndex, "ReferenceItems"); + //referenceItemsSize = CriUtfTable.GetSizeForUtfFieldForRow(synthTableUtf, this.CueList[i].ReferenceIndex, "ReferenceItems"); + //referenceCorrection = referenceItemsSize - 2; + referenceCorrection += 4 * (ulong)this.SeqList[i-1].NumTracks;// * (this.CueList[i].CueId - this.CueList[i-1].CueId); + } + break; + default: + throw new FormatException(String.Format(" Unexpected ReferenceType: '{0}' for CueIndex: '{1}.' Please report to VGMToolbox thread at hcs64.com forums, see link in 'Other' menu item.", this.CueList[i].ReferenceType.ToString("D"), i.ToString("D"))); + } + + if (referenceItemsSize != 0) + { + // get wave form info + this.CueList[i].WaveformIndex = ParseFile.ReadUshortBE(fs, (long)(referenceItemsOffset + referenceCorrection)); + + // get Streaming flag, 0 = in ACB files, 1 = in AWB file + dummy = CriUtfTable.GetUtfFieldForRow(waveformTableUtf, this.CueList[i].WaveformIndex, "Streaming"); + + if (dummy != null) // check to see if this Id actually exists in the WaveformIndex + { + isStreaming = (byte)dummy; + this.CueList[i].IsStreaming = isStreaming == 0 ? false : true; + + // get waveform id and encode type from corresponding waveform + var waveformId = CriUtfTable.GetUtfFieldForRow(waveformTableUtf, this.CueList[i].WaveformIndex, "Id"); + + if (waveformId != null) + { + // early revisions of ACB spec used "Id" + this.CueList[i].WaveformId = (ushort)waveformId; + } + else + { + // later versions using "MemoryAwbId" and "StreamingAwbId" + if (this.CueList[i].IsStreaming) + { + this.CueList[i].WaveformId = (ushort)CriUtfTable.GetUtfFieldForRow(waveformTableUtf, this.CueList[i].WaveformIndex, "StreamAwbId"); + } + else + { + this.CueList[i].WaveformId = (ushort)CriUtfTable.GetUtfFieldForRow(waveformTableUtf, this.CueList[i].WaveformIndex, "MemoryAwbId"); + } + } + + this.CueList[i].EncodeType = (byte)CriUtfTable.GetUtfFieldForRow(waveformTableUtf, this.CueList[i].WaveformIndex, "EncodeType"); + + + // update flag + this.CueList[i].IsWaveformIdentified = true; + } // if (dummy != null) + } + } + } + + protected void ExtractAwbWithCueNames(string destinationFolder, AwbToExtract whichAwb) + { + CriUtfTable waveformTableUtf; + byte encodeType; + string rawFileName; + + CriAfs2Archive awb; + string rawFileFormat = "{0}_{1}{2}"; + + if (whichAwb == AwbToExtract.Internal) + { + awb = this.InternalAwb; + } + else + { + awb = this.ExternalAwb; + } + + using (FileStream fs = File.Open(awb.SourceFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + // use files names for internal AWB + foreach (string key in this.CueNamesToWaveforms.Keys) + { + // extract file + ParseFile.ExtractChunkToFile64(fs, + (ulong)awb.Files[this.CueNamesToWaveforms[key]].FileOffsetByteAligned, + (ulong)awb.Files[this.CueNamesToWaveforms[key]].FileLength, + Path.Combine(destinationFolder, FileUtil.CleanFileName(key)), false, false); + } + + //------------------------------------- + // extract any items without a CueName + //------------------------------------- + using (FileStream acbStream = File.Open(this.SourceFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + waveformTableUtf = new CriUtfTable(); + waveformTableUtf.Initialize(acbStream, (long)this.WaveformTableOffset); + } + + // get list of unextracted files + var unextractedFiles = awb.Files.Keys.Where(x => !this.CueNamesToWaveforms.ContainsValue(x)); + foreach (ushort key in unextractedFiles) + { + encodeType = (byte)CriUtfTable.GetUtfFieldForRow(waveformTableUtf, key, "EncodeType"); + + rawFileName = String.Format(rawFileFormat, + Path.GetFileNameWithoutExtension(awb.SourceFile), key.ToString("D5"), + CriAcbFile.GetFileExtensionForEncodeType(encodeType)); + + // extract file + ParseFile.ExtractChunkToFile64(fs, + (ulong)awb.Files[key].FileOffsetByteAligned, + (ulong)awb.Files[key].FileLength, + Path.Combine(destinationFolder, rawFileName), false, false); + } + } + } + + protected void ExtractAllUsingCueList(string destinationFolder) + { + CriUtfTable waveformTableUtf; + ushort waveformIndex; + byte encodeType; + string rawFileName; + string rawFileFormat = "{0}.{1}{2}"; + + FileStream internalFs = null; + FileStream externalFs = null; + + ArrayList internalIdsExtracted = new ArrayList(); + ArrayList externalIdsExtracted = new ArrayList(); + + string acbAwbDestinationFolder = Path.Combine(destinationFolder, ACB_AWB_EXTRACTION_FOLDER); + string extAwbDestinationFolder = Path.Combine(destinationFolder, EXT_AWB_EXTRACTION_FOLDER); + + try + { + // open streams + if (this.InternalAwb != null) + { + internalFs = File.Open(this.InternalAwb.SourceFile, FileMode.Open, FileAccess.Read, FileShare.Read); + } + + if (this.ExternalAwb != null) + { + externalFs = File.Open(this.ExternalAwb.SourceFile, FileMode.Open, FileAccess.Read, FileShare.Read); + } + + // loop through cues and extract + + for (int i = 0; i < CueList.Length; i++) + { + CriAcbCueRecord cue = CueList[i]; + + if (cue.IsWaveformIdentified) + { + if (cue.IsStreaming) // external AWB file + { + ParseFile.ExtractChunkToFile64(externalFs, + (ulong)this.ExternalAwb.Files[cue.WaveformId].FileOffsetByteAligned, + (ulong)this.ExternalAwb.Files[cue.WaveformId].FileLength, + Path.Combine(extAwbDestinationFolder, FileUtil.CleanFileName(cue.CueName)), false, false); + + externalIdsExtracted.Add(cue.WaveformId); + } + else // internal AWB file (inside ACB) + { + ParseFile.ExtractChunkToFile64(internalFs, + (ulong)this.InternalAwb.Files[cue.WaveformId].FileOffsetByteAligned, + (ulong)this.InternalAwb.Files[cue.WaveformId].FileLength, + Path.Combine(acbAwbDestinationFolder, FileUtil.CleanFileName(cue.CueName)), false, false); + + internalIdsExtracted.Add(cue.WaveformId); + } + } // if (cue.IsWaveformIdentified) + } + + // extract leftovers + using (FileStream acbStream = File.Open(this.SourceFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + waveformTableUtf = new CriUtfTable(); + waveformTableUtf.Initialize(acbStream, (long)this.WaveformTableOffset); + } + + if (this.ExternalAwb != null) + { + var unextractedExternalFiles = this.ExternalAwb.Files.Keys.Where(x => !externalIdsExtracted.Contains(x)); + foreach (ushort key in unextractedExternalFiles) + { + waveformIndex = GetWaveformRowIndexForWaveformId(waveformTableUtf, key, true); + + encodeType = (byte)CriUtfTable.GetUtfFieldForRow(waveformTableUtf, waveformIndex, "EncodeType"); + + rawFileName = String.Format(rawFileFormat, + Path.GetFileName(this.ExternalAwb.SourceFile), key.ToString("D5"), + CriAcbFile.GetFileExtensionForEncodeType(encodeType)); + + // extract file + ParseFile.ExtractChunkToFile64(externalFs, + (ulong)this.ExternalAwb.Files[key].FileOffsetByteAligned, + (ulong)this.ExternalAwb.Files[key].FileLength, + Path.Combine(extAwbDestinationFolder, rawFileName), false, false); + } + } + + if (this.InternalAwb != null) + { + var unextractedInternalFiles = this.InternalAwb.Files.Keys.Where(x => !internalIdsExtracted.Contains(x)); + foreach (ushort key in unextractedInternalFiles) + { + waveformIndex = GetWaveformRowIndexForWaveformId(waveformTableUtf, key, false); + + encodeType = (byte)CriUtfTable.GetUtfFieldForRow(waveformTableUtf, waveformIndex, "EncodeType"); + + rawFileName = String.Format(rawFileFormat, + Path.GetFileName(this.InternalAwb.SourceFile), key.ToString("D5"), + CriAcbFile.GetFileExtensionForEncodeType(encodeType)); + + // extract file + ParseFile.ExtractChunkToFile64(internalFs, + (ulong)this.InternalAwb.Files[key].FileOffsetByteAligned, + (ulong)this.InternalAwb.Files[key].FileLength, + Path.Combine(acbAwbDestinationFolder, rawFileName), false, false); + } + } + } + catch (Exception ex) + { + throw ex; + } + finally + { + if (internalFs != null) + { + internalFs.Close(); + internalFs.Dispose(); + } + + if (externalFs != null) + { + externalFs.Close(); + externalFs.Dispose(); + } + + } + } + + protected CriAfs2Archive InitializeExternalAwbArchive() + { + CriAfs2Archive afs2 = null; + + string awbDirectory; + string awbMask; + string acbBaseFileName; + string[] awbFiles; + + byte[] awbHashStored; + byte[] awbHashCalculated; + + awbDirectory = Path.GetDirectoryName(Path.GetFullPath(this.SourceFile)); + + // try format 1 + acbBaseFileName = Path.GetFileNameWithoutExtension(this.SourceFile); + awbMask = String.Format(AWB_FORMAT1, acbBaseFileName); + awbFiles = Directory.GetFiles(awbDirectory, awbMask, SearchOption.TopDirectoryOnly); + + if (awbFiles.Length < 1) + { + // try format 2 + awbMask = String.Format(AWB_FORMAT2, acbBaseFileName); + awbFiles = Directory.GetFiles(awbDirectory, awbMask, SearchOption.TopDirectoryOnly); + } + + if (awbFiles.Length < 1) + { + // try format 3 + awbMask = String.Format(AWB_FORMAT3, acbBaseFileName); + awbFiles = Directory.GetFiles(awbDirectory, awbMask, SearchOption.TopDirectoryOnly); + } + + // file not found + if (awbFiles.Length < 1) + { + throw new FileNotFoundException(String.Format("Cannot find AWB file. Please verify corresponding AWB file is named '{0}', '{1}', or '{2}'.", + String.Format(AWB_FORMAT1, acbBaseFileName), String.Format(AWB_FORMAT2, acbBaseFileName), String.Format(AWB_FORMAT3, acbBaseFileName))); + } + + if (awbFiles.Length > 1) + { + throw new FileNotFoundException(String.Format("More than one matching AWB file for this ACB. Please verify only one AWB file is named '{1}' or '{2}'.", + String.Format(AWB_FORMAT1, acbBaseFileName), String.Format(AWB_FORMAT2, acbBaseFileName))); + } + + // initialize AFS2 file + using (FileStream fs = File.Open(awbFiles[0], FileMode.Open, FileAccess.Read, FileShare.Read)) + { + awbHashStored = this.StreamAwbHash; + + if (awbHashStored.Length == 0x10) // MD5, hash in newer format unknown + { + // validate MD5 checksum + /* + awbHashCalculated = ByteConversion.GetBytesFromHexString(ChecksumUtil.GetMd5OfFullFile(fs)); + + if (!ParseFile.CompareSegment(awbHashCalculated, 0, awbHashStored)) + { + throw new FormatException(String.Format("AWB file, <{0}>, did not match checksum inside ACB file.", Path.GetFileName(fs.Name))); + } + Validate my dick */ + } + + afs2 = new CriAfs2Archive(fs, 0); + } + + return afs2; + } + + public static ushort GetWaveformRowIndexForWaveformId(CriUtfTable utfTable, ushort waveformId, bool isStreamed) + { + ushort ret = ushort.MaxValue; + + for (int i = 0; i < utfTable.NumberOfRows; i++) + { + var tempId = CriUtfTable.GetUtfFieldForRow(utfTable, i, "Id"); + + // newer archives use different labels + if (tempId == null) + { + if (isStreamed) + { + tempId = CriUtfTable.GetUtfFieldForRow(utfTable, i, "StreamAwbId"); + } + else + { + tempId = CriUtfTable.GetUtfFieldForRow(utfTable, i, "MemoryAwbId"); + } + } + + if ((ushort)tempId == waveformId) + { + ret = (ushort)i; + } + } + + return ret; + } + + public static string GetFileExtensionForEncodeType(byte encodeType) + { + string ext; + + switch (encodeType) + { + case WAVEFORM_ENCODE_TYPE_ADX: + ext = ".adx"; + break; + case WAVEFORM_ENCODE_TYPE_HCA: + ext = ".hca"; + break; + case WAVEFORM_ENCODE_TYPE_ATRAC3: + ext = ".at3"; + break; + case WAVEFORM_ENCODE_TYPE_ATRAC9: + ext = ".at9"; + break; + case WAVEFORM_ENCODE_TYPE_VAG: + ext = ".vag"; + break; + case WAVEFORM_ENCODE_TYPE_BCWAV: + ext = ".bcwav"; + break; + case WAVEFORM_ENCODE_TYPE_NINTENDO_DSP: + ext = ".dsp"; + break; + default: + ext = String.Format(".EncodeType-{0}.bin", encodeType.ToString("D2")); + break; + } + + return ext; + } + } +} diff --git a/acb_extract/src/CriAfs2Archive.cs b/acb_extract/src/CriAfs2Archive.cs new file mode 100644 index 0000000..1d1aab6 --- /dev/null +++ b/acb_extract/src/CriAfs2Archive.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +//using VGMToolbox.util; + +namespace vgm_acb.extract +{ + public class CriAfs2File + { + public ushort CueId { set; get; } + public long FileOffsetRaw { set; get; } + public long FileOffsetByteAligned { set; get; } + public long FileLength { set; get; } + + // public string FileName { set; get; } // comes from ACB for now, maybe included in archive later? + } + + public class CriAfs2Archive + { + public static readonly byte[] SIGNATURE = new byte[] { 0x41, 0x46, 0x53, 0x32 }; + public const string EXTRACTION_FOLDER_FORMAT = "_vgmt_awb_ext_{0}"; + + public string SourceFile { set; get; } + public byte[] MagicBytes { set; get; } + public byte[] Version { set; get; } + public uint FileCount { set; get; } + public uint ByteAlignment { set; get; } + public Dictionary Files { set; get; } + //public CriAfs2File[] Files { set; get; } + + public CriAfs2Archive(FileStream fs, long offset) + { + ushort previousCueId = ushort.MaxValue; + + if (IsCriAfs2Archive(fs, offset)) + { + this.SourceFile = fs.Name; + long afs2FileSize = fs.Length; + + this.MagicBytes = ParseFile.ParseSimpleOffset(fs, offset, SIGNATURE.Length); + this.Version = ParseFile.ParseSimpleOffset(fs, offset + 4, 4); + this.FileCount = ParseFile.ReadUintLE(fs, offset + 8); + + // setup offset field size + int offsetFieldSize = this.Version[1]; // known values: 2 and 4. 4 is most common. I've only seen 2 in 'se_enemy_gurdon_galaga_bee.acb' from Sonic Lost World. + uint offsetMask = 0; + + for (int j = 0; j < offsetFieldSize; j++) + { + offsetMask |= (uint)((byte)0xFF << (j * 8)); + } + + if (this.FileCount > ushort.MaxValue) + { + throw new FormatException(String.Format("ERROR, file count exceeds max value for ushort. Please report this at official feedback forums (see 'Other' menu item).", fs.Name)); + } + + this.ByteAlignment = ParseFile.ReadUintLE(fs, offset + 0xC); + this.Files = new Dictionary((int)this.FileCount); + + CriAfs2File dummy; + + for (ushort i = 0; i < this.FileCount; i++) + { + dummy = new CriAfs2File(); + + dummy.CueId = ParseFile.ReadUshortLE(fs, offset + (0x10 + (2 * i))); + dummy.FileOffsetRaw = ParseFile.ReadUintLE(fs, offset + (0x10 + (this.FileCount * 2) + (offsetFieldSize * i))); + + // mask off unneeded info + dummy.FileOffsetRaw &= offsetMask; + + // add offset + dummy.FileOffsetRaw += offset; // for AFS2 files inside of other files (ACB, etc.) + + // set file offset to byte alignment + if ((dummy.FileOffsetRaw % this.ByteAlignment) != 0) + { + dummy.FileOffsetByteAligned = MathUtil.RoundUpToByteAlignment(dummy.FileOffsetRaw, this.ByteAlignment); + } + else + { + dummy.FileOffsetByteAligned = dummy.FileOffsetRaw; + } + + //--------------- + // set file size + //--------------- + // last file will use final offset entry + if (i == this.FileCount - 1) + { + dummy.FileLength = (ParseFile.ReadUintLE(fs, offset + (0x10 + (this.FileCount * 2) + ((offsetFieldSize) * i)) + offsetFieldSize) + offset) - dummy.FileOffsetByteAligned; + } + + // else set length for previous cue id + if (previousCueId != ushort.MaxValue) + { + this.Files[previousCueId].FileLength = dummy.FileOffsetRaw - this.Files[previousCueId].FileOffsetByteAligned; + } + + this.Files.Add(dummy.CueId, dummy); + previousCueId = dummy.CueId; + } // for (uint i = 0; i < this.FileCount; i++) + + } + else + { + throw new FormatException(String.Format("AFS2 magic bytes not found at offset: 0x{0}.", offset.ToString("X8"))); + } + } + + public static bool IsCriAfs2Archive(FileStream fs, long offset) + { + bool ret = false; + byte[] checkBytes = ParseFile.ParseSimpleOffset(fs, offset, SIGNATURE.Length); + + if (ParseFile.CompareSegment(checkBytes, 0, SIGNATURE)) + { + ret = true; + + } + + return ret; + } + + public void ExtractAll() + { + string baseExtractionFolder = Path.Combine(Path.GetDirectoryName(this.SourceFile), + String.Format(EXTRACTION_FOLDER_FORMAT, Path.GetFileNameWithoutExtension(this.SourceFile))); + + this.ExtractAllRaw(baseExtractionFolder); + } + + public void ExtractAllRaw(string destinationFolder) + { + string rawFileFormat = "{0}.{1}.bin"; + + using (FileStream fs = File.Open(this.SourceFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + + foreach (ushort key in this.Files.Keys) + { + ParseFile.ExtractChunkToFile64(fs, + (ulong)this.Files[key].FileOffsetByteAligned, + (ulong)this.Files[key].FileLength, + Path.Combine(destinationFolder, + String.Format(rawFileFormat, Path.GetFileName(this.SourceFile), key.ToString("D5"))), false, false); + } + } + } + + } +} diff --git a/acb_extract/src/CriUtfTable.cs b/acb_extract/src/CriUtfTable.cs new file mode 100644 index 0000000..84e8f7d --- /dev/null +++ b/acb_extract/src/CriUtfTable.cs @@ -0,0 +1,864 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Text; + +//using VGMToolbox.util; + +namespace vgm_acb.extract +{ + public class CriField + { + public byte Type { set; get; } + public string Name { set; get; } + public object Value { set; get; } + public ulong Offset { set; get; } + public ulong Size { set; get; } + + public override string ToString() + { + StringBuilder sb = new StringBuilder(); + string value = String.Empty; + + sb.AppendFormat("0x{0} {1} {2} = {3}", + this.Offset.ToString("X8"), + this.Type.ToString("X2"), + this.Name, + this.GetStringValue()); + return sb.ToString(); + } + + public string ToString(FileStream fs) + { + StringBuilder sb = new StringBuilder(); + string value = String.Empty; + + sb.AppendFormat("0x{0} {1} {2} = {3}", + this.Offset.ToString("X8"), + this.Type.ToString("X2"), + this.Name, + this.GetStringValue(fs)); + return sb.ToString(); + } + + public string GetStringValue(FileStream utfStream = null) + { + ulong numericValue; + byte maskedType = (byte)(this.Type & CriUtfTable.COLUMN_TYPE_MASK); + string stringValue = String.Empty; + + switch (maskedType) + { + case CriUtfTable.COLUMN_TYPE_DATA: + if (utfStream == null) + { + stringValue = this.Value.ToString(); + } + else + { + if (CriUtfTable.IsUtfTable(utfStream, (long)this.Offset)) + { + CriUtfTable newUtf = new CriUtfTable(); + newUtf.Initialize(utfStream, (long)this.Offset); + } + } + break; + case CriUtfTable.COLUMN_TYPE_STRING: + stringValue = (string)this.Value; + break; + case CriUtfTable.COLUMN_TYPE_FLOAT: + stringValue = Convert.ToSingle(this.Value).ToString(); + break; + case CriUtfTable.COLUMN_TYPE_8BYTE: + case CriUtfTable.COLUMN_TYPE_4BYTE2: + case CriUtfTable.COLUMN_TYPE_4BYTE: + case CriUtfTable.COLUMN_TYPE_2BYTE2: + case CriUtfTable.COLUMN_TYPE_2BYTE: + case CriUtfTable.COLUMN_TYPE_1BYTE2: + case CriUtfTable.COLUMN_TYPE_1BYTE: + numericValue = Convert.ToUInt64(this.Value); + stringValue = String.Format("0x{0}", numericValue.ToString("X8")); + break; + + } + + return stringValue; + } + } + + public class CriUtfTocFileInfo + { + public string DirName { set; get; } + public string FileName { set; get; } + public uint FileSize { set; get; } + public uint ExtractSize { set; get; } + public ulong FileOffset { set; get; } + } + + public class CriUtfTable + { + // thanks to hcs for all of this!!! + + #region Constants + + public static readonly byte[] SIGNATURE_BYTES = new byte[] { 0x40, 0x55, 0x54, 0x46 }; + public const string LCG_SEED_KEY = "SEED"; + public const string LCG_INCREMENT_KEY = "INC"; + public const string TEMP_UTF_TABLE_FILE = "VGMT_UTF.BIN"; + + public const byte COLUMN_STORAGE_MASK = 0xF0; + public const byte COLUMN_STORAGE_PERROW = 0x50; + public const byte COLUMN_STORAGE_CONSTANT = 0x30; + public const byte COLUMN_STORAGE_CONSTANT2 = 0x70; + public const byte COLUMN_STORAGE_ZERO = 0x10; + + // I suspect that "type 2" is signed + public const byte COLUMN_TYPE_MASK = 0x0F; + public const byte COLUMN_TYPE_DATA = 0x0B; + public const byte COLUMN_TYPE_STRING = 0x0A; + // 0x09 double? + public const byte COLUMN_TYPE_FLOAT = 0x08; + // 0x07 signed 8byte? + public const byte COLUMN_TYPE_8BYTE = 0x06; + public const byte COLUMN_TYPE_4BYTE2 = 0x05; + public const byte COLUMN_TYPE_4BYTE = 0x04; + public const byte COLUMN_TYPE_2BYTE2 = 0x03; + public const byte COLUMN_TYPE_2BYTE = 0x02; + public const byte COLUMN_TYPE_1BYTE2 = 0x01; + public const byte COLUMN_TYPE_1BYTE = 0x00; + + #endregion + + #region Members + + public string SourceFile { set; get; } + public long BaseOffset { set; get; } + public string UtfTableFile { set; get; } + + public byte[] MagicBytes { set; get; } + public bool IsEncrypted { set; get; } + public uint TableSize { set; get; } + + public ushort Unknown1 { set; get; } + + public uint RowOffset { set; get; } + public uint StringTableOffset { set; get; } + public uint DataOffset { set; get; } + + public uint TableNameOffset { set; get; } + public string TableName { set; get; } + + public ushort NumberOfFields { set; get; } + + public ushort RowSize { set; get; } + public uint NumberOfRows { set; get; } + + public Dictionary[] Rows { set; get; } + + public CriUtfReader UtfReader { set; get; } + + #endregion + + public void Initialize(FileStream fs, long offset) + { + this.SourceFile = fs.Name; + this.BaseOffset = offset; + + try + { + this.MagicBytes = ParseFile.ParseSimpleOffset(fs, this.BaseOffset, 4); + + // check if file is decrypted and get decryption keys if needed + this.checkEncryption(fs); + + // write (decrypted) utf header to file + this.UtfTableFile = this.WriteTableToTempFile(fs, offset); + + this.IsEncrypted = false; // since we've decrypted to a temp file + this.UtfReader.IsEncrypted = false; + + if (ParseFile.CompareSegment(this.MagicBytes, 0, SIGNATURE_BYTES)) + { + using (FileStream utfTableStream = File.Open(this.UtfTableFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + // read header + this.initializeUtfHeader(utfTableStream); + + // initialize rows + this.Rows = new Dictionary[this.NumberOfRows]; + + // read schema + if (this.TableSize > 0) + { + this.initializeUtfSchema(fs, utfTableStream, 0x20); + } + } + } + else + { + //Dictionary foo = GetKeysForEncryptedUtfTable(this.MagicBytes); + throw new FormatException(String.Format("@UTF signature not found at offset <0x{0}>.", offset.ToString("X8"))); + } + } + catch (Exception ex) + { + throw ex; + } + finally + { + if (!String.IsNullOrEmpty(this.UtfTableFile)) + { + File.Delete(this.UtfTableFile); + } + } + + } + + public override string ToString() + { + CriField field; + StringBuilder sb = new StringBuilder(); + + using (FileStream fs = File.Open(this.SourceFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + for (int i = 0; i < this.Rows.Length; i++) + { + var d = this.Rows[i]; + + foreach (var key in d.Keys) + { + field = d[key]; + sb.AppendLine(field.ToString(fs)); + } + } + } + return sb.ToString(); + } + + private void checkEncryption(FileStream fs) + { + if (ParseFile.CompareSegment(this.MagicBytes, 0, SIGNATURE_BYTES)) + { + this.IsEncrypted = false; + this.UtfReader = new CriUtfReader(); + } + else + { + this.IsEncrypted = true; + Dictionary lcgKeys = GetKeysForEncryptedUtfTable(this.MagicBytes); + + if (lcgKeys.Count != 2) + { + throw new FormatException(String.Format("Unable to decrypt UTF table at offset: 0x{0}", this.BaseOffset.ToString("X8"))); + } + else + { + this.UtfReader = new CriUtfReader(lcgKeys[LCG_SEED_KEY], lcgKeys[LCG_INCREMENT_KEY], this.IsEncrypted); + } + + this.MagicBytes = this.UtfReader.GetBytes(fs, this.BaseOffset, 4, 0); + } + + } + + private void initializeUtfHeader(FileStream UtfTableFs) + { + this.TableSize = ParseFile.ReadUintBE(UtfTableFs, 4); + + this.Unknown1 = ParseFile.ReadUshortBE(UtfTableFs, 8); + + this.RowOffset = (uint)ParseFile.ReadUshortBE(UtfTableFs, 0xA) + 8; + this.StringTableOffset = ParseFile.ReadUintBE(UtfTableFs, 0xC) + 8; + this.DataOffset = ParseFile.ReadUintBE(UtfTableFs, 0x10) + 8; + + this.TableNameOffset = ParseFile.ReadUintBE(UtfTableFs, 0x14); + this.TableName = ParseFile.ReadAsciiString(UtfTableFs, this.StringTableOffset + this.TableNameOffset); + + this.NumberOfFields = ParseFile.ReadUshortBE(UtfTableFs, 0x18); + + this.RowSize = ParseFile.ReadUshortBE(UtfTableFs, 0x1A); + this.NumberOfRows = ParseFile.ReadUintBE(UtfTableFs, 0x1C); + } + + private void initializeUtfSchema(FileStream SourceFs, FileStream UtfTableFs, long schemaOffset) + { + long nameOffset; + + long constantOffset; + + long dataOffset; + long dataSize; + + long rowDataOffset; + long rowDataSize; + + long currentOffset = schemaOffset; + long currentRowBase; + long currentRowOffset = 0; + + CriField field; + + for (uint i = 0; i < this.NumberOfRows; i++) + { + //if (i == 0x1a2a) + //{ + // int yuuuu = 1; + //} + //try + //{ + currentOffset = schemaOffset; + currentRowBase = this.RowOffset + (this.RowSize * i); + currentRowOffset = 0; + this.Rows[i] = new Dictionary(); + + // parse fields + for (ushort j = 0; j < this.NumberOfFields; j++) + { + field = new CriField(); + + field.Type = ParseFile.ReadByte(UtfTableFs, currentOffset); + nameOffset = ParseFile.ReadUintBE(UtfTableFs, currentOffset + 1); + field.Name = ParseFile.ReadAsciiString(UtfTableFs, this.StringTableOffset + nameOffset); + + // each row will have a constant + if (((field.Type & COLUMN_STORAGE_MASK) == COLUMN_STORAGE_CONSTANT) || + ((field.Type & COLUMN_STORAGE_MASK) == COLUMN_STORAGE_CONSTANT2)) + { + // capture offset of constant + constantOffset = currentOffset + 5; + + // read the constant depending on the type + switch (field.Type & COLUMN_TYPE_MASK) + { + case COLUMN_TYPE_STRING: + dataOffset = ParseFile.ReadUintBE(UtfTableFs, constantOffset); + field.Value = ParseFile.ReadAsciiString(UtfTableFs, this.StringTableOffset + dataOffset); + currentOffset += 4; + break; + case COLUMN_TYPE_8BYTE: + field.Value = ParseFile.ReadUlongBE(UtfTableFs, constantOffset); + currentOffset += 8; + break; + case COLUMN_TYPE_DATA: + dataOffset = ParseFile.ReadUintBE(UtfTableFs, constantOffset); + dataSize = ParseFile.ReadUintBE(UtfTableFs, constantOffset + 4); + field.Offset = (ulong)(this.BaseOffset + this.DataOffset + dataOffset); + field.Size = (ulong)dataSize; + + // don't think this is encrypted, need to check + field.Value = ParseFile.ParseSimpleOffset(SourceFs, (long)field.Offset, (int)dataSize); + currentOffset += 8; + break; + case COLUMN_TYPE_FLOAT: + field.Value = ParseFile.ReadFloatBE(UtfTableFs, constantOffset); + currentOffset += 4; + break; + case COLUMN_TYPE_4BYTE2: + field.Value = ParseFile.ReadInt32BE(UtfTableFs, constantOffset); + currentOffset += 4; + break; + case COLUMN_TYPE_4BYTE: + field.Value = ParseFile.ReadUintBE(UtfTableFs, constantOffset); + currentOffset += 4; + break; + case COLUMN_TYPE_2BYTE2: + field.Value = ParseFile.ReadInt16BE(UtfTableFs, constantOffset); + currentOffset += 2; + break; + case COLUMN_TYPE_2BYTE: + field.Value = ParseFile.ReadUshortBE(UtfTableFs, constantOffset); + currentOffset += 2; + break; + case COLUMN_TYPE_1BYTE2: + field.Value = ParseFile.ReadSByte(UtfTableFs, constantOffset); + currentOffset += 1; + break; + case COLUMN_TYPE_1BYTE: + field.Value = ParseFile.ReadByte(UtfTableFs, constantOffset); + currentOffset += 1; + break; + default: + throw new FormatException(String.Format("Unknown COLUMN TYPE at offset: 0x{0}", currentOffset.ToString("X8"))); + + } // switch (field.Type & COLUMN_TYPE_MASK) + } + else if ((field.Type & COLUMN_STORAGE_MASK) == COLUMN_STORAGE_PERROW) + { + // read the constant depending on the type + switch (field.Type & COLUMN_TYPE_MASK) + { + case COLUMN_TYPE_STRING: + rowDataOffset = ParseFile.ReadUintBE(UtfTableFs, currentRowBase + currentRowOffset); + field.Value = ParseFile.ReadAsciiString(UtfTableFs, this.StringTableOffset + rowDataOffset); + currentRowOffset += 4; + break; + case COLUMN_TYPE_8BYTE: + field.Value = ParseFile.ReadUlongBE(UtfTableFs, currentRowBase + currentRowOffset); + currentRowOffset += 8; + break; + case COLUMN_TYPE_DATA: + rowDataOffset = ParseFile.ReadUintBE(UtfTableFs, currentRowBase + currentRowOffset); + rowDataSize = ParseFile.ReadUintBE(UtfTableFs, currentRowBase + currentRowOffset + 4); + field.Offset = (ulong)(this.BaseOffset + this.DataOffset + rowDataOffset); + field.Size = (ulong)rowDataSize; + + // don't think this is encrypted + field.Value = ParseFile.ParseSimpleOffset(SourceFs, (long)field.Offset, (int)rowDataSize); + currentRowOffset += 8; + break; + case COLUMN_TYPE_FLOAT: + field.Value = ParseFile.ReadFloatBE(UtfTableFs, currentRowBase + currentRowOffset); + currentRowOffset += 4; + break; + case COLUMN_TYPE_4BYTE2: + field.Value = ParseFile.ReadInt32BE(UtfTableFs, currentRowBase + currentRowOffset); + currentRowOffset += 4; + break; + case COLUMN_TYPE_4BYTE: + field.Value = ParseFile.ReadUintBE(UtfTableFs, currentRowBase + currentRowOffset); + currentRowOffset += 4; + break; + case COLUMN_TYPE_2BYTE2: + field.Value = ParseFile.ReadInt16BE(UtfTableFs, currentRowBase + currentRowOffset); + currentRowOffset += 2; + break; + case COLUMN_TYPE_2BYTE: + field.Value = ParseFile.ReadUshortBE(UtfTableFs, currentRowBase + currentRowOffset); + currentRowOffset += 2; + break; + case COLUMN_TYPE_1BYTE2: + field.Value = ParseFile.ReadSByte(UtfTableFs, currentRowBase + currentRowOffset); + currentRowOffset += 1; + break; + case COLUMN_TYPE_1BYTE: + field.Value = ParseFile.ReadByte(UtfTableFs, currentRowBase + currentRowOffset); + currentRowOffset += 1; + break; + default: + throw new FormatException(String.Format("Unknown COLUMN TYPE at offset: 0x{0}", currentOffset.ToString("X8"))); + + } // switch (field.Type & COLUMN_TYPE_MASK) + + } // if ((fields[i].Type & COLUMN_STORAGE_MASK) == COLUMN_STORAGE_CONSTANT) + + // add field to dictionary + this.Rows[i].Add(field.Name, field); + + // move to next field + currentOffset += 5; // sizeof(CriField.Type + CriField.NameOffset) + + } // for (ushort j = 0; j < this.NumberOfFields; j++) + //} + //catch (Exception ex) + //{ + // int xxxx = 1; + //} + } // for (uint i = 0; i < this.NumberOfRows; i++) + } + + public string WriteTableToTempFile(FileStream fs, long offsetToUtfTable) + { + string pathToUtf = null; + + int totalBytesRead = 0; + int maxRead; + byte[] buffer = new byte[Constants.FileReadChunkSize]; + + int tableSize = (int)this.UtfReader.ReadUint(fs, offsetToUtfTable, 4) + 8; + + pathToUtf = Path.Combine(Path.GetTempPath(), TEMP_UTF_TABLE_FILE); + + if (this.IsEncrypted) + { + using (FileStream tempStream = File.Open(pathToUtf, FileMode.Create, FileAccess.Write, FileShare.Read)) + { + while (totalBytesRead < tableSize) + { + if ((tableSize - totalBytesRead) > buffer.Length) + { + maxRead = buffer.Length; + } + else + { + maxRead = (tableSize - totalBytesRead); + } + + buffer = this.UtfReader.GetBytes(fs, offsetToUtfTable, maxRead, totalBytesRead); + tempStream.Write(buffer, 0, maxRead); + totalBytesRead += maxRead; + } + } + } + else + { + if (File.Exists(pathToUtf)) + { + File.Delete(pathToUtf); + } + + ParseFile.ExtractChunkToFile64(fs, (ulong)offsetToUtfTable, (ulong)tableSize, pathToUtf, false, false); + } + + return pathToUtf; + } + + public static object GetUtfFieldForRow(CriUtfTable utfTable, int rowIndex, string key) + { + object ret = null; + + if (utfTable.Rows.GetLength(0) > rowIndex) + { + if (utfTable.Rows[rowIndex].ContainsKey(key)) + { + ret = utfTable.Rows[rowIndex][key].Value; + } + } + + return ret; + } + + public static ulong GetOffsetForUtfFieldForRow(CriUtfTable utfTable, int rowIndex, string key) + { + ulong ret = 0; + + if (utfTable.Rows.GetLength(0) > rowIndex) + { + if (utfTable.Rows[rowIndex].ContainsKey(key)) + { + ret = utfTable.Rows[rowIndex][key].Offset; + } + } + + return ret; + } + + public static ulong GetSizeForUtfFieldForRow(CriUtfTable utfTable, int rowIndex, string key) + { + ulong ret = 0; + + if (utfTable.Rows.GetLength(0) > rowIndex) + { + if (utfTable.Rows[rowIndex].ContainsKey(key)) + { + ret = utfTable.Rows[rowIndex][key].Size; + } + } + + return ret; + } + + public static Dictionary GetKeysForEncryptedUtfTable(byte[] encryptedUtfSignature) + { + Dictionary keys = new Dictionary(); + byte m, t; + + byte[] xorBytes = new byte[SIGNATURE_BYTES.Length]; + bool keysFound = false; + + for (byte seed = 0; seed <= byte.MaxValue; seed++) + { + if (!keysFound) + { + // match first char + if ((encryptedUtfSignature[0] ^ seed) == SIGNATURE_BYTES[0]) + { + for (byte increment = 0; increment <= byte.MaxValue; increment++) + { + if (!keysFound) + { + m = (byte)(seed * increment); + + if ((encryptedUtfSignature[1] ^ m) == SIGNATURE_BYTES[1]) + { + t = increment; + + for (int j = 2; j < SIGNATURE_BYTES.Length; j++) + { + m *= t; + + if ((encryptedUtfSignature[j] ^ m) != SIGNATURE_BYTES[j]) + { + break; + } + else if (j == (SIGNATURE_BYTES.Length - 1)) + { + keys.Add(LCG_SEED_KEY, seed); + keys.Add(LCG_INCREMENT_KEY, increment); + keysFound = true; + } + } + } // if ((encryptedUtfSignature[1] ^ m) == SIGNATURE_BYTES[1]) + } + else + { + break; + } // if (!keysFound) + } // for (byte inc = 0; inc <= byte.MaxValue; inc++) + } // if ((encryptedUtfSignature[0] ^ mult) == SIGNATURE_BYTES[0]) + } // if (!keysFound) + else + { + break; + } + } // for (byte mult = 0; mult <= byte.MaxValue; mult++) + + return keys; + } + + public static bool IsUtfTable(FileStream fs, long offset) + { + bool ret = false; + CriUtfTable utf = new CriUtfTable(); + + utf.SourceFile = fs.Name; + utf.BaseOffset = offset; + + try + { + utf.MagicBytes = ParseFile.ParseSimpleOffset(fs, utf.BaseOffset, 4); + + // check if file is decrypted and get decryption keys if needed + utf.checkEncryption(fs); + + // write (decrypted) utf header to file + utf.UtfTableFile = utf.WriteTableToTempFile(fs, offset); + + utf.IsEncrypted = false; // since we've decrypted to a temp file + utf.UtfReader.IsEncrypted = false; + + if (ParseFile.CompareSegment(utf.MagicBytes, 0, SIGNATURE_BYTES)) + { + ret = true; + } + else + { + ret = false; + } + } + catch (Exception ex) + { + throw ex; + } + finally + { + if (!String.IsNullOrEmpty(utf.UtfTableFile)) + { + File.Delete(utf.UtfTableFile); + } + } + + return ret; + } + } + + public class CriUtfReader + { + public byte Seed { set; get; } + public byte Increment { set; get; } + public bool IsEncrypted { set; get; } + + public byte CurrentXor { set; get; } + public long CurrentUtfOffset { set; get; } + + public byte CurrentStringXor { set; get; } + public long CurrentUtfStringOffset { set; get; } + + public CriUtfReader() + { + this.IsEncrypted = false; + } + + public CriUtfReader(byte seed, byte increment, bool isEncrypted) + { + this.Seed = seed; + this.Increment = increment; + this.IsEncrypted = isEncrypted; + } + + public byte[] GetBytes(FileStream fs, long BaseOffset, int Size, long UtfOffset) + { + byte[] ret; + + ret = ParseFile.ParseSimpleOffset(fs, BaseOffset + UtfOffset, Size); + + if (this.IsEncrypted) + { + if (UtfOffset < this.CurrentUtfOffset) + { + // reset, maybe add some sort of index later? + this.CurrentUtfOffset = 0; + } + + if (this.CurrentUtfOffset == 0) + { + // reset or initialize + this.CurrentXor = this.Seed; + } + + // catch up to this offset + for (long j = this.CurrentUtfOffset; j < UtfOffset; j++) + { + if (j > 0) + { + this.CurrentXor *= this.Increment; + } + + this.CurrentUtfOffset++; + } + + // decrypt this offset + for (long i = 0; i < Size; i++) + { + // first byte of UTF table must be XOR'd with the seed + if ((this.CurrentUtfOffset != 0) || (i > 0)) + { + this.CurrentXor *= this.Increment; + } + + ret[i] ^= this.CurrentXor; + this.CurrentUtfOffset++; + } + } + + return ret; + } + + public string ReadAsciiString(FileStream fs, long BaseOffset, long UtfOffset) + { + byte encryptedByte; + byte decryptedByte; + + StringBuilder asciiVal = new StringBuilder(); + byte xorByte; + long fileSize = fs.Length; + + if (this.IsEncrypted) + { + fs.Position = (BaseOffset + UtfOffset); + + if (UtfOffset < this.CurrentUtfStringOffset) + { + // reset, maybe add some sort of index later? + this.CurrentUtfStringOffset = 0; + } + + if (this.CurrentUtfStringOffset == 0) + { + // reset or initialize + this.CurrentStringXor = this.Seed; + } + + for (long j = this.CurrentUtfStringOffset; j < UtfOffset; j++) + { + if (j > 0) + { + this.CurrentStringXor *= this.Increment; + } + + this.CurrentUtfStringOffset++; + } + + for (long i = UtfOffset; i < (fileSize - (BaseOffset + UtfOffset)); i++) + { + this.CurrentStringXor *= this.Increment; + this.CurrentUtfStringOffset++; + + encryptedByte = (byte)fs.ReadByte(); + decryptedByte = (byte)(encryptedByte ^ this.CurrentStringXor); + + if (decryptedByte == 0) + { + break; + } + else + { + asciiVal.Append(Convert.ToChar(decryptedByte)); + } + } + } + else + { + asciiVal.Append(ParseFile.ReadAsciiString(fs, BaseOffset + UtfOffset)); + } + + return asciiVal.ToString(); + } + + public byte ReadByte(FileStream fs, long BaseOffset, long UtfOffset) + { + return this.GetBytes(fs, BaseOffset, 1, UtfOffset)[0]; + } + + public SByte ReadSByte(FileStream fs, long BaseOffset, long UtfOffset) + { + return (SByte)this.GetBytes(fs, BaseOffset, 1, UtfOffset)[0]; + } + + public ushort ReadUshort(FileStream fs, long BaseOffset, long UtfOffset) + { + byte[] temp = this.GetBytes(fs, BaseOffset, 2, UtfOffset); + + if (BitConverter.IsLittleEndian) + { + Array.Reverse(temp); + } + return BitConverter.ToUInt16(temp, 0); + } + + public short ReadShort(FileStream fs, long BaseOffset, long UtfOffset) + { + byte[] temp = this.GetBytes(fs, BaseOffset, 2, UtfOffset); + + if (BitConverter.IsLittleEndian) + { + Array.Reverse(temp); + } + return BitConverter.ToInt16(temp, 0); + } + + public uint ReadUint(FileStream fs, long BaseOffset, long UtfOffset) + { + byte[] temp = this.GetBytes(fs, BaseOffset, 4, UtfOffset); + + if (BitConverter.IsLittleEndian) + { + Array.Reverse(temp); + } + return BitConverter.ToUInt32(temp, 0); + } + + public int ReadInt(FileStream fs, long BaseOffset, long UtfOffset) + { + byte[] temp = this.GetBytes(fs, BaseOffset, 4, UtfOffset); + + if (BitConverter.IsLittleEndian) + { + Array.Reverse(temp); + } + return BitConverter.ToInt32(temp, 0); + } + + public ulong ReadUlong(FileStream fs, long BaseOffset, long UtfOffset) + { + byte[] temp = this.GetBytes(fs, BaseOffset, 8, UtfOffset); + + if (BitConverter.IsLittleEndian) + { + Array.Reverse(temp); + } + return BitConverter.ToUInt64(temp, 0); + } + + public float ReadFloat(FileStream fs, long BaseOffset, long UtfOffset) + { + byte[] temp = this.GetBytes(fs, BaseOffset, 4, UtfOffset); + + if (BitConverter.IsLittleEndian) + { + Array.Reverse(temp); + } + return BitConverter.ToSingle(temp, 0); + } + } +} diff --git a/acb_extract/src/FileUtil.cs b/acb_extract/src/FileUtil.cs new file mode 100644 index 0000000..c2c8294 --- /dev/null +++ b/acb_extract/src/FileUtil.cs @@ -0,0 +1,667 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; + +namespace vgm_acb.extract +{ + public sealed class FileUtil + { + private FileUtil() { } + + /// + /// Reads data into a complete array, throwing an EndOfStreamException + /// if the stream runs out of data first, or if an IOException + /// naturally occurs. + /// + /// The stream to read data from + /// The array to read bytes into. The array + /// will be completely filled from the stream, so an appropriate + /// size must be given. + public static void ReadWholeArray(Stream stream, byte[] data) + { + ReadWholeArray(stream, data, data.Length); + } + + public static void ReadWholeArray(Stream stream, byte[] data, int pLength) + { + int offset = 0; + int remaining = pLength; + while (remaining > 0) + { + int read = stream.Read(data, offset, remaining); + if (read <= 0) + { + throw new EndOfStreamException( + String.Format(CultureInfo.CurrentCulture, "End of stream reached with {0} bytes left to read", remaining)); + } + + remaining -= read; + offset += read; + } + } + + /// + /// Replaces 0x00 with 0x20 in an array of bytes. + /// + /// Array of bytes to alter. + /// Original array with 0x00 replaced by 0x20. + public static byte[] ReplaceNullByteWithSpace(byte[] value) + { + for (int i = 0; i < value.Length; i++) + { + if (value[i] == 0x00) + { + value[i] = 0x20; + } + } + + return value; + } + + /// + /// Returns the count of files contained in the input directories and their subdirectories. + /// + /// Paths to count files within. + /// Number of files in the incoming directories and their subdirectories. + public static int GetFileCount(string[] paths) + { + return GetFileCount(paths, true); + } + + public static int GetFileCount(string[] paths, bool includeSubdirs) + { + int totalFileCount = 0; + + foreach (string path in paths) + { + if (File.Exists(path)) + { + totalFileCount++; + } + else if (Directory.Exists(path)) + { + if (includeSubdirs) + { + totalFileCount += Directory.GetFiles(path, "*.*", SearchOption.AllDirectories).Length; + } + else + { + totalFileCount += Directory.GetFiles(path, "*.*", SearchOption.TopDirectoryOnly).Length; + } + } + } + + return totalFileCount; + } + + public static string CleanFileName(string pDirtyFileName) + { + foreach (char c in Path.GetInvalidFileNameChars()) + { + pDirtyFileName = pDirtyFileName.Replace(c, '_'); + } + + return pDirtyFileName; + } + + public static void UpdateTextField( + string pFilePath, + string pFieldValue, + int pOffset, + int pMaxLength) + { + System.Text.Encoding enc = System.Text.Encoding.ASCII; + + using (BinaryWriter bw = + new BinaryWriter(File.Open(pFilePath, FileMode.Open, FileAccess.ReadWrite))) + { + byte[] newBytes = new byte[pMaxLength]; + byte[] convertedBytes = enc.GetBytes(pFieldValue); + + int numBytesToCopy = + convertedBytes.Length <= pMaxLength ? convertedBytes.Length : pMaxLength; + Array.ConstrainedCopy(convertedBytes, 0, newBytes, 0, numBytesToCopy); + + bw.Seek(pOffset, SeekOrigin.Begin); + bw.Write(newBytes); + } + } + + public static void UpdateChunk( + string pFilePath, + int pOffset, + byte[] value) + { + using (BinaryWriter bw = + new BinaryWriter(File.Open(pFilePath, FileMode.Open, FileAccess.ReadWrite))) + { + bw.Seek(pOffset, SeekOrigin.Begin); + bw.Write(value); + } + } + + public static void ReplaceFileChunk( + string pSourceFilePath, + long pSourceOffset, + long pLength, + string pDestinationFilePath, + long pDestinationOffset) + { + int read = 0; + long maxread; + int totalBytes = 0; + byte[] bytes = new byte[Constants.FileReadChunkSize]; + + using (BinaryWriter bw = + new BinaryWriter(File.Open(pDestinationFilePath, FileMode.Open, FileAccess.ReadWrite))) + { + using (BinaryReader br = + new BinaryReader(File.Open(pSourceFilePath, FileMode.Open, FileAccess.Read))) + { + br.BaseStream.Position = pSourceOffset; + bw.BaseStream.Position = pDestinationOffset; + + maxread = pLength > bytes.Length ? bytes.Length : pLength; + + while ((read = br.Read(bytes, 0, (int)maxread)) > 0) + { + bw.Write(bytes, 0, read); + totalBytes += read; + + maxread = (pLength - totalBytes) > bytes.Length ? bytes.Length : (pLength - totalBytes); + } + } + } + } + + public static void ZeroOutFileChunk(string pPath, long pOffset, int pLength) + { + int bytesToWrite = pLength; + byte[] bytes; + + int maxWrite = bytesToWrite > Constants.FileReadChunkSize ? Constants.FileReadChunkSize : bytesToWrite; + + using (BinaryWriter bw = + new BinaryWriter(File.Open(pPath, FileMode.Open, FileAccess.Write))) + { + bw.BaseStream.Position = pOffset; + + while (bytesToWrite > 0) + { + bytes = new byte[maxWrite]; + bw.Write(bytes); + bytesToWrite -= maxWrite; + maxWrite = bytesToWrite > bytes.Length ? bytes.Length : bytesToWrite; + } + } + } + + public static void TrimFileToLength(string path, long totalLength) + { + string fullPath = Path.GetFullPath(path); + + if (File.Exists(fullPath)) + { + string destinationPath = Path.ChangeExtension(fullPath, ".trimmed"); + + using (FileStream fs = File.OpenRead(path)) + { + ParseFile.ExtractChunkToFile(fs, 0, totalLength, destinationPath); + } + + File.Copy(destinationPath, path, true); + File.Delete(destinationPath); + } + } + + public static string RemoveChunkFromFile(string path, long startingOffset, long length) + { + string fullPath = Path.GetFullPath(path); + + int bytesRead; + byte[] bytes = new byte[1024]; + + string ret = String.Empty; + + if (File.Exists(fullPath)) + { + string destinationPath = Path.ChangeExtension(fullPath, ".cut"); + + using (FileStream sourceFs = File.OpenRead(fullPath)) + { + // extract initial chunk + ParseFile.ExtractChunkToFile(sourceFs, 0, startingOffset, destinationPath); + + // append remainder + using (FileStream outFs = File.Open(destinationPath, FileMode.Append, FileAccess.Write)) + { + sourceFs.Position = startingOffset + length; + bytesRead = sourceFs.Read(bytes, 0, bytes.Length); + + while (bytesRead > 0) + { + outFs.Write(bytes, 0, bytesRead); + bytesRead = sourceFs.Read(bytes, 0, bytes.Length); + } + } + + ret = destinationPath; + } + } + + return ret; + } + + public static string RemoveAllChunksFromFile(FileStream fs, byte[] chunkToRemove) + { + int bytesRead; + + long currentReadOffset = 0; + long totalBytesRead = 0; + int maxReadSize = 0; + long currentChunkSize; + + byte[] bytes = new byte[Constants.FileReadChunkSize]; + long[] offsets = ParseFile.GetAllOffsets(fs, 0, chunkToRemove, false, -1, -1, true); + + string destinationPath = Path.ChangeExtension(fs.Name, ".cut"); + + using (FileStream destinationFs = File.OpenWrite(destinationPath)) + { + for (int i = 0; i < offsets.Length; i++) + { + // move position + fs.Position = currentReadOffset; + + // get length of current size to write + currentChunkSize = offsets[i] - currentReadOffset; + + // calculcate max cut size for this loop iteration + maxReadSize = (currentChunkSize - totalBytesRead) > (long)bytes.Length ? bytes.Length : (int)(currentChunkSize - totalBytesRead); + + while ((bytesRead = fs.Read(bytes, 0, maxReadSize)) > 0) + { + destinationFs.Write(bytes, 0, bytesRead); + totalBytesRead += (long)bytesRead; + + maxReadSize = (currentChunkSize - totalBytesRead) > (long)bytes.Length ? bytes.Length : (int)(currentChunkSize - totalBytesRead); + } + + totalBytesRead = 0; + currentReadOffset = offsets[i] + chunkToRemove.Length; + } + + //////////////////////////// + // write remainder of file + //////////////////////////// + // move position + fs.Position = currentReadOffset; + + // get length of current size to write + currentChunkSize = fs.Length - currentReadOffset; + + // calculcate max cut size + maxReadSize = (currentChunkSize - totalBytesRead) > (long)bytes.Length ? bytes.Length : (int)(currentChunkSize - totalBytesRead); + + while ((bytesRead = fs.Read(bytes, 0, maxReadSize)) > 0) + { + destinationFs.Write(bytes, 0, bytesRead); + totalBytesRead += (long)bytesRead; + + maxReadSize = (currentChunkSize - totalBytesRead) > (long)bytes.Length ? bytes.Length : (int)(currentChunkSize - totalBytesRead); + } + } + + return destinationPath; + } + + public static string RemoveAllChunksFromFile(string path, byte[] chunkToRemove) + { + string destinationPath; + + using (FileStream fs = File.OpenRead(path)) + { + destinationPath = RemoveAllChunksFromFile(fs, chunkToRemove); + } + + return destinationPath; + } + + public static bool ExecuteExternalProgram( + string pathToExecuatable, + string arguments, + string workingDirectory, + out string standardOut, + out string standardError) + { + Process externalExecutable; + bool isSuccess = false; + + standardOut = String.Empty; + standardError = String.Empty; + + using (externalExecutable = new Process()) + { + externalExecutable.StartInfo = new ProcessStartInfo(pathToExecuatable, arguments); + externalExecutable.StartInfo.WorkingDirectory = workingDirectory; + externalExecutable.StartInfo.UseShellExecute = false; + externalExecutable.StartInfo.CreateNoWindow = true; + + externalExecutable.StartInfo.RedirectStandardOutput = true; + externalExecutable.StartInfo.RedirectStandardError = true; + isSuccess = externalExecutable.Start(); + + standardOut = externalExecutable.StandardOutput.ReadToEnd(); + standardError = externalExecutable.StandardError.ReadToEnd(); + externalExecutable.WaitForExit(); + } + + return isSuccess; + } + + public static void AddHeaderToFile(byte[] headerBytes, string sourceFile, string destinationFile) + { + int bytesRead; + byte[] readBuffer = new byte[Constants.FileReadChunkSize]; + + using (FileStream destinationStream = File.Open(destinationFile, FileMode.CreateNew, FileAccess.Write)) + { + // write header + destinationStream.Write(headerBytes, 0, headerBytes.Length); + + // write the source file + using (FileStream sourceStream = File.Open(sourceFile, FileMode.Open, FileAccess.Read)) + { + while ((bytesRead = sourceStream.Read(readBuffer, 0, readBuffer.Length)) > 0) + { + destinationStream.Write(readBuffer, 0, bytesRead); + } + } + } + } + + public static void RenameFileUsingInternalName(string path, + long offset, int length, byte[] terminatorBytes, bool maintainFileExtension) + { + string destinationDirectory = Path.GetDirectoryName(path); + string destinationFile; + string originalExtension; + + int nameLength; + byte[] nameByteArray; + + using (FileStream fs = File.OpenRead(path)) + { + if (terminatorBytes != null) + { + nameLength = ParseFile.GetSegmentLength(fs, (int)offset, terminatorBytes); + } + else + { + nameLength = length; + } + + if (nameLength < 1) + { + throw new ArgumentOutOfRangeException("Name Length", "Name Length is less than 1."); + } + + if (maintainFileExtension) + { + originalExtension = Path.GetExtension(path); + } + + nameByteArray = ParseFile.ParseSimpleOffset(fs, offset, nameLength); + destinationFile = ByteConversion.GetAsciiText(FileUtil.ReplaceNullByteWithSpace(nameByteArray)).Trim(); + destinationFile = Path.Combine(destinationDirectory, destinationFile); + + if (maintainFileExtension) + { + originalExtension = Path.GetExtension(path); + destinationFile = Path.ChangeExtension(destinationFile, originalExtension); + } + } + + // try to copy using the new name + if (!path.Equals(destinationFile)) + { + try + { + if (!Directory.Exists(Path.GetDirectoryName(destinationFile))) + { + Directory.CreateDirectory(Path.GetDirectoryName(destinationFile)); + } + + if (File.Exists(destinationFile)) + { + string[] sameNamedFiles = Directory.GetFiles(Path.GetDirectoryName(destinationFile), Path.GetFileNameWithoutExtension(destinationFile) + "*"); + + // rename to prevent overwrite + destinationFile = Path.Combine(Path.GetDirectoryName(destinationFile), String.Format("{0}_{1}{2}", Path.GetFileNameWithoutExtension(destinationFile), sameNamedFiles.Length.ToString("D4"), Path.GetExtension(destinationFile))); + } + + File.Copy(path, destinationFile); + File.Delete(path); + } + catch (Exception ex) + { + throw new Exception(ex.Message, ex); + } + } + } + + public static void InterleaveFiles(string[] sourceFiles, uint interleaveValue, + long startOffest, byte[] paddingBytes, string destinationFile) + { + long currentOffset = 0; + long maxLength = 0; + long sizeDifference = 0; + long bytesToWrite; + long bytesRemaining; + + FileStream[] inputStreams = new FileStream[sourceFiles.Length]; + long[] fileLengths = new long[sourceFiles.Length]; + destinationFile = Path.GetFullPath(destinationFile); + + // open destination file for writing + using (FileStream destinationStream = File.OpenWrite(destinationFile)) + { + try + { + // build input streams array + for (int i = 0; i < sourceFiles.Length; i++) + { + inputStreams[i] = File.OpenRead(sourceFiles[i]); + fileLengths[i] = inputStreams[i].Length; + + // get max file length + if (maxLength < fileLengths[i]) + { + maxLength = fileLengths[i]; + } + } + + // write out blocks + currentOffset = startOffest; + + while (currentOffset < maxLength) + { + for (int i = 0; i < sourceFiles.Length; i++) + { + if (currentOffset + interleaveValue < fileLengths[i]) + { + // write from file + destinationStream.Write( + ParseFile.ParseSimpleOffset(inputStreams[i], currentOffset, (int)interleaveValue), + 0, (int)interleaveValue); + } + else if (currentOffset < fileLengths[i]) + { + // write some from file, some from padding + sizeDifference = (currentOffset + interleaveValue) - fileLengths[i]; + + // write from file + destinationStream.Write( + ParseFile.ParseSimpleOffset(inputStreams[i], currentOffset, (int)(interleaveValue - sizeDifference)), + 0, (int)(int)(interleaveValue - sizeDifference)); + + // write padding bytes + bytesRemaining = sizeDifference; + + while (bytesRemaining > 0) + { + bytesToWrite = bytesRemaining > paddingBytes.Length ? paddingBytes.Length : bytesRemaining; + destinationStream.Write(paddingBytes, 0, (int)bytesToWrite); + bytesRemaining -= bytesToWrite; + } + } + else + { + // write padding bytes + bytesRemaining = interleaveValue; + + while (bytesRemaining > 0) + { + bytesToWrite = bytesRemaining > paddingBytes.Length ? paddingBytes.Length : bytesRemaining; + destinationStream.Write(paddingBytes, 0, (int)bytesToWrite); + bytesRemaining -= bytesToWrite; + } + } + } + + // increment offset + currentOffset += interleaveValue; + } + } + catch (Exception ex) + { + throw new Exception(ex.Message, ex); + } + finally + { + // close all readers + for (int i = 0; i < inputStreams.Length; i++) + { + if (inputStreams[i].CanRead) + { + inputStreams[i].Close(); + inputStreams[i].Dispose(); + } + } + } + + + } + } + + public static string GetNonDuplicateFileName(string destinationFile) + { + if (File.Exists(destinationFile)) + { + string[] sameNamedFiles = Directory.GetFiles(Path.GetDirectoryName(destinationFile), Path.GetFileNameWithoutExtension(destinationFile) + "*"); + + // rename to prevent overwrite + destinationFile = Path.Combine(Path.GetDirectoryName(destinationFile), String.Format("{0}_{1}{2}", Path.GetFileNameWithoutExtension(destinationFile), sameNamedFiles.Length.ToString("D4"), Path.GetExtension(destinationFile))); + } + + return destinationFile; + } + + public static string[] SplitFile(string sourceFile, long startingOffset, ulong chunkSizeInBytes, string outputFolder) + { + string[] outputFileList; + string outputFileName; + ArrayList outputFiles = new ArrayList(); + + long fileLength; + ulong currentOffset; + + int chunkCount; + + using (FileStream sourceStream = File.OpenRead(sourceFile)) + { + // get file length + fileLength = sourceStream.Length; + + // init counters + chunkCount = 1; + currentOffset = (ulong)startingOffset; + + while (currentOffset < (ulong)fileLength) + { + // construct output file name + outputFileName = Path.Combine(outputFolder, + String.Format("{0}.{1}", + Path.GetFileName(sourceFile), + chunkCount.ToString("D3"))); + + ParseFile.ExtractChunkToFile64(sourceStream, currentOffset, chunkSizeInBytes, + outputFileName, true, true); + + + outputFiles.Add(outputFileName); + currentOffset += chunkSizeInBytes; + chunkCount++; + } + } + + outputFileList = (string[])outputFiles.ToArray(typeof(string)); + return outputFileList; + } + + public static long? GetFileSize(string path) + { + long? fileSize; + + try + { + FileInfo fi = new FileInfo(Path.GetFullPath(path)); + fileSize = fi.Length; + } + catch (Exception) + { + fileSize = null; + } + + return fileSize; + } + + public static void CreateFileFromByteArray(string destinationFile, byte[] sourceBytes) + { + string destinationDirectory = Path.GetDirectoryName(destinationFile); + + if (!Directory.Exists(destinationDirectory)) + { + Directory.CreateDirectory(destinationDirectory); + } + + using (FileStream outStream = File.Open(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read)) + { + outStream.Write(sourceBytes, 0, sourceBytes.Length); + } + } + + public static void CreateFileFromString(string destinationFile, string sourceText) + { + string destinationDirectory = Path.GetDirectoryName(destinationFile); + + if (!Directory.Exists(destinationDirectory)) + { + Directory.CreateDirectory(destinationDirectory); + } + + using ( FileStream outStream = File.Open(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read)) + { + using (StreamWriter sw = new StreamWriter(outStream, Encoding.ASCII)) + { + sw.Write(sourceText); + } + } + } + } +} diff --git a/acb_extract/src/MathUtil.cs b/acb_extract/src/MathUtil.cs new file mode 100644 index 0000000..898be48 --- /dev/null +++ b/acb_extract/src/MathUtil.cs @@ -0,0 +1,26 @@ +using System; + +namespace vgm_acb.extract +{ + public class MathUtil + { + public static long RoundUpToByteAlignment(long valueToRound, long byteAlignment) + { + long roundedValue = -1; + + roundedValue = (valueToRound + byteAlignment - 1) / byteAlignment * byteAlignment; + + return roundedValue; + } + + public static ulong RoundUpToByteAlignment(ulong valueToRound, ulong byteAlignment) + { + ulong roundedValue; + + roundedValue = (valueToRound + byteAlignment - 1) / byteAlignment * byteAlignment; + + return roundedValue; + } + } + +} diff --git a/acb_extract/src/OffsetDescription.cs b/acb_extract/src/OffsetDescription.cs new file mode 100644 index 0000000..486479c --- /dev/null +++ b/acb_extract/src/OffsetDescription.cs @@ -0,0 +1,9 @@ +namespace vgm_acb.extract +{ + public class OffsetDescription + { + public string OffsetValue { set; get; } + public string OffsetSize { set; get; } + public string OffsetByteOrder { set; get; } + } +} diff --git a/acb_extract/src/ParseFile.cs b/acb_extract/src/ParseFile.cs new file mode 100644 index 0000000..eb2afd5 --- /dev/null +++ b/acb_extract/src/ParseFile.cs @@ -0,0 +1,864 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; + +namespace vgm_acb.extract +{ + /// + /// Class for Parsing Files. + /// + public sealed class ParseFile + { + public const string LogFileName = "vgmt_extraction_log.txt"; + public const string SnakeBiteBatchFileName = "vgmt_extraction_log.bat"; + public const string VirtualFileSystemExtractionFolder = "vgmt_vfs_cut"; + /// + /// Extract a section from the incoming byte array. + /// + /// Bytes to extract from. + /// Offset to begin cutting from. + /// Number of bytes to cut. + /// Byte array containing the extracted section. + public static byte[] ParseSimpleOffset(byte[] sourceArray, int startingOffset, int lengthToCut) + { + byte[] ret = new byte[lengthToCut]; + uint j = 0; + + for (int i = startingOffset; i < startingOffset + lengthToCut; i++) + { + ret[j] = sourceArray[i]; + j++; + } + + return ret; + } + + /// + /// Extract a section from the incoming stream. + /// + /// Stream to extract the chunk from. + /// Offset to begin cutting from. + /// Number of bytes to cut. + /// Byte array containing the extracted section. + public static byte[] ParseSimpleOffset(Stream stream, int startingOffset, int lengthToCut) + { + byte[] ret = new byte[lengthToCut]; + long currentStreamPosition = stream.Position; + + stream.Seek((long)startingOffset, SeekOrigin.Begin); + BinaryReader br = new BinaryReader(stream); + ret = br.ReadBytes((int)lengthToCut); + + stream.Position = currentStreamPosition; + + return ret; + } + + /// + /// Extract a section from the incoming stream. + /// + /// Stream to extract the chunk from. + /// Offset to begin cutting from. + /// Number of bytes to cut. + /// Byte array containing the extracted section. + public static byte[] ParseSimpleOffset(Stream stream, long startingOffset, int lengthToCut) + { + byte[] ret = new byte[lengthToCut]; + long currentStreamPosition = stream.Position; + + stream.Seek(startingOffset, SeekOrigin.Begin); + BinaryReader br = new BinaryReader(stream); + ret = br.ReadBytes(lengthToCut); + + stream.Position = currentStreamPosition; + + return ret; + } + + public static byte[] SimpleArrayCopy(byte[] SourceArray, long offset, long length) + { + byte[] value = new byte[length]; + Array.Copy(SourceArray, offset, value, 0, length); + return value; + } + + public static float ReadFloatLE(Stream inStream, long offset) + { + return BitConverter.ToSingle(ParseSimpleOffset(inStream, offset, 4), 0); + } + + public static float ReadFloatBE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + Array.Reverse(val); + + return BitConverter.ToSingle(val, 0); + } + + public static uint ReadUintLE(Stream inStream, long offset) + { + return BitConverter.ToUInt32(ParseSimpleOffset(inStream, offset, 4), 0); + } + + public static uint ReadUintLE(byte[] inBytes, long offset) + { + byte[] val = new byte[4]; + + Array.Copy(inBytes, offset, val, 0, 4); + + return BitConverter.ToUInt32(val, 0); + } + + public static uint ReadUintBE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + Array.Reverse(val); + + return BitConverter.ToUInt32(val, 0); + } + + public static uint ReadUintBE(byte[] inBytes, long offset) + { + byte[] val = new byte[4]; + + Array.Copy(inBytes, offset, val, 0, 4); + Array.Reverse(val); + + return BitConverter.ToUInt32(val, 0); + } + + public static ushort ReadUshortLE(Stream inStream, long offset) + { + return BitConverter.ToUInt16(ParseSimpleOffset(inStream, offset, 2), 0); + } + + public static ushort ReadUshortLE(byte[] inBytes, long offset) + { + byte[] val = new byte[2]; + + Array.Copy(inBytes, offset, val, 0, 2); + + return BitConverter.ToUInt16(val, 0); + } + + public static ushort ReadUshortBE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 2); + Array.Reverse(val); + + return BitConverter.ToUInt16(val, 0); + } + + public static ushort ReadUshortBE(byte[] inBytes, long offset) + { + byte[] val = new byte[2]; + + Array.Copy(inBytes, offset, val, 0, 2); + Array.Reverse(val); + + return BitConverter.ToUInt16(val, 0); + } + + public static byte ReadByte(Stream inStream, long offset) + { + return (byte)ParseSimpleOffset(inStream, offset, 1)[0]; + } + + public static SByte ReadSByte(Stream inStream, long offset) + { + return (SByte)ParseSimpleOffset(inStream, offset, 1)[0]; + } + + public static int ReadInt32LE(Stream inStream, long offset) + { + return BitConverter.ToInt32(ParseSimpleOffset(inStream, offset, 4), 0); + } + + public static int ReadInt32BE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + Array.Reverse(val); + + return BitConverter.ToInt32(val, 0); + } + + public static short ReadInt16LE(Stream inStream, long offset) + { + return BitConverter.ToInt16(ParseSimpleOffset(inStream, offset, 2), 0); + } + + public static short ReadInt16BE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 2); + Array.Reverse(val); + + return BitConverter.ToInt16(val, 0); + } + + public static int ReadInt24LE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + val[3] = 0; + + return BitConverter.ToInt32(val, 0); + } + + public static int ReadInt24BE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + val[3] = 0; + Array.Reverse(val); + + return (BitConverter.ToInt32(val, 0) >> 8); + } + + public static uint ReadUint24LE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + val[3] = 0; + + return BitConverter.ToUInt32(val, 0); + } + + public static uint ReadUint24BE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + val[3] = 0; + Array.Reverse(val); + + return (BitConverter.ToUInt32(val, 0) >> 8); + } + + public static ulong ReadUlongLE(Stream inStream, long offset) + { + return BitConverter.ToUInt64(ParseSimpleOffset(inStream, offset, 8), 0); + } + + public static ulong ReadUlongLE(byte[] inBytes, long offset) + { + byte[] val = new byte[8]; + + Array.Copy(inBytes, offset, val, 0, 8); + + return BitConverter.ToUInt64(val, 0); + } + + public static ulong ReadUlongBE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 8); + Array.Reverse(val); + + return BitConverter.ToUInt64(val, 0); + } + + public static void ExtractChunkToFile(Stream stream, long startingOffset, long length, + string filePath) + { + ExtractChunkToFile(stream, startingOffset, length, filePath, false, false); + } + + public static void ExtractChunkToFile(Stream stream, long startingOffset, long length, + string filePath, bool outputLogFile) + { + ExtractChunkToFile(stream, startingOffset, length, filePath, outputLogFile, false); + } + + /// + /// Extracts a section of the incoming stream to a file. + /// + /// Stream to extract from. + /// Offset to begin the cut. + /// Number of bytes to cut. + /// File path to output the extracted chunk to. + public static void ExtractChunkToFile( + Stream stream, + long startingOffset, + long length, + string filePath, + bool outputLogFile, + bool outputSnakebiteBatchFile) + { + bool makeBatchFile = (outputSnakebiteBatchFile && (stream is FileStream)); + StringBuilder logInfo = new StringBuilder(); + StringBuilder snakeBiteBatch = new StringBuilder(); + BinaryWriter bw = null; + string fullFilePath = Path.GetFullPath(filePath); + string fullOutputDirectory = Path.GetDirectoryName(fullFilePath); + + // create output folder if needed + if (!Directory.Exists(fullOutputDirectory)) + { + Directory.CreateDirectory(fullOutputDirectory); + + if (outputLogFile) + { + logInfo.AppendLine("Created Directory: " + fullOutputDirectory); + } + } + + // check if file exists and change name as needed + if (File.Exists(fullFilePath)) + { + int fileCount = Directory.GetFiles(fullOutputDirectory, (Path.GetFileNameWithoutExtension(fullFilePath) + "*" + Path.GetExtension(fullFilePath)), SearchOption.TopDirectoryOnly).Length; + fullFilePath = Path.Combine(fullOutputDirectory, String.Format("{0}_{1}{2}", Path.GetFileNameWithoutExtension(fullFilePath), fileCount.ToString("X3"), Path.GetExtension(fullFilePath))); + } + + try + { + bw = new BinaryWriter(File.Open(fullFilePath, FileMode.Create, FileAccess.Write)); + + int read = 0; + long totalBytes = 0; + byte[] bytes = new byte[Constants.FileReadChunkSize]; + stream.Seek((long)startingOffset, SeekOrigin.Begin); + + int maxread = length > (long)bytes.Length ? bytes.Length : (int)length; + + while ((read = stream.Read(bytes, 0, maxread)) > 0) + { + bw.Write(bytes, 0, read); + totalBytes += (long)read; + + maxread = (length - totalBytes) > (long)bytes.Length ? bytes.Length : (int)(length - totalBytes); + } + + if (outputLogFile) + { + logInfo.AppendLine( + String.Format("Extracted - Offset: 0x{0} Length: 0x{1} File: {2}", + startingOffset.ToString("X8"), + length.ToString("X8"), + Path.GetFileName(fullFilePath))); + + using (StreamWriter logWriter = new StreamWriter(Path.Combine(fullOutputDirectory, ParseFile.LogFileName), true)) + { + logWriter.Write(logInfo.ToString()); + } + } + + if (makeBatchFile) + { + snakeBiteBatch.AppendLine( + String.Format("snakebite.exe \"{0}\" \"{1}\" 0x{2} 0x{3}", + Path.GetFileName(((FileStream)stream).Name), + Path.GetFileNameWithoutExtension(((FileStream)stream).Name) + Path.DirectorySeparatorChar + Path.GetFileName(fullFilePath), + startingOffset.ToString("X8"), + (startingOffset + length - 1).ToString("X8"))); + + using (StreamWriter batchWriter = new StreamWriter(Path.Combine(fullOutputDirectory, ParseFile.SnakeBiteBatchFileName), true)) + { + batchWriter.Write(snakeBiteBatch.ToString()); + } + } + } + finally + { + if (bw != null) + { + bw.Close(); + } + } + } + + public static void ExtractChunkToFile64( + Stream stream, + ulong startingOffset, + ulong length, + string filePath, + bool outputLogFile, + bool outputSnakebiteBatchFile) + { + bool makeBatchFile = (outputSnakebiteBatchFile && (stream is FileStream)); + StringBuilder logInfo = new StringBuilder(); + StringBuilder snakeBiteBatch = new StringBuilder(); + BinaryWriter bw = null; + string fullFilePath = Path.GetFullPath(filePath); + string fullOutputDirectory = Path.GetDirectoryName(fullFilePath); + + // create output folder if needed + if (!Directory.Exists(fullOutputDirectory)) + { + Directory.CreateDirectory(fullOutputDirectory); + + if (outputLogFile) + { + logInfo.AppendLine("Created Directory: " + fullOutputDirectory); + } + } + + // check if file exists and change name as needed + if (File.Exists(fullFilePath)) + { + int fileCount = Directory.GetFiles(fullOutputDirectory, (Path.GetFileName(fullFilePath) + "*"), SearchOption.TopDirectoryOnly).Length; + fullFilePath = Path.Combine(fullOutputDirectory, String.Format("{0}_{1}{2}", Path.GetFileNameWithoutExtension(fullFilePath), fileCount.ToString("X3"), Path.GetExtension(fullFilePath))); + } + + try + { + bw = new BinaryWriter(File.Open(fullFilePath, FileMode.Create, FileAccess.Write)); + + int read = 0; + ulong totalBytes = 0; + byte[] bytes = new byte[Constants.FileReadChunkSize]; + + // reset location + stream.Seek(0, SeekOrigin.Begin); + + // move stream pointer in long size chunks + while (startingOffset > long.MaxValue) // + { + stream.Seek(long.MaxValue, SeekOrigin.Current); + startingOffset -= long.MaxValue; + } + + // less than a long now, should be ok + stream.Seek((long)startingOffset, SeekOrigin.Current); + + int maxread = length > (ulong)bytes.Length ? bytes.Length : (int)length; + + while ((read = stream.Read(bytes, 0, maxread)) > 0) + { + bw.Write(bytes, 0, read); + totalBytes += (ulong)read; + + maxread = (length - totalBytes) > (ulong)bytes.Length ? bytes.Length : (int)(length - totalBytes); + } + + if (outputLogFile) + { + logInfo.AppendLine( + String.Format("Extracted - Offset: 0x{0} Length: 0x{1} File: {2}", + startingOffset.ToString("X8"), + length.ToString("X8"), + Path.GetFileName(fullFilePath))); + + using (StreamWriter logWriter = new StreamWriter(Path.Combine(fullOutputDirectory, ParseFile.LogFileName), true)) + { + logWriter.Write(logInfo.ToString()); + } + } + + if (makeBatchFile) + { + snakeBiteBatch.AppendLine( + String.Format("snakebite.exe \"{0}\" \"{1}\" 0x{2} 0x{3}", + Path.GetFileName(((FileStream)stream).Name), + Path.GetFileNameWithoutExtension(((FileStream)stream).Name) + Path.DirectorySeparatorChar + Path.GetFileName(fullFilePath), + startingOffset.ToString("X8"), + (startingOffset + length - 1).ToString("X8"))); + + using (StreamWriter batchWriter = new StreamWriter(Path.Combine(fullOutputDirectory, ParseFile.SnakeBiteBatchFileName), true)) + { + batchWriter.Write(snakeBiteBatch.ToString()); + } + } + } + finally + { + if (bw != null) + { + bw.Close(); + } + } + } + + public static byte[] ExtractChunkToFile64ReturningHash( + Stream stream, + ulong startingOffset, + ulong length, + string filePath, + HashAlgorithm checksumAlgorithm, + bool outputLogFile, + bool outputSnakebiteBatchFile) + { + byte[] checksumHash; + bool makeBatchFile = (outputSnakebiteBatchFile && (stream is FileStream)); + StringBuilder logInfo = new StringBuilder(); + StringBuilder snakeBiteBatch = new StringBuilder(); + //BinaryWriter bw = null; + string fullFilePath = Path.GetFullPath(filePath); + string fullOutputDirectory = Path.GetDirectoryName(fullFilePath); + + // create output folder if needed + if (!Directory.Exists(fullOutputDirectory)) + { + Directory.CreateDirectory(fullOutputDirectory); + + if (outputLogFile) + { + logInfo.AppendLine("Created Directory: " + fullOutputDirectory); + } + } + + // check if file exists and change name as needed + if (File.Exists(fullFilePath)) + { + int fileCount = Directory.GetFiles(fullOutputDirectory, (Path.GetFileName(fullFilePath) + "*"), SearchOption.TopDirectoryOnly).Length; + fullFilePath = Path.Combine(fullOutputDirectory, String.Format("{0}_{1}{2}", Path.GetFileNameWithoutExtension(fullFilePath), fileCount.ToString("X3"), Path.GetExtension(fullFilePath))); + } + + using (FileStream fs = File.Open(fullFilePath, FileMode.Create, FileAccess.Write)) + using (CryptoStream hashStream = new CryptoStream(fs, checksumAlgorithm, CryptoStreamMode.Write)) + { + int read = 0; + ulong totalBytes = 0; + byte[] bytes = new byte[Constants.FileReadChunkSize]; + + // reset location + stream.Seek(0, SeekOrigin.Begin); + + // move stream pointer in long size chunks + while (startingOffset > long.MaxValue) // + { + stream.Seek(long.MaxValue, SeekOrigin.Current); + startingOffset -= long.MaxValue; + } + + // less than a long now, should be ok + stream.Seek((long)startingOffset, SeekOrigin.Current); + + int maxread = length > (ulong)bytes.Length ? bytes.Length : (int)length; + + while ((read = stream.Read(bytes, 0, maxread)) > 0) + { + hashStream.Write(bytes, 0, read); + totalBytes += (ulong)read; + + maxread = (length - totalBytes) > (ulong)bytes.Length ? bytes.Length : (int)(length - totalBytes); + } + + hashStream.FlushFinalBlock(); + checksumHash = checksumAlgorithm.Hash; + + #region Log Files + if (outputLogFile) + { + logInfo.AppendLine( + String.Format("Extracted - Offset: 0x{0} Length: 0x{1} File: {2}", + startingOffset.ToString("X8"), + length.ToString("X8"), + Path.GetFileName(fullFilePath))); + + using (StreamWriter logWriter = new StreamWriter(Path.Combine(fullOutputDirectory, ParseFile.LogFileName), true)) + { + logWriter.Write(logInfo.ToString()); + } + } + + if (makeBatchFile) + { + snakeBiteBatch.AppendLine( + String.Format("snakebite.exe \"{0}\" \"{1}\" 0x{2} 0x{3}", + Path.GetFileName(((FileStream)stream).Name), + Path.GetFileNameWithoutExtension(((FileStream)stream).Name) + Path.DirectorySeparatorChar + Path.GetFileName(fullFilePath), + startingOffset.ToString("X8"), + (startingOffset + length - 1).ToString("X8"))); + + using (StreamWriter batchWriter = new StreamWriter(Path.Combine(fullOutputDirectory, ParseFile.SnakeBiteBatchFileName), true)) + { + batchWriter.Write(snakeBiteBatch.ToString()); + } + } + #endregion + + return checksumHash; + } + } + + public static string ReadAsciiString(Stream inStream, long offset) + { + long streamLength = inStream.Length; + int stringLength = 0; + int codePage; + + int dummy; + byte[] stringBytes; + string ret = String.Empty; + + // move pointer + inStream.Position = offset; + + // read until NULL + for (long i = offset; i <= streamLength; i++) + { + dummy = inStream.ReadByte(); + + if (dummy > 0) + { + stringLength++; + } + else if (dummy <= 0) + { + break; + } + } + + // parse string + stringBytes = ParseSimpleOffset(inStream, offset, stringLength); + codePage = ByteConversion.GetPredictedCodePageForTags(stringBytes); + ret = ByteConversion.GetEncodedText(stringBytes, codePage); + //ret = ByteConversion.GetAsciiText(stringBytes); + + return ret; + } + + /// + /// Compare bytes at input offset to target bytes. + /// + /// Bytes to compare. + /// Offset to begin comparison of pBytes to pTarget. + /// Target bytes to compare. + /// True if the bytes at pOffset match the pTarget bytes. + public static bool CompareSegment(byte[] sourceArray, int offset, byte[] target) + { + bool ret = true; + uint j = 0; + + if (sourceArray.Length > 0) + { + for (int i = offset; i < target.Length; i++) + { + if (sourceArray[i] != target[j]) + { + ret = false; + break; + } + + j++; + } + } + else + { + ret = false; + } + + return ret; + } + /// + /// Get the length from the input offset to the location of the input terminator bytes or zero. + /// + /// Bytes to check. + /// Offset at which to begin searching for the terminator bytes. + /// Bytes to search for. + /// Length of distance between offset and terminator or zero if not found. + public static int GetSegmentLength(byte[] sourceBytes, int searchOffset, byte[] terminatorBytes) + { + int ret; + bool terminatorFound = false; + int i = searchOffset; + + while (i < sourceBytes.Length) + { + // first char match + if (sourceBytes[i] == terminatorBytes[0]) + { + if (CompareSegment(sourceBytes, i, terminatorBytes)) + { + terminatorFound = true; + break; + } + } + + i++; + } // while (!terminatorFound) + + if (terminatorFound) + { + ret = i - searchOffset; + } + else + { + ret = 0; + } + + return ret; + } + + /// + /// Get the length from the input offset to the location of the input terminator bytes or zero. + /// + /// Stream to check. + /// Offset at which to begin searching for the terminator bytes. + /// Bytes to search for. + /// Length of distance between offset and terminator or zero if not found. + public static int GetSegmentLength(Stream stream, int searchOffset, byte[] terminatorBytes) + { + int ret; + bool terminatorFound = false; + int i = searchOffset; + byte[] checkBytes = new byte[terminatorBytes.Length]; + + while (i < stream.Length) + { + stream.Seek(i, SeekOrigin.Begin); + stream.Read(checkBytes, 0, 1); + + // first char match + if (checkBytes[0] == terminatorBytes[0]) + { + stream.Seek(i, SeekOrigin.Begin); + stream.Read(checkBytes, 0, terminatorBytes.Length); + + if (CompareSegment(checkBytes, 0, terminatorBytes)) + { + terminatorFound = true; + break; + } + } + + i++; + } // while (!terminatorFound) + + if (terminatorFound) + { + ret = i - searchOffset; + } + else + { + ret = 0; + } + + return ret; + } + public static Dictionary GetAllOffsets(Stream stream, long startingOffset, + byte[][] searchByteArrays, bool returnStreamToIncomingPosition) + { + long initialStreamPosition = 0; + + if (returnStreamToIncomingPosition) + { + initialStreamPosition = stream.Position; + } + + int maxSearchBytesLength = 0; + long absoluteOffset = startingOffset; + long relativeOffset; + byte[] checkBytes = new byte[Constants.FileReadChunkSize]; + byte[] compareBytes; + + Dictionary tempOutput = new Dictionary(); + Dictionary ret = new Dictionary(); + + foreach (byte[] b in searchByteArrays) + { + tempOutput.Add(b, new ArrayList()); + + if (b.Length > maxSearchBytesLength) + { + maxSearchBytesLength = b.Length; + } + } + + while (absoluteOffset < stream.Length) + { + stream.Position = absoluteOffset; + stream.Read(checkBytes, 0, Constants.FileReadChunkSize); + relativeOffset = 0; + + while (relativeOffset < Constants.FileReadChunkSize) + { + foreach (byte[] searchBytes in searchByteArrays) + { + if ((relativeOffset + searchBytes.Length) < checkBytes.Length) + { + compareBytes = new byte[searchBytes.Length]; + Array.Copy(checkBytes, relativeOffset, compareBytes, 0, searchBytes.Length); + + if (CompareSegment(compareBytes, 0, searchBytes)) + { + tempOutput[searchBytes].Add(absoluteOffset + relativeOffset); + break; + } + } + } // foreach (byte[] searchBytes in searchByteArrays) + + relativeOffset++; + } + + absoluteOffset += Constants.FileReadChunkSize - maxSearchBytesLength; + } + + // return stream to incoming position + if (returnStreamToIncomingPosition) + { + stream.Position = initialStreamPosition; + } + + // sort offsets and add to output + foreach (byte[] key in tempOutput.Keys) + { + tempOutput[key].Sort(); + ret.Add(key, (long[])tempOutput[key].ToArray(typeof(long))); + } + + return ret; + } + + public static long[] GetAllOffsets(Stream stream, long startingOffset, + byte[] searchBytes, bool doOffsetModulo, long offsetModuloDivisor, + long offsetModuloResult, bool returnStreamToIncomingPosition) + { + long initialStreamPosition = 0; + + if (returnStreamToIncomingPosition) + { + initialStreamPosition = stream.Position; + } + + long absoluteOffset = startingOffset; + long relativeOffset; + long actualOffset; + byte[] checkBytes = new byte[Constants.FileReadChunkSize]; + int checkBytesRead; + byte[] compareBytes; + + ArrayList offsetList = new ArrayList(); + long[] ret; + + while (absoluteOffset < stream.Length) + { + stream.Position = absoluteOffset; + checkBytesRead = stream.Read(checkBytes, 0, Constants.FileReadChunkSize); + relativeOffset = 0; + + while (relativeOffset < checkBytesRead) + { + actualOffset = absoluteOffset + relativeOffset; + + if ((!doOffsetModulo) || + (actualOffset % offsetModuloDivisor == offsetModuloResult)) + { + if ((relativeOffset + searchBytes.Length) < checkBytes.Length) + { + compareBytes = new byte[searchBytes.Length]; + Array.Copy(checkBytes, relativeOffset, compareBytes, 0, searchBytes.Length); + + if (CompareSegment(compareBytes, 0, searchBytes)) + { + offsetList.Add(actualOffset); + } + } + } + + relativeOffset++; + } + + absoluteOffset += Constants.FileReadChunkSize - searchBytes.Length; + } + + // return stream to incoming position + if (returnStreamToIncomingPosition) + { + stream.Position = initialStreamPosition; + } + + ret = (long[])offsetList.ToArray(typeof(long)); + + return ret; + } + } +} diff --git a/acb_extract/src/Program.cs b/acb_extract/src/Program.cs new file mode 100644 index 0000000..ace98b3 --- /dev/null +++ b/acb_extract/src/Program.cs @@ -0,0 +1,17 @@ +using System; +using System.IO; + +namespace vgm_acb.extract +{ + static class extract + { + static void Main(string[] args) { + foreach (string s in args) { + using (FileStream fs = File.Open(s, FileMode.Open, FileAccess.Read, FileShare.Read)) { + CriAcbFile acb = new CriAcbFile(fs, 0, false); + acb.ExtractAll(); + } + } + } + } +} diff --git a/update_bgm.zsh b/update_bgm.zsh new file mode 100755 index 0000000..3310fce --- /dev/null +++ b/update_bgm.zsh @@ -0,0 +1,11 @@ +#!/bin/zsh + +mkdir bgm +find Bgm -name \*.adx -delete +for i in Bgm/*.acb; +do + mono acb_extract.exe "$i" +done +find Bgm -name \*.adx | parallel ffmpeg -i {} -acodec libopus -ab 192k -af aresample=48000 bgm/{/.}.opus -n +../scripts/adx_parse.py $(find Bgm -name \*.adx) +cp loop.json bgm/BgmLoop.json diff --git a/update_movie.zsh b/update_movie.zsh new file mode 100755 index 0000000..df1c6ea --- /dev/null +++ b/update_movie.zsh @@ -0,0 +1,6 @@ +#!/bin/zsh + +mkdir movie +find Movie -name \*.adx -or -name \*.m2v -delete +find Movie -name \*.usm -and -not -iname mini_radio\* | parallel mono usm_extract.exe {} +find Movie -name \*.usm -and -not -iname mini_radio\* | parallel ffmpeg -i {.}.adx -i {.}.m2v -c:a libopus -b:a 196k -af aresample=48000 -c:v libvpx -b:v 1M -threads 1 movie/{/.}.webm -n diff --git a/update_se.zsh b/update_se.zsh new file mode 100755 index 0000000..0e86fe4 --- /dev/null +++ b/update_se.zsh @@ -0,0 +1,14 @@ +#!/bin/zsh + +mkdir se +find Se -name \*.wav -delete +find Se -name \*.hca -delete +for i in Se/*.acb; +do + I=$(basename -s .acb $i) + IFS='_'; read -r char sec <<< "$I" + mkdir -p se/${char}/${sec} + mono acb_extract.exe "$i" + ./clHCA Se/_vgmt_acb_ext_${I}/acb/awb/*.hca + find Se/_vgmt_acb_ext_${I}/acb/awb -name \*.wav | parallel ffmpeg -i {} -acodec libopus -ab 192k -af aresample=48000 se/${char}/${sec}/{/.}.opus -n +done diff --git a/update_voice.zsh b/update_voice.zsh new file mode 100755 index 0000000..17467a1 --- /dev/null +++ b/update_voice.zsh @@ -0,0 +1,18 @@ +#!/bin/zsh + +mkdir voice +find Voice -name \*.wav -delete +find Voice -name \*.hca -delete +for i in Voice/*.acb; +do + I=$(basename -s .acb $i) + IFS='_'; read -r char sec <<< "$I" + mkdir -p voice/${char}/${sec} + mono acb_extract.exe "$i" + ./clHCA Voice/_vgmt_acb_ext_${I}/**/*.hca + OUTDIR="voice/${char}/${sec}/" + #OUTFILE="$(basename -s .wav ${ + #FIXME + find Voice/_vgmt_acb_ext_${I}/ -name \*.wav | parallel ffmpeg -i {} -acodec libopus -ab 192k -af aresample=48000 voice/${char}/${sec}/{/.}.opus -n +done +#rename 's/\/\d{5}_([^\/]+.opus)$/\/$1/' voice/**/*.opus diff --git a/usm_extract/src/CriUsmStream.cs b/usm_extract/src/CriUsmStream.cs new file mode 100644 index 0000000..1dd57f5 --- /dev/null +++ b/usm_extract/src/CriUsmStream.cs @@ -0,0 +1,222 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using VGMToolbox.util; + +namespace vgm_usm.extract +{ + public class CriUsmStream : MpegStream + { + public const string DefaultAudioExtension = ".adx"; + public const string DefaultVideoExtension = ".m2v"; + public const string HcaAudioExtension = ".hca"; + + static readonly byte[] HCA_SIG_BYTES = new byte[] { 0x48, 0x43, 0x41, 0x00 }; + + protected static readonly byte[] ALP_BYTES = new byte[] { 0x40, 0x41, 0x4C, 0x50 }; + protected static readonly byte[] CRID_BYTES = new byte[] { 0x43, 0x52, 0x49, 0x44 }; + protected static readonly byte[] SFV_BYTES = new byte[] { 0x40, 0x53, 0x46, 0x56 }; + protected static readonly byte[] SFA_BYTES = new byte[] { 0x40, 0x53, 0x46, 0x41 }; + protected static readonly byte[] SBT_BYTES = new byte[] { 0x40, 0x53, 0x42, 0x54 }; + protected static readonly byte[] CUE_BYTES = new byte[] { 0x40, 0x43, 0x55, 0x45 }; + + protected static readonly byte[] UTF_BYTES = new byte[] { 0x40, 0x55, 0x54, 0x46 }; + + protected static readonly byte[] HEADER_END_BYTES = + new byte[] { 0x23, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x20, + 0x45, 0x4E, 0x44, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, + 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x00 }; + + protected static readonly byte[] METADATA_END_BYTES = + new byte[] { 0x23, 0x4D, 0x45, 0x54, 0x41, 0x44, 0x41, 0x54, + 0x41, 0x20, 0x45, 0x4E, 0x44, 0x20, 0x20, 0x20, + 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, + 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x00 }; + + protected static readonly byte[] CONTENTS_END_BYTES = + new byte[] { 0x23, 0x43, 0x4F, 0x4E, 0x54, 0x45, 0x4E, 0x54, + 0x53, 0x20, 0x45, 0x4E, 0x44, 0x20, 0x20, 0x20, + 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, + 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x00 }; + + public CriUsmStream(string path) + : base(path) + { + this.UsesSameIdForMultipleAudioTracks = true; + this.FileExtensionAudio = DefaultAudioExtension; + this.FileExtensionVideo = DefaultVideoExtension; + + base.BlockIdDictionary.Clear(); + base.BlockIdDictionary[BitConverter.ToUInt32(ALP_BYTES, 0)] = new BlockSizeStruct(PacketSizeType.SizeBytes, 4); // @ALP + base.BlockIdDictionary[BitConverter.ToUInt32(CRID_BYTES, 0)] = new BlockSizeStruct(PacketSizeType.SizeBytes, 4); // CRID + base.BlockIdDictionary[BitConverter.ToUInt32(SFV_BYTES, 0)] = new BlockSizeStruct(PacketSizeType.SizeBytes, 4); // @SFV + base.BlockIdDictionary[BitConverter.ToUInt32(SFA_BYTES, 0)] = new BlockSizeStruct(PacketSizeType.SizeBytes, 4); // @SFA + base.BlockIdDictionary[BitConverter.ToUInt32(SBT_BYTES, 0)] = new BlockSizeStruct(PacketSizeType.SizeBytes, 4); // @SBT + base.BlockIdDictionary[BitConverter.ToUInt32(CUE_BYTES, 0)] = new BlockSizeStruct(PacketSizeType.SizeBytes, 4); // @CUE + } + + protected override byte[] GetPacketStartBytes() { return CRID_BYTES; } + + protected override int GetAudioPacketHeaderSize(Stream readStream, long currentOffset) + { + UInt16 checkBytes; + OffsetDescription od = new OffsetDescription(); + + od.OffsetByteOrder = Constants.BigEndianByteOrder; + od.OffsetSize = "2"; + od.OffsetValue = "8"; + + checkBytes = (UInt16)ParseFile.GetVaryingByteValueAtRelativeOffset(readStream, od, currentOffset); + + return checkBytes; + } + protected override int GetVideoPacketHeaderSize(Stream readStream, long currentOffset) + { + UInt16 checkBytes; + OffsetDescription od = new OffsetDescription(); + + od.OffsetByteOrder = Constants.BigEndianByteOrder; + od.OffsetSize = "2"; + od.OffsetValue = "8"; + + checkBytes = (UInt16)ParseFile.GetVaryingByteValueAtRelativeOffset(readStream, od, currentOffset); + + return checkBytes; + } + + protected override bool IsThisAnAudioBlock(byte[] blockToCheck) + { + return ParseFile.CompareSegment(blockToCheck, 0, SFA_BYTES); + } + protected override bool IsThisAVideoBlock(byte[] blockToCheck) + { + return ParseFile.CompareSegment(blockToCheck, 0, SFV_BYTES); + } + + protected override byte GetStreamId(Stream readStream, long currentOffset) + { + byte streamId; + + streamId = ParseFile.ParseSimpleOffset(readStream, currentOffset + 0xC, 1)[0]; + + return streamId; + } + + protected override int GetAudioPacketFooterSize(Stream readStream, long currentOffset) + { + UInt16 checkBytes; + OffsetDescription od = new OffsetDescription(); + + od.OffsetByteOrder = Constants.BigEndianByteOrder; + od.OffsetSize = "2"; + od.OffsetValue = "0xA"; + + checkBytes = (UInt16)ParseFile.GetVaryingByteValueAtRelativeOffset(readStream, od, currentOffset); + + return checkBytes; + } + + protected override int GetVideoPacketFooterSize(Stream readStream, long currentOffset) + { + UInt16 checkBytes; + OffsetDescription od = new OffsetDescription(); + + od.OffsetByteOrder = Constants.BigEndianByteOrder; + od.OffsetSize = "2"; + od.OffsetValue = "0xA"; + + checkBytes = (UInt16)ParseFile.GetVaryingByteValueAtRelativeOffset(readStream, od, currentOffset); + + return checkBytes; + } + + protected override void DoFinalTasks(FileStream sourceFileStream, Dictionary outputFiles, bool addHeader) + { + long headerEndOffset; + long metadataEndOffset; + long headerSize; + + long footerOffset; + long footerSize; + + string sourceFileName; + string workingFile; + string fileExtension; + string destinationFileName; + + foreach (uint streamId in outputFiles.Keys) + { + sourceFileName = outputFiles[streamId].Name; + + //-------------------------- + // get header size + //-------------------------- + headerEndOffset = ParseFile.GetNextOffset(outputFiles[streamId], 0, HEADER_END_BYTES); + metadataEndOffset = ParseFile.GetNextOffset(outputFiles[streamId], 0, METADATA_END_BYTES); + + if (metadataEndOffset > headerEndOffset) + { + headerSize = metadataEndOffset + METADATA_END_BYTES.Length; + } + else + { + headerSize = headerEndOffset + METADATA_END_BYTES.Length; + } + + //----------------- + // get footer size + //----------------- + footerOffset = ParseFile.GetNextOffset(outputFiles[streamId], 0, CONTENTS_END_BYTES) - headerSize; + footerSize = outputFiles[streamId].Length - footerOffset; + + //------------------------------------------ + // check data to adjust extension if needed + //------------------------------------------ + if (this.IsThisAnAudioBlock(BitConverter.GetBytes(streamId & 0xFFFFFFF0))) // may need to change mask if more than 0xF streams + { + byte[] checkBytes = ParseFile.ParseSimpleOffset(outputFiles[streamId], headerSize, 4); + + if (ParseFile.CompareSegment(checkBytes, 0, SofdecStream.AixSignatureBytes)) + { + fileExtension = SofdecStream.AixAudioExtension; + } + else if (checkBytes[0] == 0x80) + { + fileExtension = SofdecStream.AdxAudioExtension; + } + else if (ParseFile.CompareSegment(checkBytes, 0, HCA_SIG_BYTES)) + { + fileExtension = HcaAudioExtension; + } + else + { + fileExtension = ".bin"; + } + } + else + { + fileExtension = Path.GetExtension(sourceFileName); + } + + outputFiles[streamId].Close(); + outputFiles[streamId].Dispose(); + + workingFile = FileUtil.RemoveChunkFromFile(sourceFileName, 0, headerSize); + File.Copy(workingFile, sourceFileName, true); + File.Delete(workingFile); + + workingFile = FileUtil.RemoveChunkFromFile(sourceFileName, footerOffset, footerSize); + destinationFileName = Path.ChangeExtension(sourceFileName, fileExtension); + File.Copy(workingFile, destinationFileName, true); + File.Delete(workingFile); + + if ((sourceFileName != destinationFileName) && (File.Exists(sourceFileName))) + { + File.Delete(sourceFileName); + } + } + } + } +} diff --git a/usm_extract/src/Mpeg1Stream.cs b/usm_extract/src/Mpeg1Stream.cs new file mode 100644 index 0000000..14b824a --- /dev/null +++ b/usm_extract/src/Mpeg1Stream.cs @@ -0,0 +1,39 @@ +using System; +using System.IO; + +namespace vgm_usm.extract +{ + public class Mpeg1Stream : MpegStream + { + public const string DefaultAudioExtension = ".mp2"; + public const string DefaultVideoExtension = ".m1v"; + + public Mpeg1Stream(string path) + : base(path) + { + this.FileExtensionAudio = DefaultAudioExtension; + this.FileExtensionVideo = DefaultVideoExtension; + + base.BlockIdDictionary[BitConverter.ToUInt32(MpegStream.PacketStartBytes, 0)] = new BlockSizeStruct(PacketSizeType.Static, 0xC); // Pack Header + } + + protected override int GetAudioPacketHeaderSize(Stream readStream, long currentOffset) + { + int paddingByteCount = 0; + readStream.Position = currentOffset + 6; + + // skip stuffing bytes + while (readStream.ReadByte() == 0xFF) + { + paddingByteCount++; + } + + return paddingByteCount + 7; + } + + protected override int GetVideoPacketHeaderSize(Stream readStream, long currentOffset) + { + return 0xC; + } + } +} diff --git a/usm_extract/src/MpegStream.cs b/usm_extract/src/MpegStream.cs new file mode 100644 index 0000000..d6c81b5 --- /dev/null +++ b/usm_extract/src/MpegStream.cs @@ -0,0 +1,503 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +using VGMToolbox.util; + +namespace vgm_usm.extract +{ + public abstract class MpegStream + { + protected static readonly byte[] PacketStartBytes = new byte[] { 0x00, 0x00, 0x01, 0xBA }; + protected static readonly byte[] PacketEndBytes = new byte[] { 0x00, 0x00, 0x01, 0xB9 }; + + public MpegStream(string path) + { + this.FilePath = path; + this.UsesSameIdForMultipleAudioTracks = false; + this.SubTitleExtractionSupported = false; + this.BlockSizeIsLittleEndian = false; + + //******************** + // Add Slice Packets + //******************** + byte[] sliceBytes; + uint sliceBytesValue; + BlockSizeStruct blockSize = new BlockSizeStruct(PacketSizeType.Static, 0xE); + + for (byte i = 0; i <= 0xAF; i++) + { + sliceBytes = new byte[] { 0x00, 0x00, 0x01, i }; + sliceBytesValue = BitConverter.ToUInt32(sliceBytes, 0); + this.BlockIdDictionary.Add(sliceBytesValue, blockSize); + } + } + + public enum PacketSizeType + { + Static, + SizeBytes, + Eof + } + + public struct MpegDemuxOptions + { + public bool AddHeader { set; get; } + } + + public struct BlockSizeStruct + { + public PacketSizeType SizeType; + public int Size; + + public BlockSizeStruct(PacketSizeType sizeTypeValue, int sizeValue) + { + this.SizeType = sizeTypeValue; + this.Size = sizeValue; + } + } + + public struct DemuxOptionsStruct + { + public bool ExtractVideo { set; get; } + public bool ExtractAudio { set; get; } + + public bool AddHeader { set; get; } + public bool SplitAudioStreams { set; get; } + public bool AddPlaybackHacks { set; get; } + } + + #region Dictionary Initialization + + protected Dictionary BlockIdDictionary = + new Dictionary + { + //******************** + // System Packets + //******************** + {BitConverter.ToUInt32(MpegStream.PacketEndBytes, 0), new BlockSizeStruct(PacketSizeType.Eof, -1)}, // Program End + {BitConverter.ToUInt32(MpegStream.PacketStartBytes, 0), new BlockSizeStruct(PacketSizeType.Static, 0xE)}, // Pack Header + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xBB }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // System Header, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xBD }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Private Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xBE }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Padding Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xBF }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Private Stream, two bytes following equal length (Big Endian) + + //**************************** + // Audio Streams + //**************************** + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC0 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC1 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC2 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC3 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC4 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC5 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC6 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC7 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC8 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC9 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xCA }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xCB }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xCC }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xCD }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xCE }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xCF }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD0 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD1 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD2 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD3 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD4 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD5 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD6 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD7 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD8 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD9 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xDA }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xDB }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xDC }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xDD }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xDE }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xDF }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian) + + //**************************** + // Video Streams + //**************************** + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE0 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE1 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE2 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE3 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE4 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE5 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE6 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE7 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE8 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE9 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xEA }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xEB }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xEC }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xED }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xEE }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + {BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xEF }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian) + }; + #endregion + + public string FilePath { get; set; } + public string FileExtensionAudio { get; set; } + public string FileExtensionVideo { get; set; } + + protected Dictionary StreamIdFileType = new Dictionary(); + + public bool UsesSameIdForMultipleAudioTracks { set; get; } // for PMF/PAM/DVD, who use 000001BD for all audio tracks + public bool SubTitleExtractionSupported { set; get; } // assume not supported. + + public bool BlockSizeIsLittleEndian { set; get; } + + protected virtual byte[] GetPacketStartBytes() { return MpegStream.PacketStartBytes; } + + protected virtual byte[] GetPacketEndBytes() { return MpegStream.PacketEndBytes; } + + protected abstract int GetAudioPacketHeaderSize(Stream readStream, long currentOffset); + + protected virtual int GetAudioPacketSubHeaderSize(Stream readStream, long currentOffset, byte streamId) { return 0; } + + protected abstract int GetVideoPacketHeaderSize(Stream readStream, long currentOffset); + + protected virtual int GetAudioPacketFooterSize(Stream readStream, long currentOffset) { return 0; } + + protected virtual int GetVideoPacketFooterSize(Stream readStream, long currentOffset) { return 0; } + + protected virtual bool IsThisAnAudioBlock(byte[] blockToCheck) + { + return ((blockToCheck[3] >= 0xC0) && + (blockToCheck[3] <= 0xDF)); + } + + protected virtual bool IsThisAVideoBlock(byte[] blockToCheck) + { + return ((blockToCheck[3] >= 0xE0) && (blockToCheck[3] <= 0xEF)); + } + + protected virtual bool IsThisASubPictureBlock(byte[] blockToCheck) + { + return ((blockToCheck[3] >= 0xE0) && (blockToCheck[3] <= 0xEF)); + } + + protected virtual string GetAudioFileExtension(Stream readStream, long currentOffset) + { + return this.FileExtensionAudio; + } + + protected virtual string GetVideoFileExtension(Stream readStream, long currentOffset) + { + return this.FileExtensionVideo; + } + + protected virtual byte GetStreamId(Stream readStream, long currentOffset) { return 0; } + + protected virtual long GetStartOffset(Stream readStream, long currentOffset) { return 0; } + + protected virtual void DoFinalTasks(FileStream sourceFileStream, Dictionary outputFiles, bool addHeader) + { + + } + + public virtual void DemultiplexStreams(DemuxOptionsStruct demuxOptions) + { + using (FileStream fs = File.OpenRead(this.FilePath)) + { + long fileSize = fs.Length; + long currentOffset = 0; + + byte[] currentBlockId; + uint currentBlockIdVal; + byte[] currentBlockIdNaming; + + BlockSizeStruct blockStruct = new BlockSizeStruct(); + byte[] blockSizeArray; + uint blockSize; + + int audioBlockSkipSize; + int videoBlockSkipSize; + + int audioBlockFooterSize; + int videoBlockFooterSize; + + int cutSize; + + bool eofFlagFound = false; + + Dictionary streamOutputWriters = new Dictionary(); + string outputFileName; + + byte streamId = 0; // for types that have multiple streams in the same block ID + uint currentStreamKey; // hash key for each file + bool isAudioBlock; + string audioFileExtension; + + // look for first packet + currentOffset = this.GetStartOffset(fs, currentOffset); + currentOffset = ParseFile.GetNextOffset(fs, currentOffset, this.GetPacketStartBytes()); + + if (currentOffset != -1) + { + while (currentOffset < fileSize) + { +#if DEBUG + //if (currentOffset == 0x414080e) + //{ + // int gggg = 1; + //} + + // hack for bad data (ni no kuni s09.pam) + //if ((currentOffset & 1) == 1) + //{ + // currentOffset = MathUtil.RoundUpToByteAlignment(currentOffset, 0x800); + //} +#endif + + try + { + // get the current block + currentBlockId = ParseFile.ParseSimpleOffset(fs, currentOffset, 4); + + // get value to use as key to hash table + currentBlockIdVal = BitConverter.ToUInt32(currentBlockId, 0); + + if (BlockIdDictionary.ContainsKey(currentBlockIdVal)) + { + // get info about this block type + blockStruct = BlockIdDictionary[currentBlockIdVal]; + + switch (blockStruct.SizeType) + { + ///////////////////// + // Static Block Size + ///////////////////// + case PacketSizeType.Static: + currentOffset += blockStruct.Size; // skip this block + break; + + ////////////////// + // End of Stream + ////////////////// + case PacketSizeType.Eof: + eofFlagFound = true; // set EOF block found so we can exit the loop + break; + + ////////////////////// + // Varying Block Size + ////////////////////// + case PacketSizeType.SizeBytes: + + // Get the block size + blockSizeArray = ParseFile.ParseSimpleOffset(fs, currentOffset + currentBlockId.Length, blockStruct.Size); + + if (!this.BlockSizeIsLittleEndian) + { + Array.Reverse(blockSizeArray); + } + + switch (blockStruct.Size) + { + case 4: + blockSize = (uint)BitConverter.ToUInt32(blockSizeArray, 0); + break; + case 2: + blockSize = (uint)BitConverter.ToUInt16(blockSizeArray, 0); + break; + case 1: + blockSize = (uint)blockSizeArray[0]; + break; + default: + throw new ArgumentOutOfRangeException(String.Format("Unhandled size block size.{0}", Environment.NewLine)); + } + + + // if block type is audio or video, extract it + isAudioBlock = this.IsThisAnAudioBlock(currentBlockId); + + if ((demuxOptions.ExtractAudio && isAudioBlock) || + (demuxOptions.ExtractVideo && this.IsThisAVideoBlock(currentBlockId))) + { + // reset stream id + streamId = 0; + + // if audio block, get the stream number from the queue + if (isAudioBlock && this.UsesSameIdForMultipleAudioTracks) + { + streamId = this.GetStreamId(fs, currentOffset); + currentStreamKey = (streamId | currentBlockIdVal); + } + else + { + currentStreamKey = currentBlockIdVal; + } + + // check if we've already started parsing this stream + if (!streamOutputWriters.ContainsKey(currentStreamKey)) + { + // convert block id to little endian for naming + currentBlockIdNaming = BitConverter.GetBytes(currentStreamKey); + Array.Reverse(currentBlockIdNaming); + + // build output file name + outputFileName = Path.GetFileNameWithoutExtension(this.FilePath); + //outputFileName = outputFileName + "_" + BitConverter.ToUInt32(currentBlockIdNaming, 0).ToString("X8"); + + // add proper extension + if (this.IsThisAnAudioBlock(currentBlockId)) + { + audioFileExtension = this.GetAudioFileExtension(fs, currentOffset); + outputFileName += audioFileExtension; + + if (!this.StreamIdFileType.ContainsKey(streamId)) + { + this.StreamIdFileType.Add(streamId, audioFileExtension); + } + } + else + { + this.FileExtensionVideo = this.GetVideoFileExtension(fs, currentOffset); + outputFileName += this.FileExtensionVideo; + } + + // add output directory + outputFileName = Path.Combine(Path.GetDirectoryName(this.FilePath), outputFileName); + + // add an output stream for writing + streamOutputWriters[currentStreamKey] = new FileStream(outputFileName, FileMode.Create, FileAccess.ReadWrite); + } + + // write the block + if (this.IsThisAnAudioBlock(currentBlockId)) + { + // write audio + audioBlockSkipSize = this.GetAudioPacketHeaderSize(fs, currentOffset) + GetAudioPacketSubHeaderSize(fs, currentOffset, streamId); + audioBlockFooterSize = this.GetAudioPacketFooterSize(fs, currentOffset); + cutSize = (int)(blockSize - audioBlockSkipSize - audioBlockFooterSize); + if (cutSize > 0) + { + streamOutputWriters[currentStreamKey].Write(ParseFile.ParseSimpleOffset(fs, currentOffset + currentBlockId.Length + blockSizeArray.Length + audioBlockSkipSize, (int)(blockSize - audioBlockSkipSize)), 0, cutSize); + } +#if DEBUG + //else + //{ + // int aaa = 1; + //} +#endif + } + else + { + // write video + videoBlockSkipSize = this.GetVideoPacketHeaderSize(fs, currentOffset); + videoBlockFooterSize = this.GetVideoPacketFooterSize(fs, currentOffset); + cutSize = (int)(blockSize - videoBlockSkipSize - videoBlockFooterSize); + if (cutSize > 0) + { + streamOutputWriters[currentStreamKey].Write(ParseFile.ParseSimpleOffset(fs, currentOffset + currentBlockId.Length + blockSizeArray.Length + videoBlockSkipSize, (int)(blockSize - videoBlockSkipSize)), 0, cutSize); + } +#if DEBUG + //else + //{ + // int vvv = 1; + //} +#endif + } + } + + // move to next block + currentOffset += currentBlockId.Length + blockSizeArray.Length + blockSize; + blockSizeArray = new byte[] { }; + break; + default: + break; + } + } + else // this is an undexpected block type + { + this.closeAllWriters(streamOutputWriters); + Array.Reverse(currentBlockId); + throw new FormatException(String.Format("Block ID at 0x{0} not found in table: 0x{1}", currentOffset.ToString("X8"), BitConverter.ToUInt32(currentBlockId, 0).ToString("X8"))); + } + + // exit loop if EOF block found + if (eofFlagFound) + { + break; + } + } + catch (Exception _ex) + { + this.closeAllWriters(streamOutputWriters); + throw new Exception(String.Format("Error parsing file at offset {0), '{1}'", currentOffset.ToString("X8"), _ex.Message), _ex); + } + } // while (currentOffset < fileSize) + } + else + { + this.closeAllWriters(streamOutputWriters); + throw new FormatException(String.Format("Cannot find Pack Header for file: {0}{1}", Path.GetFileName(this.FilePath), Environment.NewLine)); + } + + /////////////////////////////////// + // Perform any final tasks needed + /////////////////////////////////// + this.DoFinalTasks(fs, streamOutputWriters, demuxOptions.AddHeader); + + ////////////////////////// + // close all open writers + ////////////////////////// + this.closeAllWriters(streamOutputWriters); + + } // using (FileStream fs = File.OpenRead(path)) + } + + private void closeAllWriters(Dictionary writers) + { + ////////////////////////// + // close all open writers + ////////////////////////// + foreach (uint b in writers.Keys) + { + if (writers[b].CanRead) + { + writers[b].Close(); + writers[b].Dispose(); + } + } + } + + public static int GetMpegStreamType(string path) + { + int mpegType = -1; + + using (FileStream fs = File.OpenRead(path)) + { + // look for first packet + long currentOffset = ParseFile.GetNextOffset(fs, 0, MpegStream.PacketStartBytes); + + if (currentOffset != -1) + { + currentOffset += 4; + fs.Position = currentOffset; + byte idByte = (byte)fs.ReadByte(); + + if ((int)ByteConversion.GetHighNibble(idByte) == 2) + { + mpegType = 1; + } + else if ((int)ByteConversion.GetHighNibble(idByte) == 4) + { + mpegType = 2; + } + } + else + { + throw new FormatException(String.Format("Cannot find Pack Header for file: {0}{1}", Path.GetFileName(path), Environment.NewLine)); + } + } + + return mpegType; + } + } +} diff --git a/usm_extract/src/Program.cs b/usm_extract/src/Program.cs new file mode 100644 index 0000000..97baba6 --- /dev/null +++ b/usm_extract/src/Program.cs @@ -0,0 +1,21 @@ +using System; +using System.IO; + +namespace vgm_usm.extract +{ + static class extract + { + static void Main(string[] args) { + MpegStream.DemuxOptionsStruct demuxOptions = new MpegStream.DemuxOptionsStruct(); + demuxOptions.ExtractVideo = true; + demuxOptions.ExtractAudio = true; + demuxOptions.AddHeader = true; + demuxOptions.SplitAudioStreams = true; + demuxOptions.AddPlaybackHacks = true; + foreach (string s in args) { + CriUsmStream usm = new CriUsmStream(s); + usm.DemultiplexStreams(demuxOptions); + } + } + } +} diff --git a/usm_extract/src/SofdecStream.cs b/usm_extract/src/SofdecStream.cs new file mode 100644 index 0000000..2a49e96 --- /dev/null +++ b/usm_extract/src/SofdecStream.cs @@ -0,0 +1,58 @@ +using System; +using System.IO; + +using VGMToolbox.util; + +namespace vgm_usm.extract +{ + public class SofdecStream : Mpeg1Stream + { + new public const string DefaultVideoExtension = ".m2v"; + + public const string AdxAudioExtension = ".adx"; + public const string AixAudioExtension = ".aix"; + public const string Ac3AudioExtension = ".ac3"; + + public static readonly byte[] AixSignatureBytes = new byte[] { 0x41, 0x49, 0x58, 0x46 }; + public static readonly byte[] Ac3SignatureBytes = new byte[] { 0x0B, 0x77 }; + + public SofdecStream(string path): base(path) + { + this.FileExtensionAudio = AdxAudioExtension; + this.FileExtensionVideo = DefaultVideoExtension; + } + + protected override string GetAudioFileExtension(Stream readStream, long currentOffset) + { + string fileExtension; + byte[] checkBytes, checkBytesAc3; + + int headerSize = this.GetAudioPacketHeaderSize(readStream, currentOffset); + checkBytes = ParseFile.ParseSimpleOffset(readStream, (currentOffset + 6 + headerSize), 4); + + if (ParseFile.CompareSegment(checkBytes, 0, AixSignatureBytes)) + { + fileExtension = AixAudioExtension; + } + else if (checkBytes[0] == 0x80) + { + fileExtension = AdxAudioExtension; + } + else + { + checkBytesAc3 = ParseFile.ParseSimpleOffset(readStream, (currentOffset + 6 + headerSize), 2); + + if (ParseFile.CompareSegment(checkBytesAc3, 0, Ac3SignatureBytes)) + { + fileExtension = Ac3AudioExtension; + } + else + { + fileExtension = ".bin"; + } + } + + return fileExtension; + } + } +} diff --git a/usm_extract/src/util/ByteConversion.cs b/usm_extract/src/util/ByteConversion.cs new file mode 100644 index 0000000..26710b6 --- /dev/null +++ b/usm_extract/src/util/ByteConversion.cs @@ -0,0 +1,266 @@ +using System; +using System.Globalization; +using System.Text; + +namespace VGMToolbox.util +{ + /// + /// Class containing static text conversion functions. + /// + public sealed class ByteConversion + { + /// + /// Codepage value for Shift JIS (Jp) + /// + public const int CodePageJapan = 932; + /// + /// Codepage value for Cyrillic (US) + /// + public const int CodePageUnitedStates = 1251; + + /// + /// Codepage value for OEM DOS + /// + public const int CodePageOEM = 437; + + private ByteConversion() { } + + /// + /// Get string from bytes. + /// + /// Bytes to convert to a string. + /// Codepage to use in converting bytes. + /// String encoding using the input Codepage. + public static string GetEncodedText(byte[] value, int codePage) + { + //return Encoding.Unicode.GetString(value); + return System.Text.Encoding.GetEncoding(codePage).GetString(value); + } + + /// + /// Get text encoded in Shift JIS + /// + /// Bytes to decode. + /// String encoded using the Shift JIS codepage. + public static string GetJapaneseEncodedText(byte[] value) + { + return GetEncodedText(value, CodePageJapan); + } + /// + /// Get text encoded in Cyrillic + /// + /// Bytes to decode. + /// String encoded using the Cyrillic codepage. + public static string GetUnitedStatesEncodedText(byte[] value) + { + return GetEncodedText(value, CodePageUnitedStates); + } + + /// + /// Get text encoded in ASCII + /// + /// Bytes to decode. + /// String encoded using ASCII. + public static string GetAsciiText(byte[] value) + { + System.Text.Encoding ascii = System.Text.Encoding.ASCII; + return ascii.GetString(value); + } + + /// + /// Get text encoded in ASCII, stops at null. + /// + /// Bytes to decode. + /// Offet within byte array of string. + /// String encoded using ASCII. + public static string GetAsciiText(byte[] value, long offset) + { + StringBuilder sb = new StringBuilder(); + System.Text.Encoding ascii = System.Text.Encoding.ASCII; + + for (long i = offset; i < value.Length; i++) + { + if (value[i] == 0) + { + break; + } + else + { + sb.Append((char)value[i]); + } + } + + return sb.ToString(); + } + + public static string GetUtf16LeText(byte[] value) + { + System.Text.Encoding encoding = System.Text.Encoding.Unicode; + return encoding.GetString(value); + } + + + /// + /// Convert input string to a long. Works for Decimal and Hexidecimal (use 0x prefix). + /// + /// String containing a Decimal and Hexidecimal number. + /// Long representing the input string. + public static long GetLongValueFromString(string value) + { + long ret; + bool isNegative = false; + string parseValue; + + if (value.StartsWith("-")) + { + parseValue = value.Substring(1); + isNegative = true; + } + else + { + parseValue = value; + } + + if (parseValue.StartsWith("0x", StringComparison.CurrentCultureIgnoreCase)) + { + parseValue = parseValue.Substring(2); + ret = long.Parse(parseValue, System.Globalization.NumberStyles.HexNumber, null); + } + else + { + ret = long.Parse(parseValue, System.Globalization.NumberStyles.Integer, null); + } + + if (isNegative) + { + ret *= -1; + } + + return ret; + } + + /// + /// Get the UInt32 Value of the Incoming Byte Array, which is in Big Endian order. + /// + /// Bytes to convert. + /// The UInt32 Value of the Incoming Byte Array. + public static UInt32 GetUInt32BigEndian(byte[] value) + { + byte[] workingArray = new byte[value.Length]; + Array.Copy(value, 0, workingArray, 0, value.Length); + + if (BitConverter.IsLittleEndian) + { + Array.Reverse(workingArray); + } + return BitConverter.ToUInt32(workingArray, 0); + } + + /// + /// Get the UInt16 Value of the Incoming Byte Array, which is in Big Endian order. + /// + /// Bytes to convert. + /// The UInt16 Value of the Incoming Byte Array. + public static UInt16 GetUInt16BigEndian(byte[] value) + { + byte[] workingArray = new byte[value.Length]; + Array.Copy(value, 0, workingArray, 0, value.Length); + + if (BitConverter.IsLittleEndian) + { + Array.Reverse(workingArray); + } + return BitConverter.ToUInt16(workingArray, 0); + } + + /// + /// Predicts Code Page between Cyrillic and Shift-JIS based on whether high ASCII is included or not. + /// + /// Bytes containing the tags in an unknown language. + /// Integer representing the predicted code page. + public static int GetPredictedCodePageForTags(byte[] tagBytes) + { + int predictedCodePage = CodePageUnitedStates; + + foreach (byte b in tagBytes) + { + if ((int)b > 0x7F) + { + predictedCodePage = CodePageJapan; + break; + } + } + + return predictedCodePage; + } + + public static byte GetHighNibble(byte value) + { + return (byte)(((value) >> 4) & 0x0F); + } + + public static byte GetLowNibble(byte value) + { + return (byte)((value) & 0x0F); ; + } + + public static byte[] GetBytesFromHexString(string hexValue) + { + int j = 0; + byte[] bytes = new byte[hexValue.Length / 2]; + + // convert the string to bytes + for (int i = 0; i < hexValue.Length; i += 2) + { + bytes[j] = BitConverter.GetBytes(Int16.Parse(hexValue.Substring(i, 2), System.Globalization.NumberStyles.AllowHexSpecifier, CultureInfo.CurrentCulture))[0]; + j++; + } + + return bytes; + } + + public static byte[] GetBytesBigEndian(uint value) + { + byte[] ret = BitConverter.GetBytes(value); + Array.Reverse(ret); + return ret; + } + + public static DateTime GetDateTimeFromFAT32Date(int value) + { + short xDate = (short)(value >> 0x10); + short xTime = (short)(value & 0xFFFF); + + if (xDate == 0 && xTime == 0) + { + return DateTime.Now; + } + else + { + return new DateTime( + (((xDate & 0xFE00) >> 9) + 0x7BC), + ((xDate & 0x1E0) >> 5), + (xDate & 0x1F), + ((xTime & 0xF800) >> 0xB), + ((xTime & 0x7E0) >> 5), + ((xTime & 0x1F) * 2)); + } + } + + public static bool IsZeroFilledByteArray(byte[] value) + { + bool ret = true; + + for (int i = 0; i < value.Length; i++) + { + if (value[i] != 0) + { + ret = false; + break; + } + } + + return ret; + } + } +} diff --git a/usm_extract/src/util/ByteSearchCalculatingOffsetDescription.cs b/usm_extract/src/util/ByteSearchCalculatingOffsetDescription.cs new file mode 100644 index 0000000..143afa1 --- /dev/null +++ b/usm_extract/src/util/ByteSearchCalculatingOffsetDescription.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace VGMToolbox.util +{ + public class ByteSearchCalculatingOffsetDescription : CalculatingOffsetDescription + { + public const string START_OF_STRING = "start of"; + public const string END_OF_STRING = "end of"; + + public string RelativeLocationToByteString { set; get; } + public string ByteString { set; get; } + public bool TreatByteStringAsHex { set; get; } + } +} diff --git a/usm_extract/src/util/CalculatingOffsetDescription.cs b/usm_extract/src/util/CalculatingOffsetDescription.cs new file mode 100644 index 0000000..6d4c2e7 --- /dev/null +++ b/usm_extract/src/util/CalculatingOffsetDescription.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace VGMToolbox.util +{ + public class CalculatingOffsetDescription : OffsetDescription + { + public const string OFFSET_VARIABLE_STRING = "$V"; + + public string CalculationString { set; get; } + } +} diff --git a/usm_extract/src/util/ChecksumUtil.cs b/usm_extract/src/util/ChecksumUtil.cs new file mode 100644 index 0000000..0fadf4a --- /dev/null +++ b/usm_extract/src/util/ChecksumUtil.cs @@ -0,0 +1,196 @@ +// +// Copyright (c) 2008 All Rights Reserved +// +// +// +// +// Contains the ChecksumUtil class. +using System; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; + +using ICSharpCode.SharpZipLib.Checksums; +using Ionic.Zlib; + +namespace VGMToolbox.util +{ + /// + /// Class containing static functions related to checksum generation. + /// + public sealed class ChecksumUtil + { + /// + /// Prevents a default instance of the ChecksumUtil class from being created. + /// + private ChecksumUtil() + { + } + + /// + /// Get the CRC32 checksum of the input stream. + /// + /// File Stream for which to generate the checksum. + /// String containing the hexidecimal representation of the CRC32 of the input stream. + public static string GetCrc32OfFullFile(FileStream stream) + { + // get incoming stream position + long initialStreamPosition = stream.Position; + + // move to zero position + stream.Seek(0, SeekOrigin.Begin); + + // calculate CRC32 + CRC32 crc32 = new CRC32(); + int ret = crc32.GetCrc32(stream); + + // return stream to incoming position + stream.Position = initialStreamPosition; + + return ret.ToString("X8", CultureInfo.InvariantCulture); + } + + /// + /// Get the MD5 checksum of the input stream. + /// + /// File Stream for which to generate the checksum. + /// String containing the hexidecimal representation of the MD5 of the input stream. + public static string GetMd5OfFullFile(FileStream stream) + { + MD5CryptoServiceProvider hashMd5 = new MD5CryptoServiceProvider(); + + stream.Seek(0, SeekOrigin.Begin); + hashMd5.ComputeHash(stream); + return ParseFile.ByteArrayToString(hashMd5.Hash); + } + + public static byte[] GetSha1(byte[] dataBlock) + { + SHA1CryptoServiceProvider sha1Hash = new SHA1CryptoServiceProvider(); + sha1Hash.ComputeHash(dataBlock); + return sha1Hash.Hash; + } + + /// + /// Get the SHA1 checksum of the input stream. + /// + /// File Stream for which to generate the checksum. + /// String containing the hexidecimal representation of the SHA1 of the input stream. + public static string GetSha1OfFullFile(FileStream stream) + { + SHA1CryptoServiceProvider sha1Hash = new SHA1CryptoServiceProvider(); + + stream.Seek(0, SeekOrigin.Begin); + sha1Hash.ComputeHash(stream); + return ParseFile.ByteArrayToString(sha1Hash.Hash); + } + + /// + /// Get the SHA-512 checksum of the input stream. + /// + /// File Stream for which to generate the checksum. + /// String containing the hexidecimal representation of the SHA-512 of the input stream. + public static string GetSha512OfFullFile(FileStream stream) + { + SHA512CryptoServiceProvider sha512 = new SHA512CryptoServiceProvider(); + + stream.Seek(0, SeekOrigin.Begin); + sha512.ComputeHash(stream); + return ParseFile.ByteArrayToString(sha512.Hash); + } + + /// + /// Adds a chunk of data to the input CRC32 generator. + /// + /// Stream to read data from. + /// Offset to begin reading from. + /// Number of bytes to read. + /// CRC32 generator. + public static void AddChunkToChecksum(Stream stream, int startingOffset, int length, ref Crc32 checksumGenerator) + { + int remaining = length; + byte[] data = new byte[4096]; + int read; + int offset = startingOffset; + + stream.Seek((long)startingOffset, SeekOrigin.Begin); + + while (remaining > 0) + { + if (remaining < 4096) + { + read = stream.Read(data, 0, remaining); + } + else + { + read = stream.Read(data, 0, 4096); + } + + if (read <= 0) + { + throw new EndOfStreamException( + String.Format( + CultureInfo.CurrentCulture, + "End of stream reached with {0} bytes left to read", + remaining)); + } + + checksumGenerator.Update(data, 0, read); + remaining -= read; + offset += read; + } + } + + /// + /// Adds a chunk of data to the input CRC32/MD5/SHA1 generator. + /// + /// Stream to read data from. + /// Offset to begin reading from. + /// Number of bytes to read. + /// CRC32 generator. + /// MD5 generator. + /// SHA1 generator. + public static void AddChunkToChecksum( + Stream sourceStream, + int startingOffset, + int length, + ref Crc32 checksumGeneratorCrc32, + ref CryptoStream checksumStreamMd5, + ref CryptoStream checksumStreamSha1) + { + int remaining = length; + byte[] data = new byte[4096]; + int read; + int offset = startingOffset; + + sourceStream.Seek((long)startingOffset, SeekOrigin.Begin); + + while (remaining > 0) + { + if (remaining < 4096) + { + read = sourceStream.Read(data, 0, remaining); + } + else + { + read = sourceStream.Read(data, 0, 4096); + } + + if (read <= 0) + { + throw new EndOfStreamException( + String.Format( + CultureInfo.CurrentCulture, + "End of stream reached with {0} bytes left to read", + remaining)); + } + + checksumGeneratorCrc32.Update(data, 0, read); + checksumStreamMd5.Write(data, 0, read); + checksumStreamSha1.Write(data, 0, read); + remaining -= read; + offset += read; + } + } + } +} diff --git a/usm_extract/src/util/CompressionUtil.cs b/usm_extract/src/util/CompressionUtil.cs new file mode 100644 index 0000000..fdab150 --- /dev/null +++ b/usm_extract/src/util/CompressionUtil.cs @@ -0,0 +1,496 @@ +using System; +using System.IO; +using System.Reflection; +using Ionic.Zip; +using Ionic.Zlib; +using SevenZip; + +namespace VGMToolbox.util +{ + /// + /// Class containing static functions for compresson related tasks + /// + public sealed class CompressionUtil + { + /// + /// File extension used to output decompressed zlib data. + /// + public const string ZlibDecompressOutputExtension = ".zlibx"; + + /// + /// File extension used to output compressed zlib data. + /// + public const string ZlibCompressOutputExtension = ".zlib"; + + /// + /// File extension used to output decompressed gzip data. + /// + public const string GzipDecompressOutputExtension = ".gzipx"; + + /// + /// File extension used to output compressed gzip data. + /// + public const string GzipCompressOutputExtension = ".gz"; + + /// + /// Path to the included 7z.dll file. + /// + public static readonly string SevenZipDll = + Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "7z.dll"); + + /// + /// Prevents a default instance of the CompressionUtil class from being created + /// + private CompressionUtil() + { + } + + /// + /// Get a list of files inside an archive file. + /// + /// Path to the archive file. + /// An array of strings containing a list of files inside the archive. + /// Must be an archive supported by 7z.dll. + public static string[] GetFileList(string path) + { + string archivePath = Path.GetFullPath(path); + string[] filenames = null; + SevenZipExtractor sevenZipExtractor = null; + + if (File.Exists(archivePath)) + { + SevenZipExtractor.SetLibraryPath(SevenZipDll); + + try + { + sevenZipExtractor = new SevenZipExtractor(archivePath); + filenames = new string[sevenZipExtractor.ArchiveFileNames.Count]; + + int i = 0; + foreach (string f in sevenZipExtractor.ArchiveFileNames) + { + filenames[i++] = f; + } + } + catch (System.ArgumentException) + { + // ignore unsupported formats + } + finally + { + if (sevenZipExtractor != null) + { + sevenZipExtractor.Dispose(); + } + } + } + + return filenames; + } + + /// + /// Get an uppercase list of files inside an archive file. Must be an archive supported by 7z.dll. + /// + /// Path to the archive file. + /// An array of strings containing an uppercase list of files inside the archive. + /// Must be an archive supported by 7z.dll. + public static string[] GetUpperCaseFileList(string path) + { + string archivePath = Path.GetFullPath(path); + string[] filenames = null; + SevenZipExtractor sevenZipExtractor = null; + + if (File.Exists(archivePath)) + { + SevenZipExtractor.SetLibraryPath(SevenZipDll); + + try + { + sevenZipExtractor = new SevenZipExtractor(archivePath); + filenames = new string[sevenZipExtractor.ArchiveFileNames.Count]; + + int i = 0; + foreach (string f in sevenZipExtractor.ArchiveFileNames) + { + filenames[i++] = f.ToUpper(); + } + } + catch (System.ArgumentException) + { + // ignore unsupported formats + } + finally + { + if (sevenZipExtractor != null) + { + sevenZipExtractor.Dispose(); + } + } + } + + return filenames; + } + + public static bool Is7zSupportedArchive(string pArchivePath) + { + string archivePath = Path.GetFullPath(pArchivePath); + SevenZipExtractor sevenZipExtractor = null; + bool ret = false; + + if (File.Exists(archivePath)) + { + try + { + SevenZipExtractor.SetLibraryPath(SevenZipDll); + sevenZipExtractor = new SevenZipExtractor(archivePath); + sevenZipExtractor.Check(); + + ret = true; + } + catch (Exception) + { + ret = false; + } + finally + { + if (sevenZipExtractor != null) + { + sevenZipExtractor.Dispose(); + } + } + } + + return ret; + } + + /// + /// Extracts a file from an archive. The file will be output to a subfolder in the archive's directory; + /// named with the original archive name. + /// + /// Path to the archive file. + /// Name of file to extract. + /// Must be an archive supported by 7z.dll. + public static void ExtractFileFromArchive(string archivePath, string fileName) + { + ExtractFileFromArchive(archivePath, fileName, String.Empty); + } + + /// + /// Extracts a file from an archive. + /// + /// Path to the archive file. + /// Name of file to extract. + /// Folder to output the file to. If empty or null, + /// the file will be output to a subfolder in the archive's directory; + /// named with the original archive name. + /// Must be an archive supported by 7z.dll. + public static void ExtractFileFromArchive(string pArchivePath, string pFileName, string pOutputPath) + { + string archivePath = Path.GetFullPath(pArchivePath); + SevenZipExtractor sevenZipExtractor = null; + string outputDir; + + if (File.Exists(archivePath)) + { + try + { + if (!String.IsNullOrEmpty(pOutputPath)) + { + outputDir = pOutputPath; + } + else + { + outputDir = Path.Combine(Path.GetDirectoryName(archivePath), Path.GetFileNameWithoutExtension(archivePath)); + } + + if (!Directory.Exists(outputDir)) + { + Directory.CreateDirectory(outputDir); + } + + SevenZipExtractor.SetLibraryPath(SevenZipDll); + sevenZipExtractor = new SevenZipExtractor(archivePath); + sevenZipExtractor.ExtractFile(pFileName, outputDir, true); + } + catch (Exception ex) + { + throw new Exception(ex.Message); + } + finally + { + if (sevenZipExtractor != null) + { + sevenZipExtractor.Dispose(); + } + } + } + } + + public static void ExtractAllFilesFromArchive(string pArchivePath, string pOutputPath) + { + string archivePath = Path.GetFullPath(pArchivePath); + SevenZipExtractor sevenZipExtractor = null; + string outputDir; + + if (File.Exists(archivePath)) + { + try + { + if (!String.IsNullOrEmpty(pOutputPath)) + { + outputDir = pOutputPath; + } + else + { + outputDir = Path.Combine(Path.GetDirectoryName(archivePath), Path.GetFileNameWithoutExtension(archivePath)); + } + + if (!Directory.Exists(outputDir)) + { + Directory.CreateDirectory(outputDir); + } + + SevenZipExtractor.SetLibraryPath(SevenZipDll); + sevenZipExtractor = new SevenZipExtractor(archivePath); + sevenZipExtractor.ExtractArchive(outputDir); + } + catch (Exception ex) + { + throw new Exception(ex.Message); + } + finally + { + if (sevenZipExtractor != null) + { + sevenZipExtractor.Dispose(); + } + } + } + } + + /// + /// Compress input folder (recursively) with 7zip Ultra compression. + /// + /// Source directory to compress. + /// Fully rooted output archive name. + public static void CompressFolderWith7zip(string pSourcePath, string pArchiveName) + { + SevenZipCompressor.SetLibraryPath(SevenZipDll); + SevenZipCompressor compressor = new SevenZipCompressor(); + compressor.CompressionLevel = SevenZip.CompressionLevel.Ultra; + compressor.CompressDirectory(pSourcePath, pArchiveName, true); + } + + /// + /// Extract all files from the input .zip compressed file. + /// + /// Fully rooted path to the .zip file. + /// Fully rooted path to the output folder to place the deompressed files. + public static void ExtractAllFilesFromZipFile(string pZipFilePath, string pOutputFolder) + { + using (ZipFile zip = ZipFile.Read(pZipFilePath)) + { + zip.ExtractAll(pOutputFolder); + } + } + + /// + /// Add a file to the input .zip file. + /// + /// Fully rooted path to the .zip file to add files to. + /// Fully rooted path to the file to insert. + /// Path within the .zip file to insert the new file as. + public static void AddFileToZipFile(string pZipFileName, string pNewEntrySourceFileName, string pNewEntryDestinationName) + { + ZipFile zf; + + // create or open zip file + if (File.Exists(pZipFileName)) + { + zf = ZipFile.Read(pZipFileName); + } + else + { + zf = new ZipFile(pZipFileName); + } + + zf.CompressionLevel = Ionic.Zlib.CompressionLevel.BestCompression; + + using (FileStream fs = File.OpenRead(pNewEntrySourceFileName)) + { + zf.AddEntry(Path.GetFileName(pNewEntryDestinationName), Path.GetDirectoryName(pNewEntryDestinationName), fs); + zf.Save(); + } + } + + /// + /// Decompress a zlib compressed section of a stream to file. Data must begin at offset 0. + /// + /// Stream containing zlib compressed bytes. + /// Fully rooted output file name to output the decompressed data to. + public static void DecompressZlibStreamToFile(Stream pStream, string pOutputFilePath) + { + DecompressZlibStreamToFile(pStream, pOutputFilePath, 0); + } + + /// + /// Decompress a zlib compressed section of a stream to file. + /// + /// Stream containing zlib compressed bytes. + /// Fully rooted output file name to output the decompressed data to. + /// Offset to begin reading data from. + public static void DecompressZlibStreamToFile(Stream pStream, string pOutputFilePath, long pStartingOffset) + { + using (FileStream outFs = new FileStream(pOutputFilePath, FileMode.Create, FileAccess.Write)) + { + using (BinaryWriter bw = new BinaryWriter(outFs)) + { + pStream.Position = pStartingOffset; + + using (ZlibStream zs = new ZlibStream(pStream, CompressionMode.Decompress, true)) + { + int read; + byte[] data = new byte[Constants.FileReadChunkSize]; + + while ((read = zs.Read(data, 0, data.Length)) > 0) + { + bw.Write(data, 0, read); + } + } + } + } + } + + /// + /// Compress the incoming stream to a file using zlib compression. + /// + /// Stream to compress. + /// Path to the file to output. + public static void CompressStreamToZlibFile(Stream pStream, string pOutputFilePath) + { + CompressStreamToZlibFile(pStream, pOutputFilePath, 0); + } + + /// + /// Compress the incoming stream to a file using zlib compression starting at the incoming offset. + /// + /// Stream to compress. + /// Path to the file to output. + /// Offset within the stream to start compressing. + public static void CompressStreamToZlibFile(Stream pStream, string pOutputFilePath, long pStartingOffset) + { + using (FileStream outFs = new FileStream(pOutputFilePath, FileMode.Create, FileAccess.Write)) + { + using (BinaryReader br = new BinaryReader(pStream)) + { + pStream.Position = pStartingOffset; + + using (ZlibStream zs = new ZlibStream(outFs, CompressionMode.Compress, Ionic.Zlib.CompressionLevel.BestCompression, true)) + { + int read; + byte[] data = new byte[Constants.FileReadChunkSize]; + + while ((read = br.Read(data, 0, data.Length)) > 0) + { + zs.Write(data, 0, read); + } + + zs.Flush(); + } + } + } + } + + /// + /// Compress an entire file using gzip compression. + /// + /// Fully rooted file name of file to compress. + public static void GzipEntireFile(string pFileName) + { + string tempFileName; + + if (File.Exists(pFileName)) + { + using (FileStream fs = File.OpenRead(pFileName)) + { + tempFileName = Path.GetTempFileName(); + + using (FileStream outFs = File.OpenWrite(tempFileName)) + { + using (GZipStream gs = new GZipStream(outFs, CompressionMode.Compress, Ionic.Zlib.CompressionLevel.BestCompression)) + { + int read; + byte[] data = new byte[Constants.FileReadChunkSize]; + + while ((read = fs.Read(data, 0, data.Length)) > 0) + { + gs.Write(data, 0, read); + } + } + } + } + + File.Delete(pFileName); + File.Move(tempFileName, pFileName); + } + } + + /// + /// Decompress a gzip compressed section of a stream to file. + /// + /// Stream containing gzip compressed bytes. + /// Fully rooted output file name to output the decompressed data to. + /// Offset to begin reading data from. + public static void DecompressGzipStreamToFile(Stream pStream, string pOutputFilePath, long pStartingOffset) + { + using (FileStream outFs = new FileStream(pOutputFilePath, FileMode.Create, FileAccess.Write)) + { + using (BinaryWriter bw = new BinaryWriter(outFs)) + { + pStream.Position = pStartingOffset; + + using (GZipStream gs = new GZipStream(pStream, CompressionMode.Decompress, true)) + { + int read; + byte[] data = new byte[Constants.FileReadChunkSize]; + + while ((read = gs.Read(data, 0, data.Length)) > 0) + { + bw.Write(data, 0, read); + } + } + } + } + } + + /// + /// Compress the incoming stream to a file using gzip compression starting at the incoming offset. + /// + /// Stream to compress. + /// Path to the file to output. + /// Offset within the stream to start compressing. + public static void CompressStreamToGzipFile(Stream pStream, string pOutputFilePath, long pStartingOffset) + { + using (FileStream outFs = new FileStream(pOutputFilePath, FileMode.Create, FileAccess.Write)) + { + using (BinaryReader br = new BinaryReader(pStream)) + { + pStream.Position = pStartingOffset; + + using (GZipStream gs = new GZipStream(outFs, CompressionMode.Compress, Ionic.Zlib.CompressionLevel.BestCompression, true)) + { + int read; + byte[] data = new byte[Constants.FileReadChunkSize]; + + while ((read = br.Read(data, 0, data.Length)) > 0) + { + gs.Write(data, 0, read); + } + + gs.Flush(); + } + } + } + } + } +} diff --git a/usm_extract/src/util/Constants.cs b/usm_extract/src/util/Constants.cs new file mode 100644 index 0000000..69507f2 --- /dev/null +++ b/usm_extract/src/util/Constants.cs @@ -0,0 +1,453 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Windows.Forms; + +namespace VGMToolbox.util +{ + /// + /// Struct containing criteria used to find offsets. + /// + public struct FindOffsetStruct + { + private string searchString; + private string startingOffset; + private bool treatSearchStringAsHex; + private bool cutFile; + private string searchStringOffset; + private string cutSize; + private string cutSizeOffsetSize; + private bool isCutSizeAnOffset; + private string outputFileExtension; + private bool isLittleEndian; + private bool useTerminatorForCutSize; + private string terminatorString; + private bool treatTerminatorStringAsHex; + private bool includeTerminatorLength; + private string extraCutSizeBytes; + + public bool DoSearchStringModulo + { + set; + get; + } + + public string SearchStringModuloDivisor + { + set; + get; + } + + public string SearchStringModuloResult + { + set; + get; + } + + public bool DoTerminatorModulo + { + set; + get; + } + + public string TerminatorStringModuloDivisor + { + set; + get; + } + + public string TerminatorStringModuloResult + { + set; + get; + } + + public string MinimumSize + { + set; + get; + } + + public string SearchString + { + get { return searchString; } + set { searchString = value; } + } + + /// + /// Gets or sets offset to being searching at + /// + public string StartingOffset + { + set { startingOffset = value; } + get { return startingOffset; } + } + + /// + /// Gets or sets flag to indicate search string is a hex value. + /// + public bool TreatSearchStringAsHex + { + get { return treatSearchStringAsHex; } + set { treatSearchStringAsHex = value; } + } + + /// + /// Gets or sets flag to cut the file when the offset is found. + /// + public bool CutFile + { + get { return cutFile; } + set { cutFile = value; } + } + + /// + /// Gets or sets offset within destination file Search String would reside. + /// + public string SearchStringOffset + { + get { return searchStringOffset; } + set { searchStringOffset = value; } + } + + /// + /// Gets or sets size to cut from file + /// + public string CutSize + { + get { return cutSize; } + set { cutSize = value; } + } + + /// + /// Gets or sets size of offset holding cut size + /// + public string CutSizeOffsetSize + { + get { return cutSizeOffsetSize; } + set { cutSizeOffsetSize = value; } + } + + /// + /// Gets or sets flag indicating that cut size is an offset. + /// + public bool IsCutSizeAnOffset + { + get { return isCutSizeAnOffset; } + set { isCutSizeAnOffset = value; } + } + + /// + /// Gets or sets file extension to use for cut files. + /// + public string OutputFileExtension + { + get { return outputFileExtension; } + set { outputFileExtension = value; } + } + + /// + /// Gets or sets flag indicating that offset based cut size is stored in Little Endian byte order. + /// + public bool IsLittleEndian + { + get { return isLittleEndian; } + set { isLittleEndian = value; } + } + + /// + /// Gets or sets flag indicating that a terminator should be used to determine the cut size. + /// + + public bool UseLengthMultiplier { set; get; } + public string LengthMultiplier { set; get; } + + public bool UseTerminatorForCutSize + { + get { return useTerminatorForCutSize; } + set { useTerminatorForCutSize = value; } + } + + /// + /// Gets or sets terminator string to search for. + /// + public string TerminatorString + { + get { return terminatorString; } + set { terminatorString = value; } + } + + /// + /// Gets or sets flag indicating that Terminator String is hex. + /// + public bool TreatTerminatorStringAsHex + { + get { return treatTerminatorStringAsHex; } + set { treatTerminatorStringAsHex = value; } + } + + /// + /// Gets or sets flag indicating that the length of the terminator should be included in the cut size. + /// + public bool IncludeTerminatorLength + { + get { return includeTerminatorLength; } + set { includeTerminatorLength = value; } + } + + public bool CutToEofIfTerminatorNotFound { set; get; } + + /// + /// Gets or sets additional bytes to include in the cut size. + /// + public string ExtraCutSizeBytes + { + get { return extraCutSizeBytes; } + set { extraCutSizeBytes = value; } + } + + public string OutputFolder { get; set; } + } + + /// + /// Struct used to send messages conveying progress. + /// + public struct ProgressStruct + { + /// + /// File name to display in progress bar. + /// + private string fileName; + + /// + /// Error message to display in output window. + /// + private string errorMessage; + + /// + /// Generic message to display in output window. + /// + private string genericMessage; + + /// + /// New tree node to add to a TreeView. + /// + private TreeNode newNode; + + /// + /// Gets or sets fileName. + /// + public string FileName + { + get { return fileName; } + set { fileName = value; } + } + + /// + /// Gets or sets errorMessage. + /// + public string ErrorMessage + { + get { return errorMessage; } + set { errorMessage = value; } + } + + /// + /// Gets or sets genericMessage. + /// + public string GenericMessage + { + get { return genericMessage; } + set { genericMessage = value; } + } + + /// + /// Gets or sets newNode. + /// + public TreeNode NewNode + { + get { return newNode; } + set { newNode = value; } + } + + /// + /// Reset this node's values + /// + public void Clear() + { + fileName = String.Empty; + errorMessage = String.Empty; + genericMessage = String.Empty; + newNode = null; + } + } + + /// + /// Struct used to allow TreeView to select a specific form and modify the originating node upon completion of a task. + /// + public struct NodeTagStruct + { + /// + /// Class name of the Form this node will bring to focus. + /// + private string formClass; + + /// + /// Object type this node represents. + /// + private string objectType; + + /// + /// File path of the file this node represents. + /// + private string filePath; + + /// + /// Gets or sets formClass + /// + public string FormClass + { + get { return formClass; } + set { formClass = value; } + } + + /// + /// Gets or sets objectType + /// + public string ObjectType + { + get { return objectType; } + set { objectType = value; } + } + + /// + /// Gets or sets filePath + /// + public string FilePath + { + get { return filePath; } + set { filePath = value; } + } + } + + public enum VfsFileRecordRelativeOffsetLocationType + { + FileRecordStart, + FileRecordEnd + } + + public struct VfsExtractionStruct + { + // header size + public bool UseStaticHeaderSize { set; get; } + public string StaticHeaderSize { set; get; } + public bool UseHeaderSizeOffset { set; get; } + public OffsetDescription HeaderSizeOffsetDescription { set; get; } + public bool ReadHeaderToEof { set; get; } + + // file count + public bool UseStaticFileCount { set; get; } + public string StaticFileCount { set; get; } + public bool UseFileCountOffset { set; get; } + public OffsetDescription FileCountOffsetDescription { set; get; } + + // file record basic information + public string FileRecordsStartOffset { set; get; } + public string FileRecordSize { set; get; } + + // file offset + public bool UseFileOffsetOffset { set; get; } + public CalculatingOffsetDescription FileOffsetOffsetDescription { set; get; } + + public bool UsePreviousFilesSizeToDetermineOffset { set; get; } + public string BeginCuttingFilesAtOffset { set; get; } + public bool UseByteAlignmentValue { set; get; } + public string ByteAlignmentValue { set; get; } + + // file length/size + public bool UseFileLengthOffset { set; get; } + public CalculatingOffsetDescription FileLengthOffsetDescription { set; get; } + + public bool UseLocationOfNextFileToDetermineLength { set; get; } + + // file name + public bool FileNameIsPresent { set; get; } + + public bool UseStaticFileNameOffsetWithinRecord { set; get; } + public string StaticFileNameOffsetWithinRecord { set; get; } + + public bool UseAbsoluteFileNameOffset { set; get; } + public OffsetDescription AbsoluteFileNameOffsetDescription { set; get; } + + public bool UseRelativeFileNameOffset { set; get; } + public OffsetDescription RelativeFileNameOffsetDescription { set; get; } + public VfsFileRecordRelativeOffsetLocationType FileRecordNameRelativeOffsetLocation { set; get; } + + // name size + public bool UseStaticFileNameLength { set; get; } + public string StaticFileNameLength { set; get; } + + public bool UseFileNameTerminatorString { set; get; } + public string FileNameTerminatorString { set; get; } + } + + public struct SimpleFileExtractionStruct + { + public string FilePath { set; get; } + public long FileOffset { set; get; } + public long FileLength { set; get; } + public long FileNameLength { set; get; } + + public void Clear() + { + this.FilePath = String.Empty; + this.FileOffset = -1; + this.FileLength = -1; + this.FileNameLength = -1; + } + } + + /// + /// Class containing universal constants. + /// + public sealed class Constants + { + /// + /// Chunk size to use when reading from files. Used to grab maximum buffer + /// size without using the large object heap which has poor collection. + /// + public const int FileReadChunkSize = 71680; + + /// + /// Constant used to send an ignore the value message to the progress bar. + /// + public const int IgnoreProgress = -1; + + /// + /// Constant used to send a generic message to the progress bar. + /// + public const int ProgressMessageOnly = -2; + + /// + /// Text description to use when describing a Big Endian option + /// + public const string BigEndianByteOrder = "Big Endian"; + + /// + /// Text description to use when describing a Little Endian option + /// + public const string LittleEndianByteOrder = "Little Endian"; + + public static readonly byte[] RiffHeaderBytes = new byte[] { 0x52, 0x49, 0x46, 0x46 }; + public static readonly byte[] RiffDataBytes = new byte[] { 0x64, 0x61, 0x74, 0x61 }; + public static readonly byte[] RiffWaveBytes = new byte[] { 0x57, 0x41, 0x56, 0x45 }; + public static readonly byte[] RiffFmtBytes = new byte[] { 0x66, 0x6D, 0x74, 0x20 }; + + public static readonly byte[] NullByteArray = new byte[] { 0x00 }; + + public const string StringNullTerminator = "\0"; + + // empty constructor + private Constants() { } + } +} diff --git a/usm_extract/src/util/FileUtil.cs b/usm_extract/src/util/FileUtil.cs new file mode 100644 index 0000000..e380a77 --- /dev/null +++ b/usm_extract/src/util/FileUtil.cs @@ -0,0 +1,706 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; + +namespace VGMToolbox.util +{ + public sealed class FileUtil + { + private FileUtil() { } + + /// + /// Reads data into a complete array, throwing an EndOfStreamException + /// if the stream runs out of data first, or if an IOException + /// naturally occurs. + /// + /// The stream to read data from + /// The array to read bytes into. The array + /// will be completely filled from the stream, so an appropriate + /// size must be given. + public static void ReadWholeArray(Stream stream, byte[] data) + { + ReadWholeArray(stream, data, data.Length); + } + + public static void ReadWholeArray(Stream stream, byte[] data, int pLength) + { + int offset = 0; + int remaining = pLength; + while (remaining > 0) + { + int read = stream.Read(data, offset, remaining); + if (read <= 0) + { + throw new EndOfStreamException( + String.Format(CultureInfo.CurrentCulture, "End of stream reached with {0} bytes left to read", remaining)); + } + + remaining -= read; + offset += read; + } + } + + /// + /// Replaces 0x00 with 0x20 in an array of bytes. + /// + /// Array of bytes to alter. + /// Original array with 0x00 replaced by 0x20. + public static byte[] ReplaceNullByteWithSpace(byte[] value) + { + for (int i = 0; i < value.Length; i++) + { + if (value[i] == 0x00) + { + value[i] = 0x20; + } + } + + return value; + } + + /// + /// Returns the count of files contained in the input directories and their subdirectories. + /// + /// Paths to count files within. + /// Number of files in the incoming directories and their subdirectories. + public static int GetFileCount(string[] paths) + { + return GetFileCount(paths, true); + } + + public static int GetFileCount(string[] paths, bool includeSubdirs) + { + int totalFileCount = 0; + + foreach (string path in paths) + { + if (File.Exists(path)) + { + totalFileCount++; + } + else if (Directory.Exists(path)) + { + if (includeSubdirs) + { + totalFileCount += Directory.GetFiles(path, "*.*", SearchOption.AllDirectories).Length; + } + else + { + totalFileCount += Directory.GetFiles(path, "*.*", SearchOption.TopDirectoryOnly).Length; + } + } + } + + return totalFileCount; + } + + public static string CleanFileName(string pDirtyFileName) + { + foreach (char c in Path.GetInvalidFileNameChars()) + { + pDirtyFileName = pDirtyFileName.Replace(c, '_'); + } + + return pDirtyFileName; + } + + public static void UpdateTextField( + string pFilePath, + string pFieldValue, + int pOffset, + int pMaxLength) + { + System.Text.Encoding enc = System.Text.Encoding.ASCII; + + using (BinaryWriter bw = + new BinaryWriter(File.Open(pFilePath, FileMode.Open, FileAccess.ReadWrite))) + { + byte[] newBytes = new byte[pMaxLength]; + byte[] convertedBytes = enc.GetBytes(pFieldValue); + + int numBytesToCopy = + convertedBytes.Length <= pMaxLength ? convertedBytes.Length : pMaxLength; + Array.ConstrainedCopy(convertedBytes, 0, newBytes, 0, numBytesToCopy); + + bw.Seek(pOffset, SeekOrigin.Begin); + bw.Write(newBytes); + } + } + + public static void UpdateChunk( + string pFilePath, + int pOffset, + byte[] value) + { + using (BinaryWriter bw = + new BinaryWriter(File.Open(pFilePath, FileMode.Open, FileAccess.ReadWrite))) + { + bw.Seek(pOffset, SeekOrigin.Begin); + bw.Write(value); + } + } + + public static void ReplaceFileChunk( + string pSourceFilePath, + long pSourceOffset, + long pLength, + string pDestinationFilePath, + long pDestinationOffset) + { + int read = 0; + long maxread; + int totalBytes = 0; + byte[] bytes = new byte[Constants.FileReadChunkSize]; + + using (BinaryWriter bw = + new BinaryWriter(File.Open(pDestinationFilePath, FileMode.Open, FileAccess.ReadWrite))) + { + using (BinaryReader br = + new BinaryReader(File.Open(pSourceFilePath, FileMode.Open, FileAccess.Read))) + { + br.BaseStream.Position = pSourceOffset; + bw.BaseStream.Position = pDestinationOffset; + + maxread = pLength > bytes.Length ? bytes.Length : pLength; + + while ((read = br.Read(bytes, 0, (int)maxread)) > 0) + { + bw.Write(bytes, 0, read); + totalBytes += read; + + maxread = (pLength - totalBytes) > bytes.Length ? bytes.Length : (pLength - totalBytes); + } + } + } + } + + public static void ZeroOutFileChunk(string pPath, long pOffset, int pLength) + { + int bytesToWrite = pLength; + byte[] bytes; + + int maxWrite = bytesToWrite > Constants.FileReadChunkSize ? Constants.FileReadChunkSize : bytesToWrite; + + using (BinaryWriter bw = + new BinaryWriter(File.Open(pPath, FileMode.Open, FileAccess.Write))) + { + bw.BaseStream.Position = pOffset; + + while (bytesToWrite > 0) + { + bytes = new byte[maxWrite]; + bw.Write(bytes); + bytesToWrite -= maxWrite; + maxWrite = bytesToWrite > bytes.Length ? bytes.Length : bytesToWrite; + } + } + } + + public static void TrimFileToLength(string path, long totalLength) + { + string fullPath = Path.GetFullPath(path); + + if (File.Exists(fullPath)) + { + string destinationPath = Path.ChangeExtension(fullPath, ".trimmed"); + + using (FileStream fs = File.OpenRead(path)) + { + ParseFile.ExtractChunkToFile(fs, 0, totalLength, destinationPath); + } + + File.Copy(destinationPath, path, true); + File.Delete(destinationPath); + } + } + + public static string RemoveChunkFromFile(string path, long startingOffset, long length) + { + string fullPath = Path.GetFullPath(path); + + int bytesRead; + byte[] bytes = new byte[1024]; + + string ret = String.Empty; + + if (File.Exists(fullPath)) + { + string destinationPath = Path.ChangeExtension(fullPath, ".cut"); + + using (FileStream sourceFs = File.OpenRead(fullPath)) + { + // extract initial chunk + ParseFile.ExtractChunkToFile(sourceFs, 0, startingOffset, destinationPath); + + // append remainder + using (FileStream outFs = File.Open(destinationPath, FileMode.Append, FileAccess.Write)) + { + sourceFs.Position = startingOffset + length; + bytesRead = sourceFs.Read(bytes, 0, bytes.Length); + + while (bytesRead > 0) + { + outFs.Write(bytes, 0, bytesRead); + bytesRead = sourceFs.Read(bytes, 0, bytes.Length); + } + } + + ret = destinationPath; + } + } + + return ret; + } + + public static string RemoveAllChunksFromFile(FileStream fs, byte[] chunkToRemove) + { + int bytesRead; + + long currentReadOffset = 0; + long totalBytesRead = 0; + int maxReadSize = 0; + long currentChunkSize; + + byte[] bytes = new byte[Constants.FileReadChunkSize]; + long[] offsets = ParseFile.GetAllOffsets(fs, 0, chunkToRemove, false, -1, -1, true); + + string destinationPath = Path.ChangeExtension(fs.Name, ".cut"); + + using (FileStream destinationFs = File.OpenWrite(destinationPath)) + { + for (int i = 0; i < offsets.Length; i++) + { + // move position + fs.Position = currentReadOffset; + + // get length of current size to write + currentChunkSize = offsets[i] - currentReadOffset; + + // calculcate max cut size for this loop iteration + maxReadSize = (currentChunkSize - totalBytesRead) > (long)bytes.Length ? bytes.Length : (int)(currentChunkSize - totalBytesRead); + + while ((bytesRead = fs.Read(bytes, 0, maxReadSize)) > 0) + { + destinationFs.Write(bytes, 0, bytesRead); + totalBytesRead += (long)bytesRead; + + maxReadSize = (currentChunkSize - totalBytesRead) > (long)bytes.Length ? bytes.Length : (int)(currentChunkSize - totalBytesRead); + } + + totalBytesRead = 0; + currentReadOffset = offsets[i] + chunkToRemove.Length; + } + + //////////////////////////// + // write remainder of file + //////////////////////////// + // move position + fs.Position = currentReadOffset; + + // get length of current size to write + currentChunkSize = fs.Length - currentReadOffset; + + // calculcate max cut size + maxReadSize = (currentChunkSize - totalBytesRead) > (long)bytes.Length ? bytes.Length : (int)(currentChunkSize - totalBytesRead); + + while ((bytesRead = fs.Read(bytes, 0, maxReadSize)) > 0) + { + destinationFs.Write(bytes, 0, bytesRead); + totalBytesRead += (long)bytesRead; + + maxReadSize = (currentChunkSize - totalBytesRead) > (long)bytes.Length ? bytes.Length : (int)(currentChunkSize - totalBytesRead); + } + } + + return destinationPath; + } + + public static string RemoveAllChunksFromFile(string path, byte[] chunkToRemove) + { + string destinationPath; + + using (FileStream fs = File.OpenRead(path)) + { + destinationPath = RemoveAllChunksFromFile(fs, chunkToRemove); + } + + return destinationPath; + } + + public static bool ExecuteExternalProgram( + string pathToExecuatable, + string arguments, + string workingDirectory, + out string standardOut, + out string standardError) + { + Process externalExecutable; + bool isSuccess = false; + + standardOut = String.Empty; + standardError = String.Empty; + + using (externalExecutable = new Process()) + { + externalExecutable.StartInfo = new ProcessStartInfo(pathToExecuatable, arguments); + externalExecutable.StartInfo.WorkingDirectory = workingDirectory; + externalExecutable.StartInfo.UseShellExecute = false; + externalExecutable.StartInfo.CreateNoWindow = true; + + externalExecutable.StartInfo.RedirectStandardOutput = true; + externalExecutable.StartInfo.RedirectStandardError = true; + isSuccess = externalExecutable.Start(); + + standardOut = externalExecutable.StandardOutput.ReadToEnd(); + standardError = externalExecutable.StandardError.ReadToEnd(); + externalExecutable.WaitForExit(); + } + + return isSuccess; + } + + public static void AddHeaderToFile(byte[] headerBytes, string sourceFile, string destinationFile) + { + int bytesRead; + byte[] readBuffer = new byte[Constants.FileReadChunkSize]; + + using (FileStream destinationStream = File.Open(destinationFile, FileMode.CreateNew, FileAccess.Write)) + { + // write header + destinationStream.Write(headerBytes, 0, headerBytes.Length); + + // write the source file + using (FileStream sourceStream = File.Open(sourceFile, FileMode.Open, FileAccess.Read)) + { + while ((bytesRead = sourceStream.Read(readBuffer, 0, readBuffer.Length)) > 0) + { + destinationStream.Write(readBuffer, 0, bytesRead); + } + } + } + } + + public static void RenameFileUsingInternalName(string path, + long offset, int length, byte[] terminatorBytes, bool maintainFileExtension) + { + string destinationDirectory = Path.GetDirectoryName(path); + string destinationFile; + string originalExtension; + + int nameLength; + byte[] nameByteArray; + + using (FileStream fs = File.OpenRead(path)) + { + if (terminatorBytes != null) + { + nameLength = ParseFile.GetSegmentLength(fs, (int)offset, terminatorBytes); + } + else + { + nameLength = length; + } + + if (nameLength < 1) + { + throw new ArgumentOutOfRangeException("Name Length", "Name Length is less than 1."); + } + + if (maintainFileExtension) + { + originalExtension = Path.GetExtension(path); + } + + nameByteArray = ParseFile.ParseSimpleOffset(fs, offset, nameLength); + destinationFile = ByteConversion.GetAsciiText(FileUtil.ReplaceNullByteWithSpace(nameByteArray)).Trim(); + destinationFile = Path.Combine(destinationDirectory, destinationFile); + + if (maintainFileExtension) + { + originalExtension = Path.GetExtension(path); + destinationFile = Path.ChangeExtension(destinationFile, originalExtension); + } + } + + // try to copy using the new name + if (!path.Equals(destinationFile)) + { + try + { + if (!Directory.Exists(Path.GetDirectoryName(destinationFile))) + { + Directory.CreateDirectory(Path.GetDirectoryName(destinationFile)); + } + + if (File.Exists(destinationFile)) + { + string[] sameNamedFiles = Directory.GetFiles(Path.GetDirectoryName(destinationFile), Path.GetFileNameWithoutExtension(destinationFile) + "*"); + + // rename to prevent overwrite + destinationFile = Path.Combine(Path.GetDirectoryName(destinationFile), String.Format("{0}_{1}{2}", Path.GetFileNameWithoutExtension(destinationFile), sameNamedFiles.Length.ToString("D4"), Path.GetExtension(destinationFile))); + } + + File.Copy(path, destinationFile); + File.Delete(path); + } + catch (Exception ex) + { + throw new Exception(ex.Message, ex); + } + } + } + + public static void InterleaveFiles(string[] sourceFiles, uint interleaveValue, + long startOffest, byte[] paddingBytes, string destinationFile) + { + long currentOffset = 0; + long maxLength = 0; + long sizeDifference = 0; + long bytesToWrite; + long bytesRemaining; + + FileStream[] inputStreams = new FileStream[sourceFiles.Length]; + long[] fileLengths = new long[sourceFiles.Length]; + destinationFile = Path.GetFullPath(destinationFile); + + // open destination file for writing + using (FileStream destinationStream = File.OpenWrite(destinationFile)) + { + try + { + // build input streams array + for (int i = 0; i < sourceFiles.Length; i++) + { + inputStreams[i] = File.OpenRead(sourceFiles[i]); + fileLengths[i] = inputStreams[i].Length; + + // get max file length + if (maxLength < fileLengths[i]) + { + maxLength = fileLengths[i]; + } + } + + // write out blocks + currentOffset = startOffest; + + while (currentOffset < maxLength) + { + for (int i = 0; i < sourceFiles.Length; i++) + { + if (currentOffset + interleaveValue < fileLengths[i]) + { + // write from file + destinationStream.Write( + ParseFile.ParseSimpleOffset(inputStreams[i], currentOffset, (int)interleaveValue), + 0, (int)interleaveValue); + } + else if (currentOffset < fileLengths[i]) + { + // write some from file, some from padding + sizeDifference = (currentOffset + interleaveValue) - fileLengths[i]; + + // write from file + destinationStream.Write( + ParseFile.ParseSimpleOffset(inputStreams[i], currentOffset, (int)(interleaveValue - sizeDifference)), + 0, (int)(int)(interleaveValue - sizeDifference)); + + // write padding bytes + bytesRemaining = sizeDifference; + + while (bytesRemaining > 0) + { + bytesToWrite = bytesRemaining > paddingBytes.Length ? paddingBytes.Length : bytesRemaining; + destinationStream.Write(paddingBytes, 0, (int)bytesToWrite); + bytesRemaining -= bytesToWrite; + } + } + else + { + // write padding bytes + bytesRemaining = interleaveValue; + + while (bytesRemaining > 0) + { + bytesToWrite = bytesRemaining > paddingBytes.Length ? paddingBytes.Length : bytesRemaining; + destinationStream.Write(paddingBytes, 0, (int)bytesToWrite); + bytesRemaining -= bytesToWrite; + } + } + } + + // increment offset + currentOffset += interleaveValue; + } + } + catch (Exception ex) + { + throw new Exception(ex.Message, ex); + } + finally + { + // close all readers + for (int i = 0; i < inputStreams.Length; i++) + { + if (inputStreams[i].CanRead) + { + inputStreams[i].Close(); + inputStreams[i].Dispose(); + } + } + } + + + } + } + + public static string GetNonDuplicateFileName(string destinationFile) + { + if (File.Exists(destinationFile)) + { + string[] sameNamedFiles = Directory.GetFiles(Path.GetDirectoryName(destinationFile), Path.GetFileNameWithoutExtension(destinationFile) + "*"); + + // rename to prevent overwrite + destinationFile = Path.Combine(Path.GetDirectoryName(destinationFile), String.Format("{0}_{1}{2}", Path.GetFileNameWithoutExtension(destinationFile), sameNamedFiles.Length.ToString("D4"), Path.GetExtension(destinationFile))); + } + + return destinationFile; + } + + public static string[] SplitFile(string sourceFile, long startingOffset, ulong chunkSizeInBytes, string outputFolder) + { + string[] outputFileList; + string outputFileName; + ArrayList outputFiles = new ArrayList(); + + long fileLength; + ulong currentOffset; + + int chunkCount; + + using (FileStream sourceStream = File.OpenRead(sourceFile)) + { + // get file length + fileLength = sourceStream.Length; + + // init counters + chunkCount = 1; + currentOffset = (ulong)startingOffset; + + while (currentOffset < (ulong)fileLength) + { + // construct output file name + outputFileName = Path.Combine(outputFolder, + String.Format("{0}.{1}", + Path.GetFileName(sourceFile), + chunkCount.ToString("D3"))); + + ParseFile.ExtractChunkToFile64(sourceStream, currentOffset, chunkSizeInBytes, + outputFileName, true, true); + + + outputFiles.Add(outputFileName); + currentOffset += chunkSizeInBytes; + chunkCount++; + } + } + + outputFileList = (string[])outputFiles.ToArray(typeof(string)); + return outputFileList; + } + + public static long? GetFileSize(string path) + { + long? fileSize; + + try + { + FileInfo fi = new FileInfo(Path.GetFullPath(path)); + fileSize = fi.Length; + } + catch (Exception) + { + fileSize = null; + } + + return fileSize; + } + + public static void CreateFileFromByteArray(string destinationFile, byte[] sourceBytes) + { + string destinationDirectory = Path.GetDirectoryName(destinationFile); + + if (!Directory.Exists(destinationDirectory)) + { + Directory.CreateDirectory(destinationDirectory); + } + + using (FileStream outStream = File.Open(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read)) + { + outStream.Write(sourceBytes, 0, sourceBytes.Length); + } + } + + public static void CreateFileFromString(string destinationFile, string sourceText) + { + string destinationDirectory = Path.GetDirectoryName(destinationFile); + + if (!Directory.Exists(destinationDirectory)) + { + Directory.CreateDirectory(destinationDirectory); + } + + using ( FileStream outStream = File.Open(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read)) + { + using (StreamWriter sw = new StreamWriter(outStream, Encoding.ASCII)) + { + sw.Write(sourceText); + } + } + } + + public static string GetStringFromFileChunk(FileStream fs, ulong offset, ulong size) + { + StringBuilder sb = new StringBuilder(); + + int read = 0; + ulong totalBytes = 0; + byte[] bytes = new byte[Constants.FileReadChunkSize]; + + // reset location + fs.Seek(0, SeekOrigin.Begin); + + // move stream pointer in long size chunks + while (offset > long.MaxValue) // + { + fs.Seek(long.MaxValue, SeekOrigin.Current); + offset -= long.MaxValue; + } + + // less than a long now, should be ok + fs.Seek((long)offset, SeekOrigin.Current); + + int maxread = size > (ulong)bytes.Length ? bytes.Length : (int)size; + + while ((read = fs.Read(bytes, 0, maxread)) > 0) + { + for (int i = 0; i < read; i++) + { + sb.Append(bytes[i].ToString("X2")); + } + + totalBytes += (ulong)read; + + maxread = (size - totalBytes) > (ulong)bytes.Length ? bytes.Length : (int)(size - totalBytes); + } + + return sb.ToString(); + + } + } +} \ No newline at end of file diff --git a/usm_extract/src/util/IDeepCopy.cs b/usm_extract/src/util/IDeepCopy.cs new file mode 100644 index 0000000..1bf2857 --- /dev/null +++ b/usm_extract/src/util/IDeepCopy.cs @@ -0,0 +1,7 @@ +namespace VGMToolbox.util +{ + public interface IDeepCopy + { + T DeepCopy(); + } +} diff --git a/usm_extract/src/util/MathUtil.cs b/usm_extract/src/util/MathUtil.cs new file mode 100644 index 0000000..6191386 --- /dev/null +++ b/usm_extract/src/util/MathUtil.cs @@ -0,0 +1,26 @@ +using System; + +namespace VGMToolbox.util +{ + public class MathUtil + { + public static long RoundUpToByteAlignment(long valueToRound, long byteAlignment) + { + long roundedValue = -1; + + roundedValue = (valueToRound + byteAlignment - 1) / byteAlignment * byteAlignment; + + return roundedValue; + } + + public static ulong RoundUpToByteAlignment(ulong valueToRound, ulong byteAlignment) + { + ulong roundedValue; + + roundedValue = (valueToRound + byteAlignment - 1) / byteAlignment * byteAlignment; + + return roundedValue; + } + } + +} diff --git a/usm_extract/src/util/OffsetDescription.cs b/usm_extract/src/util/OffsetDescription.cs new file mode 100644 index 0000000..1aac12f --- /dev/null +++ b/usm_extract/src/util/OffsetDescription.cs @@ -0,0 +1,9 @@ +namespace VGMToolbox.util +{ + public class OffsetDescription + { + public string OffsetValue { set; get; } + public string OffsetSize { set; get; } + public string OffsetByteOrder { set; get; } + } +} diff --git a/usm_extract/src/util/ParseFile.cs b/usm_extract/src/util/ParseFile.cs new file mode 100644 index 0000000..f6d37f4 --- /dev/null +++ b/usm_extract/src/util/ParseFile.cs @@ -0,0 +1,1826 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; + +namespace VGMToolbox.util +{ + /// + /// Class for Parsing Files. + /// + public sealed class ParseFile + { + public const string LogFileName = "vgmt_extraction_log.txt"; + public const string SnakeBiteBatchFileName = "vgmt_extraction_log.bat"; + public const string VirtualFileSystemExtractionFolder = "vgmt_vfs_cut"; + + /// + /// Prevents a default instance of the ParseFile class from being created. + /// + private ParseFile() + { + } + + /// + /// Extract a section from the incoming byte array. + /// + /// Bytes to extract from. + /// Offset to begin cutting from. + /// Number of bytes to cut. + /// Byte array containing the extracted section. + public static byte[] ParseSimpleOffset(byte[] sourceArray, int startingOffset, int lengthToCut) + { + byte[] ret = new byte[lengthToCut]; + uint j = 0; + + for (int i = startingOffset; i < startingOffset + lengthToCut; i++) + { + ret[j] = sourceArray[i]; + j++; + } + + return ret; + } + + /// + /// Extract a section from the incoming stream. + /// + /// Stream to extract the chunk from. + /// Offset to begin cutting from. + /// Number of bytes to cut. + /// Byte array containing the extracted section. + public static byte[] ParseSimpleOffset(Stream stream, int startingOffset, int lengthToCut) + { + byte[] ret = new byte[lengthToCut]; + long currentStreamPosition = stream.Position; + + stream.Seek((long)startingOffset, SeekOrigin.Begin); + BinaryReader br = new BinaryReader(stream); + ret = br.ReadBytes((int)lengthToCut); + + stream.Position = currentStreamPosition; + + return ret; + } + + /// + /// Extract a section from the incoming stream. + /// + /// Stream to extract the chunk from. + /// Offset to begin cutting from. + /// Number of bytes to cut. + /// Byte array containing the extracted section. + public static byte[] ParseSimpleOffset(Stream stream, long startingOffset, int lengthToCut) + { + byte[] ret = new byte[lengthToCut]; + long currentStreamPosition = stream.Position; + + stream.Seek(startingOffset, SeekOrigin.Begin); + BinaryReader br = new BinaryReader(stream); + ret = br.ReadBytes(lengthToCut); + + stream.Position = currentStreamPosition; + + return ret; + } + + /// + /// Get the length from the input offset to the location of the input terminator bytes or zero. + /// + /// Bytes to check. + /// Offset at which to begin searching for the terminator bytes. + /// Bytes to search for. + /// Length of distance between offset and terminator or zero if not found. + public static int GetSegmentLength(byte[] sourceBytes, int searchOffset, byte[] terminatorBytes) + { + int ret; + bool terminatorFound = false; + int i = searchOffset; + + while (i < sourceBytes.Length) + { + // first char match + if (sourceBytes[i] == terminatorBytes[0]) + { + if (CompareSegment(sourceBytes, i, terminatorBytes)) + { + terminatorFound = true; + break; + } + } + + i++; + } // while (!terminatorFound) + + if (terminatorFound) + { + ret = i - searchOffset; + } + else + { + ret = 0; + } + + return ret; + } + + /// + /// Get the length from the input offset to the location of the input terminator bytes or zero. + /// + /// Stream to check. + /// Offset at which to begin searching for the terminator bytes. + /// Bytes to search for. + /// Length of distance between offset and terminator or zero if not found. + public static int GetSegmentLength(Stream stream, int searchOffset, byte[] terminatorBytes) + { + int ret; + bool terminatorFound = false; + int i = searchOffset; + byte[] checkBytes = new byte[terminatorBytes.Length]; + + while (i < stream.Length) + { + stream.Seek(i, SeekOrigin.Begin); + stream.Read(checkBytes, 0, 1); + + // first char match + if (checkBytes[0] == terminatorBytes[0]) + { + stream.Seek(i, SeekOrigin.Begin); + stream.Read(checkBytes, 0, terminatorBytes.Length); + + if (CompareSegment(checkBytes, 0, terminatorBytes)) + { + terminatorFound = true; + break; + } + } + + i++; + } // while (!terminatorFound) + + if (terminatorFound) + { + ret = i - searchOffset; + } + else + { + ret = 0; + } + + return ret; + } + + /// + /// Get the offset of the first instance of pSearchBytes after the input offset. + /// + /// Stream to search. + /// Offset to begin searching from. + /// Bytes to search for. + /// Returns the offset of the first instance of pSearchBytes after the input offset or -1 otherwise. + public static long GetNextOffset(Stream stream, long startingOffset, byte[] searchBytes) + { + return GetNextOffset(stream, startingOffset, searchBytes, true); + } + + public static long GetNextOffsetMasked(Stream stream, long startingOffset, byte[] searchBytes, byte[] searchMask) + { + return GetNextOffsetMasked(stream, startingOffset, searchBytes, searchMask, true); + } + + public static long GetNextOffset(Stream stream, long startingOffset, + byte[] searchBytes, bool returnStreamToIncomingPosition) + { + long initialStreamPosition = 0; + + if (returnStreamToIncomingPosition) + { + initialStreamPosition = stream.Position; + } + + bool itemFound = false; + long absoluteOffset = startingOffset; + long relativeOffset; + byte[] checkBytes = new byte[Constants.FileReadChunkSize]; + byte[] compareBytes; + + long ret = -1; + + while (!itemFound && (absoluteOffset < stream.Length)) + { + stream.Position = absoluteOffset; + stream.Read(checkBytes, 0, Constants.FileReadChunkSize); + relativeOffset = 0; + + while (!itemFound && (relativeOffset < Constants.FileReadChunkSize)) + { + if ((relativeOffset + searchBytes.Length) < checkBytes.Length) + { + compareBytes = new byte[searchBytes.Length]; + Array.Copy(checkBytes, relativeOffset, compareBytes, 0, searchBytes.Length); + + if (CompareSegment(compareBytes, 0, searchBytes)) + { + itemFound = true; + ret = absoluteOffset + relativeOffset; + break; + } + } + + relativeOffset++; + } + + absoluteOffset += Constants.FileReadChunkSize - searchBytes.Length; + } + + // return stream to incoming position + if (returnStreamToIncomingPosition) + { + stream.Position = initialStreamPosition; + } + + return ret; + } + + public static long GetNextOffsetMasked(Stream stream, long startingOffset, + byte[] searchBytes, byte[] searchMask, bool returnStreamToIncomingPosition) + { + long initialStreamPosition = 0; + + if (returnStreamToIncomingPosition) + { + initialStreamPosition = stream.Position; + } + + bool itemFound = false; + long absoluteOffset = startingOffset; + long relativeOffset; + byte[] checkBytes = new byte[Constants.FileReadChunkSize]; + byte[] compareBytes; + + long ret = -1; + + while (!itemFound && (absoluteOffset < stream.Length)) + { + stream.Position = absoluteOffset; + stream.Read(checkBytes, 0, Constants.FileReadChunkSize); + relativeOffset = 0; + + while (!itemFound && (relativeOffset < Constants.FileReadChunkSize)) + { + if ((relativeOffset + searchBytes.Length) < checkBytes.Length) + { + compareBytes = new byte[searchBytes.Length]; + Array.Copy(checkBytes, relativeOffset, compareBytes, 0, searchBytes.Length); + for (int idx = 0; idx < searchMask.Length; ++idx) + { + compareBytes[idx] &= searchMask[idx]; + } + + if (CompareSegment(compareBytes, 0, searchBytes)) + { + itemFound = true; + ret = absoluteOffset + relativeOffset; + break; + } + } + + relativeOffset++; + } + + absoluteOffset += Constants.FileReadChunkSize - searchBytes.Length; + } + + // return stream to incoming position + if (returnStreamToIncomingPosition) + { + stream.Position = initialStreamPosition; + } + + return ret; + } + + public static long GetNextOffset(Stream stream, long startingOffset, + byte[] searchBytes, bool doOffsetModulo, long offsetModuloDivisor, + long offsetModuloResult) + { + return GetNextOffset(stream, startingOffset, searchBytes, + doOffsetModulo, offsetModuloDivisor, offsetModuloResult, true); + } + + public static long GetNextOffset(Stream stream, long startingOffset, + byte[] searchBytes, bool doOffsetModulo, long offsetModuloDivisor, + long offsetModuloResult, bool returnStreamToIncomingPosition) + { + long initialStreamPosition = 0; + + if (returnStreamToIncomingPosition) + { + initialStreamPosition = stream.Position; + } + + bool itemFound = false; + long absoluteOffset = startingOffset; + long relativeOffset; + long actualOffset; + byte[] checkBytes = new byte[Constants.FileReadChunkSize]; + int checkBytesRead; + byte[] compareBytes; + + long ret = -1; + + while (!itemFound && (absoluteOffset < stream.Length)) + { + stream.Position = absoluteOffset; + checkBytesRead = stream.Read(checkBytes, 0, Constants.FileReadChunkSize); + relativeOffset = 0; + + while (!itemFound && (relativeOffset < checkBytesRead)) + { + actualOffset = absoluteOffset + relativeOffset; + + if ((!doOffsetModulo) || + (actualOffset % offsetModuloDivisor == offsetModuloResult)) + { + if ((relativeOffset + searchBytes.Length) < checkBytes.Length) + { + compareBytes = new byte[searchBytes.Length]; + Array.Copy(checkBytes, relativeOffset, compareBytes, 0, searchBytes.Length); + + if (CompareSegment(compareBytes, 0, searchBytes)) + { + itemFound = true; + ret = actualOffset; + break; + } + } + } + + relativeOffset++; + } + + absoluteOffset += Constants.FileReadChunkSize - searchBytes.Length; + } + + // return stream to incoming position + if (returnStreamToIncomingPosition) + { + stream.Position = initialStreamPosition; + } + + return ret; + } + + public static Dictionary GetAllOffsets(Stream stream, long startingOffset, + byte[][] searchByteArrays, bool returnStreamToIncomingPosition) + { + long initialStreamPosition = 0; + + if (returnStreamToIncomingPosition) + { + initialStreamPosition = stream.Position; + } + + int maxSearchBytesLength = 0; + long absoluteOffset = startingOffset; + long relativeOffset; + byte[] checkBytes = new byte[Constants.FileReadChunkSize]; + byte[] compareBytes; + + Dictionary tempOutput = new Dictionary(); + Dictionary ret = new Dictionary(); + + foreach (byte[] b in searchByteArrays) + { + tempOutput.Add(b, new ArrayList()); + + if (b.Length > maxSearchBytesLength) + { + maxSearchBytesLength = b.Length; + } + } + + while (absoluteOffset < stream.Length) + { + stream.Position = absoluteOffset; + stream.Read(checkBytes, 0, Constants.FileReadChunkSize); + relativeOffset = 0; + + while (relativeOffset < Constants.FileReadChunkSize) + { + foreach (byte[] searchBytes in searchByteArrays) + { + if ((relativeOffset + searchBytes.Length) < checkBytes.Length) + { + compareBytes = new byte[searchBytes.Length]; + Array.Copy(checkBytes, relativeOffset, compareBytes, 0, searchBytes.Length); + + if (CompareSegment(compareBytes, 0, searchBytes)) + { + tempOutput[searchBytes].Add(absoluteOffset + relativeOffset); + break; + } + } + } // foreach (byte[] searchBytes in searchByteArrays) + + relativeOffset++; + } + + absoluteOffset += Constants.FileReadChunkSize - maxSearchBytesLength; + } + + // return stream to incoming position + if (returnStreamToIncomingPosition) + { + stream.Position = initialStreamPosition; + } + + // sort offsets and add to output + foreach (byte[] key in tempOutput.Keys) + { + tempOutput[key].Sort(); + ret.Add(key, (long[])tempOutput[key].ToArray(typeof(long))); + } + + return ret; + } + + public static long[] GetAllOffsets(Stream stream, long startingOffset, + byte[] searchBytes, bool doOffsetModulo, long offsetModuloDivisor, + long offsetModuloResult, bool returnStreamToIncomingPosition) + { + long initialStreamPosition = 0; + + if (returnStreamToIncomingPosition) + { + initialStreamPosition = stream.Position; + } + + long absoluteOffset = startingOffset; + long relativeOffset; + long actualOffset; + byte[] checkBytes = new byte[Constants.FileReadChunkSize]; + int checkBytesRead; + byte[] compareBytes; + + ArrayList offsetList = new ArrayList(); + long[] ret; + + while (absoluteOffset < stream.Length) + { + stream.Position = absoluteOffset; + checkBytesRead = stream.Read(checkBytes, 0, Constants.FileReadChunkSize); + relativeOffset = 0; + + while (relativeOffset < checkBytesRead) + { + actualOffset = absoluteOffset + relativeOffset; + + if ((!doOffsetModulo) || + (actualOffset % offsetModuloDivisor == offsetModuloResult)) + { + if ((relativeOffset + searchBytes.Length) < checkBytes.Length) + { + compareBytes = new byte[searchBytes.Length]; + Array.Copy(checkBytes, relativeOffset, compareBytes, 0, searchBytes.Length); + + if (CompareSegment(compareBytes, 0, searchBytes)) + { + offsetList.Add(actualOffset); + } + } + } + + relativeOffset++; + } + + absoluteOffset += Constants.FileReadChunkSize - searchBytes.Length; + } + + // return stream to incoming position + if (returnStreamToIncomingPosition) + { + stream.Position = initialStreamPosition; + } + + ret = (long[])offsetList.ToArray(typeof(long)); + + return ret; + } + + /// + /// Search for bytes string within file offset [startingOffset - maxOffset]. + /// + /// Stream to search for bytes. + /// Offset to begin searching at. + /// Maximum offset to include in search. + /// Bytes to search for. + /// Return stream to incoming position. + /// + public static long GetNextOffsetWithLimit(Stream stream, long startingOffset, + long maxOffset, byte[] searchBytes, bool returnStreamToIncomingPosition) + { + long initialStreamPosition = 0; + + if (returnStreamToIncomingPosition) + { + initialStreamPosition = stream.Position; + } + + bool itemFound = false; + long absoluteOffset = startingOffset; + long relativeOffset; + byte[] checkBytes = new byte[Constants.FileReadChunkSize]; + byte[] compareBytes; + + long ret = -1; + + while (!itemFound && (absoluteOffset < maxOffset)) + { + stream.Position = absoluteOffset; + stream.Read(checkBytes, 0, Constants.FileReadChunkSize); + relativeOffset = 0; + + while (!itemFound && (relativeOffset < Constants.FileReadChunkSize)) + { + if ((relativeOffset + searchBytes.Length) < checkBytes.Length) + { + compareBytes = new byte[searchBytes.Length]; + Array.Copy(checkBytes, relativeOffset, compareBytes, 0, searchBytes.Length); + + if (CompareSegment(compareBytes, 0, searchBytes)) + { + itemFound = true; + ret = absoluteOffset + relativeOffset; + break; + } + } + + relativeOffset++; + } + + absoluteOffset += Constants.FileReadChunkSize - searchBytes.Length; + } + + // return stream to incoming position + if (returnStreamToIncomingPosition) + { + stream.Position = initialStreamPosition; + } + + return ret; + } + + public static long GetNextOffsetWithLimitMasked(Stream stream, long startingOffset, + long maxOffset, byte[] searchBytes, byte[] searchMask, bool returnStreamToIncomingPosition) + { + long initialStreamPosition = 0; + + if (returnStreamToIncomingPosition) + { + initialStreamPosition = stream.Position; + } + + bool itemFound = false; + long absoluteOffset = startingOffset; + long relativeOffset; + byte[] checkBytes = new byte[Constants.FileReadChunkSize]; + byte[] compareBytes; + + long ret = -1; + + while (!itemFound && (absoluteOffset < maxOffset)) + { + stream.Position = absoluteOffset; + stream.Read(checkBytes, 0, Constants.FileReadChunkSize); + relativeOffset = 0; + + while (!itemFound && (relativeOffset < Constants.FileReadChunkSize)) + { + if ((relativeOffset + searchBytes.Length) < checkBytes.Length) + { + compareBytes = new byte[searchBytes.Length]; + Array.Copy(checkBytes, relativeOffset, compareBytes, 0, searchBytes.Length); + for (int idx = 0; idx < searchMask.Length; ++idx) + { + compareBytes[idx] &= searchMask[idx]; + } + + if (CompareSegment(compareBytes, 0, searchBytes)) + { + itemFound = true; + ret = absoluteOffset + relativeOffset; + break; + } + } + + relativeOffset++; + } + + absoluteOffset += Constants.FileReadChunkSize - searchBytes.Length; + } + + // return stream to incoming position + if (returnStreamToIncomingPosition) + { + stream.Position = initialStreamPosition; + } + + return ret; + } + + /// + /// Get the offset of the first instance of pSearchBytes after the input offset. + /// + /// Byte array to search. + /// Offset to begin searching from. + /// Bytes to search for. + /// Returns the offset of the first instance of pSearchBytes after the input offset or -1 otherwise. + public static long GetNextOffset(byte[] bufferToSearch, long offset, byte[] searchValue) + { + bool itemFound = false; + long absoluteOffset = offset; + byte[] compareBytes; + + long ret = -1; + + while (!itemFound && (absoluteOffset < (bufferToSearch.Length - searchValue.Length))) + { + compareBytes = new byte[searchValue.Length]; + Array.Copy(bufferToSearch, absoluteOffset, compareBytes, 0, searchValue.Length); + + if (CompareSegment(compareBytes, 0, searchValue)) + { + itemFound = true; + ret = absoluteOffset; + break; + } + + absoluteOffset++; + } + + return ret; + } + + /// + /// Get the offset of the first instance of pSearchBytes before the input offset. + /// + /// Stream to search. + /// Offset to begin searching from. + /// Bytes to search for. + /// Returns the offset of the first instance of pSearchBytes before the input offset or -1 otherwise. + public static long GetPreviousOffset(Stream stream, long offset, byte[] searchBytes) + { + long initialStreamPosition = stream.Position; + + bool itemFound = false; + long relativeOffset; + byte[] checkBytes = new byte[Constants.FileReadChunkSize]; + byte[] compareBytes; + + long ret = -1; + + long absoluteOffset = offset - Constants.FileReadChunkSize + searchBytes.Length; + while (!itemFound && (absoluteOffset > -1)) + { + stream.Position = absoluteOffset; + relativeOffset = stream.Read(checkBytes, 0, Constants.FileReadChunkSize); + + while (!itemFound && (relativeOffset > -1)) + { + if ((relativeOffset + searchBytes.Length) <= checkBytes.Length) + { + compareBytes = new byte[searchBytes.Length]; + Array.Copy(checkBytes, relativeOffset, compareBytes, 0, searchBytes.Length); + + if (CompareSegment(compareBytes, 0, searchBytes)) + { + itemFound = true; + ret = absoluteOffset + relativeOffset; + + if (ret == offset) + { + ret -= searchBytes.Length; + } + + break; + } + } + + relativeOffset--; + } + + absoluteOffset = absoluteOffset - Constants.FileReadChunkSize + searchBytes.Length; + } + + // return stream to incoming position + stream.Position = initialStreamPosition; + + return ret; + } + + /// + /// Compare bytes at input offset to target bytes. + /// + /// Bytes to compare. + /// Offset to begin comparison of pBytes to pTarget. + /// Target bytes to compare. + /// True if the bytes at pOffset match the pTarget bytes. + public static bool CompareSegment(byte[] sourceArray, int offset, byte[] target) + { + bool ret = true; + uint j = 0; + + if (sourceArray.Length > 0) + { + for (int i = offset; i < target.Length; i++) + { + if (sourceArray[i] != target[j]) + { + ret = false; + break; + } + + j++; + } + } + else + { + ret = false; + } + + return ret; + } + + /// + /// Compare bytes at input offset to target bytes. + /// + /// Bytes to compare. + /// Offset to begin comparison of pBytes to pTarget. + /// Target bytes to compare. + /// True if the bytes at pOffset match the pTarget bytes. + public static bool CompareSegment(byte[] sourceArray, long offset, byte[] target) + { + bool ret = true; + uint j = 0; + for (long i = offset; i < target.Length; i++) + { + if (sourceArray[i] != target[j]) + { + ret = false; + break; + } + + j++; + } + + return ret; + } + + public static bool CompareSegmentUsingSourceOffset(byte[] sourceArray, int sourceOffset, + int numBytesToCompare, byte[] target) + { + bool ret = true; + uint j = 0; + + if (sourceArray.Length > 0) + { + for (int i = sourceOffset; + (i < sourceOffset + target.Length) || (i < sourceOffset + numBytesToCompare); i++) + { + if (sourceArray[i] != target[j]) + { + ret = false; + break; + } + + j++; + } + } + else + { + ret = false; + } + + return ret; + } + + public static void ExtractChunkToFile(Stream stream, long startingOffset, long length, + string filePath) + { + ExtractChunkToFile(stream, startingOffset, length, filePath, false, false); + } + + public static void ExtractChunkToFile(Stream stream, long startingOffset, long length, + string filePath, bool outputLogFile) + { + ExtractChunkToFile(stream, startingOffset, length, filePath, outputLogFile, false); + } + + /// + /// Extracts a section of the incoming stream to a file. + /// + /// Stream to extract from. + /// Offset to begin the cut. + /// Number of bytes to cut. + /// File path to output the extracted chunk to. + public static void ExtractChunkToFile( + Stream stream, + long startingOffset, + long length, + string filePath, + bool outputLogFile, + bool outputSnakebiteBatchFile) + { + bool makeBatchFile = (outputSnakebiteBatchFile && (stream is FileStream)); + StringBuilder logInfo = new StringBuilder(); + StringBuilder snakeBiteBatch = new StringBuilder(); + BinaryWriter bw = null; + string fullFilePath = Path.GetFullPath(filePath); + string fullOutputDirectory = Path.GetDirectoryName(fullFilePath); + + // create output folder if needed + if (!Directory.Exists(fullOutputDirectory)) + { + Directory.CreateDirectory(fullOutputDirectory); + + if (outputLogFile) + { + logInfo.AppendLine("Created Directory: " + fullOutputDirectory); + } + } + + // check if file exists and change name as needed + if (File.Exists(fullFilePath)) + { + int fileCount = Directory.GetFiles(fullOutputDirectory, (Path.GetFileNameWithoutExtension(fullFilePath) + "*" + Path.GetExtension(fullFilePath)), SearchOption.TopDirectoryOnly).Length; + fullFilePath = Path.Combine(fullOutputDirectory, String.Format("{0}_{1}{2}", Path.GetFileNameWithoutExtension(fullFilePath), fileCount.ToString("X3"), Path.GetExtension(fullFilePath))); + } + + try + { + bw = new BinaryWriter(File.Open(fullFilePath, FileMode.Create, FileAccess.Write)); + + int read = 0; + long totalBytes = 0; + byte[] bytes = new byte[Constants.FileReadChunkSize]; + stream.Seek((long)startingOffset, SeekOrigin.Begin); + + int maxread = length > (long)bytes.Length ? bytes.Length : (int)length; + + while ((read = stream.Read(bytes, 0, maxread)) > 0) + { + bw.Write(bytes, 0, read); + totalBytes += (long)read; + + maxread = (length - totalBytes) > (long)bytes.Length ? bytes.Length : (int)(length - totalBytes); + } + + if (outputLogFile) + { + logInfo.AppendLine( + String.Format("Extracted - Offset: 0x{0} Length: 0x{1} File: {2}", + startingOffset.ToString("X8"), + length.ToString("X8"), + Path.GetFileName(fullFilePath))); + + using (StreamWriter logWriter = new StreamWriter(Path.Combine(fullOutputDirectory, ParseFile.LogFileName), true)) + { + logWriter.Write(logInfo.ToString()); + } + } + + if (makeBatchFile) + { + snakeBiteBatch.AppendLine( + String.Format("snakebite.exe \"{0}\" \"{1}\" 0x{2} 0x{3}", + Path.GetFileName(((FileStream)stream).Name), + Path.GetFileNameWithoutExtension(((FileStream)stream).Name) + Path.DirectorySeparatorChar + Path.GetFileName(fullFilePath), + startingOffset.ToString("X8"), + (startingOffset + length - 1).ToString("X8"))); + + using (StreamWriter batchWriter = new StreamWriter(Path.Combine(fullOutputDirectory, ParseFile.SnakeBiteBatchFileName), true)) + { + batchWriter.Write(snakeBiteBatch.ToString()); + } + } + } + finally + { + if (bw != null) + { + bw.Close(); + } + } + } + + public static void ExtractChunkToFile64( + Stream stream, + ulong startingOffset, + ulong length, + string filePath, + bool outputLogFile, + bool outputSnakebiteBatchFile) + { + bool makeBatchFile = (outputSnakebiteBatchFile && (stream is FileStream)); + StringBuilder logInfo = new StringBuilder(); + StringBuilder snakeBiteBatch = new StringBuilder(); + BinaryWriter bw = null; + string fullFilePath = Path.GetFullPath(filePath); + string fullOutputDirectory = Path.GetDirectoryName(fullFilePath); + + // create output folder if needed + if (!Directory.Exists(fullOutputDirectory)) + { + Directory.CreateDirectory(fullOutputDirectory); + + if (outputLogFile) + { + logInfo.AppendLine("Created Directory: " + fullOutputDirectory); + } + } + + // check if file exists and change name as needed + if (File.Exists(fullFilePath)) + { + int fileCount = Directory.GetFiles(fullOutputDirectory, (Path.GetFileName(fullFilePath) + "*"), SearchOption.TopDirectoryOnly).Length; + fullFilePath = Path.Combine(fullOutputDirectory, String.Format("{0}_{1}{2}", Path.GetFileNameWithoutExtension(fullFilePath), fileCount.ToString("X3"), Path.GetExtension(fullFilePath))); + } + + try + { + bw = new BinaryWriter(File.Open(fullFilePath, FileMode.Create, FileAccess.Write)); + + int read = 0; + ulong totalBytes = 0; + byte[] bytes = new byte[Constants.FileReadChunkSize]; + + // reset location + stream.Seek(0, SeekOrigin.Begin); + + // move stream pointer in long size chunks + while (startingOffset > long.MaxValue) // + { + stream.Seek(long.MaxValue, SeekOrigin.Current); + startingOffset -= long.MaxValue; + } + + // less than a long now, should be ok + stream.Seek((long)startingOffset, SeekOrigin.Current); + + int maxread = length > (ulong)bytes.Length ? bytes.Length : (int)length; + + while ((read = stream.Read(bytes, 0, maxread)) > 0) + { + bw.Write(bytes, 0, read); + totalBytes += (ulong)read; + + maxread = (length - totalBytes) > (ulong)bytes.Length ? bytes.Length : (int)(length - totalBytes); + } + + if (outputLogFile) + { + logInfo.AppendLine( + String.Format("Extracted - Offset: 0x{0} Length: 0x{1} File: {2}", + startingOffset.ToString("X8"), + length.ToString("X8"), + Path.GetFileName(fullFilePath))); + + using (StreamWriter logWriter = new StreamWriter(Path.Combine(fullOutputDirectory, ParseFile.LogFileName), true)) + { + logWriter.Write(logInfo.ToString()); + } + } + + if (makeBatchFile) + { + snakeBiteBatch.AppendLine( + String.Format("snakebite.exe \"{0}\" \"{1}\" 0x{2} 0x{3}", + Path.GetFileName(((FileStream)stream).Name), + Path.GetFileNameWithoutExtension(((FileStream)stream).Name) + Path.DirectorySeparatorChar + Path.GetFileName(fullFilePath), + startingOffset.ToString("X8"), + (startingOffset + length - 1).ToString("X8"))); + + using (StreamWriter batchWriter = new StreamWriter(Path.Combine(fullOutputDirectory, ParseFile.SnakeBiteBatchFileName), true)) + { + batchWriter.Write(snakeBiteBatch.ToString()); + } + } + } + finally + { + if (bw != null) + { + bw.Close(); + } + } + } + + public static byte[] ExtractChunkToFile64ReturningHash( + Stream stream, + ulong startingOffset, + ulong length, + string filePath, + HashAlgorithm checksumAlgorithm, + bool outputLogFile, + bool outputSnakebiteBatchFile) + { + byte[] checksumHash; + bool makeBatchFile = (outputSnakebiteBatchFile && (stream is FileStream)); + StringBuilder logInfo = new StringBuilder(); + StringBuilder snakeBiteBatch = new StringBuilder(); + //BinaryWriter bw = null; + string fullFilePath = Path.GetFullPath(filePath); + string fullOutputDirectory = Path.GetDirectoryName(fullFilePath); + + // create output folder if needed + if (!Directory.Exists(fullOutputDirectory)) + { + Directory.CreateDirectory(fullOutputDirectory); + + if (outputLogFile) + { + logInfo.AppendLine("Created Directory: " + fullOutputDirectory); + } + } + + // check if file exists and change name as needed + if (File.Exists(fullFilePath)) + { + int fileCount = Directory.GetFiles(fullOutputDirectory, (Path.GetFileName(fullFilePath) + "*"), SearchOption.TopDirectoryOnly).Length; + fullFilePath = Path.Combine(fullOutputDirectory, String.Format("{0}_{1}{2}", Path.GetFileNameWithoutExtension(fullFilePath), fileCount.ToString("X3"), Path.GetExtension(fullFilePath))); + } + + using (FileStream fs = File.Open(fullFilePath, FileMode.Create, FileAccess.Write)) + using (CryptoStream hashStream = new CryptoStream(fs, checksumAlgorithm, CryptoStreamMode.Write)) + { + int read = 0; + ulong totalBytes = 0; + byte[] bytes = new byte[Constants.FileReadChunkSize]; + + // reset location + stream.Seek(0, SeekOrigin.Begin); + + // move stream pointer in long size chunks + while (startingOffset > long.MaxValue) // + { + stream.Seek(long.MaxValue, SeekOrigin.Current); + startingOffset -= long.MaxValue; + } + + // less than a long now, should be ok + stream.Seek((long)startingOffset, SeekOrigin.Current); + + int maxread = length > (ulong)bytes.Length ? bytes.Length : (int)length; + + while ((read = stream.Read(bytes, 0, maxread)) > 0) + { + hashStream.Write(bytes, 0, read); + totalBytes += (ulong)read; + + maxread = (length - totalBytes) > (ulong)bytes.Length ? bytes.Length : (int)(length - totalBytes); + } + + hashStream.FlushFinalBlock(); + checksumHash = checksumAlgorithm.Hash; + + #region Log Files + if (outputLogFile) + { + logInfo.AppendLine( + String.Format("Extracted - Offset: 0x{0} Length: 0x{1} File: {2}", + startingOffset.ToString("X8"), + length.ToString("X8"), + Path.GetFileName(fullFilePath))); + + using (StreamWriter logWriter = new StreamWriter(Path.Combine(fullOutputDirectory, ParseFile.LogFileName), true)) + { + logWriter.Write(logInfo.ToString()); + } + } + + if (makeBatchFile) + { + snakeBiteBatch.AppendLine( + String.Format("snakebite.exe \"{0}\" \"{1}\" 0x{2} 0x{3}", + Path.GetFileName(((FileStream)stream).Name), + Path.GetFileNameWithoutExtension(((FileStream)stream).Name) + Path.DirectorySeparatorChar + Path.GetFileName(fullFilePath), + startingOffset.ToString("X8"), + (startingOffset + length - 1).ToString("X8"))); + + using (StreamWriter batchWriter = new StreamWriter(Path.Combine(fullOutputDirectory, ParseFile.SnakeBiteBatchFileName), true)) + { + batchWriter.Write(snakeBiteBatch.ToString()); + } + } + #endregion + + return checksumHash; + } + } + + public static void AppendChunkToFile( + Stream stream, + long startingOffset, + long length, + string filePath) + { + BinaryWriter bw = null; + string fullFilePath = Path.GetFullPath(filePath); + string fullOutputDirectory = Path.GetDirectoryName(fullFilePath); + + // create directory if needed + if (!Directory.Exists(fullOutputDirectory)) + { + Directory.CreateDirectory(fullOutputDirectory); + } + + try + { + bw = new BinaryWriter(File.Open(fullFilePath, FileMode.Append, FileAccess.Write)); + + int read = 0; + long totalBytes = 0; + byte[] bytes = new byte[Constants.FileReadChunkSize]; + stream.Seek((long)startingOffset, SeekOrigin.Begin); + + int maxread = length > (long)bytes.Length ? bytes.Length : (int)length; + + while ((read = stream.Read(bytes, 0, maxread)) > 0) + { + bw.Write(bytes, 0, read); + totalBytes += (long)read; + + maxread = (length - totalBytes) > (long)bytes.Length ? bytes.Length : (int)(length - totalBytes); + } + } + finally + { + if (bw != null) + { + bw.Close(); + } + } + } + + + /// + /// Convert the input bytes to a string containing the hex values. + /// + /// Bytes to convert to a string. + /// String of hex values that represent the incoming byte array. + public static string ByteArrayToString(byte[] value) + { + StringBuilder checksum = new StringBuilder(); + + // Loop through each byte of the hashed data + // and format each one as a hexadecimal string. + for (int i = 0; i < value.Length; i++) + { + checksum.Append(value[i].ToString("X2", CultureInfo.InvariantCulture)); + } + + return checksum.ToString(); + } + + /// + /// Find an offset and cut the file based on incoming criteria. + /// + /// Path of file to search. + /// Struct containing search criteria. + /// Output messages. + /// Directory that extracted files were output into. + public static string FindOffsetAndCutFile(string sourcePath, FindOffsetStruct searchCriteria, out string messages, bool outputLog, bool outputBatchFile) + { + int i; + int j = 0; + byte[] searchBytes; + byte[] terminatorBytes = null; + System.Text.Encoding enc = System.Text.Encoding.ASCII; + + long cutStart; + long cutSize = 0; + long cutSizeOffset; + byte[] cutSizeBytes; + long lengthMultiplier; + long minimumCutSize = -1; + + long previousPosition; + string outputFolder; + string outputFile; + int chunkCount = 0; + + long offset; + long searchStringModuloDivisor = 0; + long searchStringModuloResult = 0; + long previousOffset; + + long terminatorOffset; + long terminatorModuloDivisor = 0; + long terminatorModuloResult = 0; + + bool skipCut; + + StringBuilder ret = new StringBuilder(); + + // create search bytes + if (searchCriteria.TreatSearchStringAsHex) + { + searchBytes = ByteConversion.GetBytesFromHexString(searchCriteria.SearchString); + + // convert the search string to bytes + for (i = 0; i < searchCriteria.SearchString.Length; i += 2) + { + searchBytes[j] = BitConverter.GetBytes(Int16.Parse(searchCriteria.SearchString.Substring(i, 2), System.Globalization.NumberStyles.AllowHexSpecifier, CultureInfo.CurrentCulture))[0]; + j++; + } + } + else + { + searchBytes = enc.GetBytes(searchCriteria.SearchString); + } + + // parse minimum cut size + if (!String.IsNullOrEmpty(searchCriteria.MinimumSize)) + { + minimumCutSize = VGMToolbox.util.ByteConversion.GetLongValueFromString(searchCriteria.MinimumSize); + } + + // parse Search String modulo information + if (searchCriteria.DoSearchStringModulo) + { + searchStringModuloDivisor = VGMToolbox.util.ByteConversion.GetLongValueFromString(searchCriteria.SearchStringModuloDivisor); + searchStringModuloResult = VGMToolbox.util.ByteConversion.GetLongValueFromString(searchCriteria.SearchStringModuloResult); + } + + // create terminator bytes + j = 0; + if (searchCriteria.TreatTerminatorStringAsHex) + { + terminatorBytes = ByteConversion.GetBytesFromHexString(searchCriteria.TerminatorString); + } + else if (!String.IsNullOrEmpty(searchCriteria.TerminatorString)) + { + terminatorBytes = enc.GetBytes(searchCriteria.TerminatorString); + } + + FileInfo fi = new FileInfo(sourcePath); + + using (FileStream fs = File.Open(Path.GetFullPath(sourcePath), FileMode.Open, FileAccess.Read)) + { + ret.AppendFormat("[{0}]", sourcePath); + ret.Append(Environment.NewLine); + + // setup starting offset + previousOffset = + String.IsNullOrEmpty(searchCriteria.StartingOffset) ? 0 : VGMToolbox.util.ByteConversion.GetLongValueFromString(searchCriteria.StartingOffset); + + // build output folder path + if (String.IsNullOrEmpty(searchCriteria.OutputFolder)) + { + outputFolder = Path.GetFullPath( + Path.Combine( + Path.GetDirectoryName(sourcePath), + Path.GetFileNameWithoutExtension(sourcePath) + "_CUT")); + } + else + { + outputFolder = searchCriteria.OutputFolder; + } + + // search for our string + // while ((offset = ParseFile.GetNextOffset(fs, previousOffset, searchBytes)) != -1) + while ((offset = ParseFile.GetNextOffset(fs, previousOffset, searchBytes, + searchCriteria.DoSearchStringModulo, searchStringModuloDivisor, + searchStringModuloResult)) != -1) + { + // do cut file tasks + if (searchCriteria.CutFile) + { + skipCut = false; + + cutStart = offset - VGMToolbox.util.ByteConversion.GetLongValueFromString(searchCriteria.SearchStringOffset); + + // determine cut size from value at offset + if (searchCriteria.IsCutSizeAnOffset) + { + cutSizeOffset = cutStart + VGMToolbox.util.ByteConversion.GetLongValueFromString(searchCriteria.CutSize); + previousPosition = fs.Position; + cutSizeBytes = ParseFile.ParseSimpleOffset( + fs, + cutSizeOffset, + (int)VGMToolbox.util.ByteConversion.GetLongValueFromString(searchCriteria.CutSizeOffsetSize)); + fs.Position = previousPosition; + + if (!searchCriteria.IsLittleEndian) + { + Array.Reverse(cutSizeBytes); + } + + switch (cutSizeBytes.Length) + { + case 1: + cutSize = cutSizeBytes[0]; + break; + case 2: + cutSize = BitConverter.ToInt16(cutSizeBytes, 0); + break; + case 4: + cutSize = BitConverter.ToInt32(cutSizeBytes, 0); + break; + default: + cutSize = 0; + break; + } + + if (searchCriteria.UseLengthMultiplier) + { + lengthMultiplier = VGMToolbox.util.ByteConversion.GetLongValueFromString(searchCriteria.LengthMultiplier); + cutSize *= lengthMultiplier; + } + } + else if (searchCriteria.UseTerminatorForCutSize) // look for terminator + { + if (cutStart >= 0) + { + if (searchCriteria.DoTerminatorModulo) + { + terminatorModuloDivisor = VGMToolbox.util.ByteConversion.GetLongValueFromString(searchCriteria.TerminatorStringModuloDivisor); + terminatorModuloResult = VGMToolbox.util.ByteConversion.GetLongValueFromString(searchCriteria.TerminatorStringModuloResult); + } + + terminatorOffset = GetNextOffset(fs, offset + searchBytes.Length, terminatorBytes, + searchCriteria.DoTerminatorModulo, terminatorModuloDivisor, + terminatorModuloResult); + + // cut to EOF if terminator not found + if (searchCriteria.CutToEofIfTerminatorNotFound && (terminatorOffset == -1)) + { + cutSize = fs.Length - cutStart; + } + else + { + cutSize = terminatorOffset - cutStart; + } + + if (searchCriteria.IncludeTerminatorLength) + { + cutSize += terminatorBytes.Length; + } + } + } + else // static size + { + cutSize = VGMToolbox.util.ByteConversion.GetLongValueFromString(searchCriteria.CutSize); + } + + outputFile = String.Format(CultureInfo.InvariantCulture, "{0}_{1}{2}", Path.GetFileNameWithoutExtension(sourcePath), chunkCount.ToString("X8", CultureInfo.InvariantCulture), searchCriteria.OutputFileExtension); + + if (cutStart < 0) + { + ret.AppendFormat( + CultureInfo.CurrentCulture, + " Warning: For string found at: 0x{0}, cut begin is less than 0 ({1})...Skipping", + offset.ToString("X8", CultureInfo.InvariantCulture), + cutStart.ToString("X8", CultureInfo.InvariantCulture)); + ret.Append(Environment.NewLine); + + skipCut = true; + } + else if (cutSize < 1) + { + ret.AppendFormat( + CultureInfo.CurrentCulture, + " Warning: For string found at: 0x{0}, cut size is less than 1 ({1})...Skipping", + offset.ToString("X8", CultureInfo.InvariantCulture), + cutSize.ToString("X8", CultureInfo.InvariantCulture)); + ret.Append(Environment.NewLine); + + skipCut = true; + } + else if ((cutStart + cutSize) > fi.Length) + { + ret.AppendFormat( + CultureInfo.CurrentCulture, + " Warning: For string found at: 0x{0}, total file end will go past the end of the file ({1})", + offset.ToString("X8", CultureInfo.InvariantCulture), + (cutStart + cutSize).ToString("X8", CultureInfo.InvariantCulture)); + ret.Append(Environment.NewLine); + } + + if (skipCut) + { + previousOffset = offset + 1; + } + else + { + if (!String.IsNullOrEmpty(searchCriteria.ExtraCutSizeBytes)) + { + cutSize += (long)VGMToolbox.util.ByteConversion.GetLongValueFromString(searchCriteria.ExtraCutSizeBytes); + } + + // check minimum cut size + if (((minimumCutSize > 0) && (cutSize >= minimumCutSize)) || + (minimumCutSize < 1)) + { + ParseFile.ExtractChunkToFile(fs, cutStart, cutSize, Path.Combine(outputFolder, outputFile), outputLog, outputBatchFile); + + ret.AppendFormat( + CultureInfo.CurrentCulture, + " Extracted [{3}] begining at 0x{0}, for string found at: 0x{1}, with size 0x{2}", + cutStart.ToString("X8", CultureInfo.InvariantCulture), + offset.ToString("X8", CultureInfo.InvariantCulture), + cutSize.ToString("X8", CultureInfo.InvariantCulture), + outputFile); + ret.Append(Environment.NewLine); + + chunkCount++; + } + + previousOffset = cutStart + cutSize; + } + } + else // just output text + { + // just append the offset + ret.AppendFormat(CultureInfo.CurrentCulture, " String found at: 0x{0}", offset.ToString("X8", CultureInfo.InvariantCulture)); + ret.Append(Environment.NewLine); + + previousOffset = offset + searchBytes.Length; + } + } + } + + messages = ret.ToString(); + return outputFolder; + } + + public static long GetVaryingByteValueAtOffset(Stream inStream, string valueOffset, string valueLength, + Boolean valueIsLittleEndian) + { + long newValueOffset; + long newValueLength; + + newValueOffset = VGMToolbox.util.ByteConversion.GetLongValueFromString(valueOffset); + newValueLength = VGMToolbox.util.ByteConversion.GetLongValueFromString(valueLength); + + return GetVaryingByteValueAtOffset(inStream, newValueOffset, newValueLength, valueIsLittleEndian); + } + + public static long GetVaryingByteValueAtAbsoluteOffset(Stream inStream, OffsetDescription offsetInfo) + { + return GetVaryingByteValueAtAbsoluteOffset(inStream, offsetInfo, false); + } + + public static long GetVaryingByteValueAtAbsoluteOffset(Stream inStream, OffsetDescription offsetInfo, bool allowNegativeOffset) + { + long newValueOffset; + long newValueLength; + + newValueOffset = ByteConversion.GetLongValueFromString(offsetInfo.OffsetValue); + newValueLength = ByteConversion.GetLongValueFromString(offsetInfo.OffsetSize); + + return GetVaryingByteValueAtOffset(inStream, newValueOffset, newValueLength, offsetInfo.OffsetByteOrder.Equals(Constants.LittleEndianByteOrder), allowNegativeOffset); + } + + public static long GetVaryingByteValueAtRelativeOffset(Stream inStream, OffsetDescription offsetInfo, long currentOffset) + { + long newValueOffset; + long newValueLength; + + newValueOffset = currentOffset + ByteConversion.GetLongValueFromString(offsetInfo.OffsetValue); + newValueLength = ByteConversion.GetLongValueFromString(offsetInfo.OffsetSize); + + return GetVaryingByteValueAtOffset(inStream, newValueOffset, newValueLength, offsetInfo.OffsetByteOrder.Equals(Constants.LittleEndianByteOrder)); + } + + public static long GetVaryingByteValueAtOffset(Stream inStream, long valueOffset, long valueLength, + Boolean valueIsLittleEndian) + { + return GetVaryingByteValueAtOffset(inStream, valueOffset, valueLength, + valueIsLittleEndian, false); + } + + public static long GetVaryingByteValueAtOffset(Stream inStream, long valueOffset, long valueLength, + Boolean valueIsLittleEndian, bool allowNegativeOffset) + { + long newValue; + + if (allowNegativeOffset && (valueOffset < 0)) + { + valueOffset = inStream.Length + valueOffset; + } + + byte[] newValueBytes = ParseFile.ParseSimpleOffset(inStream, valueOffset, (int)valueLength); + + if (!valueIsLittleEndian) + { + Array.Reverse(newValueBytes); + } + + switch (newValueBytes.Length) + { + case 1: + newValue = newValueBytes[0]; + break; + case 2: + newValue = BitConverter.ToUInt16(newValueBytes, 0); + break; + case 4: + newValue = BitConverter.ToUInt32(newValueBytes, 0); + break; + default: + newValue = -1; + break; + } + + return newValue; + } + + public static string RemoveLeadingPathSeparator(string pathToCheck) + { + char firstChar; + string ret = pathToCheck; + + if (!String.IsNullOrEmpty(pathToCheck) && pathToCheck.Length > 0) + { + firstChar = pathToCheck.ToCharArray()[0]; + + if (firstChar.Equals(Path.DirectorySeparatorChar) || + firstChar.Equals(Path.AltDirectorySeparatorChar) || + firstChar.Equals(Path.VolumeSeparatorChar)) + { + ret = pathToCheck.Substring(1); + } + } + + return ret; + } + + public static string RemoveIllegalCharactersFromPath(string path) + { + char[] illegalFileNameChars = Path.GetInvalidFileNameChars(); + char[] illegalPathChars = Path.GetInvalidPathChars(); + + foreach (char c in path.ToCharArray()) + { + for (int i = 0; i < illegalFileNameChars.Length; i++) + { + if (c == illegalFileNameChars[i]) + { + path.Replace(illegalFileNameChars[i], '_'); + continue; + } + } + + for (int i = 0; i < illegalPathChars.Length; i++) + { + if (c == illegalPathChars[i]) + { + path.Replace(illegalPathChars[i], '_'); + continue; + } + } + } + + return path.Trim(); + } + + public static string TrimFileNameToNullBytes(string path) + { + byte[] stringBytes; + int nameLength = path.Length; + + stringBytes = Encoding.ASCII.GetBytes(path); + + for (int i = 0; i < stringBytes.Length; i++) + { + if (stringBytes[i] == 0) + { + nameLength = i; + break; + } + } + + path = path.Substring(0, nameLength); + + return path; + } + + public static byte[] SimpleArrayCopy(byte[] SourceArray, long offset, long length) + { + byte[] value = new byte[length]; + Array.Copy(SourceArray, offset, value, 0, length); + return value; + } + + public static float ReadFloatLE(Stream inStream, long offset) + { + return BitConverter.ToSingle(ParseSimpleOffset(inStream, offset, 4), 0); + } + + public static float ReadFloatBE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + Array.Reverse(val); + + return BitConverter.ToSingle(val, 0); + } + + public static uint ReadUintLE(Stream inStream, long offset) + { + return BitConverter.ToUInt32(ParseSimpleOffset(inStream, offset, 4), 0); + } + + public static uint ReadUintLE(byte[] inBytes, long offset) + { + byte[] val = new byte[4]; + + Array.Copy(inBytes, offset, val, 0, 4); + + return BitConverter.ToUInt32(val, 0); + } + + public static uint ReadUintBE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + Array.Reverse(val); + + return BitConverter.ToUInt32(val, 0); + } + + public static uint ReadUintBE(byte[] inBytes, long offset) + { + byte[] val = new byte[4]; + + Array.Copy(inBytes, offset, val, 0, 4); + Array.Reverse(val); + + return BitConverter.ToUInt32(val, 0); + } + + public static ushort ReadUshortLE(Stream inStream, long offset) + { + return BitConverter.ToUInt16(ParseSimpleOffset(inStream, offset, 2), 0); + } + + public static ushort ReadUshortLE(byte[] inBytes, long offset) + { + byte[] val = new byte[2]; + + Array.Copy(inBytes, offset, val, 0, 2); + + return BitConverter.ToUInt16(val, 0); + } + + public static ushort ReadUshortBE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 2); + Array.Reverse(val); + + return BitConverter.ToUInt16(val, 0); + } + + public static ushort ReadUshortBE(byte[] inBytes, long offset) + { + byte[] val = new byte[2]; + + Array.Copy(inBytes, offset, val, 0, 2); + Array.Reverse(val); + + return BitConverter.ToUInt16(val, 0); + } + + public static byte ReadByte(Stream inStream, long offset) + { + return (byte)ParseSimpleOffset(inStream, offset, 1)[0]; + } + + public static SByte ReadSByte(Stream inStream, long offset) + { + return (SByte)ParseSimpleOffset(inStream, offset, 1)[0]; + } + + public static int ReadInt32LE(Stream inStream, long offset) + { + return BitConverter.ToInt32(ParseSimpleOffset(inStream, offset, 4), 0); + } + + public static int ReadInt32BE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + Array.Reverse(val); + + return BitConverter.ToInt32(val, 0); + } + + public static short ReadInt16LE(Stream inStream, long offset) + { + return BitConverter.ToInt16(ParseSimpleOffset(inStream, offset, 2), 0); + } + + public static short ReadInt16BE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 2); + Array.Reverse(val); + + return BitConverter.ToInt16(val, 0); + } + + public static int ReadInt24LE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + val[3] = 0; + + return BitConverter.ToInt32(val, 0); + } + + public static int ReadInt24BE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + val[3] = 0; + Array.Reverse(val); + + return (BitConverter.ToInt32(val, 0) >> 8); + } + + public static uint ReadUint24LE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + val[3] = 0; + + return BitConverter.ToUInt32(val, 0); + } + + public static uint ReadUint24BE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 4); + val[3] = 0; + Array.Reverse(val); + + return (BitConverter.ToUInt32(val, 0) >> 8); + } + + public static ulong ReadUlongLE(Stream inStream, long offset) + { + return BitConverter.ToUInt64(ParseSimpleOffset(inStream, offset, 8), 0); + } + + public static ulong ReadUlongLE(byte[] inBytes, long offset) + { + byte[] val = new byte[8]; + + Array.Copy(inBytes, offset, val, 0, 8); + + return BitConverter.ToUInt64(val, 0); + } + + public static ulong ReadUlongBE(Stream inStream, long offset) + { + byte[] val = ParseSimpleOffset(inStream, offset, 8); + Array.Reverse(val); + + return BitConverter.ToUInt64(val, 0); + } + + public static string ReadAsciiString(Stream inStream, long offset) + { + long streamLength = inStream.Length; + int stringLength = 0; + int codePage; + + int dummy; + byte[] stringBytes; + string ret = String.Empty; + + // move pointer + inStream.Position = offset; + + // read until NULL + for (long i = offset; i <= streamLength; i++) + { + dummy = inStream.ReadByte(); + + if (dummy > 0) + { + stringLength++; + } + else if (dummy <= 0) + { + break; + } + } + + // parse string + stringBytes = ParseSimpleOffset(inStream, offset, stringLength); + codePage = ByteConversion.GetPredictedCodePageForTags(stringBytes); + ret = ByteConversion.GetEncodedText(stringBytes, codePage); + //ret = ByteConversion.GetAsciiText(stringBytes); + + return ret; + } + } +} diff --git a/usm_extract/src/util/RiffCalculatingOffsetDescription.cs b/usm_extract/src/util/RiffCalculatingOffsetDescription.cs new file mode 100644 index 0000000..b56487e --- /dev/null +++ b/usm_extract/src/util/RiffCalculatingOffsetDescription.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace VGMToolbox.util +{ + public class RiffCalculatingOffsetDescription : CalculatingOffsetDescription + { + public const string START_OF_STRING = "start of"; + public const string END_OF_STRING = "end of"; + + public string RelativeLocationToRiffChunkString { set; get; } + public string RiffChunkString { set; get; } + } +} diff --git a/usm_extract/usm_extract.csproj b/usm_extract/usm_extract.csproj new file mode 100644 index 0000000..d8fd316 --- /dev/null +++ b/usm_extract/usm_extract.csproj @@ -0,0 +1,86 @@ + + + + Debug + AnyCPU + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83} + Exe + Properties + usm_extract + usm_extract + v4.5 + 512 + + + + + 3.5 + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + bin\x86\Debug\ + DEBUG;TRACE + full + x86 + prompt + 4 + false + + + bin\x86\Release\ + TRACE + true + pdbonly + x86 + prompt + 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/usm_extract/usm_extract.sln b/usm_extract/usm_extract.sln new file mode 100644 index 0000000..7dee767 --- /dev/null +++ b/usm_extract/usm_extract.sln @@ -0,0 +1,23 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2012 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "usm_extract", "usm_extract.csproj", "{3951D6E5-61BE-4D7C-A464-F44D98E9CE83}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + Debug|x86 = Debug|x86 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Debug|x86.ActiveCfg = Debug|x86 + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Debug|x86.Build.0 = Debug|x86 + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Release|Any CPU.Build.0 = Release|Any CPU + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Release|x86.ActiveCfg = Release|x86 + {3951D6E5-61BE-4D7C-A464-F44D98E9CE83}.Release|x86.Build.0 = Release|x86 + EndGlobalSection +EndGlobal