xdu/media-scripts
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

initial commit of media shit

Browse source
This commit is contained in:
louis 2020-02-18 18:44:58 -05:00
commit 8c0a5e3aa7
38 changed files with 9195 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

222
usm_extract/src/CriUsmStream.cs Normal file
View file

@ -0,0 +1,222 @@
using System;
using System.Collections.Generic;
using System.IO;
using VGMToolbox.util;
namespace vgm_usm.extract
{
public class CriUsmStream : MpegStream
{
public const string DefaultAudioExtension = ".adx";
public const string DefaultVideoExtension = ".m2v";
public const string HcaAudioExtension = ".hca";
static readonly byte[] HCA_SIG_BYTES = new byte[] { 0x48, 0x43, 0x41, 0x00 };
protected static readonly byte[] ALP_BYTES = new byte[] { 0x40, 0x41, 0x4C, 0x50 };
protected static readonly byte[] CRID_BYTES = new byte[] { 0x43, 0x52, 0x49, 0x44 };
protected static readonly byte[] SFV_BYTES = new byte[] { 0x40, 0x53, 0x46, 0x56 };
protected static readonly byte[] SFA_BYTES = new byte[] { 0x40, 0x53, 0x46, 0x41 };
protected static readonly byte[] SBT_BYTES = new byte[] { 0x40, 0x53, 0x42, 0x54 };
protected static readonly byte[] CUE_BYTES = new byte[] { 0x40, 0x43, 0x55, 0x45 };
protected static readonly byte[] UTF_BYTES = new byte[] { 0x40, 0x55, 0x54, 0x46 };
protected static readonly byte[] HEADER_END_BYTES =
new byte[] { 0x23, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x20,
0x45, 0x4E, 0x44, 0x20, 0x20, 0x20, 0x20, 0x20,
0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D,
0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x00 };
protected static readonly byte[] METADATA_END_BYTES =
new byte[] { 0x23, 0x4D, 0x45, 0x54, 0x41, 0x44, 0x41, 0x54,
0x41, 0x20, 0x45, 0x4E, 0x44, 0x20, 0x20, 0x20,
0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D,
0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x00 };
protected static readonly byte[] CONTENTS_END_BYTES =
new byte[] { 0x23, 0x43, 0x4F, 0x4E, 0x54, 0x45, 0x4E, 0x54,
0x53, 0x20, 0x45, 0x4E, 0x44, 0x20, 0x20, 0x20,
0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D,
0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x3D, 0x00 };
public CriUsmStream(string path)
: base(path)
{
this.UsesSameIdForMultipleAudioTracks = true;
this.FileExtensionAudio = DefaultAudioExtension;
this.FileExtensionVideo = DefaultVideoExtension;
base.BlockIdDictionary.Clear();
base.BlockIdDictionary[BitConverter.ToUInt32(ALP_BYTES, 0)] = new BlockSizeStruct(PacketSizeType.SizeBytes, 4); // @ALP
base.BlockIdDictionary[BitConverter.ToUInt32(CRID_BYTES, 0)] = new BlockSizeStruct(PacketSizeType.SizeBytes, 4); // CRID
base.BlockIdDictionary[BitConverter.ToUInt32(SFV_BYTES, 0)] = new BlockSizeStruct(PacketSizeType.SizeBytes, 4); // @SFV
base.BlockIdDictionary[BitConverter.ToUInt32(SFA_BYTES, 0)] = new BlockSizeStruct(PacketSizeType.SizeBytes, 4); // @SFA
base.BlockIdDictionary[BitConverter.ToUInt32(SBT_BYTES, 0)] = new BlockSizeStruct(PacketSizeType.SizeBytes, 4); // @SBT
base.BlockIdDictionary[BitConverter.ToUInt32(CUE_BYTES, 0)] = new BlockSizeStruct(PacketSizeType.SizeBytes, 4); // @CUE
}
protected override byte[] GetPacketStartBytes() { return CRID_BYTES; }
protected override int GetAudioPacketHeaderSize(Stream readStream, long currentOffset)
{
UInt16 checkBytes;
OffsetDescription od = new OffsetDescription();
od.OffsetByteOrder = Constants.BigEndianByteOrder;
od.OffsetSize = "2";
od.OffsetValue = "8";
checkBytes = (UInt16)ParseFile.GetVaryingByteValueAtRelativeOffset(readStream, od, currentOffset);
return checkBytes;
}
protected override int GetVideoPacketHeaderSize(Stream readStream, long currentOffset)
{
UInt16 checkBytes;
OffsetDescription od = new OffsetDescription();
od.OffsetByteOrder = Constants.BigEndianByteOrder;
od.OffsetSize = "2";
od.OffsetValue = "8";
checkBytes = (UInt16)ParseFile.GetVaryingByteValueAtRelativeOffset(readStream, od, currentOffset);
return checkBytes;
}
protected override bool IsThisAnAudioBlock(byte[] blockToCheck)
{
return ParseFile.CompareSegment(blockToCheck, 0, SFA_BYTES);
}
protected override bool IsThisAVideoBlock(byte[] blockToCheck)
{
return ParseFile.CompareSegment(blockToCheck, 0, SFV_BYTES);
}
protected override byte GetStreamId(Stream readStream, long currentOffset)
{
byte streamId;
streamId = ParseFile.ParseSimpleOffset(readStream, currentOffset + 0xC, 1)[0];
return streamId;
}
protected override int GetAudioPacketFooterSize(Stream readStream, long currentOffset)
{
UInt16 checkBytes;
OffsetDescription od = new OffsetDescription();
od.OffsetByteOrder = Constants.BigEndianByteOrder;
od.OffsetSize = "2";
od.OffsetValue = "0xA";
checkBytes = (UInt16)ParseFile.GetVaryingByteValueAtRelativeOffset(readStream, od, currentOffset);
return checkBytes;
}
protected override int GetVideoPacketFooterSize(Stream readStream, long currentOffset)
{
UInt16 checkBytes;
OffsetDescription od = new OffsetDescription();
od.OffsetByteOrder = Constants.BigEndianByteOrder;
od.OffsetSize = "2";
od.OffsetValue = "0xA";
checkBytes = (UInt16)ParseFile.GetVaryingByteValueAtRelativeOffset(readStream, od, currentOffset);
return checkBytes;
}
protected override void DoFinalTasks(FileStream sourceFileStream, Dictionary<uint, FileStream> outputFiles, bool addHeader)
{
long headerEndOffset;
long metadataEndOffset;
long headerSize;
long footerOffset;
long footerSize;
string sourceFileName;
string workingFile;
string fileExtension;
string destinationFileName;
foreach (uint streamId in outputFiles.Keys)
{
sourceFileName = outputFiles[streamId].Name;
//--------------------------
// get header size
//--------------------------
headerEndOffset = ParseFile.GetNextOffset(outputFiles[streamId], 0, HEADER_END_BYTES);
metadataEndOffset = ParseFile.GetNextOffset(outputFiles[streamId], 0, METADATA_END_BYTES);
if (metadataEndOffset > headerEndOffset)
{
headerSize = metadataEndOffset + METADATA_END_BYTES.Length;
}
else
{
headerSize = headerEndOffset + METADATA_END_BYTES.Length;
}
//-----------------
// get footer size
//-----------------
footerOffset = ParseFile.GetNextOffset(outputFiles[streamId], 0, CONTENTS_END_BYTES) - headerSize;
footerSize = outputFiles[streamId].Length - footerOffset;
//------------------------------------------
// check data to adjust extension if needed
//------------------------------------------
if (this.IsThisAnAudioBlock(BitConverter.GetBytes(streamId & 0xFFFFFFF0))) // may need to change mask if more than 0xF streams
{
byte[] checkBytes = ParseFile.ParseSimpleOffset(outputFiles[streamId], headerSize, 4);
if (ParseFile.CompareSegment(checkBytes, 0, SofdecStream.AixSignatureBytes))
{
fileExtension = SofdecStream.AixAudioExtension;
}
else if (checkBytes[0] == 0x80)
{
fileExtension = SofdecStream.AdxAudioExtension;
}
else if (ParseFile.CompareSegment(checkBytes, 0, HCA_SIG_BYTES))
{
fileExtension = HcaAudioExtension;
}
else
{
fileExtension = ".bin";
}
}
else
{
fileExtension = Path.GetExtension(sourceFileName);
}
outputFiles[streamId].Close();
outputFiles[streamId].Dispose();
workingFile = FileUtil.RemoveChunkFromFile(sourceFileName, 0, headerSize);
File.Copy(workingFile, sourceFileName, true);
File.Delete(workingFile);
workingFile = FileUtil.RemoveChunkFromFile(sourceFileName, footerOffset, footerSize);
destinationFileName = Path.ChangeExtension(sourceFileName, fileExtension);
File.Copy(workingFile, destinationFileName, true);
File.Delete(workingFile);
if ((sourceFileName != destinationFileName) && (File.Exists(sourceFileName)))
{
File.Delete(sourceFileName);
}
}
}
}
}

39
usm_extract/src/Mpeg1Stream.cs Normal file
View file

@ -0,0 +1,39 @@
using System;
using System.IO;
namespace vgm_usm.extract
{
public class Mpeg1Stream : MpegStream
{
public const string DefaultAudioExtension = ".mp2";
public const string DefaultVideoExtension = ".m1v";
public Mpeg1Stream(string path)
: base(path)
{
this.FileExtensionAudio = DefaultAudioExtension;
this.FileExtensionVideo = DefaultVideoExtension;
base.BlockIdDictionary[BitConverter.ToUInt32(MpegStream.PacketStartBytes, 0)] = new BlockSizeStruct(PacketSizeType.Static, 0xC); // Pack Header
}
protected override int GetAudioPacketHeaderSize(Stream readStream, long currentOffset)
{
int paddingByteCount = 0;
readStream.Position = currentOffset + 6;
// skip stuffing bytes
while (readStream.ReadByte() == 0xFF)
{
paddingByteCount++;
}
return paddingByteCount + 7;
}
protected override int GetVideoPacketHeaderSize(Stream readStream, long currentOffset)
{
return 0xC;
}
}
}

503
usm_extract/src/MpegStream.cs Normal file
View file

@ -0,0 +1,503 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using VGMToolbox.util;
namespace vgm_usm.extract
{
public abstract class MpegStream
{
protected static readonly byte[] PacketStartBytes = new byte[] { 0x00, 0x00, 0x01, 0xBA };
protected static readonly byte[] PacketEndBytes = new byte[] { 0x00, 0x00, 0x01, 0xB9 };
public MpegStream(string path)
{
this.FilePath = path;
this.UsesSameIdForMultipleAudioTracks = false;
this.SubTitleExtractionSupported = false;
this.BlockSizeIsLittleEndian = false;
//********************
// Add Slice Packets
//********************
byte[] sliceBytes;
uint sliceBytesValue;
BlockSizeStruct blockSize = new BlockSizeStruct(PacketSizeType.Static, 0xE);
for (byte i = 0; i <= 0xAF; i++)
{
sliceBytes = new byte[] { 0x00, 0x00, 0x01, i };
sliceBytesValue = BitConverter.ToUInt32(sliceBytes, 0);
this.BlockIdDictionary.Add(sliceBytesValue, blockSize);
}
}
public enum PacketSizeType
{
Static,
SizeBytes,
Eof
}
public struct MpegDemuxOptions
{
public bool AddHeader { set; get; }
}
public struct BlockSizeStruct
{
public PacketSizeType SizeType;
public int Size;
public BlockSizeStruct(PacketSizeType sizeTypeValue, int sizeValue)
{
this.SizeType = sizeTypeValue;
this.Size = sizeValue;
}
}
public struct DemuxOptionsStruct
{
public bool ExtractVideo { set; get; }
public bool ExtractAudio { set; get; }
public bool AddHeader { set; get; }
public bool SplitAudioStreams { set; get; }
public bool AddPlaybackHacks { set; get; }
}
#region Dictionary Initialization
protected Dictionary<uint, BlockSizeStruct> BlockIdDictionary =
new Dictionary<uint, BlockSizeStruct>
{
//********************
// System Packets
//********************
{BitConverter.ToUInt32(MpegStream.PacketEndBytes, 0), new BlockSizeStruct(PacketSizeType.Eof, -1)}, // Program End
{BitConverter.ToUInt32(MpegStream.PacketStartBytes, 0), new BlockSizeStruct(PacketSizeType.Static, 0xE)}, // Pack Header
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xBB }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // System Header, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xBD }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Private Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xBE }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Padding Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xBF }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Private Stream, two bytes following equal length (Big Endian)
//****************************
// Audio Streams
//****************************
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC0 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC1 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC2 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC3 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC4 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC5 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC6 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC7 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC8 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xC9 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xCA }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xCB }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xCC }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xCD }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xCE }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xCF }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD0 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD1 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD2 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD3 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD4 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD5 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD6 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD7 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD8 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xD9 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xDA }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xDB }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xDC }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xDD }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xDE }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xDF }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Audio Stream, two bytes following equal length (Big Endian)
//****************************
// Video Streams
//****************************
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE0 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE1 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE2 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE3 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE4 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE5 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE6 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE7 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE8 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xE9 }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xEA }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xEB }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xEC }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xED }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xEE }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
{BitConverter.ToUInt32(new byte[] { 0x00, 0x00, 0x01, 0xEF }, 0), new BlockSizeStruct(PacketSizeType.SizeBytes, 2)}, // Video Stream, two bytes following equal length (Big Endian)
};
#endregion
public string FilePath { get; set; }
public string FileExtensionAudio { get; set; }
public string FileExtensionVideo { get; set; }
protected Dictionary<byte, string> StreamIdFileType = new Dictionary<byte, string>();
public bool UsesSameIdForMultipleAudioTracks { set; get; } // for PMF/PAM/DVD, who use 000001BD for all audio tracks
public bool SubTitleExtractionSupported { set; get; } // assume not supported.
public bool BlockSizeIsLittleEndian { set; get; }
protected virtual byte[] GetPacketStartBytes() { return MpegStream.PacketStartBytes; }
protected virtual byte[] GetPacketEndBytes() { return MpegStream.PacketEndBytes; }
protected abstract int GetAudioPacketHeaderSize(Stream readStream, long currentOffset);
protected virtual int GetAudioPacketSubHeaderSize(Stream readStream, long currentOffset, byte streamId) { return 0; }
protected abstract int GetVideoPacketHeaderSize(Stream readStream, long currentOffset);
protected virtual int GetAudioPacketFooterSize(Stream readStream, long currentOffset) { return 0; }
protected virtual int GetVideoPacketFooterSize(Stream readStream, long currentOffset) { return 0; }
protected virtual bool IsThisAnAudioBlock(byte[] blockToCheck)
{
return ((blockToCheck[3] >= 0xC0) &&
(blockToCheck[3] <= 0xDF));
}
protected virtual bool IsThisAVideoBlock(byte[] blockToCheck)
{
return ((blockToCheck[3] >= 0xE0) && (blockToCheck[3] <= 0xEF));
}
protected virtual bool IsThisASubPictureBlock(byte[] blockToCheck)
{
return ((blockToCheck[3] >= 0xE0) && (blockToCheck[3] <= 0xEF));
}
protected virtual string GetAudioFileExtension(Stream readStream, long currentOffset)
{
return this.FileExtensionAudio;
}
protected virtual string GetVideoFileExtension(Stream readStream, long currentOffset)
{
return this.FileExtensionVideo;
}
protected virtual byte GetStreamId(Stream readStream, long currentOffset) { return 0; }
protected virtual long GetStartOffset(Stream readStream, long currentOffset) { return 0; }
protected virtual void DoFinalTasks(FileStream sourceFileStream, Dictionary<uint, FileStream> outputFiles, bool addHeader)
{
}
public virtual void DemultiplexStreams(DemuxOptionsStruct demuxOptions)
{
using (FileStream fs = File.OpenRead(this.FilePath))
{
long fileSize = fs.Length;
long currentOffset = 0;
byte[] currentBlockId;
uint currentBlockIdVal;
byte[] currentBlockIdNaming;
BlockSizeStruct blockStruct = new BlockSizeStruct();
byte[] blockSizeArray;
uint blockSize;
int audioBlockSkipSize;
int videoBlockSkipSize;
int audioBlockFooterSize;
int videoBlockFooterSize;
int cutSize;
bool eofFlagFound = false;
Dictionary<uint, FileStream> streamOutputWriters = new Dictionary<uint, FileStream>();
string outputFileName;
byte streamId = 0; // for types that have multiple streams in the same block ID
uint currentStreamKey; // hash key for each file
bool isAudioBlock;
string audioFileExtension;
// look for first packet
currentOffset = this.GetStartOffset(fs, currentOffset);
currentOffset = ParseFile.GetNextOffset(fs, currentOffset, this.GetPacketStartBytes());
if (currentOffset != -1)
{
while (currentOffset < fileSize)
{
#if DEBUG
//if (currentOffset == 0x414080e)
//{
// int gggg = 1;
//}
// hack for bad data (ni no kuni s09.pam)
//if ((currentOffset & 1) == 1)
//{
// currentOffset = MathUtil.RoundUpToByteAlignment(currentOffset, 0x800);
//}
#endif
try
{
// get the current block
currentBlockId = ParseFile.ParseSimpleOffset(fs, currentOffset, 4);
// get value to use as key to hash table
currentBlockIdVal = BitConverter.ToUInt32(currentBlockId, 0);
if (BlockIdDictionary.ContainsKey(currentBlockIdVal))
{
// get info about this block type
blockStruct = BlockIdDictionary[currentBlockIdVal];
switch (blockStruct.SizeType)
{
/////////////////////
// Static Block Size
/////////////////////
case PacketSizeType.Static:
currentOffset += blockStruct.Size; // skip this block
break;
//////////////////
// End of Stream
//////////////////
case PacketSizeType.Eof:
eofFlagFound = true; // set EOF block found so we can exit the loop
break;
//////////////////////
// Varying Block Size
//////////////////////
case PacketSizeType.SizeBytes:
// Get the block size
blockSizeArray = ParseFile.ParseSimpleOffset(fs, currentOffset + currentBlockId.Length, blockStruct.Size);
if (!this.BlockSizeIsLittleEndian)
{
Array.Reverse(blockSizeArray);
}
switch (blockStruct.Size)
{
case 4:
blockSize = (uint)BitConverter.ToUInt32(blockSizeArray, 0);
break;
case 2:
blockSize = (uint)BitConverter.ToUInt16(blockSizeArray, 0);
break;
case 1:
blockSize = (uint)blockSizeArray[0];
break;
default:
throw new ArgumentOutOfRangeException(String.Format("Unhandled size block size.{0}", Environment.NewLine));
}
// if block type is audio or video, extract it
isAudioBlock = this.IsThisAnAudioBlock(currentBlockId);
if ((demuxOptions.ExtractAudio && isAudioBlock) ||
(demuxOptions.ExtractVideo && this.IsThisAVideoBlock(currentBlockId)))
{
// reset stream id
streamId = 0;
// if audio block, get the stream number from the queue
if (isAudioBlock && this.UsesSameIdForMultipleAudioTracks)
{
streamId = this.GetStreamId(fs, currentOffset);
currentStreamKey = (streamId | currentBlockIdVal);
}
else
{
currentStreamKey = currentBlockIdVal;
}
// check if we've already started parsing this stream
if (!streamOutputWriters.ContainsKey(currentStreamKey))
{
// convert block id to little endian for naming
currentBlockIdNaming = BitConverter.GetBytes(currentStreamKey);
Array.Reverse(currentBlockIdNaming);
// build output file name
outputFileName = Path.GetFileNameWithoutExtension(this.FilePath);
//outputFileName = outputFileName + "_" + BitConverter.ToUInt32(currentBlockIdNaming, 0).ToString("X8");
// add proper extension
if (this.IsThisAnAudioBlock(currentBlockId))
{
audioFileExtension = this.GetAudioFileExtension(fs, currentOffset);
outputFileName += audioFileExtension;
if (!this.StreamIdFileType.ContainsKey(streamId))
{
this.StreamIdFileType.Add(streamId, audioFileExtension);
}
}
else
{
this.FileExtensionVideo = this.GetVideoFileExtension(fs, currentOffset);
outputFileName += this.FileExtensionVideo;
}
// add output directory
outputFileName = Path.Combine(Path.GetDirectoryName(this.FilePath), outputFileName);
// add an output stream for writing
streamOutputWriters[currentStreamKey] = new FileStream(outputFileName, FileMode.Create, FileAccess.ReadWrite);
}
// write the block
if (this.IsThisAnAudioBlock(currentBlockId))
{
// write audio
audioBlockSkipSize = this.GetAudioPacketHeaderSize(fs, currentOffset) + GetAudioPacketSubHeaderSize(fs, currentOffset, streamId);
audioBlockFooterSize = this.GetAudioPacketFooterSize(fs, currentOffset);
cutSize = (int)(blockSize - audioBlockSkipSize - audioBlockFooterSize);
if (cutSize > 0)
{
streamOutputWriters[currentStreamKey].Write(ParseFile.ParseSimpleOffset(fs, currentOffset + currentBlockId.Length + blockSizeArray.Length + audioBlockSkipSize, (int)(blockSize - audioBlockSkipSize)), 0, cutSize);
}
#if DEBUG
//else
//{
// int aaa = 1;
//}
#endif
}
else
{
// write video
videoBlockSkipSize = this.GetVideoPacketHeaderSize(fs, currentOffset);
videoBlockFooterSize = this.GetVideoPacketFooterSize(fs, currentOffset);
cutSize = (int)(blockSize - videoBlockSkipSize - videoBlockFooterSize);
if (cutSize > 0)
{
streamOutputWriters[currentStreamKey].Write(ParseFile.ParseSimpleOffset(fs, currentOffset + currentBlockId.Length + blockSizeArray.Length + videoBlockSkipSize, (int)(blockSize - videoBlockSkipSize)), 0, cutSize);
}
#if DEBUG
//else
//{
// int vvv = 1;
//}
#endif
}
}
// move to next block
currentOffset += currentBlockId.Length + blockSizeArray.Length + blockSize;
blockSizeArray = new byte[] { };
break;
default:
break;
}
}
else // this is an undexpected block type
{
this.closeAllWriters(streamOutputWriters);
Array.Reverse(currentBlockId);
throw new FormatException(String.Format("Block ID at 0x{0} not found in table: 0x{1}", currentOffset.ToString("X8"), BitConverter.ToUInt32(currentBlockId, 0).ToString("X8")));
}
// exit loop if EOF block found
if (eofFlagFound)
{
break;
}
}
catch (Exception _ex)
{
this.closeAllWriters(streamOutputWriters);
throw new Exception(String.Format("Error parsing file at offset {0), '{1}'", currentOffset.ToString("X8"), _ex.Message), _ex);
}
} // while (currentOffset < fileSize)
}
else
{
this.closeAllWriters(streamOutputWriters);
throw new FormatException(String.Format("Cannot find Pack Header for file: {0}{1}", Path.GetFileName(this.FilePath), Environment.NewLine));
}
///////////////////////////////////
// Perform any final tasks needed
///////////////////////////////////
this.DoFinalTasks(fs, streamOutputWriters, demuxOptions.AddHeader);
//////////////////////////
// close all open writers
//////////////////////////
this.closeAllWriters(streamOutputWriters);
} // using (FileStream fs = File.OpenRead(path))
}
private void closeAllWriters(Dictionary<uint, FileStream> writers)
{
//////////////////////////
// close all open writers
//////////////////////////
foreach (uint b in writers.Keys)
{
if (writers[b].CanRead)
{
writers[b].Close();
writers[b].Dispose();
}
}
}
public static int GetMpegStreamType(string path)
{
int mpegType = -1;
using (FileStream fs = File.OpenRead(path))
{
// look for first packet
long currentOffset = ParseFile.GetNextOffset(fs, 0, MpegStream.PacketStartBytes);
if (currentOffset != -1)
{
currentOffset += 4;
fs.Position = currentOffset;
byte idByte = (byte)fs.ReadByte();
if ((int)ByteConversion.GetHighNibble(idByte) == 2)
{
mpegType = 1;
}
else if ((int)ByteConversion.GetHighNibble(idByte) == 4)
{
mpegType = 2;
}
}
else
{
throw new FormatException(String.Format("Cannot find Pack Header for file: {0}{1}", Path.GetFileName(path), Environment.NewLine));
}
}
return mpegType;
}
}
}

21
usm_extract/src/Program.cs Normal file
View file

@ -0,0 +1,21 @@
using System;
using System.IO;
namespace vgm_usm.extract
{
static class extract
{
static void Main(string[] args) {
MpegStream.DemuxOptionsStruct demuxOptions = new MpegStream.DemuxOptionsStruct();
demuxOptions.ExtractVideo = true;
demuxOptions.ExtractAudio = true;
demuxOptions.AddHeader = true;
demuxOptions.SplitAudioStreams = true;
demuxOptions.AddPlaybackHacks = true;
foreach (string s in args) {
CriUsmStream usm = new CriUsmStream(s);
usm.DemultiplexStreams(demuxOptions);
}
}
}
}

58
usm_extract/src/SofdecStream.cs Normal file
View file

@ -0,0 +1,58 @@
using System;
using System.IO;
using VGMToolbox.util;
namespace vgm_usm.extract
{
public class SofdecStream : Mpeg1Stream
{
new public const string DefaultVideoExtension = ".m2v";
public const string AdxAudioExtension = ".adx";
public const string AixAudioExtension = ".aix";
public const string Ac3AudioExtension = ".ac3";
public static readonly byte[] AixSignatureBytes = new byte[] { 0x41, 0x49, 0x58, 0x46 };
public static readonly byte[] Ac3SignatureBytes = new byte[] { 0x0B, 0x77 };
public SofdecStream(string path): base(path)
{
this.FileExtensionAudio = AdxAudioExtension;
this.FileExtensionVideo = DefaultVideoExtension;
}
protected override string GetAudioFileExtension(Stream readStream, long currentOffset)
{
string fileExtension;
byte[] checkBytes, checkBytesAc3;
int headerSize = this.GetAudioPacketHeaderSize(readStream, currentOffset);
checkBytes = ParseFile.ParseSimpleOffset(readStream, (currentOffset + 6 + headerSize), 4);
if (ParseFile.CompareSegment(checkBytes, 0, AixSignatureBytes))
{
fileExtension = AixAudioExtension;
}
else if (checkBytes[0] == 0x80)
{
fileExtension = AdxAudioExtension;
}
else
{
checkBytesAc3 = ParseFile.ParseSimpleOffset(readStream, (currentOffset + 6 + headerSize), 2);
if (ParseFile.CompareSegment(checkBytesAc3, 0, Ac3SignatureBytes))
{
fileExtension = Ac3AudioExtension;
}
else
{
fileExtension = ".bin";
}
}
return fileExtension;
}
}
}

266
usm_extract/src/util/ByteConversion.cs Normal file
View file

@ -0,0 +1,266 @@
using System;
using System.Globalization;
using System.Text;
namespace VGMToolbox.util
{
/// <summary>
/// Class containing static text conversion functions.
/// </summary>
public sealed class ByteConversion
{
/// <summary>
/// Codepage value for Shift JIS (Jp)
/// </summary>
public const int CodePageJapan = 932;
/// <summary>
/// Codepage value for Cyrillic (US)
/// </summary>
public const int CodePageUnitedStates = 1251;
/// <summary>
/// Codepage value for OEM DOS
/// </summary>
public const int CodePageOEM = 437;
private ByteConversion() { }
/// <summary>
/// Get string from bytes.
/// </summary>
/// <param name="pBytes">Bytes to convert to a string.</param>
/// <param name="codePage">Codepage to use in converting bytes.</param>
/// <returns>String encoding using the input Codepage.</returns>
public static string GetEncodedText(byte[] value, int codePage)
{
//return Encoding.Unicode.GetString(value);
return System.Text.Encoding.GetEncoding(codePage).GetString(value);
}
/// <summary>
/// Get text encoded in Shift JIS
/// </summary>
/// <param name="pBytes">Bytes to decode.</param>
/// <returns>String encoded using the Shift JIS codepage.</returns>
public static string GetJapaneseEncodedText(byte[] value)
{
return GetEncodedText(value, CodePageJapan);
}
/// <summary>
/// Get text encoded in Cyrillic
/// </summary>
/// <param name="pBytes">Bytes to decode.</param>
/// <returns>String encoded using the Cyrillic codepage.</returns>
public static string GetUnitedStatesEncodedText(byte[] value)
{
return GetEncodedText(value, CodePageUnitedStates);
}
/// <summary>
/// Get text encoded in ASCII
/// </summary>
/// <param name="pBytes">Bytes to decode.</param>
/// <returns>String encoded using ASCII.</returns>
public static string GetAsciiText(byte[] value)
{
System.Text.Encoding ascii = System.Text.Encoding.ASCII;
return ascii.GetString(value);
}
/// <summary>
/// Get text encoded in ASCII, stops at null.
/// </summary>
/// <param name="pBytes">Bytes to decode.</param>
/// <param name="offset">Offet within byte array of string.</param>
/// <returns>String encoded using ASCII.</returns>
public static string GetAsciiText(byte[] value, long offset)
{
StringBuilder sb = new StringBuilder();
System.Text.Encoding ascii = System.Text.Encoding.ASCII;
for (long i = offset; i < value.Length; i++)
{
if (value[i] == 0)
{
break;
}
else
{
sb.Append((char)value[i]);
}
}
return sb.ToString();
}
public static string GetUtf16LeText(byte[] value)
{
System.Text.Encoding encoding = System.Text.Encoding.Unicode;
return encoding.GetString(value);
}
/// <summary>
/// Convert input string to a long. Works for Decimal and Hexidecimal (use 0x prefix).
/// </summary>
/// <param name="pStringNumber">String containing a Decimal and Hexidecimal number.</param>
/// <returns>Long representing the input string.</returns>
public static long GetLongValueFromString(string value)
{
long ret;
bool isNegative = false;
string parseValue;
if (value.StartsWith("-"))
{
parseValue = value.Substring(1);
isNegative = true;
}
else
{
parseValue = value;
}
if (parseValue.StartsWith("0x", StringComparison.CurrentCultureIgnoreCase))
{
parseValue = parseValue.Substring(2);
ret = long.Parse(parseValue, System.Globalization.NumberStyles.HexNumber, null);
}
else
{
ret = long.Parse(parseValue, System.Globalization.NumberStyles.Integer, null);
}
if (isNegative)
{
ret *= -1;
}
return ret;
}
/// <summary>
/// Get the UInt32 Value of the Incoming Byte Array, which is in Big Endian order.
/// </summary>
/// <param name="pBytes">Bytes to convert.</param>
/// <returns>The UInt32 Value of the Incoming Byte Array.</returns>
public static UInt32 GetUInt32BigEndian(byte[] value)
{
byte[] workingArray = new byte[value.Length];
Array.Copy(value, 0, workingArray, 0, value.Length);
if (BitConverter.IsLittleEndian)
{
Array.Reverse(workingArray);
}
return BitConverter.ToUInt32(workingArray, 0);
}
/// <summary>
/// Get the UInt16 Value of the Incoming Byte Array, which is in Big Endian order.
/// </summary>
/// <param name="pBytes">Bytes to convert.</param>
/// <returns>The UInt16 Value of the Incoming Byte Array.</returns>
public static UInt16 GetUInt16BigEndian(byte[] value)
{
byte[] workingArray = new byte[value.Length];
Array.Copy(value, 0, workingArray, 0, value.Length);
if (BitConverter.IsLittleEndian)
{
Array.Reverse(workingArray);
}
return BitConverter.ToUInt16(workingArray, 0);
}
/// <summary>
/// Predicts Code Page between Cyrillic and Shift-JIS based on whether high ASCII is included or not.
/// </summary>
/// <param name="tagBytes">Bytes containing the tags in an unknown language.</param>
/// <returns>Integer representing the predicted code page.</returns>
public static int GetPredictedCodePageForTags(byte[] tagBytes)
{
int predictedCodePage = CodePageUnitedStates;
foreach (byte b in tagBytes)
{
if ((int)b > 0x7F)
{
predictedCodePage = CodePageJapan;
break;
}
}
return predictedCodePage;
}
public static byte GetHighNibble(byte value)
{
return (byte)(((value) >> 4) & 0x0F);
}
public static byte GetLowNibble(byte value)
{
return (byte)((value) & 0x0F); ;
}
public static byte[] GetBytesFromHexString(string hexValue)
{
int j = 0;
byte[] bytes = new byte[hexValue.Length / 2];
// convert the string to bytes
for (int i = 0; i < hexValue.Length; i += 2)
{
bytes[j] = BitConverter.GetBytes(Int16.Parse(hexValue.Substring(i, 2), System.Globalization.NumberStyles.AllowHexSpecifier, CultureInfo.CurrentCulture))[0];
j++;
}
return bytes;
}
public static byte[] GetBytesBigEndian(uint value)
{
byte[] ret = BitConverter.GetBytes(value);
Array.Reverse(ret);
return ret;
}
public static DateTime GetDateTimeFromFAT32Date(int value)
{
short xDate = (short)(value >> 0x10);
short xTime = (short)(value & 0xFFFF);
if (xDate == 0 && xTime == 0)
{
return DateTime.Now;
}
else
{
return new DateTime(
(((xDate & 0xFE00) >> 9) + 0x7BC),
((xDate & 0x1E0) >> 5),
(xDate & 0x1F),
((xTime & 0xF800) >> 0xB),
((xTime & 0x7E0) >> 5),
((xTime & 0x1F) * 2));
}
}
public static bool IsZeroFilledByteArray(byte[] value)
{
bool ret = true;
for (int i = 0; i < value.Length; i++)
{
if (value[i] != 0)
{
ret = false;
break;
}
}
return ret;
}
}
}

16
usm_extract/src/util/ByteSearchCalculatingOffsetDescription.cs Normal file
View file

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace VGMToolbox.util
{
public class ByteSearchCalculatingOffsetDescription : CalculatingOffsetDescription
{
public const string START_OF_STRING = "start of";
public const string END_OF_STRING = "end of";
public string RelativeLocationToByteString { set; get; }
public string ByteString { set; get; }
public bool TreatByteStringAsHex { set; get; }
}
}

13
usm_extract/src/util/CalculatingOffsetDescription.cs Normal file
View file

@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace VGMToolbox.util
{
public class CalculatingOffsetDescription : OffsetDescription
{
public const string OFFSET_VARIABLE_STRING = "$V";
public string CalculationString { set; get; }
}
}

196
usm_extract/src/util/ChecksumUtil.cs Normal file
View file

@ -0,0 +1,196 @@
// <copyright file="ChecksumUtil.cs" company="N/A">
// Copyright (c) 2008 All Rights Reserved
// </copyright>
// <author></author>
// <email></email>
// <date></date>
// <summary>Contains the ChecksumUtil class.</summary>
using System;
using System.Globalization;
using System.IO;
using System.Security.Cryptography;
using ICSharpCode.SharpZipLib.Checksums;
using Ionic.Zlib;
namespace VGMToolbox.util
{
/// <summary>
/// Class containing static functions related to checksum generation.
/// </summary>
public sealed class ChecksumUtil
{
/// <summary>
/// Prevents a default instance of the ChecksumUtil class from being created.
/// </summary>
private ChecksumUtil()
{
}
/// <summary>
/// Get the CRC32 checksum of the input stream.
/// </summary>
/// <param name="stream">File Stream for which to generate the checksum.</param>
/// <returns>String containing the hexidecimal representation of the CRC32 of the input stream.</returns>
public static string GetCrc32OfFullFile(FileStream stream)
{
// get incoming stream position
long initialStreamPosition = stream.Position;
// move to zero position
stream.Seek(0, SeekOrigin.Begin);
// calculate CRC32
CRC32 crc32 = new CRC32();
int ret = crc32.GetCrc32(stream);
// return stream to incoming position
stream.Position = initialStreamPosition;
return ret.ToString("X8", CultureInfo.InvariantCulture);
}
/// <summary>
/// Get the MD5 checksum of the input stream.
/// </summary>
/// <param name="stream">File Stream for which to generate the checksum.</param>
/// <returns>String containing the hexidecimal representation of the MD5 of the input stream.</returns>
public static string GetMd5OfFullFile(FileStream stream)
{
MD5CryptoServiceProvider hashMd5 = new MD5CryptoServiceProvider();
stream.Seek(0, SeekOrigin.Begin);
hashMd5.ComputeHash(stream);
return ParseFile.ByteArrayToString(hashMd5.Hash);
}
public static byte[] GetSha1(byte[] dataBlock)
{
SHA1CryptoServiceProvider sha1Hash = new SHA1CryptoServiceProvider();
sha1Hash.ComputeHash(dataBlock);
return sha1Hash.Hash;
}
/// <summary>
/// Get the SHA1 checksum of the input stream.
/// </summary>
/// <param name="stream">File Stream for which to generate the checksum.</param>
/// <returns>String containing the hexidecimal representation of the SHA1 of the input stream.</returns>
public static string GetSha1OfFullFile(FileStream stream)
{
SHA1CryptoServiceProvider sha1Hash = new SHA1CryptoServiceProvider();
stream.Seek(0, SeekOrigin.Begin);
sha1Hash.ComputeHash(stream);
return ParseFile.ByteArrayToString(sha1Hash.Hash);
}
/// <summary>
/// Get the SHA-512 checksum of the input stream.
/// </summary>
/// <param name="stream">File Stream for which to generate the checksum.</param>
/// <returns>String containing the hexidecimal representation of the SHA-512 of the input stream.</returns>
public static string GetSha512OfFullFile(FileStream stream)
{
SHA512CryptoServiceProvider sha512 = new SHA512CryptoServiceProvider();
stream.Seek(0, SeekOrigin.Begin);
sha512.ComputeHash(stream);
return ParseFile.ByteArrayToString(sha512.Hash);
}
/// <summary>
/// Adds a chunk of data to the input CRC32 generator.
/// </summary>
/// <param name="stream">Stream to read data from.</param>
/// <param name="startingOffset">Offset to begin reading from.</param>
/// <param name="length">Number of bytes to read.</param>
/// <param name="checksumGenerator">CRC32 generator.</param>
public static void AddChunkToChecksum(Stream stream, int startingOffset, int length, ref Crc32 checksumGenerator)
{
int remaining = length;
byte[] data = new byte[4096];
int read;
int offset = startingOffset;
stream.Seek((long)startingOffset, SeekOrigin.Begin);
while (remaining > 0)
{
if (remaining < 4096)
{
read = stream.Read(data, 0, remaining);
}
else
{
read = stream.Read(data, 0, 4096);
}
if (read <= 0)
{
throw new EndOfStreamException(
String.Format(
CultureInfo.CurrentCulture,
"End of stream reached with {0} bytes left to read",
remaining));
}
checksumGenerator.Update(data, 0, read);
remaining -= read;
offset += read;
}
}
/// <summary>
/// Adds a chunk of data to the input CRC32/MD5/SHA1 generator.
/// </summary>
/// <param name="sourceStream">Stream to read data from.</param>
/// <param name="startingOffset">Offset to begin reading from.</param>
/// <param name="length">Number of bytes to read.</param>
/// <param name="checksumGeneratorCrc32">CRC32 generator.</param>
/// <param name="checksumStreamMd5">MD5 generator.</param>
/// <param name="checksumStreamSha1">SHA1 generator.</param>
public static void AddChunkToChecksum(
Stream sourceStream,
int startingOffset,
int length,
ref Crc32 checksumGeneratorCrc32,
ref CryptoStream checksumStreamMd5,
ref CryptoStream checksumStreamSha1)
{
int remaining = length;
byte[] data = new byte[4096];
int read;
int offset = startingOffset;
sourceStream.Seek((long)startingOffset, SeekOrigin.Begin);
while (remaining > 0)
{
if (remaining < 4096)
{
read = sourceStream.Read(data, 0, remaining);
}
else
{
read = sourceStream.Read(data, 0, 4096);
}
if (read <= 0)
{
throw new EndOfStreamException(
String.Format(
CultureInfo.CurrentCulture,
"End of stream reached with {0} bytes left to read",
remaining));
}
checksumGeneratorCrc32.Update(data, 0, read);
checksumStreamMd5.Write(data, 0, read);
checksumStreamSha1.Write(data, 0, read);
remaining -= read;
offset += read;
}
}
}
}

496
usm_extract/src/util/CompressionUtil.cs Normal file
View file

@ -0,0 +1,496 @@
using System;
using System.IO;
using System.Reflection;
using Ionic.Zip;
using Ionic.Zlib;
using SevenZip;
namespace VGMToolbox.util
{
/// <summary>
/// Class containing static functions for compresson related tasks
/// </summary>
public sealed class CompressionUtil
{
/// <summary>
/// File extension used to output decompressed zlib data.
/// </summary>
public const string ZlibDecompressOutputExtension = ".zlibx";
/// <summary>
/// File extension used to output compressed zlib data.
/// </summary>
public const string ZlibCompressOutputExtension = ".zlib";
/// <summary>
/// File extension used to output decompressed gzip data.
/// </summary>
public const string GzipDecompressOutputExtension = ".gzipx";
/// <summary>
/// File extension used to output compressed gzip data.
/// </summary>
public const string GzipCompressOutputExtension = ".gz";
/// <summary>
/// Path to the included 7z.dll file.
/// </summary>
public static readonly string SevenZipDll =
Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "7z.dll");
/// <summary>
/// Prevents a default instance of the CompressionUtil class from being created
/// </summary>
private CompressionUtil()
{
}
/// <summary>
/// Get a list of files inside an archive file.
/// </summary>
/// <param name="path">Path to the archive file.</param>
/// <returns>An array of strings containing a list of files inside the archive.</returns>
/// <remarks>Must be an archive supported by 7z.dll.</remarks>
public static string[] GetFileList(string path)
{
string archivePath = Path.GetFullPath(path);
string[] filenames = null;
SevenZipExtractor sevenZipExtractor = null;
if (File.Exists(archivePath))
{
SevenZipExtractor.SetLibraryPath(SevenZipDll);
try
{
sevenZipExtractor = new SevenZipExtractor(archivePath);
filenames = new string[sevenZipExtractor.ArchiveFileNames.Count];
int i = 0;
foreach (string f in sevenZipExtractor.ArchiveFileNames)
{
filenames[i++] = f;
}
}
catch (System.ArgumentException)
{
// ignore unsupported formats
}
finally
{
if (sevenZipExtractor != null)
{
sevenZipExtractor.Dispose();
}
}
}
return filenames;
}
/// <summary>
/// Get an uppercase list of files inside an archive file. Must be an archive supported by 7z.dll.
/// </summary>
/// <param name="path">Path to the archive file.</param>
/// <returns>An array of strings containing an uppercase list of files inside the archive.</returns>
/// <remarks>Must be an archive supported by 7z.dll.</remarks>
public static string[] GetUpperCaseFileList(string path)
{
string archivePath = Path.GetFullPath(path);
string[] filenames = null;
SevenZipExtractor sevenZipExtractor = null;
if (File.Exists(archivePath))
{
SevenZipExtractor.SetLibraryPath(SevenZipDll);
try
{
sevenZipExtractor = new SevenZipExtractor(archivePath);
filenames = new string[sevenZipExtractor.ArchiveFileNames.Count];
int i = 0;
foreach (string f in sevenZipExtractor.ArchiveFileNames)
{
filenames[i++] = f.ToUpper();
}
}
catch (System.ArgumentException)
{
// ignore unsupported formats
}
finally
{
if (sevenZipExtractor != null)
{
sevenZipExtractor.Dispose();
}
}
}
return filenames;
}
public static bool Is7zSupportedArchive(string pArchivePath)
{
string archivePath = Path.GetFullPath(pArchivePath);
SevenZipExtractor sevenZipExtractor = null;
bool ret = false;
if (File.Exists(archivePath))
{
try
{
SevenZipExtractor.SetLibraryPath(SevenZipDll);
sevenZipExtractor = new SevenZipExtractor(archivePath);
sevenZipExtractor.Check();
ret = true;
}
catch (Exception)
{
ret = false;
}
finally
{
if (sevenZipExtractor != null)
{
sevenZipExtractor.Dispose();
}
}
}
return ret;
}
/// <summary>
/// Extracts a file from an archive. The file will be output to a subfolder in the archive's directory;
/// named with the original archive name.
/// </summary>
/// <param name="archivePath">Path to the archive file.</param>
/// <param name="fileName">Name of file to extract.</param>
/// <remarks>Must be an archive supported by 7z.dll.</remarks>
public static void ExtractFileFromArchive(string archivePath, string fileName)
{
ExtractFileFromArchive(archivePath, fileName, String.Empty);
}
/// <summary>
/// Extracts a file from an archive.
/// </summary>
/// <param name="pArchivePath">Path to the archive file.</param>
/// <param name="pFileName">Name of file to extract.</param>
/// <param name="pOutputPath">Folder to output the file to. If empty or null,
/// the file will be output to a subfolder in the archive's directory;
/// named with the original archive name.</param>
/// <remarks>Must be an archive supported by 7z.dll.</remarks>
public static void ExtractFileFromArchive(string pArchivePath, string pFileName, string pOutputPath)
{
string archivePath = Path.GetFullPath(pArchivePath);
SevenZipExtractor sevenZipExtractor = null;
string outputDir;
if (File.Exists(archivePath))
{
try
{
if (!String.IsNullOrEmpty(pOutputPath))
{
outputDir = pOutputPath;
}
else
{
outputDir = Path.Combine(Path.GetDirectoryName(archivePath), Path.GetFileNameWithoutExtension(archivePath));
}
if (!Directory.Exists(outputDir))
{
Directory.CreateDirectory(outputDir);
}
SevenZipExtractor.SetLibraryPath(SevenZipDll);
sevenZipExtractor = new SevenZipExtractor(archivePath);
sevenZipExtractor.ExtractFile(pFileName, outputDir, true);
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
finally
{
if (sevenZipExtractor != null)
{
sevenZipExtractor.Dispose();
}
}
}
}
public static void ExtractAllFilesFromArchive(string pArchivePath, string pOutputPath)
{
string archivePath = Path.GetFullPath(pArchivePath);
SevenZipExtractor sevenZipExtractor = null;
string outputDir;
if (File.Exists(archivePath))
{
try
{
if (!String.IsNullOrEmpty(pOutputPath))
{
outputDir = pOutputPath;
}
else
{
outputDir = Path.Combine(Path.GetDirectoryName(archivePath), Path.GetFileNameWithoutExtension(archivePath));
}
if (!Directory.Exists(outputDir))
{
Directory.CreateDirectory(outputDir);
}
SevenZipExtractor.SetLibraryPath(SevenZipDll);
sevenZipExtractor = new SevenZipExtractor(archivePath);
sevenZipExtractor.ExtractArchive(outputDir);
}
catch (Exception ex)
{
throw new Exception(ex.Message);
}
finally
{
if (sevenZipExtractor != null)
{
sevenZipExtractor.Dispose();
}
}
}
}
/// <summary>
/// Compress input folder (recursively) with 7zip Ultra compression.
/// </summary>
/// <param name="pSourcePath">Source directory to compress.</param>
/// <param name="pArchiveName">Fully rooted output archive name.</param>
public static void CompressFolderWith7zip(string pSourcePath, string pArchiveName)
{
SevenZipCompressor.SetLibraryPath(SevenZipDll);
SevenZipCompressor compressor = new SevenZipCompressor();
compressor.CompressionLevel = SevenZip.CompressionLevel.Ultra;
compressor.CompressDirectory(pSourcePath, pArchiveName, true);
}
/// <summary>
/// Extract all files from the input .zip compressed file.
/// </summary>
/// <param name="pZipFilePath">Fully rooted path to the .zip file.</param>
/// <param name="pOutputFolder">Fully rooted path to the output folder to place the deompressed files.</param>
public static void ExtractAllFilesFromZipFile(string pZipFilePath, string pOutputFolder)
{
using (ZipFile zip = ZipFile.Read(pZipFilePath))
{
zip.ExtractAll(pOutputFolder);
}
}
/// <summary>
/// Add a file to the input .zip file.
/// </summary>
/// <param name="pZipFileName">Fully rooted path to the .zip file to add files to.</param>
/// <param name="pNewEntrySourceFileName">Fully rooted path to the file to insert.</param>
/// <param name="pNewEntryDestinationName">Path within the .zip file to insert the new file as.</param>
public static void AddFileToZipFile(string pZipFileName, string pNewEntrySourceFileName, string pNewEntryDestinationName)
{
ZipFile zf;
// create or open zip file
if (File.Exists(pZipFileName))
{
zf = ZipFile.Read(pZipFileName);
}
else
{
zf = new ZipFile(pZipFileName);
}
zf.CompressionLevel = Ionic.Zlib.CompressionLevel.BestCompression;
using (FileStream fs = File.OpenRead(pNewEntrySourceFileName))
{
zf.AddEntry(Path.GetFileName(pNewEntryDestinationName), Path.GetDirectoryName(pNewEntryDestinationName), fs);
zf.Save();
}
}
/// <summary>
/// Decompress a zlib compressed section of a stream to file. Data must begin at offset 0.
/// </summary>
/// <param name="pStream">Stream containing zlib compressed bytes.</param>
/// <param name="pOutputFilePath">Fully rooted output file name to output the decompressed data to.</param>
public static void DecompressZlibStreamToFile(Stream pStream, string pOutputFilePath)
{
DecompressZlibStreamToFile(pStream, pOutputFilePath, 0);
}
/// <summary>
/// Decompress a zlib compressed section of a stream to file.
/// </summary>
/// <param name="pStream">Stream containing zlib compressed bytes.</param>
/// <param name="pOutputFilePath">Fully rooted output file name to output the decompressed data to.</param>
/// <param name="pStartingOffset">Offset to begin reading data from.</param>
public static void DecompressZlibStreamToFile(Stream pStream, string pOutputFilePath, long pStartingOffset)
{
using (FileStream outFs = new FileStream(pOutputFilePath, FileMode.Create, FileAccess.Write))
{
using (BinaryWriter bw = new BinaryWriter(outFs))
{
pStream.Position = pStartingOffset;
using (ZlibStream zs = new ZlibStream(pStream, CompressionMode.Decompress, true))
{
int read;
byte[] data = new byte[Constants.FileReadChunkSize];
while ((read = zs.Read(data, 0, data.Length)) > 0)
{
bw.Write(data, 0, read);
}
}
}
}
}
/// <summary>
/// Compress the incoming stream to a file using zlib compression.
/// </summary>
/// <param name="pStream">Stream to compress.</param>
/// <param name="pOutputFilePath">Path to the file to output.</param>
public static void CompressStreamToZlibFile(Stream pStream, string pOutputFilePath)
{
CompressStreamToZlibFile(pStream, pOutputFilePath, 0);
}
/// <summary>
/// Compress the incoming stream to a file using zlib compression starting at the incoming offset.
/// </summary>
/// <param name="pStream">Stream to compress.</param>
/// <param name="pOutputFilePath">Path to the file to output.</param>
/// <param name="pStartingOffset">Offset within the stream to start compressing.</param>
public static void CompressStreamToZlibFile(Stream pStream, string pOutputFilePath, long pStartingOffset)
{
using (FileStream outFs = new FileStream(pOutputFilePath, FileMode.Create, FileAccess.Write))
{
using (BinaryReader br = new BinaryReader(pStream))
{
pStream.Position = pStartingOffset;
using (ZlibStream zs = new ZlibStream(outFs, CompressionMode.Compress, Ionic.Zlib.CompressionLevel.BestCompression, true))
{
int read;
byte[] data = new byte[Constants.FileReadChunkSize];
while ((read = br.Read(data, 0, data.Length)) > 0)
{
zs.Write(data, 0, read);
}
zs.Flush();
}
}
}
}
/// <summary>
/// Compress an entire file using gzip compression.
/// </summary>
/// <param name="pFileName">Fully rooted file name of file to compress.</param>
public static void GzipEntireFile(string pFileName)
{
string tempFileName;
if (File.Exists(pFileName))
{
using (FileStream fs = File.OpenRead(pFileName))
{
tempFileName = Path.GetTempFileName();
using (FileStream outFs = File.OpenWrite(tempFileName))
{
using (GZipStream gs = new GZipStream(outFs, CompressionMode.Compress, Ionic.Zlib.CompressionLevel.BestCompression))
{
int read;
byte[] data = new byte[Constants.FileReadChunkSize];
while ((read = fs.Read(data, 0, data.Length)) > 0)
{
gs.Write(data, 0, read);
}
}
}
}
File.Delete(pFileName);
File.Move(tempFileName, pFileName);
}
}
/// <summary>
/// Decompress a gzip compressed section of a stream to file.
/// </summary>
/// <param name="pStream">Stream containing gzip compressed bytes.</param>
/// <param name="pOutputFilePath">Fully rooted output file name to output the decompressed data to.</param>
/// <param name="pStartingOffset">Offset to begin reading data from.</param>
public static void DecompressGzipStreamToFile(Stream pStream, string pOutputFilePath, long pStartingOffset)
{
using (FileStream outFs = new FileStream(pOutputFilePath, FileMode.Create, FileAccess.Write))
{
using (BinaryWriter bw = new BinaryWriter(outFs))
{
pStream.Position = pStartingOffset;
using (GZipStream gs = new GZipStream(pStream, CompressionMode.Decompress, true))
{
int read;
byte[] data = new byte[Constants.FileReadChunkSize];
while ((read = gs.Read(data, 0, data.Length)) > 0)
{
bw.Write(data, 0, read);
}
}
}
}
}
/// <summary>
/// Compress the incoming stream to a file using gzip compression starting at the incoming offset.
/// </summary>
/// <param name="pStream">Stream to compress.</param>
/// <param name="pOutputFilePath">Path to the file to output.</param>
/// <param name="pStartingOffset">Offset within the stream to start compressing.</param>
public static void CompressStreamToGzipFile(Stream pStream, string pOutputFilePath, long pStartingOffset)
{
using (FileStream outFs = new FileStream(pOutputFilePath, FileMode.Create, FileAccess.Write))
{
using (BinaryReader br = new BinaryReader(pStream))
{
pStream.Position = pStartingOffset;
using (GZipStream gs = new GZipStream(outFs, CompressionMode.Compress, Ionic.Zlib.CompressionLevel.BestCompression, true))
{
int read;
byte[] data = new byte[Constants.FileReadChunkSize];
while ((read = br.Read(data, 0, data.Length)) > 0)
{
gs.Write(data, 0, read);
}
gs.Flush();
}
}
}
}
}
}

453
usm_extract/src/util/Constants.cs Normal file
View file

@ -0,0 +1,453 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Windows.Forms;
namespace VGMToolbox.util
{
/// <summary>
/// Struct containing criteria used to find offsets.
/// </summary>
public struct FindOffsetStruct
{
private string searchString;
private string startingOffset;
private bool treatSearchStringAsHex;
private bool cutFile;
private string searchStringOffset;
private string cutSize;
private string cutSizeOffsetSize;
private bool isCutSizeAnOffset;
private string outputFileExtension;
private bool isLittleEndian;
private bool useTerminatorForCutSize;
private string terminatorString;
private bool treatTerminatorStringAsHex;
private bool includeTerminatorLength;
private string extraCutSizeBytes;
public bool DoSearchStringModulo
{
set;
get;
}
public string SearchStringModuloDivisor
{
set;
get;
}
public string SearchStringModuloResult
{
set;
get;
}
public bool DoTerminatorModulo
{
set;
get;
}
public string TerminatorStringModuloDivisor
{
set;
get;
}
public string TerminatorStringModuloResult
{
set;
get;
}
public string MinimumSize
{
set;
get;
}
public string SearchString
{
get { return searchString; }
set { searchString = value; }
}
/// <summary>
/// Gets or sets offset to being searching at
/// </summary>
public string StartingOffset
{
set { startingOffset = value; }
get { return startingOffset; }
}
/// <summary>
/// Gets or sets flag to indicate search string is a hex value.
/// </summary>
public bool TreatSearchStringAsHex
{
get { return treatSearchStringAsHex; }
set { treatSearchStringAsHex = value; }
}
/// <summary>
/// Gets or sets flag to cut the file when the offset is found.
/// </summary>
public bool CutFile
{
get { return cutFile; }
set { cutFile = value; }
}
/// <summary>
/// Gets or sets offset within destination file Search String would reside.
/// </summary>
public string SearchStringOffset
{
get { return searchStringOffset; }
set { searchStringOffset = value; }
}
/// <summary>
/// Gets or sets size to cut from file
/// </summary>
public string CutSize
{
get { return cutSize; }
set { cutSize = value; }
}
/// <summary>
/// Gets or sets size of offset holding cut size
/// </summary>
public string CutSizeOffsetSize
{
get { return cutSizeOffsetSize; }
set { cutSizeOffsetSize = value; }
}
/// <summary>
/// Gets or sets flag indicating that cut size is an offset.
/// </summary>
public bool IsCutSizeAnOffset
{
get { return isCutSizeAnOffset; }
set { isCutSizeAnOffset = value; }
}
/// <summary>
/// Gets or sets file extension to use for cut files.
/// </summary>
public string OutputFileExtension
{
get { return outputFileExtension; }
set { outputFileExtension = value; }
}
/// <summary>
/// Gets or sets flag indicating that offset based cut size is stored in Little Endian byte order.
/// </summary>
public bool IsLittleEndian
{
get { return isLittleEndian; }
set { isLittleEndian = value; }
}
/// <summary>
/// Gets or sets flag indicating that a terminator should be used to determine the cut size.
/// </summary>
public bool UseLengthMultiplier { set; get; }
public string LengthMultiplier { set; get; }
public bool UseTerminatorForCutSize
{
get { return useTerminatorForCutSize; }
set { useTerminatorForCutSize = value; }
}
/// <summary>
/// Gets or sets terminator string to search for.
/// </summary>
public string TerminatorString
{
get { return terminatorString; }
set { terminatorString = value; }
}
/// <summary>
/// Gets or sets flag indicating that Terminator String is hex.
/// </summary>
public bool TreatTerminatorStringAsHex
{
get { return treatTerminatorStringAsHex; }
set { treatTerminatorStringAsHex = value; }
}
/// <summary>
/// Gets or sets flag indicating that the length of the terminator should be included in the cut size.
/// </summary>
public bool IncludeTerminatorLength
{
get { return includeTerminatorLength; }
set { includeTerminatorLength = value; }
}
public bool CutToEofIfTerminatorNotFound { set; get; }
/// <summary>
/// Gets or sets additional bytes to include in the cut size.
/// </summary>
public string ExtraCutSizeBytes
{
get { return extraCutSizeBytes; }
set { extraCutSizeBytes = value; }
}
public string OutputFolder { get; set; }
}
/// <summary>
/// Struct used to send messages conveying progress.
/// </summary>
public struct ProgressStruct
{
/// <summary>
/// File name to display in progress bar.
/// </summary>
private string fileName;
/// <summary>
/// Error message to display in output window.
/// </summary>
private string errorMessage;
/// <summary>
/// Generic message to display in output window.
/// </summary>
private string genericMessage;
/// <summary>
/// New tree node to add to a TreeView.
/// </summary>
private TreeNode newNode;
/// <summary>
/// Gets or sets fileName.
/// </summary>
public string FileName
{
get { return fileName; }
set { fileName = value; }
}
/// <summary>
/// Gets or sets errorMessage.
/// </summary>
public string ErrorMessage
{
get { return errorMessage; }
set { errorMessage = value; }
}
/// <summary>
/// Gets or sets genericMessage.
/// </summary>
public string GenericMessage
{
get { return genericMessage; }
set { genericMessage = value; }
}
/// <summary>
/// Gets or sets newNode.
/// </summary>
public TreeNode NewNode
{
get { return newNode; }
set { newNode = value; }
}
/// <summary>
/// Reset this node's values
/// </summary>
public void Clear()
{
fileName = String.Empty;
errorMessage = String.Empty;
genericMessage = String.Empty;
newNode = null;
}
}
/// <summary>
/// Struct used to allow TreeView to select a specific form and modify the originating node upon completion of a task.
/// </summary>
public struct NodeTagStruct
{
/// <summary>
/// Class name of the Form this node will bring to focus.
/// </summary>
private string formClass;
/// <summary>
/// Object type this node represents.
/// </summary>
private string objectType;
/// <summary>
/// File path of the file this node represents.
/// </summary>
private string filePath;
/// <summary>
/// Gets or sets formClass
/// </summary>
public string FormClass
{
get { return formClass; }
set { formClass = value; }
}
/// <summary>
/// Gets or sets objectType
/// </summary>
public string ObjectType
{
get { return objectType; }
set { objectType = value; }
}
/// <summary>
/// Gets or sets filePath
/// </summary>
public string FilePath
{
get { return filePath; }
set { filePath = value; }
}
}
public enum VfsFileRecordRelativeOffsetLocationType
{
FileRecordStart,
FileRecordEnd
}
public struct VfsExtractionStruct
{
// header size
public bool UseStaticHeaderSize { set; get; }
public string StaticHeaderSize { set; get; }
public bool UseHeaderSizeOffset { set; get; }
public OffsetDescription HeaderSizeOffsetDescription { set; get; }
public bool ReadHeaderToEof { set; get; }
// file count
public bool UseStaticFileCount { set; get; }
public string StaticFileCount { set; get; }
public bool UseFileCountOffset { set; get; }
public OffsetDescription FileCountOffsetDescription { set; get; }
// file record basic information
public string FileRecordsStartOffset { set; get; }
public string FileRecordSize { set; get; }
// file offset
public bool UseFileOffsetOffset { set; get; }
public CalculatingOffsetDescription FileOffsetOffsetDescription { set; get; }
public bool UsePreviousFilesSizeToDetermineOffset { set; get; }
public string BeginCuttingFilesAtOffset { set; get; }
public bool UseByteAlignmentValue { set; get; }
public string ByteAlignmentValue { set; get; }
// file length/size
public bool UseFileLengthOffset { set; get; }
public CalculatingOffsetDescription FileLengthOffsetDescription { set; get; }
public bool UseLocationOfNextFileToDetermineLength { set; get; }
// file name
public bool FileNameIsPresent { set; get; }
public bool UseStaticFileNameOffsetWithinRecord { set; get; }
public string StaticFileNameOffsetWithinRecord { set; get; }
public bool UseAbsoluteFileNameOffset { set; get; }
public OffsetDescription AbsoluteFileNameOffsetDescription { set; get; }
public bool UseRelativeFileNameOffset { set; get; }
public OffsetDescription RelativeFileNameOffsetDescription { set; get; }
public VfsFileRecordRelativeOffsetLocationType FileRecordNameRelativeOffsetLocation { set; get; }
// name size
public bool UseStaticFileNameLength { set; get; }
public string StaticFileNameLength { set; get; }
public bool UseFileNameTerminatorString { set; get; }
public string FileNameTerminatorString { set; get; }
}
public struct SimpleFileExtractionStruct
{
public string FilePath { set; get; }
public long FileOffset { set; get; }
public long FileLength { set; get; }
public long FileNameLength { set; get; }
public void Clear()
{
this.FilePath = String.Empty;
this.FileOffset = -1;
this.FileLength = -1;
this.FileNameLength = -1;
}
}
/// <summary>
/// Class containing universal constants.
/// </summary>
public sealed class Constants
{
/// <summary>
/// Chunk size to use when reading from files. Used to grab maximum buffer
/// size without using the large object heap which has poor collection.
/// </summary>
public const int FileReadChunkSize = 71680;
/// <summary>
/// Constant used to send an ignore the value message to the progress bar.
/// </summary>
public const int IgnoreProgress = -1;
/// <summary>
/// Constant used to send a generic message to the progress bar.
/// </summary>
public const int ProgressMessageOnly = -2;
/// <summary>
/// Text description to use when describing a Big Endian option
/// </summary>
public const string BigEndianByteOrder = "Big Endian";
/// <summary>
/// Text description to use when describing a Little Endian option
/// </summary>
public const string LittleEndianByteOrder = "Little Endian";
public static readonly byte[] RiffHeaderBytes = new byte[] { 0x52, 0x49, 0x46, 0x46 };
public static readonly byte[] RiffDataBytes = new byte[] { 0x64, 0x61, 0x74, 0x61 };
public static readonly byte[] RiffWaveBytes = new byte[] { 0x57, 0x41, 0x56, 0x45 };
public static readonly byte[] RiffFmtBytes = new byte[] { 0x66, 0x6D, 0x74, 0x20 };
public static readonly byte[] NullByteArray = new byte[] { 0x00 };
public const string StringNullTerminator = "\0";
// empty constructor
private Constants() { }
}
}

706
usm_extract/src/util/FileUtil.cs Normal file
View file

@ -0,0 +1,706 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
namespace VGMToolbox.util
{
public sealed class FileUtil
{
private FileUtil() { }
/// <summary>
/// Reads data into a complete array, throwing an EndOfStreamException
/// if the stream runs out of data first, or if an IOException
/// naturally occurs.
/// </summary>
/// <param name="stream">The stream to read data from</param>
/// <param name="data">The array to read bytes into. The array
/// will be completely filled from the stream, so an appropriate
/// size must be given.</param>
public static void ReadWholeArray(Stream stream, byte[] data)
{
ReadWholeArray(stream, data, data.Length);
}
public static void ReadWholeArray(Stream stream, byte[] data, int pLength)
{
int offset = 0;
int remaining = pLength;
while (remaining > 0)
{
int read = stream.Read(data, offset, remaining);
if (read <= 0)
{
throw new EndOfStreamException(
String.Format(CultureInfo.CurrentCulture, "End of stream reached with {0} bytes left to read", remaining));
}
remaining -= read;
offset += read;
}
}
/// <summary>
/// Replaces 0x00 with 0x20 in an array of bytes.
/// </summary>
/// <param name="value">Array of bytes to alter.</param>
/// <returns>Original array with 0x00 replaced by 0x20.</returns>
public static byte[] ReplaceNullByteWithSpace(byte[] value)
{
for (int i = 0; i < value.Length; i++)
{
if (value[i] == 0x00)
{
value[i] = 0x20;
}
}
return value;
}
/// <summary>
/// Returns the count of files contained in the input directories and their subdirectories.
/// </summary>
/// <param name="paths">Paths to count files within.</param>
/// <returns>Number of files in the incoming directories and their subdirectories.</returns>
public static int GetFileCount(string[] paths)
{
return GetFileCount(paths, true);
}
public static int GetFileCount(string[] paths, bool includeSubdirs)
{
int totalFileCount = 0;
foreach (string path in paths)
{
if (File.Exists(path))
{
totalFileCount++;
}
else if (Directory.Exists(path))
{
if (includeSubdirs)
{
totalFileCount += Directory.GetFiles(path, "*.*", SearchOption.AllDirectories).Length;
}
else
{
totalFileCount += Directory.GetFiles(path, "*.*", SearchOption.TopDirectoryOnly).Length;
}
}
}
return totalFileCount;
}
public static string CleanFileName(string pDirtyFileName)
{
foreach (char c in Path.GetInvalidFileNameChars())
{
pDirtyFileName = pDirtyFileName.Replace(c, '_');
}
return pDirtyFileName;
}
public static void UpdateTextField(
string pFilePath,
string pFieldValue,
int pOffset,
int pMaxLength)
{
System.Text.Encoding enc = System.Text.Encoding.ASCII;
using (BinaryWriter bw =
new BinaryWriter(File.Open(pFilePath, FileMode.Open, FileAccess.ReadWrite)))
{
byte[] newBytes = new byte[pMaxLength];
byte[] convertedBytes = enc.GetBytes(pFieldValue);
int numBytesToCopy =
convertedBytes.Length <= pMaxLength ? convertedBytes.Length : pMaxLength;
Array.ConstrainedCopy(convertedBytes, 0, newBytes, 0, numBytesToCopy);
bw.Seek(pOffset, SeekOrigin.Begin);
bw.Write(newBytes);
}
}
public static void UpdateChunk(
string pFilePath,
int pOffset,
byte[] value)
{
using (BinaryWriter bw =
new BinaryWriter(File.Open(pFilePath, FileMode.Open, FileAccess.ReadWrite)))
{
bw.Seek(pOffset, SeekOrigin.Begin);
bw.Write(value);
}
}
public static void ReplaceFileChunk(
string pSourceFilePath,
long pSourceOffset,
long pLength,
string pDestinationFilePath,
long pDestinationOffset)
{
int read = 0;
long maxread;
int totalBytes = 0;
byte[] bytes = new byte[Constants.FileReadChunkSize];
using (BinaryWriter bw =
new BinaryWriter(File.Open(pDestinationFilePath, FileMode.Open, FileAccess.ReadWrite)))
{
using (BinaryReader br =
new BinaryReader(File.Open(pSourceFilePath, FileMode.Open, FileAccess.Read)))
{
br.BaseStream.Position = pSourceOffset;
bw.BaseStream.Position = pDestinationOffset;
maxread = pLength > bytes.Length ? bytes.Length : pLength;
while ((read = br.Read(bytes, 0, (int)maxread)) > 0)
{
bw.Write(bytes, 0, read);
totalBytes += read;
maxread = (pLength - totalBytes) > bytes.Length ? bytes.Length : (pLength - totalBytes);
}
}
}
}
public static void ZeroOutFileChunk(string pPath, long pOffset, int pLength)
{
int bytesToWrite = pLength;
byte[] bytes;
int maxWrite = bytesToWrite > Constants.FileReadChunkSize ? Constants.FileReadChunkSize : bytesToWrite;
using (BinaryWriter bw =
new BinaryWriter(File.Open(pPath, FileMode.Open, FileAccess.Write)))
{
bw.BaseStream.Position = pOffset;
while (bytesToWrite > 0)
{
bytes = new byte[maxWrite];
bw.Write(bytes);
bytesToWrite -= maxWrite;
maxWrite = bytesToWrite > bytes.Length ? bytes.Length : bytesToWrite;
}
}
}
public static void TrimFileToLength(string path, long totalLength)
{
string fullPath = Path.GetFullPath(path);
if (File.Exists(fullPath))
{
string destinationPath = Path.ChangeExtension(fullPath, ".trimmed");
using (FileStream fs = File.OpenRead(path))
{
ParseFile.ExtractChunkToFile(fs, 0, totalLength, destinationPath);
}
File.Copy(destinationPath, path, true);
File.Delete(destinationPath);
}
}
public static string RemoveChunkFromFile(string path, long startingOffset, long length)
{
string fullPath = Path.GetFullPath(path);
int bytesRead;
byte[] bytes = new byte[1024];
string ret = String.Empty;
if (File.Exists(fullPath))
{
string destinationPath = Path.ChangeExtension(fullPath, ".cut");
using (FileStream sourceFs = File.OpenRead(fullPath))
{
// extract initial chunk
ParseFile.ExtractChunkToFile(sourceFs, 0, startingOffset, destinationPath);
// append remainder
using (FileStream outFs = File.Open(destinationPath, FileMode.Append, FileAccess.Write))
{
sourceFs.Position = startingOffset + length;
bytesRead = sourceFs.Read(bytes, 0, bytes.Length);
while (bytesRead > 0)
{
outFs.Write(bytes, 0, bytesRead);
bytesRead = sourceFs.Read(bytes, 0, bytes.Length);
}
}
ret = destinationPath;
}
}
return ret;
}
public static string RemoveAllChunksFromFile(FileStream fs, byte[] chunkToRemove)
{
int bytesRead;
long currentReadOffset = 0;
long totalBytesRead = 0;
int maxReadSize = 0;
long currentChunkSize;
byte[] bytes = new byte[Constants.FileReadChunkSize];
long[] offsets = ParseFile.GetAllOffsets(fs, 0, chunkToRemove, false, -1, -1, true);
string destinationPath = Path.ChangeExtension(fs.Name, ".cut");
using (FileStream destinationFs = File.OpenWrite(destinationPath))
{
for (int i = 0; i < offsets.Length; i++)
{
// move position
fs.Position = currentReadOffset;
// get length of current size to write
currentChunkSize = offsets[i] - currentReadOffset;
// calculcate max cut size for this loop iteration
maxReadSize = (currentChunkSize - totalBytesRead) > (long)bytes.Length ? bytes.Length : (int)(currentChunkSize - totalBytesRead);
while ((bytesRead = fs.Read(bytes, 0, maxReadSize)) > 0)
{
destinationFs.Write(bytes, 0, bytesRead);
totalBytesRead += (long)bytesRead;
maxReadSize = (currentChunkSize - totalBytesRead) > (long)bytes.Length ? bytes.Length : (int)(currentChunkSize - totalBytesRead);
}
totalBytesRead = 0;
currentReadOffset = offsets[i] + chunkToRemove.Length;
}
////////////////////////////
// write remainder of file
////////////////////////////
// move position
fs.Position = currentReadOffset;
// get length of current size to write
currentChunkSize = fs.Length - currentReadOffset;
// calculcate max cut size
maxReadSize = (currentChunkSize - totalBytesRead) > (long)bytes.Length ? bytes.Length : (int)(currentChunkSize - totalBytesRead);
while ((bytesRead = fs.Read(bytes, 0, maxReadSize)) > 0)
{
destinationFs.Write(bytes, 0, bytesRead);
totalBytesRead += (long)bytesRead;
maxReadSize = (currentChunkSize - totalBytesRead) > (long)bytes.Length ? bytes.Length : (int)(currentChunkSize - totalBytesRead);
}
}
return destinationPath;
}
public static string RemoveAllChunksFromFile(string path, byte[] chunkToRemove)
{
string destinationPath;
using (FileStream fs = File.OpenRead(path))
{
destinationPath = RemoveAllChunksFromFile(fs, chunkToRemove);
}
return destinationPath;
}
public static bool ExecuteExternalProgram(
string pathToExecuatable,
string arguments,
string workingDirectory,
out string standardOut,
out string standardError)
{
Process externalExecutable;
bool isSuccess = false;
standardOut = String.Empty;
standardError = String.Empty;
using (externalExecutable = new Process())
{
externalExecutable.StartInfo = new ProcessStartInfo(pathToExecuatable, arguments);
externalExecutable.StartInfo.WorkingDirectory = workingDirectory;
externalExecutable.StartInfo.UseShellExecute = false;
externalExecutable.StartInfo.CreateNoWindow = true;
externalExecutable.StartInfo.RedirectStandardOutput = true;
externalExecutable.StartInfo.RedirectStandardError = true;
isSuccess = externalExecutable.Start();
standardOut = externalExecutable.StandardOutput.ReadToEnd();
standardError = externalExecutable.StandardError.ReadToEnd();
externalExecutable.WaitForExit();
}
return isSuccess;
}
public static void AddHeaderToFile(byte[] headerBytes, string sourceFile, string destinationFile)
{
int bytesRead;
byte[] readBuffer = new byte[Constants.FileReadChunkSize];
using (FileStream destinationStream = File.Open(destinationFile, FileMode.CreateNew, FileAccess.Write))
{
// write header
destinationStream.Write(headerBytes, 0, headerBytes.Length);
// write the source file
using (FileStream sourceStream = File.Open(sourceFile, FileMode.Open, FileAccess.Read))
{
while ((bytesRead = sourceStream.Read(readBuffer, 0, readBuffer.Length)) > 0)
{
destinationStream.Write(readBuffer, 0, bytesRead);
}
}
}
}
public static void RenameFileUsingInternalName(string path,
long offset, int length, byte[] terminatorBytes, bool maintainFileExtension)
{
string destinationDirectory = Path.GetDirectoryName(path);
string destinationFile;
string originalExtension;
int nameLength;
byte[] nameByteArray;
using (FileStream fs = File.OpenRead(path))
{
if (terminatorBytes != null)
{
nameLength = ParseFile.GetSegmentLength(fs, (int)offset, terminatorBytes);
}
else
{
nameLength = length;
}
if (nameLength < 1)
{
throw new ArgumentOutOfRangeException("Name Length", "Name Length is less than 1.");
}
if (maintainFileExtension)
{
originalExtension = Path.GetExtension(path);
}
nameByteArray = ParseFile.ParseSimpleOffset(fs, offset, nameLength);
destinationFile = ByteConversion.GetAsciiText(FileUtil.ReplaceNullByteWithSpace(nameByteArray)).Trim();
destinationFile = Path.Combine(destinationDirectory, destinationFile);
if (maintainFileExtension)
{
originalExtension = Path.GetExtension(path);
destinationFile = Path.ChangeExtension(destinationFile, originalExtension);
}
}
// try to copy using the new name
if (!path.Equals(destinationFile))
{
try
{
if (!Directory.Exists(Path.GetDirectoryName(destinationFile)))
{
Directory.CreateDirectory(Path.GetDirectoryName(destinationFile));
}
if (File.Exists(destinationFile))
{
string[] sameNamedFiles = Directory.GetFiles(Path.GetDirectoryName(destinationFile), Path.GetFileNameWithoutExtension(destinationFile) + "*");
// rename to prevent overwrite
destinationFile = Path.Combine(Path.GetDirectoryName(destinationFile), String.Format("{0}_{1}{2}", Path.GetFileNameWithoutExtension(destinationFile), sameNamedFiles.Length.ToString("D4"), Path.GetExtension(destinationFile)));
}
File.Copy(path, destinationFile);
File.Delete(path);
}
catch (Exception ex)
{
throw new Exception(ex.Message, ex);
}
}
}
public static void InterleaveFiles(string[] sourceFiles, uint interleaveValue,
long startOffest, byte[] paddingBytes, string destinationFile)
{
long currentOffset = 0;
long maxLength = 0;
long sizeDifference = 0;
long bytesToWrite;
long bytesRemaining;
FileStream[] inputStreams = new FileStream[sourceFiles.Length];
long[] fileLengths = new long[sourceFiles.Length];
destinationFile = Path.GetFullPath(destinationFile);
// open destination file for writing
using (FileStream destinationStream = File.OpenWrite(destinationFile))
{
try
{
// build input streams array
for (int i = 0; i < sourceFiles.Length; i++)
{
inputStreams[i] = File.OpenRead(sourceFiles[i]);
fileLengths[i] = inputStreams[i].Length;
// get max file length
if (maxLength < fileLengths[i])
{
maxLength = fileLengths[i];
}
}
// write out blocks
currentOffset = startOffest;
while (currentOffset < maxLength)
{
for (int i = 0; i < sourceFiles.Length; i++)
{
if (currentOffset + interleaveValue < fileLengths[i])
{
// write from file
destinationStream.Write(
ParseFile.ParseSimpleOffset(inputStreams[i], currentOffset, (int)interleaveValue),
0, (int)interleaveValue);
}
else if (currentOffset < fileLengths[i])
{
// write some from file, some from padding
sizeDifference = (currentOffset + interleaveValue) - fileLengths[i];
// write from file
destinationStream.Write(
ParseFile.ParseSimpleOffset(inputStreams[i], currentOffset, (int)(interleaveValue - sizeDifference)),
0, (int)(int)(interleaveValue - sizeDifference));
// write padding bytes
bytesRemaining = sizeDifference;
while (bytesRemaining > 0)
{
bytesToWrite = bytesRemaining > paddingBytes.Length ? paddingBytes.Length : bytesRemaining;
destinationStream.Write(paddingBytes, 0, (int)bytesToWrite);
bytesRemaining -= bytesToWrite;
}
}
else
{
// write padding bytes
bytesRemaining = interleaveValue;
while (bytesRemaining > 0)
{
bytesToWrite = bytesRemaining > paddingBytes.Length ? paddingBytes.Length : bytesRemaining;
destinationStream.Write(paddingBytes, 0, (int)bytesToWrite);
bytesRemaining -= bytesToWrite;
}
}
}
// increment offset
currentOffset += interleaveValue;
}
}
catch (Exception ex)
{
throw new Exception(ex.Message, ex);
}
finally
{
// close all readers
for (int i = 0; i < inputStreams.Length; i++)
{
if (inputStreams[i].CanRead)
{
inputStreams[i].Close();
inputStreams[i].Dispose();
}
}
}
}
}
public static string GetNonDuplicateFileName(string destinationFile)
{
if (File.Exists(destinationFile))
{
string[] sameNamedFiles = Directory.GetFiles(Path.GetDirectoryName(destinationFile), Path.GetFileNameWithoutExtension(destinationFile) + "*");
// rename to prevent overwrite
destinationFile = Path.Combine(Path.GetDirectoryName(destinationFile), String.Format("{0}_{1}{2}", Path.GetFileNameWithoutExtension(destinationFile), sameNamedFiles.Length.ToString("D4"), Path.GetExtension(destinationFile)));
}
return destinationFile;
}
public static string[] SplitFile(string sourceFile, long startingOffset, ulong chunkSizeInBytes, string outputFolder)
{
string[] outputFileList;
string outputFileName;
ArrayList outputFiles = new ArrayList();
long fileLength;
ulong currentOffset;
int chunkCount;
using (FileStream sourceStream = File.OpenRead(sourceFile))
{
// get file length
fileLength = sourceStream.Length;
// init counters
chunkCount = 1;
currentOffset = (ulong)startingOffset;
while (currentOffset < (ulong)fileLength)
{
// construct output file name
outputFileName = Path.Combine(outputFolder,
String.Format("{0}.{1}",
Path.GetFileName(sourceFile),
chunkCount.ToString("D3")));
ParseFile.ExtractChunkToFile64(sourceStream, currentOffset, chunkSizeInBytes,
outputFileName, true, true);
outputFiles.Add(outputFileName);
currentOffset += chunkSizeInBytes;
chunkCount++;
}
}
outputFileList = (string[])outputFiles.ToArray(typeof(string));
return outputFileList;
}
public static long? GetFileSize(string path)
{
long? fileSize;
try
{
FileInfo fi = new FileInfo(Path.GetFullPath(path));
fileSize = fi.Length;
}
catch (Exception)
{
fileSize = null;
}
return fileSize;
}
public static void CreateFileFromByteArray(string destinationFile, byte[] sourceBytes)
{
string destinationDirectory = Path.GetDirectoryName(destinationFile);
if (!Directory.Exists(destinationDirectory))
{
Directory.CreateDirectory(destinationDirectory);
}
using (FileStream outStream = File.Open(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read))
{
outStream.Write(sourceBytes, 0, sourceBytes.Length);
}
}
public static void CreateFileFromString(string destinationFile, string sourceText)
{
string destinationDirectory = Path.GetDirectoryName(destinationFile);
if (!Directory.Exists(destinationDirectory))
{
Directory.CreateDirectory(destinationDirectory);
}
using ( FileStream outStream = File.Open(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read))
{
using (StreamWriter sw = new StreamWriter(outStream, Encoding.ASCII))
{
sw.Write(sourceText);
}
}
}
public static string GetStringFromFileChunk(FileStream fs, ulong offset, ulong size)
{
StringBuilder sb = new StringBuilder();
int read = 0;
ulong totalBytes = 0;
byte[] bytes = new byte[Constants.FileReadChunkSize];
// reset location
fs.Seek(0, SeekOrigin.Begin);
// move stream pointer in long size chunks
while (offset > long.MaxValue) //
{
fs.Seek(long.MaxValue, SeekOrigin.Current);
offset -= long.MaxValue;
}
// less than a long now, should be ok
fs.Seek((long)offset, SeekOrigin.Current);
int maxread = size > (ulong)bytes.Length ? bytes.Length : (int)size;
while ((read = fs.Read(bytes, 0, maxread)) > 0)
{
for (int i = 0; i < read; i++)
{
sb.Append(bytes[i].ToString("X2"));
}
totalBytes += (ulong)read;
maxread = (size - totalBytes) > (ulong)bytes.Length ? bytes.Length : (int)(size - totalBytes);
}
return sb.ToString();
}
}
}

7
usm_extract/src/util/IDeepCopy.cs Normal file
View file

@ -0,0 +1,7 @@
namespace VGMToolbox.util
{
public interface IDeepCopy<T>
{
T DeepCopy();
}
}

26
usm_extract/src/util/MathUtil.cs Normal file
View file

@ -0,0 +1,26 @@
using System;
namespace VGMToolbox.util
{
public class MathUtil
{
public static long RoundUpToByteAlignment(long valueToRound, long byteAlignment)
{
long roundedValue = -1;
roundedValue = (valueToRound + byteAlignment - 1) / byteAlignment * byteAlignment;
return roundedValue;
}
public static ulong RoundUpToByteAlignment(ulong valueToRound, ulong byteAlignment)
{
ulong roundedValue;
roundedValue = (valueToRound + byteAlignment - 1) / byteAlignment * byteAlignment;
return roundedValue;
}
}
}

9
usm_extract/src/util/OffsetDescription.cs Normal file
View file

@ -0,0 +1,9 @@
namespace VGMToolbox.util
{
public class OffsetDescription
{
public string OffsetValue { set; get; }
public string OffsetSize { set; get; }
public string OffsetByteOrder { set; get; }
}
}

1826
usm_extract/src/util/ParseFile.cs Normal file
View file

File diff suppressed because it is too large Load diff

15
usm_extract/src/util/RiffCalculatingOffsetDescription.cs Normal file
View file

@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace VGMToolbox.util
{
public class RiffCalculatingOffsetDescription : CalculatingOffsetDescription
{
public const string START_OF_STRING = "start of";
public const string END_OF_STRING = "end of";
public string RelativeLocationToRiffChunkString { set; get; }
public string RiffChunkString { set; get; }
}
}