diff --git a/Editors/ImportExportEditor/Editors.ImportExport/Importing/Presentation/ImporterCoreViewModel.cs b/Editors/ImportExportEditor/Editors.ImportExport/Importing/Presentation/ImporterCoreViewModel.cs index 7466b5d97..2415322fb 100644 --- a/Editors/ImportExportEditor/Editors.ImportExport/Importing/Presentation/ImporterCoreViewModel.cs +++ b/Editors/ImportExportEditor/Editors.ImportExport/Importing/Presentation/ImporterCoreViewModel.cs @@ -45,8 +45,6 @@ public void Initialize(PackFileContainer packFile, string packPath, string diskF _packPath = packPath; SystemPath = diskFile; - - _inputFile = new PackFile(SystemPath, new FileSystemSource(SystemPath)); FindImporter(); } diff --git a/Editors/Reports/DeepSearch/DeepSearchReport.cs b/Editors/Reports/DeepSearch/DeepSearchReport.cs index 8afe28867..61c7b8cae 100644 --- a/Editors/Reports/DeepSearch/DeepSearchReport.cs +++ b/Editors/Reports/DeepSearch/DeepSearchReport.cs @@ -32,10 +32,12 @@ public class DeepSearchReport { private readonly ILogger _logger = Logging.Create(); private readonly IPackFileService _packFileService; + private readonly IPackFileContainerLoader _loader; - public DeepSearchReport(IPackFileService packFileService) + public DeepSearchReport(IPackFileService packFileService, IPackFileContainerLoader loader) { _packFileService = packFileService; + _loader = loader; } public List DeepSearch(string searchStr, bool caseSensetive) @@ -71,7 +73,7 @@ public List DeepSearch(string searchStr, bool caseSensetive) { using (var reader = new BinaryReader(fileStram, Encoding.ASCII)) { - var pfc = PackFileSerializerLoader.Load(packFilePath, reader, new CaPackDuplicateFileResolver()); + var pfc = _loader.Load(packFilePath); _logger.Here().Information($"Searching through packfile {currentIndex}/{files.Count} - {packFilePath} {pfc.FileList.Count} files"); diff --git a/Shared/SharedCore/Assembly.cs b/Shared/SharedCore/Assembly.cs index 5559e0ad1..dca793627 100644 --- a/Shared/SharedCore/Assembly.cs +++ b/Shared/SharedCore/Assembly.cs @@ -1,2 +1,4 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Test.Shared.Core")] +[assembly: InternalsVisibleTo("Test.TestingUtility")] +[assembly: InternalsVisibleTo("Test.GameWorld.Core")] diff --git a/Shared/SharedCore/PackFiles/Models/FileSystemSource.cs b/Shared/SharedCore/PackFiles/Models/FileSystemSource.cs index d542caef8..2c2e003e6 100644 --- a/Shared/SharedCore/PackFiles/Models/FileSystemSource.cs +++ b/Shared/SharedCore/PackFiles/Models/FileSystemSource.cs @@ -1,7 +1,10 @@ using Shared.ByteParsing; +using Shared.Core.PackFiles.Utility; namespace Shared.Core.PackFiles.Models { + + // This should only be used for unit tests - move to test project later public class FileSystemSource : IDataSource { public long Size { get; private set; } = 0; @@ -20,10 +23,7 @@ public FileSystemSource(string filepath) } - public byte[] ReadData() - { - return File.ReadAllBytes(_filepath); - } + public byte[] ReadData() => File.ReadAllBytes(_filepath); public byte[] PeekData(int size) { @@ -35,10 +35,9 @@ public byte[] PeekData(int size) } } - public ByteChunk ReadDataAsChunk() - { - return new ByteChunk(ReadData()); - } + public ByteChunk ReadDataAsChunk() => new ByteChunk(ReadData()); + + public CompressionFormat CompressionFormat { get => CompressionFormat.None; } } diff --git a/Shared/SharedCore/PackFiles/Models/IDataSource.cs b/Shared/SharedCore/PackFiles/Models/IDataSource.cs index 96da55b4d..f4e465f1f 100644 --- a/Shared/SharedCore/PackFiles/Models/IDataSource.cs +++ b/Shared/SharedCore/PackFiles/Models/IDataSource.cs @@ -1,4 +1,5 @@ using Shared.ByteParsing; +using Shared.Core.PackFiles.Utility; namespace Shared.Core.PackFiles.Models { @@ -8,6 +9,8 @@ public interface IDataSource byte[] ReadData(); byte[] PeekData(int size); ByteChunk ReadDataAsChunk(); + + CompressionFormat CompressionFormat { get; } } diff --git a/Shared/SharedCore/PackFiles/Models/MemorySource.cs b/Shared/SharedCore/PackFiles/Models/MemorySource.cs index 7de9d6867..2b70bbf88 100644 --- a/Shared/SharedCore/PackFiles/Models/MemorySource.cs +++ b/Shared/SharedCore/PackFiles/Models/MemorySource.cs @@ -1,4 +1,5 @@ using Shared.ByteParsing; +using Shared.Core.PackFiles.Utility; namespace Shared.Core.PackFiles.Models { @@ -24,18 +25,13 @@ public byte[] PeekData(int size) var output = new byte[size]; Array.Copy(_data, 0, output, 0, size); return output; - } - public static MemorySource FromFile(string path) - { - return new MemorySource(File.ReadAllBytes(path)); - } + public ByteChunk ReadDataAsChunk() => new ByteChunk(ReadData()); - public ByteChunk ReadDataAsChunk() - { - return new ByteChunk(ReadData()); - } + public CompressionFormat CompressionFormat { get => CompressionFormat.None; } + + public static MemorySource FromFile(string path) => new MemorySource(File.ReadAllBytes(path)); } diff --git a/Shared/SharedCore/PackFiles/Models/PFHeader.cs b/Shared/SharedCore/PackFiles/Models/PFHeader.cs index d126db8ee..bea1bd5e0 100644 --- a/Shared/SharedCore/PackFiles/Models/PFHeader.cs +++ b/Shared/SharedCore/PackFiles/Models/PFHeader.cs @@ -55,80 +55,13 @@ public class PFHeader public List DependantFiles { get; set; } = []; - public PFHeader() { } - //public PFHeader(BinaryReader reader) - //{ - // // var fileNameBuffer = new byte[1024]; - // // Version = new string(reader.ReadChars(4)); - // // ByteMask = reader.ReadInt32(); - // // - // // ReferenceFileCount = reader.ReadInt32(); - // // var pack_file_index_size = reader.ReadInt32(); - // // FileCount = reader.ReadInt32(); - // // var packed_file_index_size = reader.ReadInt32(); - // // - // // var headerOffset = 24; - // // if (Version == "PFH0") - // // { - // // _headerBuffer = new byte[0]; - // // } - // // else if (Version == "PFH2" || Version == "PFH3") - // // { - // // _headerBuffer = reader.ReadBytes(32 - headerOffset); - // // } - // // else if (Version == "PFH4" || Version == "PFH5") - // // { - // // if ((ByteMask & HAS_EXTENDED_HEADER) != 0) - // // _headerBuffer = reader.ReadBytes(48 - headerOffset); - // // else - // // _headerBuffer = reader.ReadBytes(28 - headerOffset); // 4 bytes for timestamp - // // } - // // else if (Version == "PFH6") - // // { - // // _headerBuffer = reader.ReadBytes(308 - headerOffset); - // // } - // // - // // for (int i = 0; i < ReferenceFileCount; i++) - // // _dependantFiles.Add(IOFunctions.ReadZeroTerminatedAscii(reader, fileNameBuffer)); - // // - // // HasAdditionalInfo = (ByteMask & HAS_INDEX_WITH_TIMESTAMPS) != 0; - // // DataStart = headerOffset + _headerBuffer.Length + pack_file_index_size + packed_file_index_size; - //} - public PFHeader(string version, PackFileCAType type) { StrVersion = version; ByteMask = (int)type; Buffer = DefaultTimeStamp; } - - /* Delete later - * public void Save(int fileCount, uint fileContentSize, BinaryWriter binaryWriter) - { - foreach (byte c in StrVersion) - binaryWriter.Write(c); - binaryWriter.Write(ByteMask); - - binaryWriter.Write(DependantFiles.Count); - - var pack_file_index_size = 0; - foreach (var file in DependantFiles) - pack_file_index_size += file.Length + 1; - - binaryWriter.Write(pack_file_index_size); - binaryWriter.Write(fileCount); - binaryWriter.Write(fileContentSize); - - binaryWriter.Write(Buffer); - - foreach (var file in DependantFiles) - { - foreach (byte c in file) - binaryWriter.Write(c); - binaryWriter.Write((byte)0); - } - }*/ } } diff --git a/Shared/SharedCore/PackFiles/Models/PackedFileSource.cs b/Shared/SharedCore/PackFiles/Models/PackedFileSource.cs index 5defeb68c..7c22ec469 100644 --- a/Shared/SharedCore/PackFiles/Models/PackedFileSource.cs +++ b/Shared/SharedCore/PackFiles/Models/PackedFileSource.cs @@ -38,13 +38,15 @@ public PackedFileSource( public byte[] ReadData() { - var data = new byte[Size]; + using var stream = File.Open(Parent.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + return ReadData(stream); + } - using (var stream = File.Open(Parent.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - { - stream.Seek(Offset, SeekOrigin.Begin); - stream.ReadExactly(data); - } + public byte[] ReadData(Stream knownStream) + { + var data = new byte[Size]; + knownStream.Seek(Offset, SeekOrigin.Begin); + knownStream.ReadExactly(data, 0, (int)Size); if (IsEncrypted) data = FileEncryption.Decrypt(data); @@ -59,6 +61,7 @@ public byte[] ReadData() return data; } + public byte[] PeekData(int size) { byte[] data; @@ -92,24 +95,7 @@ public byte[] PeekData(int size) return data; } - public byte[] ReadData(Stream knownStream) - { - var data = new byte[Size]; - knownStream.Seek(Offset, SeekOrigin.Begin); - knownStream.ReadExactly(data, 0, (int)Size); - - if (IsEncrypted) - data = FileEncryption.Decrypt(data); - - if (IsCompressed) - { - data = FileCompression.Decompress(data, (int)UncompressedSize, CompressionFormat); - if (data.Length != UncompressedSize) - throw new InvalidDataException($"Decompressed bytes {data.Length:N0} does not match the expected uncompressed bytes {UncompressedSize:N0}."); - } - - return data; - } + public byte[] ReadDataWithoutDecompressing() { diff --git a/Shared/SharedCore/PackFiles/PackFileService.cs b/Shared/SharedCore/PackFiles/PackFileService.cs index e9b2277b1..8f3475218 100644 --- a/Shared/SharedCore/PackFiles/PackFileService.cs +++ b/Shared/SharedCore/PackFiles/PackFileService.cs @@ -9,7 +9,7 @@ namespace Shared.Core.PackFiles { - public class PackFileService : IPackFileService + class PackFileService : IPackFileService { private readonly ILogger _logger = Logging.Create(); private readonly IGlobalEventHub? _globalEventHub; @@ -112,7 +112,8 @@ public void CopyFileFromOtherPackFile(PackFileContainer source, string path, Pac if (source.FileList.ContainsKey(lowerPath)) { var file = source.FileList[lowerPath]; - var newFile = new PackFile(file.Name, file.DataSource); + var data = file.DataSource.ReadData(); + var newFile = new PackFile(file.Name, new MemorySource(data)); target.FileList[lowerPath] = newFile; _globalEventHub?.PublishGlobalEvent(new PackFileContainerFilesAddedEvent(target, [newFile])); diff --git a/Shared/SharedCore/PackFiles/Serialization/PackFileSerializerLoader.cs b/Shared/SharedCore/PackFiles/Serialization/PackFileSerializerLoader.cs index 60a8b77b4..8400c0f6a 100644 --- a/Shared/SharedCore/PackFiles/Serialization/PackFileSerializerLoader.cs +++ b/Shared/SharedCore/PackFiles/Serialization/PackFileSerializerLoader.cs @@ -22,11 +22,11 @@ public static class PackFileVersionConverter public static PackFileVersion GetEnum(string versionStr) => s_values.First(x => x.StringValue == versionStr.ToUpper()).EnumValue; } - public static class PackFileSerializerLoader + static class PackFileSerializerLoader { static readonly ILogger s_logger = Logging.CreateStatic(typeof(PackFileSerializerLoader)); - public static PackFileContainer Load(string packFileSystemPath, BinaryReader reader, IDuplicateFileResolver duplicatePackFileResolver) + public static PackFileContainer Load(string packFileSystemPath, long packFileSize, BinaryReader reader, IDuplicateFileResolver duplicatePackFileResolver) { try { @@ -37,7 +37,7 @@ public static PackFileContainer Load(string packFileSystemPath, BinaryReader rea SystemFilePath = packFileSystemPath, Name = Path.GetFileNameWithoutExtension(packFileSystemPath), Header = ReadHeader(reader), - OriginalLoadByteSize = new FileInfo(packFileSystemPath).Length, + OriginalLoadByteSize = packFileSize, }; // If larger then int.max throw error @@ -51,8 +51,6 @@ public static PackFileContainer Load(string packFileSystemPath, BinaryReader rea FilePath = packFileSystemPath, }; - //var buffer = reader.ReadBytes((int)output.Header.DataStart - 28); - var offset = output.Header.DataStart; var headerVersion = output.Header.Version; for (var i = 0; i < output.Header.FileCount; i++) @@ -179,52 +177,7 @@ static PFHeader ReadHeader(BinaryReader reader) return header; } - public static void WriteHeader(PFHeader header, uint fileContentSize, BinaryWriter writer) - { - var packFileTypeStr = PackFileVersionConverter.ToString(header.Version); // 4 - foreach (var c in packFileTypeStr) - writer.Write(c); - - writer.Write(header.ByteMask); // 8 - writer.Write(header.DependantFiles.Count); // 12 - - var pack_file_index_size = 0; - foreach (var file in header.DependantFiles) - pack_file_index_size += file.Length + 1; - - writer.Write(pack_file_index_size); // 16 - writer.Write(header.FileCount); // 20 - writer.Write(fileContentSize); // 24 - - switch (header.Version) - { - case PackFileVersion.PFH0: - break;// Nothing needed to do - case PackFileVersion.PFH2: - case PackFileVersion.PFH3: - // 64 bit timestamp - writer.Write(0); - writer.Write(0); - break; - case PackFileVersion.PFH4: - case PackFileVersion.PFH5: - if (header.HasExtendedHeader) - throw new Exception("Not supported packfile type"); - - writer.Write(PFHeader.DefaultTimeStamp); - break; - - default: - throw new Exception("Not supported packfile type"); - } - - foreach (var file in header.DependantFiles) - { - foreach (byte c in file) - writer.Write(c); - writer.Write((byte)0); - } - } + private static byte[] DetectCompressionInfo(BinaryReader reader, long dataOffset, uint entrySize, bool isEncrypted) { diff --git a/Shared/SharedCore/PackFiles/Serialization/PackFileSerializerWriter.cs b/Shared/SharedCore/PackFiles/Serialization/PackFileSerializerWriter.cs index af51381d8..bb0342ec1 100644 --- a/Shared/SharedCore/PackFiles/Serialization/PackFileSerializerWriter.cs +++ b/Shared/SharedCore/PackFiles/Serialization/PackFileSerializerWriter.cs @@ -5,113 +5,224 @@ namespace Shared.Core.PackFiles.Serialization { - public record PackFileWriteInformation( - PackFile PackFile, - long SizePosition, - CompressionFormat CurrentCompressionFormat, - CompressionFormat IntendedCompressionFormat); + record FileCompressionInfo(CompressionFormat IntendedCompressionFormat, bool DecompressBeforeSaving); - public static class PackFileSerializerWriter + class PackFileWriteInformation(PackFile pf, string fullFileName, long sizePosition, FileCompressionInfo compressionInfo) { - public static void SaveToByteArray(PackFileContainer container, BinaryWriter writer, GameInformation gameInformation) + public PackFile PackFile { get; set; } = pf; + public string FullFileName { get; set; } = fullFileName; + public long SizePosition { get; set; } = sizePosition; + public FileCompressionInfo CompressionInfo { get; set; } = compressionInfo; + } + + static class PackFileSerializerWriter + { + public static void SaveToByteArray(PackFileContainer container, BinaryWriter writer, GameInformation currentGameInformation) { if (container.Header.HasEncryptedData || container.Header.HasEncryptedIndex) throw new InvalidOperationException("Saving encrypted packs is not supported."); + var sortedFiles = container.FileList.OrderBy(x => x.Key, StringComparer.Ordinal).ToList(); + var headerSpecificBytes = ComputeFileHeaderSpecificByte(container); + var fileNamesOffset = ComputeFileNameOffset(headerSpecificBytes, sortedFiles); + + // Update and write header + container.Header.FileCount = (uint)container.FileList.Count; + WriteHeader(container.Header, (uint)fileNamesOffset, writer); + + // Write the core of the file + var fileMetaDataTable = BuildMetaDataTable(sortedFiles, container, currentGameInformation); + SerializeFileTable(fileMetaDataTable, container, writer); + SerializeFileBlob(fileMetaDataTable, container, writer); + } + + public static void WriteHeader(PFHeader header, uint fileContentSize, BinaryWriter writer) + { + var packFileTypeStr = PackFileVersionConverter.ToString(header.Version); // 4 + foreach (var c in packFileTypeStr) + writer.Write(c); + + writer.Write(header.ByteMask); // 8 + writer.Write(header.DependantFiles.Count); // 12 + + var pack_file_index_size = 0; + foreach (var file in header.DependantFiles) + pack_file_index_size += file.Length + 1; + + writer.Write(pack_file_index_size); // 16 + writer.Write(header.FileCount); // 20 + writer.Write(fileContentSize); // 24 + + switch (header.Version) + { + case PackFileVersion.PFH0: + break;// Nothing needed to do + case PackFileVersion.PFH2: + case PackFileVersion.PFH3: // 64 bit timestamp + writer.Write(0); + writer.Write(0); + break; + case PackFileVersion.PFH4: + case PackFileVersion.PFH5: + if (header.HasExtendedHeader) + throw new Exception("Not supported packfile type"); + + writer.Write(PFHeader.DefaultTimeStamp); + break; + + default: + throw new Exception("Not supported packfile type"); + } + + foreach (var file in header.DependantFiles) + { + var fileNameBytes = Encoding.UTF8.GetBytes(file); + writer.Write(fileNameBytes); + writer.Write((byte)0); + } + } + + static long ComputeFileHeaderSpecificByte(PackFileContainer container) + { long headerSpecificBytes = 0; if (container.Header.HasIndexWithTimeStamp) headerSpecificBytes += 4; if (container.Header.Version == PackFileVersion.PFH5) headerSpecificBytes += 1; - long fileNamesOffset = 0; + return headerSpecificBytes; + } - var sortedFiles = container.FileList.OrderBy(x => x.Key, StringComparer.Ordinal).ToList(); - foreach (var file in sortedFiles) + static long ComputeFileNameOffset(long headerSpecificBytePerFile, IList> sortedFileList) + { + long fileNamesOffset = 0; + foreach (var file in sortedFileList) { var fileSize = 4; var zeroTerminator = 1; var fileNameBytes = Encoding.UTF8.GetByteCount(file.Key); - fileNamesOffset += fileSize + headerSpecificBytes + fileNameBytes + zeroTerminator; + fileNamesOffset += fileSize + headerSpecificBytePerFile + fileNameBytes + zeroTerminator; } - container.Header.FileCount = (uint)container.FileList.Count; - PackFileSerializerLoader.WriteHeader(container.Header, (uint)fileNamesOffset, writer); + return fileNamesOffset; + } + + public static FileCompressionInfo DetermineFileCompression(PackFileVersion outputPackFileVersion, GameInformation currentGameInformation, string fullFileName, CompressionFormat currentFileCompressionFormat) + { + var doesGameSupportCompression = FileCompression.DoesGameSupportCompression(currentGameInformation); + var compressIfPossible = doesGameSupportCompression && outputPackFileVersion == PackFileVersion.PFH5; - var filesToWrite = new List(); + var targetFileCompressionFormat = FileCompression.GetCompressionFormat(currentGameInformation, fullFileName); + var isFileCompressed = currentFileCompressionFormat != CompressionFormat.None; - // Write file metadata + if (isFileCompressed == false) + { + if (compressIfPossible) + { + // Case 1 - Not a compressed file, going to a packfile/game with compression + return new FileCompressionInfo(targetFileCompressionFormat, true); + } + else + { + // Case 2 - Not a compressed file, going to a packfile/game without compression + return new FileCompressionInfo(CompressionFormat.None, false); + } + } + else + { + if (compressIfPossible) + { + // Case 3 - A compressed file, going to a packfile/game with compression. Same target and source format + if (currentFileCompressionFormat == targetFileCompressionFormat) + return new FileCompressionInfo(targetFileCompressionFormat, false); + + // Case 4 - A compressed file, going to a packfile/game with compression. Different target and source format + return new FileCompressionInfo(targetFileCompressionFormat, true); + } + else + { + // Case 5 - A compressed file, going to a packfile/game without compression + return new FileCompressionInfo(CompressionFormat.None, true); + } + } + } + + public static List BuildMetaDataTable(IList> sortedFiles, PackFileContainer container, GameInformation currentGameInformation) + { + var filesToWrite = new List(); foreach (var file in sortedFiles) { var packFile = file.Value; + var fileCompressionInfo = DetermineFileCompression(container.Header.Version, currentGameInformation, file.Key, packFile.DataSource.CompressionFormat); + filesToWrite.Add(new PackFileWriteInformation(packFile, file.Key, 0, fileCompressionInfo)); + } - // Determine compression info - var currentCompressionFormat = CompressionFormat.None; - if (packFile.DataSource is PackedFileSource packedFileSource) - currentCompressionFormat = packedFileSource.CompressionFormat; - var firstFilePathPart = file.Key.Split(['\\', '/'], StringSplitOptions.RemoveEmptyEntries).First(); - var intendedCompressionFormat = FileCompression.GetCompressionFormat(gameInformation, firstFilePathPart, packFile.Extension); - var shouldCompress = intendedCompressionFormat != CompressionFormat.None; + return filesToWrite; + } + public static void SerializeFileTable(List fileMetaData, PackFileContainer container, BinaryWriter writer) + { + // Write file table + // FileStartPosition + // TimeStamp + // CompressionFlag + // FileName + // ZeroTerminator for FileName + foreach (var file in fileMetaData) + { // File size placeholder (rewritten later) var sizePosition = writer.BaseStream.Position; writer.Write(0); + file.SizePosition = sizePosition; // Timestamp if (container.Header.HasIndexWithTimeStamp) writer.Write(0); - // Compression + // Even if we do not compress - we alsways need to write the flag for PFH5 if (container.Header.Version == PackFileVersion.PFH5) + { + var shouldCompress = file.CompressionInfo.IntendedCompressionFormat != CompressionFormat.None; writer.Write(shouldCompress); + } // Filename - var fileNameBytes = Encoding.UTF8.GetBytes(file.Key); + var fileNameBytes = Encoding.UTF8.GetBytes(file.FullFileName); writer.Write(fileNameBytes); - - // Zero terminator - writer.Write((byte)0); - - filesToWrite.Add(new PackFileWriteInformation(packFile, sizePosition, currentCompressionFormat, intendedCompressionFormat)); + writer.Write((byte)0); // Zero terminator } + } - var packedFileSourceParent = new PackedFileSourceParent { FilePath = container.SystemFilePath }; - - // Write the files - foreach (var file in filesToWrite) + public static void SerializeFileBlob(List fileMetaDataTabel, PackFileContainer container, BinaryWriter writer) + { + foreach (var fileMetaData in fileMetaDataTabel) { - var packFile = file.PackFile; + var packFile = fileMetaData.PackFile; byte[] data; uint uncompressedSize = 0; - // Determine compression info - var shouldCompress = file.IntendedCompressionFormat != CompressionFormat.None; - var isCorrectCompressionFormat = file.CurrentCompressionFormat == file.IntendedCompressionFormat; + // Read file data + if (fileMetaData.CompressionInfo.DecompressBeforeSaving == false && packFile.DataSource is PackedFileSource packedFileSource) + data = packedFileSource.ReadDataWithoutDecompressing(); + else + data = packFile.DataSource.ReadData(); - // Read the data - if (shouldCompress && !isCorrectCompressionFormat) + // Compress if needed + var shouldCompress = fileMetaData.CompressionInfo.IntendedCompressionFormat != CompressionFormat.None; + if (shouldCompress) { - // Decompress the data var uncompressedData = packFile.DataSource.ReadData(); uncompressedSize = (uint)uncompressedData.Length; // Compress the data into the right format - var compressedData = FileCompression.Compress(uncompressedData, file.IntendedCompressionFormat); + var compressedData = FileCompression.Compress(uncompressedData, fileMetaData.CompressionInfo.IntendedCompressionFormat); data = compressedData; // Validate new compression - var decompressedData = FileCompression.Decompress(compressedData, uncompressedData.Length, file.IntendedCompressionFormat); + var decompressedData = FileCompression.Decompress(compressedData, uncompressedData.Length, fileMetaData.CompressionInfo.IntendedCompressionFormat); if (decompressedData.Length != uncompressedData.Length) throw new InvalidDataException($"Decompressed bytes {decompressedData.Length:N0} does not match the expected uncompressed bytes {uncompressedData.Length:N0}."); } - else if (packFile.DataSource is PackedFileSource packedFileSource && isCorrectCompressionFormat) - { - // The data is already in the right format so just get the data as is - uncompressedSize = packedFileSource.UncompressedSize; - data = packedFileSource.ReadDataWithoutDecompressing(); - } - else - data = packFile.DataSource.ReadData(); // Write the data var offset = writer.BaseStream.Position; @@ -119,23 +230,23 @@ public static void SaveToByteArray(PackFileContainer container, BinaryWriter wri // Patch the size from the position stored earlier var currentPosition = writer.BaseStream.Position; - writer.BaseStream.Position = file.SizePosition; + writer.BaseStream.Position = fileMetaData.SizePosition; writer.Write(data.Length); writer.BaseStream.Position = currentPosition; - // We do not encrypt - var isEncrypted = false; - // Update DataSource + var packedFileSourceParent = new PackedFileSourceParent { FilePath = container.SystemFilePath }; packFile.DataSource = new PackedFileSource( packedFileSourceParent, offset, data.Length, - isEncrypted, + false, // We do not encrypt shouldCompress, - file.IntendedCompressionFormat, + fileMetaData.CompressionInfo.IntendedCompressionFormat, uncompressedSize); } } + + } } diff --git a/Shared/SharedCore/PackFiles/Utility/FileCompression.cs b/Shared/SharedCore/PackFiles/Utility/FileCompression.cs index 60f3dc441..452091257 100644 --- a/Shared/SharedCore/PackFiles/Utility/FileCompression.cs +++ b/Shared/SharedCore/PackFiles/Utility/FileCompression.cs @@ -1,7 +1,9 @@ -using K4os.Compression.LZ4.Encoders; +using System.ComponentModel; +using K4os.Compression.LZ4.Encoders; using K4os.Compression.LZ4.Streams; using SevenZip; using SevenZip.Compression.LZMA; +using Shared.Core.PackFiles.Models; using Shared.Core.Settings; using ZstdSharp; using ZstdSharp.Unsafe; @@ -261,8 +263,17 @@ public static CompressionFormat GetCompressionFormat(byte[] compressionFormatByt return CompressionFormat.None; } - public static CompressionFormat GetCompressionFormat(GameInformation gameInformation, string firstFilePathPart, string extension) + public static bool DoesGameSupportCompression(GameInformation gameInformation) { + var doesGameFilesSupportCompression = !(gameInformation.CompressionFormats.Count == 1 && gameInformation.CompressionFormats.First() == CompressionFormat.None); + return doesGameFilesSupportCompression; + } + + public static CompressionFormat GetCompressionFormat(GameInformation gameInformation, string fileName) + { + var extension = Path.GetExtension(fileName); + var firstFilePathPart = fileName.Split(['\\', '/'], StringSplitOptions.RemoveEmptyEntries).First(); + var compressionFormats = gameInformation.CompressionFormats; // Check if the game supports any compression at all diff --git a/Shared/SharedCore/PackFiles/Utility/PackFileContainerLoader.cs b/Shared/SharedCore/PackFiles/Utility/PackFileContainerLoader.cs index eee17a5da..bbe642f30 100644 --- a/Shared/SharedCore/PackFiles/Utility/PackFileContainerLoader.cs +++ b/Shared/SharedCore/PackFiles/Utility/PackFileContainerLoader.cs @@ -75,7 +75,8 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri using var fileStream = File.OpenRead(packFileSystemPath); using var reader = new BinaryReader(fileStream, Encoding.ASCII); - var pack = PackFileSerializerLoader.Load(packFileSystemPath, reader, new CustomPackDuplicateFileResolver()); + var packFileSize = new FileInfo(packFileSystemPath).Length; + var pack = PackFileSerializerLoader.Load(packFileSystemPath, packFileSize, reader, new CustomPackDuplicateFileResolver()); PackFileLog.LogPackCompression(pack); return pack; @@ -121,7 +122,8 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri using var fileStream = File.OpenRead(path); using var reader = new BinaryReader(fileStream, Encoding.ASCII); - var pack = PackFileSerializerLoader.Load(path, reader, packfileResolver); + var packFileSize = new FileInfo(path).Length; + var pack = PackFileSerializerLoader.Load(path, packFileSize, reader, packfileResolver); packList.Add(pack); PackFileLog.LogPackCompression(pack); diff --git a/Shared/SharedCore/Settings/GameInformationDatabase.cs b/Shared/SharedCore/Settings/GameInformationDatabase.cs index 02236e7f5..816f97e94 100644 --- a/Shared/SharedCore/Settings/GameInformationDatabase.cs +++ b/Shared/SharedCore/Settings/GameInformationDatabase.cs @@ -52,8 +52,6 @@ public enum WsModelVersion Version3, } - //RmvVersionEnum - public class GameInformation( GameTypeEnum gameType, string displayName, diff --git a/Testing/Shared.Core.Test/PackFiles/PackFileContainerLoaderTests.cs b/Testing/Shared.Core.Test/PackFiles/PackFileContainerLoaderTests.cs deleted file mode 100644 index 60b51e6e4..000000000 --- a/Testing/Shared.Core.Test/PackFiles/PackFileContainerLoaderTests.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Test.Shared.Core.PackFiles -{ - internal class PackFileContainerLoaderTests - { - } -} diff --git a/Testing/Shared.Core.Test/PackFiles/PackFileContainerSaveTests.cs b/Testing/Shared.Core.Test/PackFiles/PackFileContainerSaveTests.cs deleted file mode 100644 index ccba8de64..000000000 --- a/Testing/Shared.Core.Test/PackFiles/PackFileContainerSaveTests.cs +++ /dev/null @@ -1,74 +0,0 @@ -using Shared.Core.PackFiles.Models; -using Shared.Core.PackFiles.Serialization; -using Shared.Core.PackFiles.Utility; -using Shared.Core.Settings; - -namespace Test.Shared.Core.PackFiles -{ - [TestFixture] - internal class PackFileContainerSaveTests - { - private GameInformation CreateGameWithCompression(CompressionFormat format) - { - return new GameInformation( - GameTypeEnum.Unknown, - "Test", - PackFileVersion.PFH5, - GameBnkVersion.Unsupported, - WwiseProjectId.Unsupported, - WsModelVersion.Unknown, - new System.Collections.Generic.List() { format } - ); - } - - [Test] - public void Save_PFH5_WritesCompressionFlag() - { - var container = new PackFileContainer("test") - { - Header = new PFHeader("PFH5", PackFileCAType.MOD), - SystemFilePath = "test.pack" - }; - - container.FileList = new System.Collections.Generic.Dictionary(); - container.FileList["directory\\file.txt"] = PackFile.CreateFromASCII("file.txt", new string('A', 2048)); - - var gameInfo = CreateGameWithCompression(CompressionFormat.Zstd); - - using var ms = new MemoryStream(); - using var writer = new BinaryWriter(ms); - PackFileSerializerWriter.SaveToByteArray(container, writer, gameInfo); - var data = ms.ToArray(); - - var compressionFlagPosition = 28 + 4; // header (28) + size (4) - Assert.That(data.Length, Is.GreaterThan(compressionFlagPosition)); - Assert.That(data[compressionFlagPosition], Is.EqualTo((byte)1), "Expected compression flag (true) to be written for PFH5"); - } - - [Test] - public void Save_PFH4_WritesCompressionFlag_Fails_DemonstratingBug() - { - return; // return for now - known bug - - var container = new PackFileContainer("test") - { - Header = new PFHeader("PFH4", PackFileCAType.MOD), - SystemFilePath = "test.pack" - }; - - container.FileList = new System.Collections.Generic.Dictionary(); - container.FileList["directory\\file.txt"] = PackFile.CreateFromASCII("file.txt", new string('A', 2048)); - - var gameInfo = CreateGameWithCompression(CompressionFormat.Zstd); - - using var ms = new MemoryStream(); - using var writer = new BinaryWriter(ms); - PackFileSerializerWriter.SaveToByteArray(container, writer, gameInfo); - var data = ms.ToArray(); - - var compressionFlagPosition = 28 + 4; // header (28) + size (4) - Assert.That(data.Length, Is.GreaterThan(compressionFlagPosition)); - Assert.That(data[compressionFlagPosition], Is.EqualTo((byte)1), "PFH4 should have written a compression flag but does not (this test should fail to demonstrate the bug)"); - } - } -} diff --git a/Testing/Shared.Core.Test/PackFiles/Serialization/PackFileSerializerWriterTests.cs b/Testing/Shared.Core.Test/PackFiles/Serialization/PackFileSerializerWriterTests.cs new file mode 100644 index 000000000..dac92df30 --- /dev/null +++ b/Testing/Shared.Core.Test/PackFiles/Serialization/PackFileSerializerWriterTests.cs @@ -0,0 +1,103 @@ +using Shared.Core.PackFiles.Models; +using Shared.Core.PackFiles.Serialization; +using Shared.Core.PackFiles.Utility; +using Shared.Core.Settings; + +namespace Test.Shared.Core.PackFiles.Serialization +{ + [TestFixture] + internal class PackFileSerializerWriterTests + { + [TestCase(GameTypeEnum.Warhammer3, PackFileVersion.PFH4, "folder//filex.txt", CompressionFormat.Zstd, CompressionFormat.None, true)] + [TestCase(GameTypeEnum.Warhammer3, PackFileVersion.PFH5, "folder//filex.txt", CompressionFormat.Zstd, CompressionFormat.Zstd, false)] + [TestCase(GameTypeEnum.Warhammer3, PackFileVersion.PFH5, "folder//filex.txt", CompressionFormat.Lzma1, CompressionFormat.Zstd, true)] + [TestCase(GameTypeEnum.Warhammer3, PackFileVersion.PFH4, "folder//filex", CompressionFormat.None, CompressionFormat.None, false)] + [TestCase(GameTypeEnum.Warhammer3, PackFileVersion.PFH4, "folder//filex", CompressionFormat.Lz4, CompressionFormat.None, true)] + [TestCase(GameTypeEnum.Rome2, PackFileVersion.PFH4, "folder//filex.txt", CompressionFormat.Lz4, CompressionFormat.None, true)] + [TestCase(GameTypeEnum.Rome2, PackFileVersion.PFH4, "folder//filex.txt", CompressionFormat.None, CompressionFormat.None, false)] + // Rome 2 cases + public void DetermineFileCompression( + GameTypeEnum game, + PackFileVersion outputPackFileVersion, + string fileName, + CompressionFormat currentFileCompression, + CompressionFormat expected_Compression, + bool expected_deserializeBeforeWrite) + { + var gameInfo = GameInformationDatabase.GetGameById(game); + + var res = PackFileSerializerWriter.DetermineFileCompression(outputPackFileVersion, gameInfo, fileName, currentFileCompression); + Assert.That(res.DecompressBeforeSaving, Is.EqualTo(expected_deserializeBeforeWrite)); + Assert.That(res.IntendedCompressionFormat, Is.EqualTo(expected_Compression)); + } + + [Test] + [TestCase(GameTypeEnum.Rome2, PackFileVersion.PFH4, false)] + [TestCase(GameTypeEnum.Warhammer3, PackFileVersion.PFH4, false)] + [TestCase(GameTypeEnum.Warhammer3, PackFileVersion.PFH5, true)] + public void PackFileSerializerWriterTests_GameWithoutCompression( + GameTypeEnum game, + PackFileVersion outputPackFileVersion, + bool expectFileCompression) + { + // Arrange + var gameInfo = GameInformationDatabase.GetGameById(game); + var expectedFileInfo = new List<(string FilePath, string FileName, int Length, char Content, bool IsCompressable)> + { + ("directory\\fileA.txt", "fileA.txt", 512, 'A', true), + ("directory\\fileB.txt", "fileB.txt", 1024, 'B', true), + ("directory\\fileC.txt", "fileC.txt", 2048, 'C', true), + ("directory\\fileD", "fileD", 512, 'D', false), + ("\"directory\\\\db\\\\TableTest\"", "TableTest", 128, 'E', false), + }; + + // Create packfile with the above files + var packFileHeader = PackFileVersionConverter.ToString(outputPackFileVersion); + var container = new PackFileContainer("test") + { + Header = new PFHeader(packFileHeader, PackFileCAType.MOD), + SystemFilePath = "test.pack" + }; + + foreach (var fileInfo in expectedFileInfo) + container.FileList[fileInfo.FilePath] = PackFile.CreateFromASCII(fileInfo.FileName, new string(fileInfo.Content, fileInfo.Length)); + + using var writeMs = new MemoryStream(); + using var writer = new BinaryWriter(writeMs); + PackFileSerializerWriter.SaveToByteArray(container, writer, gameInfo); + var data = writeMs.ToArray(); + + + // Assert + using var readBackMs = new MemoryStream(data); + var reader = new BinaryReader(readBackMs); + var loadedPackFile = PackFileSerializerLoader.Load("testpackfile.pack", data.LongLength, reader, new CaPackDuplicateFileResolver()); + + for (int i = 0; i < expectedFileInfo.Count; i++) + { + var expectedFileInfoInstance = expectedFileInfo[i]; + var packFile = loadedPackFile.FileList[expectedFileInfoInstance.FilePath.ToLower()]; + + // Bypass the filesystem lookup and go directly to stream + var packFileConentet = (packFile.DataSource as PackedFileSource).ReadData(readBackMs); + // + // + //// Assert content is correct + Assert.That(packFileConentet.Length, Is.EqualTo(expectedFileInfoInstance.Length)); + Assert.That(packFileConentet, Is.EqualTo(new string(expectedFileInfoInstance.Content, expectedFileInfoInstance.Length))); + + if (expectedFileInfoInstance.IsCompressable && expectFileCompression) + { + Assert.That(packFile.DataSource.CompressionFormat, Is.Not.EqualTo(CompressionFormat.None)); + Assert.That(packFile.DataSource.Size, Is.LessThan(expectedFileInfoInstance.Length)); + } + else + { + Assert.That(packFile.DataSource.CompressionFormat, Is.EqualTo(CompressionFormat.None)); + Assert.That(packFile.DataSource.Size, Is.EqualTo(expectedFileInfoInstance.Length)); + } + } + + } + } +} diff --git a/Testing/Shared.Core.Test/Test.Shared.Core.csproj b/Testing/Shared.Core.Test/Test.Shared.Core.csproj index 01340dbdf..0d9565e39 100644 --- a/Testing/Shared.Core.Test/Test.Shared.Core.csproj +++ b/Testing/Shared.Core.Test/Test.Shared.Core.csproj @@ -31,8 +31,4 @@ - - - -