Skip to content

Fix for corruption isses #291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
donkeyProgramming merged 2 commits into from
Feb 6, 2026
Merged

Fix for corruption isses #291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f046021
PackFile handling cleanup
donkeyProgramming Jan 30, 2026
c510730
Fixed some corruption isses and added tests
donkeyProgramming Feb 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions ...s/ImportExportEditor/Editors.ImportExport/Importing/Presentation/ImporterCoreViewModel.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,6 @@ public void Initialize(PackFileContainer packFile, string packPath, string diskF
_packPath = packPath;
SystemPath = diskFile;



_inputFile = new PackFile(SystemPath, new FileSystemSource(SystemPath));
FindImporter();
}
Expand Down
6 changes: 4 additions & 2 deletions Editors/Reports/DeepSearch/DeepSearchReport.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@ public class DeepSearchReport
{
private readonly ILogger _logger = Logging.Create<DeepSearchReport>();
private readonly IPackFileService _packFileService;
private readonly IPackFileContainerLoader _loader;

public DeepSearchReport(IPackFileService packFileService)
public DeepSearchReport(IPackFileService packFileService, IPackFileContainerLoader loader)
{
_packFileService = packFileService;
_loader = loader;
}

public List<string> DeepSearch(string searchStr, bool caseSensetive)
Expand Down Expand Up @@ -71,7 +73,7 @@ public List<string> DeepSearch(string searchStr, bool caseSensetive)
{
using (var reader = new BinaryReader(fileStram, Encoding.ASCII))
{
var pfc = PackFileSerializerLoader.Load(packFilePath, reader, new CaPackDuplicateFileResolver());
var pfc = _loader.Load(packFilePath);

_logger.Here().Information($"Searching through packfile {currentIndex}/{files.Count} - {packFilePath} {pfc.FileList.Count} files");

Expand Down
2 changes: 2 additions & 0 deletions Shared/SharedCore/Assembly.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("Test.Shared.Core")]
[assembly: InternalsVisibleTo("Test.TestingUtility")]
[assembly: InternalsVisibleTo("Test.GameWorld.Core")]
15 changes: 7 additions & 8 deletions Shared/SharedCore/PackFiles/Models/FileSystemSource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
using Shared.ByteParsing;
using Shared.Core.PackFiles.Utility;

namespace Shared.Core.PackFiles.Models
{

// This should only be used for unit tests - move to test project later
public class FileSystemSource : IDataSource
{
public long Size { get; private set; } = 0;
Expand All @@ -20,10 +23,7 @@ public FileSystemSource(string filepath)
}


public byte[] ReadData()
{
return File.ReadAllBytes(_filepath);
}
public byte[] ReadData() => File.ReadAllBytes(_filepath);

public byte[] PeekData(int size)
{
Expand All @@ -35,10 +35,9 @@ public byte[] PeekData(int size)
}
}

public ByteChunk ReadDataAsChunk()
{
return new ByteChunk(ReadData());
}
public ByteChunk ReadDataAsChunk() => new ByteChunk(ReadData());

public CompressionFormat CompressionFormat { get => CompressionFormat.None; }
}


Expand Down
3 changes: 3 additions & 0 deletions Shared/SharedCore/PackFiles/Models/IDataSource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Shared.ByteParsing;
using Shared.Core.PackFiles.Utility;

namespace Shared.Core.PackFiles.Models
{
Expand All @@ -8,6 +9,8 @@ public interface IDataSource
byte[] ReadData();
byte[] PeekData(int size);
ByteChunk ReadDataAsChunk();

CompressionFormat CompressionFormat { get; }
}


Expand Down
14 changes: 5 additions & 9 deletions Shared/SharedCore/PackFiles/Models/MemorySource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Shared.ByteParsing;
using Shared.Core.PackFiles.Utility;

namespace Shared.Core.PackFiles.Models
{
Expand All @@ -24,18 +25,13 @@ public byte[] PeekData(int size)
var output = new byte[size];
Array.Copy(_data, 0, output, 0, size);
return output;

}

public static MemorySource FromFile(string path)
{
return new MemorySource(File.ReadAllBytes(path));
}
public ByteChunk ReadDataAsChunk() => new ByteChunk(ReadData());

public ByteChunk ReadDataAsChunk()
{
return new ByteChunk(ReadData());
}
public CompressionFormat CompressionFormat { get => CompressionFormat.None; }

public static MemorySource FromFile(string path) => new MemorySource(File.ReadAllBytes(path));
}


Expand Down
67 changes: 0 additions & 67 deletions Shared/SharedCore/PackFiles/Models/PFHeader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,80 +55,13 @@ public class PFHeader

public List<string> DependantFiles { get; set; } = [];


public PFHeader() { }

//public PFHeader(BinaryReader reader)
//{
// // var fileNameBuffer = new byte[1024];
// // Version = new string(reader.ReadChars(4));
// // ByteMask = reader.ReadInt32();
// //
// // ReferenceFileCount = reader.ReadInt32();
// // var pack_file_index_size = reader.ReadInt32();
// // FileCount = reader.ReadInt32();
// // var packed_file_index_size = reader.ReadInt32();
// //
// // var headerOffset = 24;
// // if (Version == "PFH0")
// // {
// // _headerBuffer = new byte[0];
// // }
// // else if (Version == "PFH2" || Version == "PFH3")
// // {
// // _headerBuffer = reader.ReadBytes(32 - headerOffset);
// // }
// // else if (Version == "PFH4" || Version == "PFH5")
// // {
// // if ((ByteMask & HAS_EXTENDED_HEADER) != 0)
// // _headerBuffer = reader.ReadBytes(48 - headerOffset);
// // else
// // _headerBuffer = reader.ReadBytes(28 - headerOffset); // 4 bytes for timestamp
// // }
// // else if (Version == "PFH6")
// // {
// // _headerBuffer = reader.ReadBytes(308 - headerOffset);
// // }
// //
// // for (int i = 0; i < ReferenceFileCount; i++)
// // _dependantFiles.Add(IOFunctions.ReadZeroTerminatedAscii(reader, fileNameBuffer));
// //
// // HasAdditionalInfo = (ByteMask & HAS_INDEX_WITH_TIMESTAMPS) != 0;
// // DataStart = headerOffset + _headerBuffer.Length + pack_file_index_size + packed_file_index_size;
//}

public PFHeader(string version, PackFileCAType type)
{
StrVersion = version;
ByteMask = (int)type;
Buffer = DefaultTimeStamp;
}

/* Delete later
* public void Save(int fileCount, uint fileContentSize, BinaryWriter binaryWriter)
{
foreach (byte c in StrVersion)
binaryWriter.Write(c);
binaryWriter.Write(ByteMask);

binaryWriter.Write(DependantFiles.Count);

var pack_file_index_size = 0;
foreach (var file in DependantFiles)
pack_file_index_size += file.Length + 1;

binaryWriter.Write(pack_file_index_size);
binaryWriter.Write(fileCount);
binaryWriter.Write(fileContentSize);

binaryWriter.Write(Buffer);

foreach (var file in DependantFiles)
{
foreach (byte c in file)
binaryWriter.Write(c);
binaryWriter.Write((byte)0);
}
}*/
}
}
34 changes: 10 additions & 24 deletions Shared/SharedCore/PackFiles/Models/PackedFileSource.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,15 @@ public PackedFileSource(

public byte[] ReadData()
{
var data = new byte[Size];
using var stream = File.Open(Parent.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
return ReadData(stream);
}

using (var stream = File.Open(Parent.FilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
stream.Seek(Offset, SeekOrigin.Begin);
stream.ReadExactly(data);
}
public byte[] ReadData(Stream knownStream)
{
var data = new byte[Size];
knownStream.Seek(Offset, SeekOrigin.Begin);
knownStream.ReadExactly(data, 0, (int)Size);

if (IsEncrypted)
data = FileEncryption.Decrypt(data);
Expand All @@ -59,6 +61,7 @@ public byte[] ReadData()
return data;
}


public byte[] PeekData(int size)
{
byte[] data;
Expand Down Expand Up @@ -92,24 +95,7 @@ public byte[] PeekData(int size)
return data;
}

public byte[] ReadData(Stream knownStream)
{
var data = new byte[Size];
knownStream.Seek(Offset, SeekOrigin.Begin);
knownStream.ReadExactly(data, 0, (int)Size);

if (IsEncrypted)
data = FileEncryption.Decrypt(data);

if (IsCompressed)
{
data = FileCompression.Decompress(data, (int)UncompressedSize, CompressionFormat);
if (data.Length != UncompressedSize)
throw new InvalidDataException($"Decompressed bytes {data.Length:N0} does not match the expected uncompressed bytes {UncompressedSize:N0}.");
}

return data;
}


public byte[] ReadDataWithoutDecompressing()
{
Expand Down
5 changes: 3 additions & 2 deletions Shared/SharedCore/PackFiles/PackFileService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

namespace Shared.Core.PackFiles
{
public class PackFileService : IPackFileService
class PackFileService : IPackFileService
{
private readonly ILogger _logger = Logging.Create<PackFileService>();
private readonly IGlobalEventHub? _globalEventHub;
Expand Down Expand Up @@ -112,7 +112,8 @@ public void CopyFileFromOtherPackFile(PackFileContainer source, string path, Pac
if (source.FileList.ContainsKey(lowerPath))
{
var file = source.FileList[lowerPath];
var newFile = new PackFile(file.Name, file.DataSource);
var data = file.DataSource.ReadData();
var newFile = new PackFile(file.Name, new MemorySource(data));
target.FileList[lowerPath] = newFile;

_globalEventHub?.PublishGlobalEvent(new PackFileContainerFilesAddedEvent(target, [newFile]));
Expand Down
55 changes: 4 additions & 51 deletions Shared/SharedCore/PackFiles/Serialization/PackFileSerializerLoader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@ public static class PackFileVersionConverter
public static PackFileVersion GetEnum(string versionStr) => s_values.First(x => x.StringValue == versionStr.ToUpper()).EnumValue;
}

public static class PackFileSerializerLoader
static class PackFileSerializerLoader
{
static readonly ILogger s_logger = Logging.CreateStatic(typeof(PackFileSerializerLoader));

public static PackFileContainer Load(string packFileSystemPath, BinaryReader reader, IDuplicateFileResolver duplicatePackFileResolver)
public static PackFileContainer Load(string packFileSystemPath, long packFileSize, BinaryReader reader, IDuplicateFileResolver duplicatePackFileResolver)
{
try
{
Expand All @@ -37,7 +37,7 @@ public static PackFileContainer Load(string packFileSystemPath, BinaryReader rea
SystemFilePath = packFileSystemPath,
Name = Path.GetFileNameWithoutExtension(packFileSystemPath),
Header = ReadHeader(reader),
OriginalLoadByteSize = new FileInfo(packFileSystemPath).Length,
OriginalLoadByteSize = packFileSize,
};

// If larger then int.max throw error
Expand All @@ -51,8 +51,6 @@ public static PackFileContainer Load(string packFileSystemPath, BinaryReader rea
FilePath = packFileSystemPath,
};

//var buffer = reader.ReadBytes((int)output.Header.DataStart - 28);

var offset = output.Header.DataStart;
var headerVersion = output.Header.Version;
for (var i = 0; i < output.Header.FileCount; i++)
Expand Down Expand Up @@ -179,52 +177,7 @@ static PFHeader ReadHeader(BinaryReader reader)
return header;
}

public static void WriteHeader(PFHeader header, uint fileContentSize, BinaryWriter writer)
{
var packFileTypeStr = PackFileVersionConverter.ToString(header.Version); // 4
foreach (var c in packFileTypeStr)
writer.Write(c);

writer.Write(header.ByteMask); // 8
writer.Write(header.DependantFiles.Count); // 12

var pack_file_index_size = 0;
foreach (var file in header.DependantFiles)
pack_file_index_size += file.Length + 1;

writer.Write(pack_file_index_size); // 16
writer.Write(header.FileCount); // 20
writer.Write(fileContentSize); // 24

switch (header.Version)
{
case PackFileVersion.PFH0:
break;// Nothing needed to do
case PackFileVersion.PFH2:
case PackFileVersion.PFH3:
// 64 bit timestamp
writer.Write(0);
writer.Write(0);
break;
case PackFileVersion.PFH4:
case PackFileVersion.PFH5:
if (header.HasExtendedHeader)
throw new Exception("Not supported packfile type");

writer.Write(PFHeader.DefaultTimeStamp);
break;

default:
throw new Exception("Not supported packfile type");
}

foreach (var file in header.DependantFiles)
{
foreach (byte c in file)
writer.Write(c);
writer.Write((byte)0);
}
}


private static byte[] DetectCompressionInfo(BinaryReader reader, long dataOffset, uint entrySize, bool isEncrypted)
{
Expand Down
Loading
Loading