Initial commit

This commit is contained in:
dennisarfan 2025-07-30 07:30:00 +07:00
commit ef3b7d68fb
30 changed files with 1568 additions and 0 deletions

View file

@ -0,0 +1,29 @@
using System.Buffers;
namespace StitchATon2.Infra.Buffers;
public class ArrayOwner<T> : IBuffer<T> where T : unmanaged
{
private readonly ArrayPool<T> _owner;
private readonly T[] _buffer;
public ArrayOwner(ArrayPool<T> owner, int size)
{
_owner = owner;
_buffer = owner.Rent(size);
}
~ArrayOwner() => Dispose();
public void Dispose()
{
_owner.Return(_buffer);
GC.SuppressFinalize(this);
}
public ref T this[int index] => ref _buffer[index];
public Span<T> Span => _buffer;
public T[] Array => _buffer;
}

8
Infra/Buffers/IBuffer.cs Normal file
View file

@ -0,0 +1,8 @@
namespace StitchATon2.Infra.Buffers;
public interface IBuffer<T> : IDisposable where T : unmanaged
{
ref T this[int index] { get; }
Span<T> Span { get; }
}

View file

@ -0,0 +1,15 @@
using System.Buffers;
namespace StitchATon2.Infra.Buffers;
public static class MemoryAllocator
{
public static IBuffer<T> Allocate<T>(int count) where T : unmanaged
=> new UnmanagedMemory<T>(count);
public static IMemoryOwner<T> AllocateManaged<T>(int count)
=> MemoryPool<T>.Shared.Rent(count);
public static ArrayOwner<T> AllocateArray<T>(int count) where T : unmanaged
=> new(ArrayPool<T>.Shared, count);
}

View file

@ -0,0 +1,28 @@
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StitchATon2.Infra.Buffers;
internal sealed unsafe class UnmanagedMemory<T> : IBuffer<T> where T : unmanaged
{
private readonly void* _pointer;
private readonly int _count;
public ref T this[int index] => ref Unsafe.AsRef<T>((T*)_pointer + index); // *((T*)_pointer + index);
public Span<T> Span => new(_pointer, _count);
public UnmanagedMemory(int count)
{
_pointer = NativeMemory.Alloc((nuint)count, (nuint)Unsafe.SizeOf<T>());
_count = count;
}
~UnmanagedMemory() => Dispose();
public void Dispose()
{
NativeMemory.Free(_pointer);
GC.SuppressFinalize(this);
}
}

33
Infra/Encoders/Crc32.cs Normal file
View file

@ -0,0 +1,33 @@
namespace StitchATon2.Infra.Encoders;
public static class Crc32
{
private static readonly Lazy<uint[]> LazyTable = new(GenerateTable);
private static uint[] Table => LazyTable.Value;
public static uint Compute(Span<byte> buffer, uint initial = 0xFFFFFFFF)
{
uint crc = initial;
foreach (var b in buffer)
{
crc = Table[(crc ^ b) & 0xFF] ^ (crc >> 8);
}
return ~crc;
}
private static uint[] GenerateTable()
{
const uint poly = 0xEDB88320;
var table = new uint[256];
for (uint i = 0; i < 256; i++)
{
uint c = i;
for (int j = 0; j < 8; j++)
{
c = (c & 1) != 0 ? (poly ^ (c >> 1)) : (c >> 1);
}
table[i] = c;
}
return table;
}
}

View file

@ -0,0 +1,143 @@
using System.Buffers.Binary;
using System.IO.Compression;
namespace StitchATon2.Infra.Encoders;
public class PngStreamEncoder : IDisposable, IAsyncDisposable
{
private const int BufferSize = 8 * 1024;
private const int FlushThreshold = 1024;
private readonly Stream _stream;
private readonly MemoryStream _memoryStream;
private readonly int _width;
private readonly int _height;
private readonly ZLibStream _zlibStream;
private bool _disposed;
private bool _shouldFlush;
public PngStreamEncoder(Stream writableStream, int width, int height)
{
_stream = writableStream;
_width = width;
_height = height;
_memoryStream = new MemoryStream(BufferSize * 2);
_zlibStream = new ZLibStream(_memoryStream, CompressionLevel.Optimal, leaveOpen: true);
_memoryStream.SetLength(8);
_memoryStream.Position = 8;
}
~PngStreamEncoder() => Dispose();
public async Task WriteHeader()
{
byte[] headerBytes = [
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, // PNG Signature
0x00, 0x00, 0x00, 0x0D, // Length
// IHDR chunk
0x49, 0x48, 0x44, 0x52, // IHDR
0x00, 0x00, 0x00, 0x00, // Reserve to write Width
0x00, 0x00, 0x00, 0x00, // Reserve to write Height
0x08, // Bit depth
0x02, // Color type
0x00, // Compression method
0x00, // Filter method
0x00, // Interlace method
0x00, 0x00, 0x00, 0x00, // Reserve to write CRC-32
];
BinaryPrimitives.WriteInt32BigEndian(headerBytes.AsSpan(16), _width);
BinaryPrimitives.WriteInt32BigEndian(headerBytes.AsSpan(20), _height);
var crc = Crc32.Compute(headerBytes.AsSpan(12, 17));
BinaryPrimitives.WriteUInt32BigEndian(headerBytes.AsSpan(29), crc);
await _stream.WriteAsync(headerBytes);
}
public async Task WriteData(Memory<byte> data)
{
_zlibStream.Write([0]);
var dataSlice = data;
while (dataSlice.Length > FlushThreshold)
{
await _zlibStream.WriteAsync(dataSlice[..FlushThreshold]);
await _zlibStream.FlushAsync();
dataSlice = dataSlice[FlushThreshold..];
if(_memoryStream.Length >= BufferSize)
await Flush();
}
if (dataSlice.Length > 0)
{
await _zlibStream.WriteAsync(dataSlice);
await _zlibStream.FlushAsync();
_shouldFlush = true;
}
}
private async Task Flush()
{
await _zlibStream.FlushAsync();
var dataSize = (int)(_memoryStream.Length - 8);
_memoryStream.Write("\0\0\0\0"u8);
_memoryStream.Position = 4;
_memoryStream.Write("IDAT"u8);
var buffer = _memoryStream.GetBuffer();
BinaryPrimitives.WriteInt32BigEndian(buffer, dataSize);
// write Crc
var crc = Crc32.Compute(buffer.AsSpan(4, dataSize + 4));
BinaryPrimitives.WriteUInt32BigEndian(buffer.AsSpan(dataSize + 8), crc);
await _stream.WriteAsync(buffer.AsMemory(0, dataSize + 12));
_memoryStream.SetLength(8);
_memoryStream.Position = 8;
_shouldFlush = false;
}
public async ValueTask WriteEndOfFile()
{
if(_shouldFlush)
await Flush();
var endChunk = new byte[] {
0x00, 0x00, 0x00, 0x00, // Length
0x49, 0x45, 0x4E, 0x44, // IEND
0xAE, 0x42, 0x60, 0x82, // Crc
};
await _stream.WriteAsync(endChunk);
await DisposeAsync();
}
public void Dispose()
{
if (!_disposed)
{
_zlibStream.Dispose();
_memoryStream.Dispose();
_disposed = true;
}
GC.SuppressFinalize(this);
}
public async ValueTask DisposeAsync()
{
if (!_disposed)
{
await _zlibStream.DisposeAsync();
await _memoryStream.DisposeAsync();
_disposed = true;
}
GC.SuppressFinalize(this);
}
}

387
Infra/ImageIntegral.cs Normal file
View file

@ -0,0 +1,387 @@
using System.Buffers;
using System.IO.MemoryMappedFiles;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using SixLabors.ImageSharp.Formats;
using SixLabors.ImageSharp.Formats.Png;
using SixLabors.ImageSharp.PixelFormats;
using StitchATon2.Infra.Buffers;
namespace StitchATon2.Infra;
public class ImageIntegral : IDisposable
{
private const int MaxProcessingQueue = 4;
private readonly string _imagePath;
private readonly string _outputDirectory;
private readonly int _width;
private readonly int _height;
private IMemoryOwner<ManualResetEventSlim>? _rowLocks;
private MemoryMappedFile? _memoryMappedFile;
private readonly object _lock = new();
private readonly ManualResetEventSlim _queueLock = new(true);
private readonly ManualResetEventSlim _initializationLock = new(false);
private volatile int _processedRows;
private volatile int _queueCounter;
public ImageIntegral(string imagePath, string outputDirectory, int width, int height)
{
_imagePath = imagePath;
_outputDirectory = outputDirectory;
_width = width;
_height = height;
}
~ImageIntegral() => Dispose();
public void Acquire(int row, Int32Pixel[] buffer, CancellationToken cancellationToken = default)
{
Acquire(row, cancellationToken);
ReadRow(row, buffer);
}
public void Acquire(int row, Span<Int32Pixel> buffer, CancellationToken cancellationToken = default)
{
Acquire(row, cancellationToken);
ReadRow(row, buffer);
}
private void Acquire(int row, CancellationToken cancellationToken)
{
if (_memoryMappedFile is null)
{
lock (_lock)
{
if (_memoryMappedFile is null)
{
Task.Factory.StartNew(() => Initialize(cancellationToken), cancellationToken);
_initializationLock.Wait(cancellationToken);
_initializationLock.Dispose();
}
}
}
_rowLocks?.Memory.Span[row].Wait(cancellationToken);
}
private void Initialize(CancellationToken cancellationToken)
{
var fileName = Path.GetFileNameWithoutExtension(_imagePath);
var path = Path.Combine(_outputDirectory, $"{fileName}.mmf");
var backedFileStream = InitializeBackedFile(path, out var header);
_processedRows = header.ProcessedRows;
if (header.ProcessedRows >= _height)
{
// When statement above is true
// then it is guaranteed that backed file is valid and fully processed
_memoryMappedFile = MemoryMappedFile.CreateFromFile(path, FileMode.Open);
_initializationLock.Set();
return;
}
var taskQueue = backedFileStream == null
? Task.CompletedTask
: AllocateBackedFile(backedFileStream, header);
taskQueue = taskQueue.ContinueWith(
_ =>
{
_memoryMappedFile = MemoryMappedFile.CreateFromFile(path, FileMode.Open);
_initializationLock.Set();
},
cancellationToken);
// initialize resource gating, all rows is expected to be locked
// if the backed file require to allocate, it should be safe to do this
// asynchronously
var rowLocks = MemoryAllocator.AllocateManaged<ManualResetEventSlim>(_height);
var rowLocksSpan = rowLocks.Memory.Span;
for (int i = 0; i < _height; i++)
{
var isOpen = i < header.ProcessedRows;
rowLocksSpan[i] = new ManualResetEventSlim(isOpen);
}
_rowLocks = rowLocks;
ProcessIntegral(taskQueue, cancellationToken);
}
private void ProcessIntegral(Task taskQueue, CancellationToken cancellationToken)
{
PngDecoderOptions decoderOptions = new()
{
PngCrcChunkHandling = PngCrcChunkHandling.IgnoreAll,
GeneralOptions = new DecoderOptions
{
MaxFrames = 1,
SkipMetadata = true,
}
};
using var fileStream = File.OpenRead(_imagePath);
using var image = PngDecoder.Instance.Decode<Rgb24>(decoderOptions, fileStream);
var imageBuffer = image.Frames.RootFrame.PixelBuffer;
var accumulator = Int32Pixel.Zero;
var buffer = MemoryAllocator.AllocateArray<Int32Pixel>(_width);
var processedRows = _processedRows;
Interlocked.Exchange(ref _queueCounter, 0);
// First row
if (processedRows == 0)
{
var sourceRow = imageBuffer.DangerousGetRowSpan(0);
for (var x = 0; x < sourceRow.Length; x++)
{
accumulator.Accumulate(sourceRow[x]);
buffer[x] = accumulator;
}
taskQueue = QueueWriterTask(taskQueue, 0, buffer.Clone(_width), cancellationToken);
processedRows++;
}
else
{
ReadRow(processedRows - 1, buffer);
}
if(cancellationToken.IsCancellationRequested)
return;
var prevBuffer = buffer;
buffer = MemoryAllocator.AllocateArray<Int32Pixel>(_width);
for (int y = processedRows; y < image.Height; y++)
{
var sourceRow = imageBuffer.DangerousGetRowSpan(y);
accumulator = (Int32Pixel)sourceRow[0];
buffer[0] = accumulator + prevBuffer[0];
// Process all other columns
for (var x = 1; x < sourceRow.Length; x++)
{
accumulator.Accumulate(sourceRow[x]);
buffer[x] = accumulator + prevBuffer[x];
}
if (_queueCounter >= MaxProcessingQueue)
{
_queueLock.Reset();
_queueLock.Wait(cancellationToken);
}
if(cancellationToken.IsCancellationRequested)
break;
var writeBuffer = prevBuffer;
Array.Copy(buffer.Array, writeBuffer.Array, image.Width);
taskQueue = QueueWriterTask(taskQueue, y, writeBuffer, cancellationToken);
prevBuffer = buffer;
buffer = MemoryAllocator.AllocateArray<Int32Pixel>(_width);
}
buffer.Dispose();
if(cancellationToken.IsCancellationRequested)
return;
taskQueue = taskQueue.ContinueWith(task =>
{
if (task.IsCompletedSuccessfully)
{
DisposeRowLocks();
_queueLock.Dispose();
}
}, cancellationToken);
taskQueue.Wait(cancellationToken);
}
private Task QueueWriterTask(
Task taskQueue,
int row,
ArrayOwner<Int32Pixel> writeBuffer,
CancellationToken cancellationToken)
{
Interlocked.Increment(ref _queueCounter);
cancellationToken.Register(writeBuffer.Dispose);
return taskQueue.ContinueWith(_ =>
{
using (var view = AcquireView(row, MemoryMappedFileAccess.Write))
view.WriteArray(0, writeBuffer.Array, 0, _width);
writeBuffer.Dispose();
_rowLocks!.Memory.Span[row].Set();
Interlocked.Increment(ref _processedRows);
using (var view = AcquireHeaderView(MemoryMappedFileAccess.Write))
view.Write(16, _processedRows);
Interlocked.Decrement(ref _queueCounter);
_queueLock.Set();
}, cancellationToken);
}
private MemoryMappedViewAccessor AcquireHeaderView(MemoryMappedFileAccess access)
=> _memoryMappedFile!.CreateViewAccessor(0, Header.Size, access);
private MemoryMappedViewAccessor AcquireView(int row, MemoryMappedFileAccess access)
{
var size = _width * Int32Pixel.Size;
var offset = row * size + Header.Size;
return _memoryMappedFile!.CreateViewAccessor(offset, size, access);
}
private void ReadRow(int row, Int32Pixel[] readBuffer)
{
using var view = AcquireView(row, MemoryMappedFileAccess.Read);
view.ReadArray(0, readBuffer, 0, _width);
}
private void ReadRow(int row, Span<Int32Pixel> buffer)
{
using var view = AcquireView(row, MemoryMappedFileAccess.Read);
view.DangerousReadSpan(0, buffer, 0, _width);
}
private void ReadRow(int row, IBuffer<Int32Pixel> buffer)
{
using var view = AcquireView(row, MemoryMappedFileAccess.Read);
view.DangerousReadSpan(0, buffer.Span, 0, _width);
}
private FileStream? InitializeBackedFile(string path, out Header header)
{
var expectedHeader = Header.CreateInitial(_width, _height);
// Expectation when file exists:
// - throws IOException when it's being processed, handle it if possible
// - returns null if file is valid
// - delete the existing file if it's not valid (modified from external)
FileStream fs;
if (File.Exists(path))
{
fs = File.OpenRead(path);
if (fs.Length < expectedHeader.Length + Header.Size)
{
fs.Dispose();
File.Delete(path);
}
Span<byte> headerBytes = stackalloc byte[Header.Size];
fs.ReadExactly(headerBytes);
header = MemoryMarshal.Cast<byte, Header>(headerBytes)[0];
var isValid = expectedHeader.Identifier == header.Identifier
&& expectedHeader.Width == header.Width
&& expectedHeader.Height == header.Height
&& expectedHeader.Length == header.Length;
fs.Dispose();
Console.WriteLine($"Image integral file found: {path}");
if (!isValid) File.Delete(path);
else return null;
}
// Expected process:
// - Initialize file creation
// - Write the header
// - Allocate initial content by Header.Length
// - Action above should be done asynchronously and return the task handle.
var fsOptions = new FileStreamOptions
{
Access = FileAccess.Write,
Share = FileShare.None,
Mode = FileMode.CreateNew,
PreallocationSize = Header.Size,
};
Console.WriteLine($"Create image integral file: {path}");
Directory.CreateDirectory(_outputDirectory);
fs = File.Open(path, fsOptions);
fs.Write(MemoryMarshal.AsBytes([expectedHeader]));
header = expectedHeader;
return fs;
}
private static async Task AllocateBackedFile(FileStream fileStream, Header header)
{
// The input filestream is expected to be empty with
// initial cursor at the beginning of the file and the content
// is pre-allocated for at least Header.Length bytes
// No other process should be accessed the file while being
// allocated.
// Allocated bytes is not necessary to be zeroed.
// const int writeBufferSize = 4 * 1024;
// using var writeBuffer = MemoryPool<byte>.Shared.Rent(writeBufferSize);
//
// var written = 0;
// while (written + writeBufferSize < header.Length)
// {
// await fileStream.WriteAsync(writeBuffer.Memory, cancellationToken);
// written += writeBufferSize;
// }
//
// if (written < header.Length)
// {
// await fileStream.WriteAsync(writeBuffer.Memory[..(header.Length - written)], cancellationToken);
// }
fileStream.SetLength(header.Length + Header.Size);
await fileStream.DisposeAsync();
}
[StructLayout(LayoutKind.Sequential)]
private struct Header
{
private const int Signature = 0x47544e49; // INTG
public static int Size => Unsafe.SizeOf<Header>();
public uint Identifier;
public int Width;
public int Height;
public int Length;
public int ProcessedRows;
public static Header CreateInitial(int width, int height) => new()
{
Identifier = Signature,
Width = width,
Height = height,
Length = width * height * Int32Pixel.Size,
ProcessedRows = 0,
};
}
private void DisposeRowLocks()
{
if (_rowLocks is { } locks)
{
_rowLocks = null;
var lockSpan = locks.Memory.Span;
for(int i = 0; i < _height; i++)
lockSpan[i].Dispose();
locks.Dispose();
}
}
public void Dispose()
{
DisposeRowLocks();
_memoryMappedFile?.Dispose();
_queueLock.Dispose();
_initializationLock.Dispose();
GC.SuppressFinalize(this);
}
}

82
Infra/Int32Pixel.cs Normal file
View file

@ -0,0 +1,82 @@
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using SixLabors.ImageSharp.PixelFormats;
namespace StitchATon2.Infra;
[StructLayout(LayoutKind.Sequential)]
public record struct Int32Pixel
{
public static int Size
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => Unsafe.SizeOf<Int32Pixel>();
}
public static Int32Pixel Zero
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => new();
}
public int R;
public int G;
public int B;
public Int32Pixel(int r, int g, int b)
{
R = r;
G = g;
B = b;
}
public void Accumulate(Int32Pixel pixel)
{
R += pixel.R;
G += pixel.G;
B += pixel.B;
}
public void Accumulate(Rgb24 pixel)
{
R += pixel.R;
G += pixel.G;
B += pixel.B;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Int32Pixel operator +(Int32Pixel a, Int32Pixel b)
{
return new Int32Pixel(a.R + b.R, a.G + b.G, a.B + b.B);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Int32Pixel operator +(Int32Pixel a, int b)
{
return new Int32Pixel(a.R + b, a.G + b, a.B + b);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Int32Pixel operator -(Int32Pixel a, Int32Pixel b)
{
return new Int32Pixel(a.R - b.R, a.G - b.G, a.B - b.B);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Int32Pixel operator /(Int32Pixel a, int b)
{
return new Int32Pixel(a.R / b, a.G / b, a.B / b);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static explicit operator Int32Pixel(Rgb24 pixel)
{
return new Int32Pixel(pixel.R, pixel.G, pixel.B);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static explicit operator Rgb24(Int32Pixel pixel)
{
return new Rgb24((byte)pixel.R, (byte)pixel.G, (byte)pixel.B);
}
}

View file

@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.10" />
</ItemGroup>
</Project>

View file

@ -0,0 +1,12 @@
namespace StitchATon2.Infra.Synchronization;
public static class TaskHelper
{
public static TaskFactory CreateTaskFactory()
{
return new TaskFactory(
TaskCreationOptions.AttachedToParent,
TaskContinuationOptions.ExecuteSynchronously
);
}
}

56
Infra/Utils.cs Normal file
View file

@ -0,0 +1,56 @@
using System.IO.MemoryMappedFiles;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using StitchATon2.Infra.Buffers;
namespace StitchATon2.Infra;
public static class Utils
{
[UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_buffer")]
private static extern ref SafeBuffer GetSafeBuffer(this UnmanagedMemoryAccessor view);
[UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_offset")]
private static extern ref long GetOffset(this UnmanagedMemoryAccessor view);
private static unsafe uint AlignedSizeOf<T>() where T : unmanaged
{
uint size = (uint)sizeof(T);
return size is 1 or 2 ? size : (uint)((size + 3) & (~3));
}
public static void DangerousReadSpan<T>(this UnmanagedMemoryAccessor view, long position, Span<T> span, int offset, int count)
where T : unmanaged
{
uint sizeOfT = AlignedSizeOf<T>();
int n = count;
long spaceLeft = view.Capacity - position;
if (spaceLeft < 0)
{
n = 0;
}
else
{
ulong spaceNeeded = (ulong)(sizeOfT * count);
if ((ulong)spaceLeft < spaceNeeded)
{
n = (int)(spaceLeft / sizeOfT);
}
}
var byteOffset = (ulong)(view.GetOffset() + position);
view.GetSafeBuffer().ReadSpan(byteOffset, span.Slice(offset, n));
}
public static ArrayOwner<T> Clone<T>(this ArrayOwner<T> arrayOwner, int length) where T : unmanaged
{
var newArrayOwner = MemoryAllocator.AllocateArray<T>(length);
Array.Copy(arrayOwner.Array, 0, newArrayOwner.Array, 0, length);
return newArrayOwner;
}
public static void CopyTo<T>(this ArrayOwner<T> arrayOwner, ArrayOwner<T> target, int length) where T : unmanaged
{
Array.Copy(arrayOwner.Array, 0, target.Array, 0, length);
}
}