From a1cb6592eb4d109a4f9ccc50239dfcb48a6c8ed7 Mon Sep 17 00:00:00 2001 From: dennisarfan Date: Thu, 31 Jul 2025 06:19:32 +0700 Subject: [PATCH] Upgrade to .net9 --- App/Controllers/ImageController.cs | 4 +- App/StitchATon2.App.csproj | 3 +- App/Utils.cs | 7 + Domain/ImageCreators/DangerousImageCreator.cs | 172 ++++++++++++++++++ Domain/StitchATon2.Domain.csproj | 2 +- Domain/Utils.cs | 30 ++- Infra/Buffers/ImmovableMemory.cs | 37 ++++ Infra/Buffers/MemoryAllocator.cs | 3 + Infra/Buffers/UnmanagedMemory.cs | 19 +- Infra/Encoders/PngPipeEncoder.cs | 135 ++++++++++++++ Infra/ImageIntegral.cs | 4 +- Infra/StitchATon2.Infra.csproj | 4 +- Infra/Synchronization/TaskHelper.cs | 10 +- Infra/Utils.cs | 17 +- global.json | 2 +- 15 files changed, 395 insertions(+), 54 deletions(-) create mode 100644 Domain/ImageCreators/DangerousImageCreator.cs create mode 100644 Infra/Buffers/ImmovableMemory.cs create mode 100644 Infra/Encoders/PngPipeEncoder.cs diff --git a/App/Controllers/ImageController.cs b/App/Controllers/ImageController.cs index 2b5386f..d2fda76 100644 --- a/App/Controllers/ImageController.cs +++ b/App/Controllers/ImageController.cs @@ -24,7 +24,7 @@ public static class ImageController await tileManager .CreateSection(dto) - .WriteToStream(response.Body, dto.OutputScale); + .DangerousWriteToPipe(response.BodyWriter, dto.OutputScale); await response.CompleteAsync(); } @@ -47,7 +47,7 @@ public static class ImageController var scale = float.Clamp(480f / int.Max(section.Width, section.Height), 0.01f, 1f); Console.WriteLine($"Generate random image for {coordinatePair} scale: {scale}"); - await section.WriteToStream(response.Body, scale); + await section.DangerousWriteToPipe(response.BodyWriter, scale); await response.CompleteAsync(); } } \ No newline at end of file diff --git a/App/StitchATon2.App.csproj b/App/StitchATon2.App.csproj index ff5d1d5..aa50324 100644 --- a/App/StitchATon2.App.csproj +++ b/App/StitchATon2.App.csproj @@ -1,11 +1,12 @@ - net8.0 + net9.0 enable enable true StitchATon2.App + true diff --git a/App/Utils.cs b/App/Utils.cs index bf13bbd..2b96e01 100644 --- a/App/Utils.cs +++ b/App/Utils.cs @@ -1,3 +1,4 @@ +using System.IO.Pipelines; using StitchATon2.App.Models; using StitchATon2.Domain; using StitchATon2.Domain.ImageCreators; @@ -19,4 +20,10 @@ public static class Utils var imageCreator = new ImageCreator(section); await imageCreator.WriteToStream(stream, scale!.Value); } + + public static async Task DangerousWriteToPipe(this GridSection section, PipeWriter pipeWriter, float? scale) + { + var imageCreator = new DangerousImageCreator(section); + await imageCreator.WriteToPipe(pipeWriter, scale!.Value); + } } \ No newline at end of file diff --git a/Domain/ImageCreators/DangerousImageCreator.cs b/Domain/ImageCreators/DangerousImageCreator.cs new file mode 100644 index 0000000..2346479 --- /dev/null +++ b/Domain/ImageCreators/DangerousImageCreator.cs @@ -0,0 +1,172 @@ +using System.IO.Pipelines; +using System.Runtime.CompilerServices; +using SixLabors.ImageSharp.PixelFormats; +using StitchATon2.Infra; +using StitchATon2.Infra.Buffers; +using StitchATon2.Infra.Encoders; +using StitchATon2.Infra.Synchronization; + +namespace StitchATon2.Domain.ImageCreators; + +public sealed class DangerousImageCreator : IDisposable +{ + private readonly GridSection _section; + private readonly IBuffer _mmfReadBuffer; + + private int FullWidth => _section.TileManager.Configuration.FullWidth; + private int FullHeight => _section.TileManager.Configuration.FullHeight; + + private int OffsetX => _section.OffsetX; + private int OffsetY => _section.OffsetY; + + private int Width => _section.Width; + private int Height => _section.Height; + + private int TileWidth => _section.TileManager.Configuration.Width; + private int TileHeight => _section.TileManager.Configuration.Height; + private Tile TileOrigin => _section.Origin; + + private int RightmostPixelIndex => _section.TileManager.Configuration.RightTileIndex; + private int BottomPixelIndex => _section.TileManager.Configuration.BottomTileIndex; + + private TileManager TileManager => _section.TileManager; + + public DangerousImageCreator(GridSection section) + { + _section = section; + _mmfReadBuffer = MemoryAllocator.Allocate(TileWidth); + } + + ~DangerousImageCreator() => Dispose(); + + public async Task WriteToPipe(PipeWriter outputPipe, float scale, CancellationToken cancellationToken = default) + { + var scaleFactor = MathF.ReciprocalEstimate(scale); + var targetWidth = (int)(Width / scaleFactor); + var targetHeight = (int)(Height / scaleFactor); + + var encoder = new PngPipeEncoder(outputPipe, targetWidth, targetHeight); + encoder.WriteHeader(); + + var outputBufferSize = targetWidth * Unsafe.SizeOf(); + + using var xLookup = Utils.BoundsMatrix(scaleFactor, targetWidth, FullWidth, OffsetX); + using var yLookup = Utils.BoundsMatrix(scaleFactor, targetHeight, FullHeight, OffsetY); + + using var yStartMap = MemoryAllocator.Allocate(targetWidth); + using var yEndMap = MemoryAllocator.Allocate(targetWidth); + + var yStart = OffsetY; + + var outputTaskQueue = TaskHelper.SynchronizedTaskFactory.StartNew(() => { }, cancellationToken); + for (var y = 0; y < targetHeight; y++) + { + var yEnd = yLookup[y]; + + var (localRow0, localOffsetY0) = int.DivRem(yStart, TileHeight); + MapRow(localRow0, localOffsetY0, xLookup, targetWidth, yStartMap); + + var (localRow1, localOffsetY1) = int.DivRem(yEnd, TileHeight); + MapRow(localRow1, localOffsetY1, xLookup, targetWidth, yEndMap); + + if (localRow0 != localRow1) + { + MapRow(localRow0, BottomPixelIndex, xLookup, targetWidth, yEndMap, true); + } + + int xStart = OffsetX, x0 = 0; + + var pxInt32 = Int32Pixel.Zero; + ref var px = ref pxInt32; + ref var rChannel = ref Unsafe.As(ref px); + ref var gChannel = ref Unsafe.Add(ref rChannel, 4); + ref var bChannel = ref Unsafe.Add(ref rChannel, 8); + + var outputBuffer = MemoryAllocator.Allocate(outputBufferSize); + ref var outputChannel = ref outputBuffer.Span[0]; + for (int x1 = 0; x1 < targetWidth; x1++) + { + var xEnd = xLookup[x1]; + + px = yEndMap[x1]; + px += yStartMap[x0]; + px -= yEndMap[x0]; + px -= yStartMap[x1]; + px /= Math.Max(1, (xEnd - xStart) * (yEnd - yStart)); + + outputChannel = rChannel; + outputChannel = ref Unsafe.Add(ref outputChannel, 1); + + outputChannel = gChannel; + outputChannel = ref Unsafe.Add(ref outputChannel, 1); + + outputChannel = bChannel; + outputChannel = ref Unsafe.Add(ref outputChannel, 1); + + xStart = xEnd; + x0 = x1; + } + + outputTaskQueue = outputTaskQueue + .ContinueWith(_ => encoder.WriteData(outputBuffer, cancellationToken: cancellationToken), cancellationToken); + + yStart = yEnd; + } + + await outputTaskQueue; + encoder.WriteEndOfFile(cancellationToken); + } + + private void MapRow( + int rowOffset, + int yOffset, + IBuffer boundsMatrix, + int count, + IBuffer destination, + bool appendMode = false) + { + var sourceMap = boundsMatrix.Span[..count]; + var currentTile = TileManager.GetAdjacent(TileOrigin, 0, rowOffset); + var xAdder = Int32Pixel.Zero; + var xOffset = 0; + var written = 0; + var destinationSpan = destination.Span; + var readBufferSpan = _mmfReadBuffer.Span; + while (true) + { + currentTile.Integral.Acquire(yOffset, readBufferSpan); + int localX; + if (appendMode) + { + while (written < sourceMap.Length && (localX = sourceMap[written] - xOffset) < TileWidth) + { + destinationSpan[written] += readBufferSpan[localX]; + destinationSpan[written] += xAdder; + written++; + } + } + else + { + while (written < sourceMap.Length && (localX = sourceMap[written] - xOffset) < TileWidth) + { + destinationSpan[written] = readBufferSpan[localX]; + destinationSpan[written] += xAdder; + written++; + } + } + + if (written >= sourceMap.Length) + break; + + xAdder += readBufferSpan[RightmostPixelIndex]; + xOffset += TileWidth; + currentTile = TileManager.GetAdjacent(currentTile, 1, 0); + } + } + + public void Dispose() + { + _mmfReadBuffer.Dispose(); + GC.SuppressFinalize(this); + } +} \ No newline at end of file diff --git a/Domain/StitchATon2.Domain.csproj b/Domain/StitchATon2.Domain.csproj index 6e68f4d..99a234b 100644 --- a/Domain/StitchATon2.Domain.csproj +++ b/Domain/StitchATon2.Domain.csproj @@ -1,7 +1,7 @@  - net8.0 + net9.0 enable enable diff --git a/Domain/Utils.cs b/Domain/Utils.cs index b3bd3d0..4cde46b 100644 --- a/Domain/Utils.cs +++ b/Domain/Utils.cs @@ -20,6 +20,7 @@ public static class Utils return (a + b - 1) / b; } + [Pure] public static (int Column, int Row) GetSBSCoordinate(string coordinate) { var column = coordinate[^1] - '0'; @@ -33,7 +34,13 @@ public static class Utils return (column, row); } - + /// + /// Performs a SIMD-accelerated calculation that generates a buffer of bounded, scaled indices. + /// + /// The amount by which to scale the sequence values. + /// The total number of scalar values to generate. + /// Upper limit (exclusive) for clamping values. + /// The offset to apply before clamping. public static IBuffer BoundsMatrix(float scaleFactor, int length, int max, int offset) { var vectorSize = DivCeil(length, Vector.Count); @@ -45,14 +52,13 @@ public static class Utils var vectorMax = new Vector(max - 1); var vectorScale = new Vector(scaleFactor); - var vectorSequence = SequenceVector(0f, 1f); + var vectorSequence = Vector.CreateSequence(0f, 1f); var seq = 0f; for (var i = 0; i < vectorSize; i++, seq += Vector.Count) { var sequence = new Vector(seq) + vectorSequence; - span[i] = Vector.Multiply(sequence, vectorScale); - span[i] = Vector.Add(span[i], vectorScale); + span[i] = Vector.FusedMultiplyAdd(sequence, vectorScale, vectorScale); span[i] = Vector.Ceiling(span[i]); } @@ -62,23 +68,9 @@ public static class Utils { resultSpan[i] = Vector.ConvertToInt32(span[i]); resultSpan[i] = Vector.Add(resultSpan[i], vectorOffset); - resultSpan[i] = Vector.Min(resultSpan[i], vectorMax); - resultSpan[i] = Vector.Max(resultSpan[i], vectorMin); + resultSpan[i] = Vector.ClampNative(resultSpan[i], vectorMin, vectorMax); } return result; } - - private static Vector SequenceVector(float start, float step) - { - var vector = Vector.Zero; - ref var reference = ref Unsafe.As, float>(ref vector); - for (var i = 0; i < Vector.Count; i++) - { - ref var current = ref Unsafe.Add(ref reference, i); - current = start + step * i; - } - - return vector; - } } \ No newline at end of file diff --git a/Infra/Buffers/ImmovableMemory.cs b/Infra/Buffers/ImmovableMemory.cs new file mode 100644 index 0000000..4400c9a --- /dev/null +++ b/Infra/Buffers/ImmovableMemory.cs @@ -0,0 +1,37 @@ +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace StitchATon2.Infra.Buffers; + +internal sealed unsafe class ImmovableMemory : MemoryManager where T : unmanaged +{ + private readonly T* _pointer; + private readonly int _length; + private bool _disposed; + + public ImmovableMemory(int count) + { + _pointer = (T*)NativeMemory.Alloc((nuint)count, (nuint)Unsafe.SizeOf()); + _length = count; + } + + protected override void Dispose(bool disposing) + { + if (!_disposed) + { + NativeMemory.Free(_pointer); + _disposed = true; + } + } + + public override Span GetSpan() + => new(_pointer, _length); + + public override MemoryHandle Pin(int elementIndex = 0) + => new(_pointer + elementIndex); + + public override void Unpin() + { + } +} \ No newline at end of file diff --git a/Infra/Buffers/MemoryAllocator.cs b/Infra/Buffers/MemoryAllocator.cs index 43c1590..75787f0 100644 --- a/Infra/Buffers/MemoryAllocator.cs +++ b/Infra/Buffers/MemoryAllocator.cs @@ -12,4 +12,7 @@ public static class MemoryAllocator public static ArrayOwner AllocateArray(int count) where T : unmanaged => new(ArrayPool.Shared, count); + + public static MemoryManager AllocateImmovable(int count) where T : unmanaged + => new ImmovableMemory(count); } \ No newline at end of file diff --git a/Infra/Buffers/UnmanagedMemory.cs b/Infra/Buffers/UnmanagedMemory.cs index 4bf6064..7f61b3d 100644 --- a/Infra/Buffers/UnmanagedMemory.cs +++ b/Infra/Buffers/UnmanagedMemory.cs @@ -3,18 +3,23 @@ using System.Runtime.InteropServices; namespace StitchATon2.Infra.Buffers; +/// +/// Provide non-thread safe anti GC contiguous memory. +/// +/// internal sealed unsafe class UnmanagedMemory : IBuffer where T : unmanaged { - private readonly void* _pointer; + private readonly T* _pointer; private readonly int _count; + private bool _disposed; - public ref T this[int index] => ref Unsafe.AsRef((T*)_pointer + index); // *((T*)_pointer + index); + public ref T this[int index] => ref Unsafe.AsRef(_pointer + index); public Span Span => new(_pointer, _count); public UnmanagedMemory(int count) { - _pointer = NativeMemory.Alloc((nuint)count, (nuint)Unsafe.SizeOf()); + _pointer = (T*)NativeMemory.Alloc((nuint)count, (nuint)Unsafe.SizeOf()); _count = count; } @@ -22,7 +27,11 @@ internal sealed unsafe class UnmanagedMemory : IBuffer where T : unmanaged public void Dispose() { - NativeMemory.Free(_pointer); - GC.SuppressFinalize(this); + if (!_disposed) + { + NativeMemory.Free(_pointer); + GC.SuppressFinalize(this); + _disposed = true; + } } } \ No newline at end of file diff --git a/Infra/Encoders/PngPipeEncoder.cs b/Infra/Encoders/PngPipeEncoder.cs new file mode 100644 index 0000000..8aef896 --- /dev/null +++ b/Infra/Encoders/PngPipeEncoder.cs @@ -0,0 +1,135 @@ +using System.Buffers; +using System.Buffers.Binary; +using System.IO.Compression; +using System.IO.Pipelines; +using StitchATon2.Infra.Buffers; + +namespace StitchATon2.Infra.Encoders; + +public class PngPipeEncoder : IDisposable +{ + private const int BufferSize = 8 * 1024; + private const int FlushThreshold = 1024; + + private readonly PipeWriter _outputPipe; + private readonly MemoryStream _memoryStream; + private readonly int _width; + private readonly int _height; + + private readonly ZLibStream _zlibStream; + private bool _disposed; + private bool _shouldFlush; + + public PngPipeEncoder(PipeWriter outputPipe, int width, int height) + { + _outputPipe = outputPipe; + _width = width; + _height = height; + _memoryStream = new MemoryStream(BufferSize * 2); + _zlibStream = new ZLibStream(_memoryStream, CompressionLevel.Optimal, leaveOpen: true); + _memoryStream.SetLength(8); + _memoryStream.Position = 8; + } + + ~PngPipeEncoder() => Dispose(); + + public void WriteHeader() + { + Span headerBytes = [ + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, // PNG Signature + 0x00, 0x00, 0x00, 0x0D, // Length + + // IHDR chunk + 0x49, 0x48, 0x44, 0x52, // IHDR + 0x00, 0x00, 0x00, 0x00, // Reserve to write Width + 0x00, 0x00, 0x00, 0x00, // Reserve to write Height + 0x08, // Bit depth + 0x02, // Color type + 0x00, // Compression method + 0x00, // Filter method + 0x00, // Interlace method + 0x00, 0x00, 0x00, 0x00, // Reserve to write CRC-32 + ]; + + BinaryPrimitives.WriteInt32BigEndian(headerBytes[16..], _width); + BinaryPrimitives.WriteInt32BigEndian(headerBytes[20..], _height); + var crc = Crc32.Compute(headerBytes.Slice(12, 17)); + + BinaryPrimitives.WriteUInt32BigEndian(headerBytes[29..], crc); + + _outputPipe.Write(headerBytes); + } + + public void WriteData(IBuffer buffer, bool disposeBuffer = true, CancellationToken cancellationToken = default) + { + _zlibStream.Write([0]); + + var dataSlice = buffer.Span; + while (dataSlice.Length > FlushThreshold) + { + _zlibStream.Write(dataSlice[..FlushThreshold]); + _zlibStream.Flush(); + dataSlice = dataSlice[FlushThreshold..]; + if(_memoryStream.Length >= BufferSize) + Flush(cancellationToken); + } + + if (dataSlice.Length > 0) + { + _zlibStream.Write(dataSlice); + _zlibStream.Flush(); + _shouldFlush = true; + } + + if(disposeBuffer) buffer.Dispose(); + } + + private void Flush(CancellationToken cancellationToken) + { + _zlibStream.Flush(); + var dataSize = (int)(_memoryStream.Length - 8); + + _memoryStream.Write("\0\0\0\0"u8); + + _memoryStream.Position = 4; + _memoryStream.Write("IDAT"u8); + + var buffer = _memoryStream.GetBuffer(); + BinaryPrimitives.WriteInt32BigEndian(buffer, dataSize); + + // write Crc + var crc = Crc32.Compute(buffer.AsSpan(4, dataSize + 4)); + BinaryPrimitives.WriteUInt32BigEndian(buffer.AsSpan(dataSize + 8), crc); + + _outputPipe.Write(buffer.AsSpan(0, dataSize + 12)); + _memoryStream.SetLength(8); + _memoryStream.Position = 8; + _shouldFlush = false; + } + + public void WriteEndOfFile(CancellationToken cancellationToken = default) + { + if(_shouldFlush) + Flush(cancellationToken); + + Span endChunk = [ + 0x00, 0x00, 0x00, 0x00, // Length + 0x49, 0x45, 0x4E, 0x44, // IEND + 0xAE, 0x42, 0x60, 0x82, // Crc + ]; + + _outputPipe.Write(endChunk); + Dispose(); + } + + public void Dispose() + { + if (!_disposed) + { + _zlibStream.Dispose(); + _memoryStream.Dispose(); + _disposed = true; + GC.SuppressFinalize(this); + } + } +} \ No newline at end of file diff --git a/Infra/ImageIntegral.cs b/Infra/ImageIntegral.cs index 49a8d49..f3d20e4 100644 --- a/Infra/ImageIntegral.cs +++ b/Infra/ImageIntegral.cs @@ -20,7 +20,7 @@ public class ImageIntegral : IDisposable private IMemoryOwner? _rowLocks; private MemoryMappedFile? _memoryMappedFile; - private readonly object _lock = new(); + private readonly Lock _lock = new(); private readonly ManualResetEventSlim _queueLock = new(true); private readonly ManualResetEventSlim _initializationLock = new(false); @@ -249,7 +249,7 @@ public class ImageIntegral : IDisposable view.DangerousReadSpan(0, buffer, 0, _width); } - private void ReadRow(int row, IBuffer buffer) + private void ReadRow(int row, ArrayOwner buffer) { using var view = AcquireView(row, MemoryMappedFileAccess.Read); view.DangerousReadSpan(0, buffer.Span, 0, _width); diff --git a/Infra/StitchATon2.Infra.csproj b/Infra/StitchATon2.Infra.csproj index 6e6dca5..8a8bbb2 100644 --- a/Infra/StitchATon2.Infra.csproj +++ b/Infra/StitchATon2.Infra.csproj @@ -1,14 +1,14 @@  - net8.0 + net9.0 enable enable true - + diff --git a/Infra/Synchronization/TaskHelper.cs b/Infra/Synchronization/TaskHelper.cs index aa10870..594c9c0 100644 --- a/Infra/Synchronization/TaskHelper.cs +++ b/Infra/Synchronization/TaskHelper.cs @@ -2,11 +2,7 @@ namespace StitchATon2.Infra.Synchronization; public static class TaskHelper { - public static TaskFactory CreateTaskFactory() - { - return new TaskFactory( - TaskCreationOptions.AttachedToParent, - TaskContinuationOptions.ExecuteSynchronously - ); - } + public static readonly TaskFactory SynchronizedTaskFactory = new( + TaskCreationOptions.LongRunning, + TaskContinuationOptions.ExecuteSynchronously); } \ No newline at end of file diff --git a/Infra/Utils.cs b/Infra/Utils.cs index a67462c..4ca34c3 100644 --- a/Infra/Utils.cs +++ b/Infra/Utils.cs @@ -7,19 +7,13 @@ namespace StitchATon2.Infra; public static class Utils { - [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_buffer")] - private static extern ref SafeBuffer GetSafeBuffer(this UnmanagedMemoryAccessor view); - - [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_offset")] - private static extern ref long GetOffset(this UnmanagedMemoryAccessor view); - private static unsafe uint AlignedSizeOf() where T : unmanaged { uint size = (uint)sizeof(T); return size is 1 or 2 ? size : (uint)((size + 3) & (~3)); } - public static void DangerousReadSpan(this UnmanagedMemoryAccessor view, long position, Span span, int offset, int count) + internal static void DangerousReadSpan(this MemoryMappedViewAccessor view, long position, Span span, int offset, int count) where T : unmanaged { uint sizeOfT = AlignedSizeOf(); @@ -38,8 +32,8 @@ public static class Utils } } - var byteOffset = (ulong)(view.GetOffset() + position); - view.GetSafeBuffer().ReadSpan(byteOffset, span.Slice(offset, n)); + var byteOffset = (ulong)(view.PointerOffset + position); + view.SafeMemoryMappedViewHandle.ReadSpan(byteOffset, span.Slice(offset, n)); } public static ArrayOwner Clone(this ArrayOwner arrayOwner, int length) where T : unmanaged @@ -48,9 +42,4 @@ public static class Utils Array.Copy(arrayOwner.Array, 0, newArrayOwner.Array, 0, length); return newArrayOwner; } - - public static void CopyTo(this ArrayOwner arrayOwner, ArrayOwner target, int length) where T : unmanaged - { - Array.Copy(arrayOwner.Array, 0, target.Array, 0, length); - } } \ No newline at end of file diff --git a/global.json b/global.json index 2ddda36..93681ff 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.0", + "version": "9.0.0", "rollForward": "latestMinor", "allowPrerelease": false }