dangerous #2

Merged
dennisarfan merged 4 commits from dangerous into main 2025-08-01 15:17:43 +00:00
15 changed files with 395 additions and 54 deletions
Showing only changes of commit a1cb6592eb - Show all commits

View file

@ -24,7 +24,7 @@ public static class ImageController
await tileManager
.CreateSection(dto)
.WriteToStream(response.Body, dto.OutputScale);
.DangerousWriteToPipe(response.BodyWriter, dto.OutputScale);
await response.CompleteAsync();
}
@ -47,7 +47,7 @@ public static class ImageController
var scale = float.Clamp(480f / int.Max(section.Width, section.Height), 0.01f, 1f);
Console.WriteLine($"Generate random image for {coordinatePair} scale: {scale}");
await section.WriteToStream(response.Body, scale);
await section.DangerousWriteToPipe(response.BodyWriter, scale);
await response.CompleteAsync();
}
}

View file

@ -1,11 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<InvariantGlobalization>true</InvariantGlobalization>
<RootNamespace>StitchATon2.App</RootNamespace>
<PublishAot>true</PublishAot>
</PropertyGroup>
<ItemGroup>

View file

@ -1,3 +1,4 @@
using System.IO.Pipelines;
using StitchATon2.App.Models;
using StitchATon2.Domain;
using StitchATon2.Domain.ImageCreators;
@ -19,4 +20,10 @@ public static class Utils
var imageCreator = new ImageCreator(section);
await imageCreator.WriteToStream(stream, scale!.Value);
}
public static async Task DangerousWriteToPipe(this GridSection section, PipeWriter pipeWriter, float? scale)
{
var imageCreator = new DangerousImageCreator(section);
await imageCreator.WriteToPipe(pipeWriter, scale!.Value);
}
}

View file

@ -0,0 +1,172 @@
using System.IO.Pipelines;
using System.Runtime.CompilerServices;
using SixLabors.ImageSharp.PixelFormats;
using StitchATon2.Infra;
using StitchATon2.Infra.Buffers;
using StitchATon2.Infra.Encoders;
using StitchATon2.Infra.Synchronization;
namespace StitchATon2.Domain.ImageCreators;
public sealed class DangerousImageCreator : IDisposable
{
private readonly GridSection _section;
private readonly IBuffer<Int32Pixel> _mmfReadBuffer;
private int FullWidth => _section.TileManager.Configuration.FullWidth;
private int FullHeight => _section.TileManager.Configuration.FullHeight;
private int OffsetX => _section.OffsetX;
private int OffsetY => _section.OffsetY;
private int Width => _section.Width;
private int Height => _section.Height;
private int TileWidth => _section.TileManager.Configuration.Width;
private int TileHeight => _section.TileManager.Configuration.Height;
private Tile TileOrigin => _section.Origin;
private int RightmostPixelIndex => _section.TileManager.Configuration.RightTileIndex;
private int BottomPixelIndex => _section.TileManager.Configuration.BottomTileIndex;
private TileManager TileManager => _section.TileManager;
public DangerousImageCreator(GridSection section)
{
_section = section;
_mmfReadBuffer = MemoryAllocator.Allocate<Int32Pixel>(TileWidth);
}
~DangerousImageCreator() => Dispose();
public async Task WriteToPipe(PipeWriter outputPipe, float scale, CancellationToken cancellationToken = default)
{
var scaleFactor = MathF.ReciprocalEstimate(scale);
var targetWidth = (int)(Width / scaleFactor);
var targetHeight = (int)(Height / scaleFactor);
var encoder = new PngPipeEncoder(outputPipe, targetWidth, targetHeight);
encoder.WriteHeader();
var outputBufferSize = targetWidth * Unsafe.SizeOf<Rgb24>();
using var xLookup = Utils.BoundsMatrix(scaleFactor, targetWidth, FullWidth, OffsetX);
using var yLookup = Utils.BoundsMatrix(scaleFactor, targetHeight, FullHeight, OffsetY);
using var yStartMap = MemoryAllocator.Allocate<Int32Pixel>(targetWidth);
using var yEndMap = MemoryAllocator.Allocate<Int32Pixel>(targetWidth);
var yStart = OffsetY;
var outputTaskQueue = TaskHelper.SynchronizedTaskFactory.StartNew(() => { }, cancellationToken);
for (var y = 0; y < targetHeight; y++)
{
var yEnd = yLookup[y];
var (localRow0, localOffsetY0) = int.DivRem(yStart, TileHeight);
MapRow(localRow0, localOffsetY0, xLookup, targetWidth, yStartMap);
var (localRow1, localOffsetY1) = int.DivRem(yEnd, TileHeight);
MapRow(localRow1, localOffsetY1, xLookup, targetWidth, yEndMap);
if (localRow0 != localRow1)
{
MapRow(localRow0, BottomPixelIndex, xLookup, targetWidth, yEndMap, true);
}
int xStart = OffsetX, x0 = 0;
var pxInt32 = Int32Pixel.Zero;
ref var px = ref pxInt32;
ref var rChannel = ref Unsafe.As<Int32Pixel, byte>(ref px);
ref var gChannel = ref Unsafe.Add(ref rChannel, 4);
ref var bChannel = ref Unsafe.Add(ref rChannel, 8);
var outputBuffer = MemoryAllocator.Allocate<byte>(outputBufferSize);
ref var outputChannel = ref outputBuffer.Span[0];
for (int x1 = 0; x1 < targetWidth; x1++)
{
var xEnd = xLookup[x1];
px = yEndMap[x1];
px += yStartMap[x0];
px -= yEndMap[x0];
px -= yStartMap[x1];
px /= Math.Max(1, (xEnd - xStart) * (yEnd - yStart));
outputChannel = rChannel;
outputChannel = ref Unsafe.Add(ref outputChannel, 1);
outputChannel = gChannel;
outputChannel = ref Unsafe.Add(ref outputChannel, 1);
outputChannel = bChannel;
outputChannel = ref Unsafe.Add(ref outputChannel, 1);
xStart = xEnd;
x0 = x1;
}
outputTaskQueue = outputTaskQueue
.ContinueWith(_ => encoder.WriteData(outputBuffer, cancellationToken: cancellationToken), cancellationToken);
yStart = yEnd;
}
await outputTaskQueue;
encoder.WriteEndOfFile(cancellationToken);
}
private void MapRow(
int rowOffset,
int yOffset,
IBuffer<int> boundsMatrix,
int count,
IBuffer<Int32Pixel> destination,
bool appendMode = false)
{
var sourceMap = boundsMatrix.Span[..count];
var currentTile = TileManager.GetAdjacent(TileOrigin, 0, rowOffset);
var xAdder = Int32Pixel.Zero;
var xOffset = 0;
var written = 0;
var destinationSpan = destination.Span;
var readBufferSpan = _mmfReadBuffer.Span;
while (true)
{
currentTile.Integral.Acquire(yOffset, readBufferSpan);
int localX;
if (appendMode)
{
while (written < sourceMap.Length && (localX = sourceMap[written] - xOffset) < TileWidth)
{
destinationSpan[written] += readBufferSpan[localX];
destinationSpan[written] += xAdder;
written++;
}
}
else
{
while (written < sourceMap.Length && (localX = sourceMap[written] - xOffset) < TileWidth)
{
destinationSpan[written] = readBufferSpan[localX];
destinationSpan[written] += xAdder;
written++;
}
}
if (written >= sourceMap.Length)
break;
xAdder += readBufferSpan[RightmostPixelIndex];
xOffset += TileWidth;
currentTile = TileManager.GetAdjacent(currentTile, 1, 0);
}
}
public void Dispose()
{
_mmfReadBuffer.Dispose();
GC.SuppressFinalize(this);
}
}

View file

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

View file

@ -20,6 +20,7 @@ public static class Utils
return (a + b - 1) / b;
}
[Pure]
public static (int Column, int Row) GetSBSCoordinate(string coordinate)
{
var column = coordinate[^1] - '0';
@ -33,7 +34,13 @@ public static class Utils
return (column, row);
}
/// <summary>
/// Performs a SIMD-accelerated calculation that generates a buffer of bounded, scaled indices.
/// </summary>
/// <param name="scaleFactor">The amount by which to scale the sequence values.</param>
/// <param name="length">The total number of scalar values to generate.</param>
/// <param name="max">Upper limit (exclusive) for clamping values.</param>
/// <param name="offset">The offset to apply before clamping.</param>
public static IBuffer<int> BoundsMatrix(float scaleFactor, int length, int max, int offset)
{
var vectorSize = DivCeil(length, Vector<float>.Count);
@ -45,14 +52,13 @@ public static class Utils
var vectorMax = new Vector<int>(max - 1);
var vectorScale = new Vector<float>(scaleFactor);
var vectorSequence = SequenceVector(0f, 1f);
var vectorSequence = Vector.CreateSequence(0f, 1f);
var seq = 0f;
for (var i = 0; i < vectorSize; i++, seq += Vector<float>.Count)
{
var sequence = new Vector<float>(seq) + vectorSequence;
span[i] = Vector.Multiply(sequence, vectorScale);
span[i] = Vector.Add(span[i], vectorScale);
span[i] = Vector.FusedMultiplyAdd(sequence, vectorScale, vectorScale);
span[i] = Vector.Ceiling(span[i]);
}
@ -62,23 +68,9 @@ public static class Utils
{
resultSpan[i] = Vector.ConvertToInt32(span[i]);
resultSpan[i] = Vector.Add(resultSpan[i], vectorOffset);
resultSpan[i] = Vector.Min(resultSpan[i], vectorMax);
resultSpan[i] = Vector.Max(resultSpan[i], vectorMin);
resultSpan[i] = Vector.ClampNative(resultSpan[i], vectorMin, vectorMax);
}
return result;
}
private static Vector<float> SequenceVector(float start, float step)
{
var vector = Vector<float>.Zero;
ref var reference = ref Unsafe.As<Vector<float>, float>(ref vector);
for (var i = 0; i < Vector<float>.Count; i++)
{
ref var current = ref Unsafe.Add(ref reference, i);
current = start + step * i;
}
return vector;
}
}

View file

@ -0,0 +1,37 @@
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StitchATon2.Infra.Buffers;
internal sealed unsafe class ImmovableMemory<T> : MemoryManager<T> where T : unmanaged
{
private readonly T* _pointer;
private readonly int _length;
private bool _disposed;
public ImmovableMemory(int count)
{
_pointer = (T*)NativeMemory.Alloc((nuint)count, (nuint)Unsafe.SizeOf<T>());
_length = count;
}
protected override void Dispose(bool disposing)
{
if (!_disposed)
{
NativeMemory.Free(_pointer);
_disposed = true;
}
}
public override Span<T> GetSpan()
=> new(_pointer, _length);
public override MemoryHandle Pin(int elementIndex = 0)
=> new(_pointer + elementIndex);
public override void Unpin()
{
}
}

View file

@ -12,4 +12,7 @@ public static class MemoryAllocator
public static ArrayOwner<T> AllocateArray<T>(int count) where T : unmanaged
=> new(ArrayPool<T>.Shared, count);
public static MemoryManager<T> AllocateImmovable<T>(int count) where T : unmanaged
=> new ImmovableMemory<T>(count);
}

View file

@ -3,26 +3,35 @@ using System.Runtime.InteropServices;
namespace StitchATon2.Infra.Buffers;
/// <summary>
/// Provide non-thread safe anti GC contiguous memory.
/// </summary>
/// <typeparam name="T"></typeparam>
internal sealed unsafe class UnmanagedMemory<T> : IBuffer<T> where T : unmanaged
{
private readonly void* _pointer;
private readonly T* _pointer;
private readonly int _count;
private bool _disposed;
public ref T this[int index] => ref Unsafe.AsRef<T>((T*)_pointer + index); // *((T*)_pointer + index);
public ref T this[int index] => ref Unsafe.AsRef<T>(_pointer + index);
public Span<T> Span => new(_pointer, _count);
public UnmanagedMemory(int count)
{
_pointer = NativeMemory.Alloc((nuint)count, (nuint)Unsafe.SizeOf<T>());
_pointer = (T*)NativeMemory.Alloc((nuint)count, (nuint)Unsafe.SizeOf<T>());
_count = count;
}
~UnmanagedMemory() => Dispose();
public void Dispose()
{
if (!_disposed)
{
NativeMemory.Free(_pointer);
GC.SuppressFinalize(this);
_disposed = true;
}
}
}

View file

@ -0,0 +1,135 @@
using System.Buffers;
using System.Buffers.Binary;
using System.IO.Compression;
using System.IO.Pipelines;
using StitchATon2.Infra.Buffers;
namespace StitchATon2.Infra.Encoders;
public class PngPipeEncoder : IDisposable
{
private const int BufferSize = 8 * 1024;
private const int FlushThreshold = 1024;
private readonly PipeWriter _outputPipe;
private readonly MemoryStream _memoryStream;
private readonly int _width;
private readonly int _height;
private readonly ZLibStream _zlibStream;
private bool _disposed;
private bool _shouldFlush;
public PngPipeEncoder(PipeWriter outputPipe, int width, int height)
{
_outputPipe = outputPipe;
_width = width;
_height = height;
_memoryStream = new MemoryStream(BufferSize * 2);
_zlibStream = new ZLibStream(_memoryStream, CompressionLevel.Optimal, leaveOpen: true);
_memoryStream.SetLength(8);
_memoryStream.Position = 8;
}
~PngPipeEncoder() => Dispose();
public void WriteHeader()
{
Span<byte> headerBytes = [
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, // PNG Signature
0x00, 0x00, 0x00, 0x0D, // Length
// IHDR chunk
0x49, 0x48, 0x44, 0x52, // IHDR
0x00, 0x00, 0x00, 0x00, // Reserve to write Width
0x00, 0x00, 0x00, 0x00, // Reserve to write Height
0x08, // Bit depth
0x02, // Color type
0x00, // Compression method
0x00, // Filter method
0x00, // Interlace method
0x00, 0x00, 0x00, 0x00, // Reserve to write CRC-32
];
BinaryPrimitives.WriteInt32BigEndian(headerBytes[16..], _width);
BinaryPrimitives.WriteInt32BigEndian(headerBytes[20..], _height);
var crc = Crc32.Compute(headerBytes.Slice(12, 17));
BinaryPrimitives.WriteUInt32BigEndian(headerBytes[29..], crc);
_outputPipe.Write(headerBytes);
}
public void WriteData(IBuffer<byte> buffer, bool disposeBuffer = true, CancellationToken cancellationToken = default)
{
_zlibStream.Write([0]);
var dataSlice = buffer.Span;
while (dataSlice.Length > FlushThreshold)
{
_zlibStream.Write(dataSlice[..FlushThreshold]);
_zlibStream.Flush();
dataSlice = dataSlice[FlushThreshold..];
if(_memoryStream.Length >= BufferSize)
Flush(cancellationToken);
}
if (dataSlice.Length > 0)
{
_zlibStream.Write(dataSlice);
_zlibStream.Flush();
_shouldFlush = true;
}
if(disposeBuffer) buffer.Dispose();
}
private void Flush(CancellationToken cancellationToken)
{
_zlibStream.Flush();
var dataSize = (int)(_memoryStream.Length - 8);
_memoryStream.Write("\0\0\0\0"u8);
_memoryStream.Position = 4;
_memoryStream.Write("IDAT"u8);
var buffer = _memoryStream.GetBuffer();
BinaryPrimitives.WriteInt32BigEndian(buffer, dataSize);
// write Crc
var crc = Crc32.Compute(buffer.AsSpan(4, dataSize + 4));
BinaryPrimitives.WriteUInt32BigEndian(buffer.AsSpan(dataSize + 8), crc);
_outputPipe.Write(buffer.AsSpan(0, dataSize + 12));
_memoryStream.SetLength(8);
_memoryStream.Position = 8;
_shouldFlush = false;
}
public void WriteEndOfFile(CancellationToken cancellationToken = default)
{
if(_shouldFlush)
Flush(cancellationToken);
Span<byte> endChunk = [
0x00, 0x00, 0x00, 0x00, // Length
0x49, 0x45, 0x4E, 0x44, // IEND
0xAE, 0x42, 0x60, 0x82, // Crc
];
_outputPipe.Write(endChunk);
Dispose();
}
public void Dispose()
{
if (!_disposed)
{
_zlibStream.Dispose();
_memoryStream.Dispose();
_disposed = true;
GC.SuppressFinalize(this);
}
}
}

View file

@ -20,7 +20,7 @@ public class ImageIntegral : IDisposable
private IMemoryOwner<ManualResetEventSlim>? _rowLocks;
private MemoryMappedFile? _memoryMappedFile;
private readonly object _lock = new();
private readonly Lock _lock = new();
private readonly ManualResetEventSlim _queueLock = new(true);
private readonly ManualResetEventSlim _initializationLock = new(false);
@ -249,7 +249,7 @@ public class ImageIntegral : IDisposable
view.DangerousReadSpan(0, buffer, 0, _width);
}
private void ReadRow(int row, IBuffer<Int32Pixel> buffer)
private void ReadRow(int row, ArrayOwner<Int32Pixel> buffer)
{
using var view = AcquireView(row, MemoryMappedFileAccess.Read);
view.DangerousReadSpan(0, buffer.Span, 0, _width);

View file

@ -1,14 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.10" />
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.11" />
</ItemGroup>
</Project>

View file

@ -2,11 +2,7 @@ namespace StitchATon2.Infra.Synchronization;
public static class TaskHelper
{
public static TaskFactory CreateTaskFactory()
{
return new TaskFactory(
TaskCreationOptions.AttachedToParent,
TaskContinuationOptions.ExecuteSynchronously
);
}
public static readonly TaskFactory SynchronizedTaskFactory = new(
TaskCreationOptions.LongRunning,
TaskContinuationOptions.ExecuteSynchronously);
}

View file

@ -7,19 +7,13 @@ namespace StitchATon2.Infra;
public static class Utils
{
[UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_buffer")]
private static extern ref SafeBuffer GetSafeBuffer(this UnmanagedMemoryAccessor view);
[UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_offset")]
private static extern ref long GetOffset(this UnmanagedMemoryAccessor view);
private static unsafe uint AlignedSizeOf<T>() where T : unmanaged
{
uint size = (uint)sizeof(T);
return size is 1 or 2 ? size : (uint)((size + 3) & (~3));
}
public static void DangerousReadSpan<T>(this UnmanagedMemoryAccessor view, long position, Span<T> span, int offset, int count)
internal static void DangerousReadSpan<T>(this MemoryMappedViewAccessor view, long position, Span<T> span, int offset, int count)
where T : unmanaged
{
uint sizeOfT = AlignedSizeOf<T>();
@ -38,8 +32,8 @@ public static class Utils
}
}
var byteOffset = (ulong)(view.GetOffset() + position);
view.GetSafeBuffer().ReadSpan(byteOffset, span.Slice(offset, n));
var byteOffset = (ulong)(view.PointerOffset + position);
view.SafeMemoryMappedViewHandle.ReadSpan(byteOffset, span.Slice(offset, n));
}
public static ArrayOwner<T> Clone<T>(this ArrayOwner<T> arrayOwner, int length) where T : unmanaged
@ -48,9 +42,4 @@ public static class Utils
Array.Copy(arrayOwner.Array, 0, newArrayOwner.Array, 0, length);
return newArrayOwner;
}
public static void CopyTo<T>(this ArrayOwner<T> arrayOwner, ArrayOwner<T> target, int length) where T : unmanaged
{
Array.Copy(arrayOwner.Array, 0, target.Array, 0, length);
}
}

View file

@ -1,6 +1,6 @@
{
"sdk": {
"version": "8.0.0",
"version": "9.0.0",
"rollForward": "latestMinor",
"allowPrerelease": false
}