Compare commits

..

8 commits
v1.0.1 ... main

Author SHA1 Message Date
2ec9d356b2 Update README.md 2025-12-04 12:04:39 +00:00
ce0a4b636b Update README.md 2025-08-01 15:28:31 +00:00
55a51335f7 Merge pull request 'dangerous' (#2) from dangerous into main
Reviewed-on: #2
2025-08-01 15:17:43 +00:00
d3dfdd6a74 solve 'edge' case and pass cancellation token 2025-08-01 22:13:13 +07:00
dennisarfan
0472bfe58e solve edge case 2025-08-01 09:51:39 +07:00
dennisarfan
741d34a5e0 dangerous release (possibly memory leak and deadlock) 2025-07-31 07:40:28 +07:00
dennisarfan
a1cb6592eb Upgrade to .net9 2025-07-31 06:19:32 +07:00
dennisarfan
eb97cfb57c Dispose buffer 2025-07-30 08:35:03 +07:00
31 changed files with 1062 additions and 361 deletions

View file

@ -6,7 +6,11 @@ namespace StitchATon2.App.Controllers;
public static class ImageController
{
public static async Task GenerateImage(HttpResponse response, GenerateImageDto dto, TileManager tileManager)
public static async Task GenerateImage(
HttpResponse response,
GenerateImageDto dto,
TileManager tileManager,
CancellationToken cancellationToken)
{
if (dto.GetErrors() is { Count: > 0 } errors)
{
@ -14,22 +18,27 @@ public static class ImageController
response.ContentType = "text/json";
var errorBody = JsonSerializer.Serialize(errors, AppJsonSerializerContext.Default.DictionaryStringListString);
response.ContentLength = errorBody.Length;
await response.WriteAsync(errorBody);
await response.WriteAsync(errorBody, cancellationToken: cancellationToken);
await response.CompleteAsync();
return;
}
response.StatusCode = 200;
response.ContentType = "image/png";
Console.WriteLine($"Generate image for {dto}");
await tileManager
.CreateSection(dto)
.WriteToStream(response.Body, dto.OutputScale);
.WriteToPipe(response.BodyWriter, dto.OutputScale, cancellationToken);
await response.CompleteAsync();
}
public static async Task GenerateRandomImage(HttpResponse response, TileManager tileManager)
public static async Task GenerateRandomImage(
HttpResponse response,
TileManager tileManager,
CancellationToken cancellationToken)
{
response.StatusCode = 200;
response.ContentType = "image/png";
@ -47,7 +56,7 @@ public static class ImageController
var scale = float.Clamp(480f / int.Max(section.Width, section.Height), 0.01f, 1f);
Console.WriteLine($"Generate random image for {coordinatePair} scale: {scale}");
await section.WriteToStream(response.Body, scale);
await section.WriteToPipe(response.BodyWriter, scale, cancellationToken);
await response.CompleteAsync();
}
}

View file

@ -69,4 +69,9 @@ public class GenerateImageDto
yield return (fieldName, $"{fieldName} must be less than or equal to {max}.");
}
}
public override string ToString()
{
return $"CoordinatePair: {CanvasRect}, Crop: [{CropOffset![0]} {CropOffset[1]} {CropSize![0]} {CropSize[1]}], OutputScale: {OutputScale}";
}
}

View file

@ -5,7 +5,7 @@ using StitchATon2.Domain;
var builder = WebApplication.CreateSlimBuilder(args);
using var tileManager = new TileManager(Configuration.Default);
var tileManager = new TileManager(Configuration.Default);
builder.Services.AddSingleton(tileManager);
builder.Services.ConfigureHttpJsonOptions(options =>

View file

@ -1,11 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<InvariantGlobalization>true</InvariantGlobalization>
<RootNamespace>StitchATon2.App</RootNamespace>
<PublishAot>true</PublishAot>
</PropertyGroup>
<ItemGroup>

View file

@ -1,3 +1,4 @@
using System.IO.Pipelines;
using StitchATon2.App.Models;
using StitchATon2.Domain;
using StitchATon2.Domain.ImageCreators;
@ -14,9 +15,23 @@ public static class Utils
dto.CropSize![0],
dto.CropSize![1]);
public static async Task WriteToStream(this GridSection section, Stream stream, float? scale)
public static async Task WriteToStream(
this GridSection section,
Stream stream,
float? scale,
CancellationToken cancellationToken = default)
{
var imageCreator = new ImageCreator(section);
await imageCreator.WriteToStream(stream, scale!.Value);
using var imageCreator = new DangerousImageCreator(section);
await imageCreator.WriteToStream(stream, scale!.Value, cancellationToken);
}
public static async Task WriteToPipe(
this GridSection section,
PipeWriter pipeWriter,
float? scale,
CancellationToken cancellationToken = default)
{
using var imageCreator = new DangerousImageCreator(section);
await imageCreator.WriteToPipe(pipeWriter, scale!.Value, cancellationToken);
}
}

View file

@ -20,25 +20,20 @@ public class Configuration
public int TileCount => Columns * Rows;
public required int ImageCacheCapacity { get; init; }
public required int IntegralCacheCapacity { get; init; }
public static Configuration Default
{
get
{
var assetPath = Environment.GetEnvironmentVariable("ASSET_PATH_RO")!;
var assetPath = Environment.GetEnvironmentVariable("ASSET_PATH_RO");
var cachePath = Path.Combine(Path.GetTempPath(), "d42df2a2-60ac-4dc3-a6b9-d4c04f2e08e6");
return new Configuration
{
AssetPath = assetPath,
AssetPath = assetPath!,
CachePath = cachePath,
Columns = 55,
Rows = 31,
Width = 720,
Height = 720,
ImageCacheCapacity = 5,
IntegralCacheCapacity = 10,
};
}
}

View file

@ -47,5 +47,10 @@ public class GridSection
(var rowOffset, OffsetY) = Math.DivRem(y0, config.Height);
Origin = tileManager.GetTile(col0 + columnOffset, row0 + rowOffset);
Console.Write($"Origin: {Origin.Coordinate} ({Origin.Column}, {Origin.Row}) ");
Console.Write($"Tile offset: [{columnOffset} {rowOffset}] ");
Console.Write($"Pixel offset: [{OffsetX} {OffsetY}] ");
Console.Write($"Size: [{Width}x{Height}]");
Console.WriteLine();
}
}

View file

@ -0,0 +1,234 @@
using System.IO.Pipelines;
using System.Runtime.CompilerServices;
using SixLabors.ImageSharp.PixelFormats;
using StitchATon2.Infra;
using StitchATon2.Infra.Buffers;
using StitchATon2.Infra.Encoders;
using StitchATon2.Infra.Synchronization;
namespace StitchATon2.Domain.ImageCreators;
public sealed class DangerousImageCreator : IDisposable
{
private readonly GridSection _section;
private readonly IBuffer<Int32Pixel> _mmfReadBuffer;
private int FullWidth => _section.TileManager.Configuration.FullWidth;
private int FullHeight => _section.TileManager.Configuration.FullHeight;
private int OffsetX => _section.OffsetX;
private int OffsetY => _section.OffsetY;
private int Width => _section.Width;
private int Height => _section.Height;
private int TileWidth => _section.TileManager.Configuration.Width;
private int TileHeight => _section.TileManager.Configuration.Height;
private Tile TileOrigin => _section.Origin;
private int RightmostPixelIndex => _section.TileManager.Configuration.RightTileIndex;
private int BottomPixelIndex => _section.TileManager.Configuration.BottomTileIndex;
private TileManager TileManager => _section.TileManager;
public DangerousImageCreator(GridSection section)
{
_section = section;
_mmfReadBuffer = MemoryAllocator.Allocate<Int32Pixel>(TileWidth);
}
~DangerousImageCreator() => Dispose();
private Task Create(float scale, Action<IBuffer<byte>> writeRowCallback, CancellationToken cancellationToken)
{
var scaleFactor = MathF.ReciprocalEstimate(scale);
var targetWidth = (int)(Width / scaleFactor);
var targetHeight = (int)(Height / scaleFactor);
if (targetHeight == 0 || targetWidth == 0)
return Task.CompletedTask;
using var xLookup = Utils.BoundsMatrix(scaleFactor, targetWidth, FullWidth, OffsetX);
using var yLookup = Utils.BoundsMatrix(scaleFactor, targetHeight, FullHeight, OffsetY);
using var yStartMap = MemoryAllocator.Allocate<Int32Pixel>(targetWidth + 1);
using var yEndMap = MemoryAllocator.Allocate<Int32Pixel>(targetWidth + 1);
var outputBufferSize = targetWidth * Unsafe.SizeOf<Rgb24>();
// Use pixel referencing to eliminate type casting
var pxInt32 = Int32Pixel.Zero;
ref var px = ref pxInt32;
ref var rChannel = ref Unsafe.As<Int32Pixel, byte>(ref px);
ref var gChannel = ref Unsafe.Add(ref rChannel, 4);
ref var bChannel = ref Unsafe.Add(ref rChannel, 8);
var taskQueue = TaskHelper.SynchronizedTaskFactory.StartNew(() => { }, cancellationToken);
for (var y = 1; y <= targetHeight; y++)
{
var yStart = yLookup[y - 1];
var yEnd = yLookup[y];
var (localRow0, localOffsetY0) = int.DivRem(yStart, TileHeight);
MapRow(localRow0, localOffsetY0, xLookup, targetWidth + 1, yStartMap);
var (localRow1, localOffsetY1) = int.DivRem(yEnd, TileHeight);
MapRow(localRow1, localOffsetY1, xLookup, targetWidth + 1, yEndMap);
// Cross row
if (localRow0 != localRow1)
MapRow(localRow0, BottomPixelIndex, xLookup, targetWidth + 1, yEndMap, true);
var outputBuffer = MemoryAllocator.Allocate<byte>(outputBufferSize);
ref var outputChannel = ref outputBuffer.Span[0];
var boxHeight = yEnd - yStart;
for (int x1 = 1, x0 = 0; x1 <= targetWidth; x0 = x1++)
{
var xStart = xLookup[x1 - 1];
var xEnd = xLookup[x1];
px = yEndMap[x1];
px += yStartMap[x0];
px -= yEndMap[x0];
px -= yStartMap[x1];
px /= Math.Max(1, (xEnd - xStart) * boxHeight);
outputChannel = rChannel;
outputChannel = ref Unsafe.AddByteOffset(ref outputChannel, 1);
outputChannel = gChannel;
outputChannel = ref Unsafe.AddByteOffset(ref outputChannel, 1);
outputChannel = bChannel;
outputChannel = ref Unsafe.AddByteOffset(ref outputChannel, 1);
}
if (cancellationToken.IsCancellationRequested)
{
outputBuffer.Dispose();
return Task.FromCanceled(cancellationToken);
}
cancellationToken.Register(outputBuffer.Dispose);
taskQueue = taskQueue
.ContinueWith(
_ => writeRowCallback.Invoke(outputBuffer),
cancellationToken,
TaskContinuationOptions.OnlyOnRanToCompletion,
TaskScheduler.Current);
}
return taskQueue;
}
public async Task WriteToPipe(PipeWriter outputPipe, float scale, CancellationToken cancellationToken = default)
{
var scaleFactor = MathF.ReciprocalEstimate(scale);
var targetWidth = (int)(Width / scaleFactor);
var targetHeight = (int)(Height / scaleFactor);
if (targetHeight == 0 || targetWidth == 0)
return;
var encoder = new PngPipeEncoder(outputPipe, targetWidth, targetHeight);
encoder.WriteHeader();
await Create(scale,
output => encoder
.WriteDataAsync(output, cancellationToken: cancellationToken)
.Wait(cancellationToken),
cancellationToken);
await encoder.WriteEndOfFileAsync(cancellationToken);
}
public async Task WriteToStream(Stream outputStream, float scale, CancellationToken cancellationToken = default)
{
var scaleFactor = MathF.ReciprocalEstimate(scale);
var targetWidth = (int)(Width / scaleFactor);
var targetHeight = (int)(Height / scaleFactor);
if (targetHeight == 0 || targetWidth == 0)
return;
var encoder = new PngStreamEncoder(outputStream, targetWidth, targetHeight);
await encoder.WriteHeader(cancellationToken);
await Create(scale,
output => encoder
.WriteDataAsync(output, cancellationToken: cancellationToken)
.Wait(cancellationToken),
cancellationToken);
await encoder.WriteEndOfFileAsync(cancellationToken);
}
private void MapRow(
int rowOffset,
int yOffset,
IBuffer<int> boundsMatrix,
int count,
IBuffer<Int32Pixel> destination,
bool appendMode = false)
{
var currentTile = TileManager.TryGetAdjacent(TileOrigin, 0, rowOffset);
if (currentTile == null)
{
if (appendMode) return;
for (var i = 0; i < count; i++)
destination[i] = Int32Pixel.Zero;
return;
}
var sourceMap = boundsMatrix.Span[..count];
var xAdder = Int32Pixel.Zero;
var xOffset = 0;
var written = 0;
var destinationSpan = destination.Span;
var readBufferSpan = _mmfReadBuffer.Span;
while (true)
{
currentTile.Integral.Acquire(yOffset, readBufferSpan);
int localX;
if (appendMode)
{
while (written < sourceMap.Length && (localX = sourceMap[written] - xOffset) < TileWidth)
{
destinationSpan[written] += readBufferSpan[localX];
destinationSpan[written] += xAdder;
written++;
}
}
else
{
while (written < sourceMap.Length && (localX = sourceMap[written] - xOffset) < TileWidth)
{
destinationSpan[written] = readBufferSpan[localX];
destinationSpan[written] += xAdder;
written++;
}
}
if (written >= sourceMap.Length)
break;
currentTile = TileManager.TryGetAdjacent(currentTile, 1, 0);
if (currentTile == null)
{
destinationSpan[written] = destinationSpan[written - 1];
break;
}
xAdder += readBufferSpan[RightmostPixelIndex];
xOffset += TileWidth;
}
}
public void Dispose()
{
_mmfReadBuffer.Dispose();
GC.SuppressFinalize(this);
}
}

View file

@ -1,155 +0,0 @@
using System.Buffers;
using System.Runtime.CompilerServices;
using SixLabors.ImageSharp.PixelFormats;
using StitchATon2.Infra;
using StitchATon2.Infra.Buffers;
using StitchATon2.Infra.Encoders;
namespace StitchATon2.Domain.ImageCreators;
public class ImageCreator
{
private readonly GridSection _section;
private int FullWidth => _section.TileManager.Configuration.FullWidth;
private int FullHeight => _section.TileManager.Configuration.FullHeight;
private int OffsetX => _section.OffsetX;
private int OffsetY => _section.OffsetY;
private int Width => _section.Width;
private int Height => _section.Height;
private int TileWidth => _section.TileManager.Configuration.Width;
private int TileHeight => _section.TileManager.Configuration.Height;
private Tile TileOrigin => _section.Origin;
private int RightmostPixelIndex => _section.TileManager.Configuration.RightTileIndex;
private int BottomPixelIndex => _section.TileManager.Configuration.BottomTileIndex;
private TileManager TileManager => _section.TileManager;
private readonly Int32Pixel[] _mmfReadBuffer;
public ImageCreator(GridSection section)
{
_section = section;
_mmfReadBuffer = ArrayPool<Int32Pixel>.Shared.Rent(TileWidth);
}
public async Task WriteToStream(Stream writableStream, float scale)
{
var scaleFactor = MathF.ReciprocalEstimate(scale);
var targetWidth = (int)(Width / scaleFactor);
var targetHeight = (int)(Height / scaleFactor);
var encoder = new PngStreamEncoder(writableStream, targetWidth, targetHeight);
await encoder.WriteHeader();
var outputBufferSize = targetWidth * Unsafe.SizeOf<Rgb24>();
using var outputBuffer = MemoryAllocator.AllocateManaged<byte>(outputBufferSize);
using var xLookup = Utils.BoundsMatrix(scaleFactor, targetWidth, FullWidth, OffsetX);
using var yLookup = Utils.BoundsMatrix(scaleFactor, targetHeight, FullHeight, OffsetY);
using var yStartMap = MemoryAllocator.Allocate<Int32Pixel>(targetWidth);
using var yEndMap = MemoryAllocator.Allocate<Int32Pixel>(targetWidth);
var yStart = OffsetY;
Task? outputTask = null;
for (var y = 0; y < targetHeight; y++)
{
var yEnd = yLookup[y];
var (localRow0, localOffsetY0) = int.DivRem(yStart, TileHeight);
MapRow(localRow0, localOffsetY0, xLookup.Span[..targetWidth], yStartMap);
var (localRow1, localOffsetY1) = int.DivRem(yEnd, TileHeight);
MapRow(localRow1, localOffsetY1, xLookup.Span[..targetWidth], yEndMap);
if (localRow0 != localRow1)
{
MapRowAppend(localRow0, BottomPixelIndex, xLookup.Span[..targetWidth], yEndMap);
}
if(outputTask != null)
await outputTask;
int xStart = OffsetX, x0 = 0;
for (int x1 = 0, i = 0; x1 < targetWidth; x1++)
{
var xEnd = xLookup[x1];
var pixel = yEndMap[x1];
pixel += yStartMap[x0];
pixel -= yEndMap[x0];
pixel -= yStartMap[x1];
pixel /= Math.Max(1, (xEnd - xStart) * (yEnd - yStart));
outputBuffer.Memory.Span[i++] = (byte)pixel.R;
outputBuffer.Memory.Span[i++] = (byte)pixel.G;
outputBuffer.Memory.Span[i++] = (byte)pixel.B;
xStart = xEnd;
x0 = x1;
}
outputTask = encoder.WriteData(outputBuffer.Memory[..outputBufferSize]);
yStart = yEnd;
}
await encoder.WriteEndOfFile();
}
private void MapRow(int rowOffset, int yOffset, Span<int> sourceMap, IBuffer<Int32Pixel> destination)
{
var currentTile = TileManager.GetAdjacent(TileOrigin, 0, rowOffset);
var xAdder = Int32Pixel.Zero;
var xOffset = 0;
var written = 0;
while (true)
{
currentTile.Integral.Acquire(yOffset, _mmfReadBuffer);
int localX;
while (written < sourceMap.Length && (localX = sourceMap[written] - xOffset) < TileWidth)
{
destination.Span[written] = _mmfReadBuffer[localX];
destination.Span[written] += xAdder;
written++;
}
if (written >= sourceMap.Length)
break;
xAdder += _mmfReadBuffer[RightmostPixelIndex];
xOffset += TileWidth;
currentTile = TileManager.GetAdjacent(currentTile, 1, 0);
}
}
private void MapRowAppend(int rowOffset, int yOffset, Span<int> sourceMap, IBuffer<Int32Pixel> destination)
{
var currentTile = TileManager.GetAdjacent(TileOrigin, 0, rowOffset);
var xAdder = Int32Pixel.Zero;
var xOffset = 0;
var written = 0;
while (true)
{
currentTile.Integral.Acquire(yOffset, _mmfReadBuffer);
int localX;
while (written < sourceMap.Length && (localX = sourceMap[written] - xOffset) < TileWidth)
{
destination.Span[written] += _mmfReadBuffer[localX];
destination.Span[written] += xAdder;
written++;
}
if (written >= sourceMap.Length)
break;
xAdder += _mmfReadBuffer[RightmostPixelIndex];
xOffset += TileWidth;
currentTile = TileManager.GetAdjacent(currentTile, 1, 0);
}
}
}

View file

@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>

View file

@ -5,29 +5,26 @@ using StitchATon2.Infra.Buffers;
namespace StitchATon2.Domain;
public sealed class TileManager : IDisposable
public sealed class TileManager
{
private readonly IMemoryOwner<Tile> _tiles;
private readonly Tile[] _tiles;
public Configuration Configuration { get; }
public TileManager(Configuration config)
{
Configuration = config;
_tiles = MemoryAllocator.AllocateManaged<Tile>(config.TileCount);
var tilesSpan = _tiles.Memory.Span;
_tiles = new Tile[Configuration.TileCount];
for (var id = 0; id < config.TileCount; id++)
tilesSpan[id] = CreateTile(id);
_tiles[id] = CreateTile(id);
Console.WriteLine("Tile manager created");
}
~TileManager() => Dispose();
private Tile CreateTile(int id)
{
var (row, column) = int.DivRem(id, Configuration.Columns);
var coordinate = $"{Utils.GetSBSNotation(++row)}{++column}";
var coordinate = $"{Utils.GetSbsNotationRow(++row)}{++column}";
return new Tile
{
Id = id,
@ -47,14 +44,14 @@ public sealed class TileManager : IDisposable
private int GetId(int column, int row) => column - 1 + (row - 1) * Configuration.Columns;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Tile GetTile(int id) => _tiles.Memory.Span[id];
public Tile GetTile(int id) => _tiles[id];
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Tile GetTile(int column, int row) => GetTile(GetId(column, row));
public Tile GetTile(string coordinate)
{
var (column, row) = Utils.GetSBSCoordinate(coordinate);
var (column, row) = Utils.GetSbsNotationCoordinate(coordinate);
return GetTile(column, row);
}
@ -99,10 +96,4 @@ public sealed class TileManager : IDisposable
cropY,
cropWidth,
cropHeight);
public void Dispose()
{
_tiles.Dispose();
GC.SuppressFinalize(this);
}
}

View file

@ -9,7 +9,7 @@ namespace StitchATon2.Domain;
public static class Utils
{
[Pure]
public static string GetSBSNotation(int row)
public static string GetSbsNotationRow(int row)
=> row <= 26
? new string([(char)(row + 'A' - 1)])
: new string(['A', (char)(row + 'A' - 27)]);
@ -20,7 +20,8 @@ public static class Utils
return (a + b - 1) / b;
}
public static (int Column, int Row) GetSBSCoordinate(string coordinate)
[Pure]
public static (int Column, int Row) GetSbsNotationCoordinate(string coordinate)
{
var column = coordinate[^1] - '0';
if(char.IsDigit(coordinate[^2]))
@ -33,26 +34,31 @@ public static class Utils
return (column, row);
}
/// <summary>
/// Performs a SIMD-accelerated calculation that generates a buffer of bounded, scaled indices.
/// </summary>
/// <param name="scaleFactor">The amount by which to scale the sequence values.</param>
/// <param name="length">The total number of scalar values to generate.</param>
/// <param name="max">Upper limit (exclusive) for clamping values.</param>
/// <param name="offset">The offset to apply before clamping.</param>
public static IBuffer<int> BoundsMatrix(float scaleFactor, int length, int max, int offset)
{
var vectorSize = DivCeil(length, Vector<float>.Count);
var vectorSize = DivCeil(length + 1, Vector<float>.Count);
using var buffer = MemoryAllocator.Allocate<Vector<float>>(vectorSize);
var span = buffer.Span;
var vectorMin = Vector<int>.Zero;
var vectorOffset = new Vector<int>(offset - 1);
var vectorOffset = new Vector<int>(offset);
var vectorMax = new Vector<int>(max - 1);
var vectorScale = new Vector<float>(scaleFactor);
var vectorSequence = SequenceVector(0f, 1f);
var vectorSequence = Vector.CreateSequence(0f, 1f);
var seq = 0f;
var seq = -1f;
for (var i = 0; i < vectorSize; i++, seq += Vector<float>.Count)
{
var sequence = new Vector<float>(seq) + vectorSequence;
span[i] = Vector.Multiply(sequence, vectorScale);
span[i] = Vector.Add(span[i], vectorScale);
span[i] = Vector.FusedMultiplyAdd(sequence, vectorScale, vectorScale);
span[i] = Vector.Ceiling(span[i]);
}
@ -62,23 +68,9 @@ public static class Utils
{
resultSpan[i] = Vector.ConvertToInt32(span[i]);
resultSpan[i] = Vector.Add(resultSpan[i], vectorOffset);
resultSpan[i] = Vector.Min(resultSpan[i], vectorMax);
resultSpan[i] = Vector.Max(resultSpan[i], vectorMin);
resultSpan[i] = Vector.ClampNative(resultSpan[i], vectorMin, vectorMax);
}
return result;
}
private static Vector<float> SequenceVector(float start, float step)
{
var vector = Vector<float>.Zero;
ref var reference = ref Unsafe.As<Vector<float>, float>(ref vector);
for (var i = 0; i < Vector<float>.Count; i++)
{
ref var current = ref Unsafe.Add(ref reference, i);
current = start + step * i;
}
return vector;
}
}

View file

@ -7,10 +7,11 @@ public class ArrayOwner<T> : IBuffer<T> where T : unmanaged
private readonly ArrayPool<T> _owner;
private readonly T[] _buffer;
public ArrayOwner(ArrayPool<T> owner, int size)
public ArrayOwner(ArrayPool<T> owner, int length)
{
_owner = owner;
_buffer = owner.Rent(size);
_buffer = owner.Rent(length);
Length = length;
}
~ArrayOwner() => Dispose();
@ -23,7 +24,10 @@ public class ArrayOwner<T> : IBuffer<T> where T : unmanaged
public ref T this[int index] => ref _buffer[index];
public Span<T> Span => _buffer;
public Span<T> Span => _buffer.AsSpan(0, Length);
public Memory<T> Memory => _buffer.AsMemory(0, Length);
public T[] Array => _buffer;
public int Length { get; }
}

View file

@ -5,4 +5,8 @@ public interface IBuffer<T> : IDisposable where T : unmanaged
ref T this[int index] { get; }
Span<T> Span { get; }
Memory<T> Memory { get; }
int Length { get; }
}

View file

@ -0,0 +1,51 @@
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StitchATon2.Infra.Buffers;
internal sealed unsafe class ImmovableMemory<T> : MemoryManager<T>, IBuffer<T> where T : unmanaged
{
internal readonly T* Pointer;
private bool _disposed;
public ImmovableMemory(int length)
{
Pointer = (T*)NativeMemory.Alloc((nuint)length, (nuint)Unsafe.SizeOf<T>());
Length = length;
}
protected override void Dispose(bool disposing)
{
if (_disposed) return;
NativeMemory.Free(Pointer);
_disposed = true;
}
public override Span<T> GetSpan()
=> _disposed
? throw new ObjectDisposedException(nameof(ImmovableMemory<T>))
: new Span<T>(Pointer, Length);
public override MemoryHandle Pin(int elementIndex = 0)
=> _disposed
? throw new ObjectDisposedException(nameof(ImmovableMemory<T>))
: new MemoryHandle(Pointer + elementIndex);
public override void Unpin()
{
}
public ref T this[int index]
{
get
{
if (_disposed) throw new ObjectDisposedException(nameof(ImmovableMemory<T>));
return ref Unsafe.AsRef<T>(Pointer + index);
}
}
public Span<T> Span => GetSpan();
public int Length { get; }
}

View file

@ -1,15 +1,36 @@
using System.Buffers;
using System.Runtime.CompilerServices;
namespace StitchATon2.Infra.Buffers;
public static class MemoryAllocator
{
public static IBuffer<T> Allocate<T>(int count) where T : unmanaged
=> new UnmanagedMemory<T>(count);
=> new ImmovableMemory<T>(count);
public static IMemoryOwner<T> AllocateManaged<T>(int count)
=> MemoryPool<T>.Shared.Rent(count);
public static ArrayOwner<T> AllocateArray<T>(int count) where T : unmanaged
=> new(ArrayPool<T>.Shared, count);
public static unsafe IBuffer<T> Clone<T>(this IBuffer<T> buffer) where T : unmanaged
{
if (buffer is not ImmovableMemory<T> unmanagedMemory)
throw new NotSupportedException();
var newBuffer = new ImmovableMemory<T>(buffer.Length);
var byteCount = (uint)(Unsafe.SizeOf<T>() * buffer.Length);
Unsafe.CopyBlock(newBuffer.Pointer, unmanagedMemory.Pointer, byteCount);
return newBuffer;
}
public static unsafe void Copy<T>(this IBuffer<T> source, IBuffer<T> destination, int count) where T : unmanaged
{
if (source is not ImmovableMemory<T> sourceBuffer || destination is not ImmovableMemory<T> destinationBuffer)
throw new NotSupportedException();
var byteCount = (uint)(Unsafe.SizeOf<T>() * count);
Unsafe.CopyBlock(destinationBuffer.Pointer, sourceBuffer.Pointer, byteCount);
}
}

View file

@ -0,0 +1,102 @@
using System.Buffers;
namespace StitchATon2.Infra.Buffers;
public class PooledMemoryStream : Stream
{
private byte[] _buffer;
private int _length;
private int _position;
private readonly ArrayPool<byte> _pool;
private bool _disposed;
public PooledMemoryStream(int initialCapacity = 1024, ArrayPool<byte>? pool = null)
{
_pool = pool ?? ArrayPool<byte>.Shared;
_buffer = _pool.Rent(initialCapacity);
}
public override bool CanRead => !_disposed;
public override bool CanSeek => !_disposed;
public override bool CanWrite => !_disposed;
public override long Length => _length;
public override long Position
{
get => _position;
set
{
if (_disposed) throw new ObjectDisposedException(nameof(PooledMemoryStream));
if (value < 0 || value > int.MaxValue) throw new ArgumentOutOfRangeException();
_position = (int)value;
}
}
public byte[] GetBuffer() => _buffer;
public ArraySegment<byte> GetWrittenSegment() => new(_buffer, 0, _length);
public override void Flush() { /* no-op */ }
public override int Read(byte[] buffer, int offset, int count)
{
if (_disposed) throw new ObjectDisposedException(nameof(PooledMemoryStream));
int available = _length - _position;
int toRead = Math.Min(count, available);
Buffer.BlockCopy(_buffer, _position, buffer, offset, toRead);
_position += toRead;
return toRead;
}
public override void Write(byte[] buffer, int offset, int count)
{
if (_disposed) throw new ObjectDisposedException(nameof(PooledMemoryStream));
EnsureCapacity(_position + count);
Buffer.BlockCopy(buffer, offset, _buffer, _position, count);
_position += count;
_length = Math.Max(_length, _position);
}
public override long Seek(long offset, SeekOrigin origin)
{
if (_disposed) throw new ObjectDisposedException(nameof(PooledMemoryStream));
int newPos = origin switch
{
SeekOrigin.Begin => (int)offset,
SeekOrigin.Current => _position + (int)offset,
SeekOrigin.End => _length + (int)offset,
_ => throw new ArgumentOutOfRangeException()
};
if (newPos < 0) throw new IOException("Negative position");
_position = newPos;
return _position;
}
public override void SetLength(long value)
{
if (_disposed) throw new ObjectDisposedException(nameof(PooledMemoryStream));
if (value < 0 || value > int.MaxValue) throw new ArgumentOutOfRangeException();
EnsureCapacity((int)value);
_length = (int)value;
if (_position > _length) _position = _length;
}
private void EnsureCapacity(int size)
{
if (size <= _buffer.Length) return;
int newSize = Math.Max(size, _buffer.Length * 2);
byte[] newBuffer = _pool.Rent(newSize);
Buffer.BlockCopy(_buffer, 0, newBuffer, 0, _length);
_pool.Return(_buffer, clearArray: true);
_buffer = newBuffer;
}
protected override void Dispose(bool disposing)
{
if (!_disposed)
{
_pool.Return(_buffer, clearArray: true);
_buffer = Array.Empty<byte>();
_disposed = true;
}
base.Dispose(disposing);
}
}

View file

@ -3,26 +3,38 @@ using System.Runtime.InteropServices;
namespace StitchATon2.Infra.Buffers;
/// <summary>
/// Provide non-thread safe anti GC contiguous memory.
/// </summary>
/// <typeparam name="T"></typeparam>
[Obsolete("Use immovable memory instead")]
internal sealed unsafe class UnmanagedMemory<T> : IBuffer<T> where T : unmanaged
{
private readonly void* _pointer;
private readonly int _count;
internal readonly T* Pointer;
private bool _disposed;
public ref T this[int index] => ref Unsafe.AsRef<T>((T*)_pointer + index); // *((T*)_pointer + index);
public Memory<T> Memory => throw new NotImplementedException();
public int Length { get; }
public Span<T> Span => new(_pointer, _count);
public ref T this[int index] => ref Unsafe.AsRef<T>(Pointer + index);
public UnmanagedMemory(int count)
public Span<T> Span => new(Pointer, Length);
public UnmanagedMemory(int length)
{
_pointer = NativeMemory.Alloc((nuint)count, (nuint)Unsafe.SizeOf<T>());
_count = count;
Pointer = (T*)NativeMemory.Alloc((nuint)length, (nuint)Unsafe.SizeOf<T>());
Length = length;
}
~UnmanagedMemory() => Dispose();
public void Dispose()
{
NativeMemory.Free(_pointer);
GC.SuppressFinalize(this);
if (!_disposed)
{
NativeMemory.Free(Pointer);
GC.SuppressFinalize(this);
_disposed = true;
}
}
}

View file

@ -15,6 +15,17 @@ public static class Crc32
return ~crc;
}
public static uint Compute(Stream stream, int count, uint initial = 0xFFFFFFFF)
{
uint crc = initial;
while (count-- > 0)
{
crc = Table[(crc ^ stream.ReadByte()) & 0xFF] ^ (crc >> 8);
}
return ~crc;
}
private static uint[] GenerateTable()
{
const uint poly = 0xEDB88320;

View file

@ -0,0 +1,144 @@
using System.Buffers;
using System.Buffers.Binary;
using System.IO.Compression;
using System.IO.Pipelines;
using StitchATon2.Infra.Buffers;
namespace StitchATon2.Infra.Encoders;
public class PngPipeEncoder : IDisposable
{
private const int BufferSize = 8 * 1024;
private const int FlushThreshold = 1024;
private const int PipeChunkThreshold = 16 * 1024;
private readonly PipeWriter _outputPipe;
private readonly MemoryStream _memoryStream;
private readonly int _width;
private readonly int _height;
private readonly ZLibStream _zlibStream;
private bool _disposed;
private bool _shouldFlush;
public PngPipeEncoder(PipeWriter outputPipe, int width, int height)
{
_outputPipe = outputPipe;
_width = width;
_height = height;
_memoryStream = new MemoryStream(BufferSize * 2);
_zlibStream = new ZLibStream(_memoryStream, CompressionLevel.Optimal, leaveOpen: true);
_memoryStream.SetLength(8);
_memoryStream.Position = 8;
}
~PngPipeEncoder() => Dispose();
public void WriteHeader()
{
Span<byte> headerBytes = [
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, // PNG Signature
0x00, 0x00, 0x00, 0x0D, // Length
// IHDR chunk
0x49, 0x48, 0x44, 0x52, // IHDR
0x00, 0x00, 0x00, 0x00, // Reserve to write Width
0x00, 0x00, 0x00, 0x00, // Reserve to write Height
0x08, // Bit depth
0x02, // Color type
0x00, // Compression method
0x00, // Filter method
0x00, // Interlace method
0x00, 0x00, 0x00, 0x00, // Reserve to write CRC-32
];
BinaryPrimitives.WriteInt32BigEndian(headerBytes[16..], _width);
BinaryPrimitives.WriteInt32BigEndian(headerBytes[20..], _height);
var crc = Crc32.Compute(headerBytes.Slice(12, 17));
BinaryPrimitives.WriteUInt32BigEndian(headerBytes[29..], crc);
_outputPipe.Write(headerBytes);
}
public async Task WriteDataAsync(IBuffer<byte> buffer, bool disposeBuffer = true, CancellationToken cancellationToken = default)
{
try
{
_zlibStream.Write([0]);
var offset = 0;
while (buffer.Length - offset > FlushThreshold)
{
_zlibStream.Write(buffer.Span.Slice(offset, FlushThreshold));
await _zlibStream.FlushAsync(cancellationToken);
offset += FlushThreshold;
if (_memoryStream.Length >= BufferSize)
await FlushAsync(cancellationToken);
}
if (buffer.Length > offset)
{
_zlibStream.Write(buffer.Span[offset..]);
await _zlibStream.FlushAsync(cancellationToken);
_shouldFlush = true;
}
}
finally
{
if(disposeBuffer)
buffer.Dispose();
}
}
private async Task FlushAsync(CancellationToken cancellationToken)
{
await _zlibStream.FlushAsync(cancellationToken);
var dataSize = (int)(_memoryStream.Length - 8);
_memoryStream.Write("\0\0\0\0"u8);
_memoryStream.Position = 4;
_memoryStream.Write("IDAT"u8);
var buffer = _memoryStream.GetBuffer();
BinaryPrimitives.WriteInt32BigEndian(buffer, dataSize);
// write Crc
var crc = Crc32.Compute(buffer.AsSpan(4, dataSize + 4));
BinaryPrimitives.WriteUInt32BigEndian(buffer.AsSpan(dataSize + 8), crc);
_outputPipe.Write(buffer.AsSpan(0, dataSize + 12));
await _outputPipe.FlushAsync(cancellationToken);
_memoryStream.SetLength(8);
_memoryStream.Position = 8;
_shouldFlush = false;
}
public async Task WriteEndOfFileAsync(CancellationToken cancellationToken = default)
{
if(_shouldFlush)
await FlushAsync(cancellationToken);
Span<byte> endChunk = [
0x00, 0x00, 0x00, 0x00, // Length
0x49, 0x45, 0x4E, 0x44, // IEND
0xAE, 0x42, 0x60, 0x82, // Crc
];
_outputPipe.Write(endChunk);
Dispose();
}
public void Dispose()
{
if (!_disposed)
{
_zlibStream.Dispose();
_memoryStream.Dispose();
_disposed = true;
GC.SuppressFinalize(this);
}
}
}

View file

@ -1,5 +1,6 @@
using System.Buffers.Binary;
using System.IO.Compression;
using StitchATon2.Infra.Buffers;
namespace StitchATon2.Infra.Encoders;
@ -30,7 +31,7 @@ public class PngStreamEncoder : IDisposable, IAsyncDisposable
~PngStreamEncoder() => Dispose();
public async Task WriteHeader()
public async Task WriteHeader(CancellationToken cancellationToken = default)
{
byte[] headerBytes = [
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, // PNG Signature
@ -54,34 +55,42 @@ public class PngStreamEncoder : IDisposable, IAsyncDisposable
BinaryPrimitives.WriteUInt32BigEndian(headerBytes.AsSpan(29), crc);
await _stream.WriteAsync(headerBytes);
await _stream.WriteAsync(headerBytes, cancellationToken);
}
public async Task WriteData(Memory<byte> data)
public async Task WriteDataAsync(IBuffer<byte> buffer, bool disposeBuffer = true, CancellationToken cancellationToken = default)
{
_zlibStream.Write([0]);
var dataSlice = data;
while (dataSlice.Length > FlushThreshold)
try
{
await _zlibStream.WriteAsync(dataSlice[..FlushThreshold]);
await _zlibStream.FlushAsync();
dataSlice = dataSlice[FlushThreshold..];
if(_memoryStream.Length >= BufferSize)
await Flush();
_zlibStream.Write([0]);
var dataSlice = buffer.Memory;
while (dataSlice.Length > FlushThreshold)
{
await _zlibStream.WriteAsync(dataSlice[..FlushThreshold], cancellationToken);
await _zlibStream.FlushAsync(cancellationToken);
dataSlice = dataSlice[FlushThreshold..];
if (_memoryStream.Length >= BufferSize)
await FlushAsync(cancellationToken);
}
if (dataSlice.Length > 0)
{
await _zlibStream.WriteAsync(dataSlice, cancellationToken);
await _zlibStream.FlushAsync(cancellationToken);
_shouldFlush = true;
}
}
if (dataSlice.Length > 0)
finally
{
await _zlibStream.WriteAsync(dataSlice);
await _zlibStream.FlushAsync();
_shouldFlush = true;
if(disposeBuffer)
buffer.Dispose();
}
}
private async Task Flush()
private async Task FlushAsync(CancellationToken cancellationToken)
{
await _zlibStream.FlushAsync();
await _zlibStream.FlushAsync(cancellationToken);
var dataSize = (int)(_memoryStream.Length - 8);
_memoryStream.Write("\0\0\0\0"u8);
@ -96,16 +105,16 @@ public class PngStreamEncoder : IDisposable, IAsyncDisposable
var crc = Crc32.Compute(buffer.AsSpan(4, dataSize + 4));
BinaryPrimitives.WriteUInt32BigEndian(buffer.AsSpan(dataSize + 8), crc);
await _stream.WriteAsync(buffer.AsMemory(0, dataSize + 12));
await _stream.WriteAsync(buffer.AsMemory(0, dataSize + 12), cancellationToken);
_memoryStream.SetLength(8);
_memoryStream.Position = 8;
_shouldFlush = false;
}
public async ValueTask WriteEndOfFile()
public async ValueTask WriteEndOfFileAsync(CancellationToken cancellationToken = default)
{
if(_shouldFlush)
await Flush();
await FlushAsync(cancellationToken);
var endChunk = new byte[] {
0x00, 0x00, 0x00, 0x00, // Length
@ -113,7 +122,7 @@ public class PngStreamEncoder : IDisposable, IAsyncDisposable
0xAE, 0x42, 0x60, 0x82, // Crc
};
await _stream.WriteAsync(endChunk);
await _stream.WriteAsync(endChunk, cancellationToken);
await DisposeAsync();
}

View file

@ -0,0 +1,166 @@
using System.Buffers;
using System.Buffers.Binary;
using System.IO.Compression;
using System.IO.Pipelines;
using StitchATon2.Infra.Buffers;
namespace StitchATon2.Infra.Encoders;
public class UnsafePngEncoder : IDisposable
{
private const int BufferSize = 8 * 1024;
private const int FlushThreshold = 1024;
private const int PipeChunkThreshold = 16 * 1024;
private readonly PipeWriter _outputPipe;
private readonly int _width;
private readonly int _height;
private MemoryHandle? _memoryHandle;
private readonly RawPointerStream _memoryStream;// = new RawPointerStream();
private ZLibStream? _zlibStream;// = new ZLibStream(_memoryStream, CompressionLevel.Optimal, leaveOpen: true);
private bool _disposed;
private bool _shouldFlush;
public UnsafePngEncoder(PipeWriter outputPipe, int width, int height)
{
_outputPipe = outputPipe;
_width = width;
_height = height;
_memoryStream = new RawPointerStream();
}
~UnsafePngEncoder() => Dispose();
public void WriteHeader()
{
Span<byte> headerBytes = [
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, // PNG Signature
0x00, 0x00, 0x00, 0x0D, // Length
// IHDR chunk
0x49, 0x48, 0x44, 0x52, // IHDR
0x00, 0x00, 0x00, 0x00, // Reserve to write Width
0x00, 0x00, 0x00, 0x00, // Reserve to write Height
0x08, // Bit depth
0x02, // Color type
0x00, // Compression method
0x00, // Filter method
0x00, // Interlace method
0x00, 0x00, 0x00, 0x00, // Reserve to write CRC-32
];
BinaryPrimitives.WriteInt32BigEndian(headerBytes[16..], _width);
BinaryPrimitives.WriteInt32BigEndian(headerBytes[20..], _height);
var crc = Crc32.Compute(headerBytes.Slice(12, 17));
BinaryPrimitives.WriteUInt32BigEndian(headerBytes[29..], crc);
_outputPipe.Write(headerBytes);
}
private unsafe void Initialize()
{
if (_memoryHandle == null)
{
var memory = _outputPipe.GetMemory(PipeChunkThreshold);
var handle = memory.Pin();
_memoryStream.Initialize((byte*)handle.Pointer, 0, memory.Length);
_memoryHandle = handle;
_memoryStream.SetLength(8);
_memoryStream.Position = 8;
_zlibStream = new ZLibStream(_memoryStream, CompressionLevel.Optimal, true);
}
}
public async Task WriteDataAsync(IBuffer<byte> buffer, bool disposeBuffer = true, CancellationToken cancellationToken = default)
{
Initialize();
_zlibStream!.Write([0]);
var offset = 0;
while (buffer.Length - offset > FlushThreshold)
{
_zlibStream.Write(buffer.Span.Slice(offset, FlushThreshold));
await _zlibStream.FlushAsync(cancellationToken);
offset += FlushThreshold;
if(_memoryStream.Length >= BufferSize)
await FlushAsync(cancellationToken);
}
if (buffer.Length > offset)
{
_zlibStream.Write(buffer.Span[offset..]);
await _zlibStream.FlushAsync(cancellationToken);
_shouldFlush = true;
}
if(disposeBuffer) buffer.Dispose();
}
private async Task FlushAsync(CancellationToken cancellationToken)
{
await _zlibStream!.FlushAsync(cancellationToken);
var dataSize = (int)(_memoryStream.Length - 8);
_memoryStream.Position = 0;
Span<byte> buffer = stackalloc byte[4];
BinaryPrimitives.WriteInt32BigEndian(buffer, dataSize);
_memoryStream.Write(buffer);
_memoryStream.Write("IDAT"u8);
_memoryStream.Position = 4;
// write Crc
var crc = Crc32.Compute(_memoryStream, dataSize + 4);
BinaryPrimitives.WriteUInt32BigEndian(buffer, crc);
_memoryStream.Write(buffer);
_outputPipe.Advance((int)_memoryStream.Length);
await _memoryStream.DisposeAsync();
_memoryHandle!.Value.Dispose();
_memoryHandle = null;
_shouldFlush = false;
}
public async Task WriteEndOfFileAsync(CancellationToken cancellationToken = default)
{
if(_shouldFlush)
await FlushAsync(cancellationToken);
Span<byte> endChunk = [
0x00, 0x00, 0x00, 0x00, // Length
0x49, 0x45, 0x4E, 0x44, // IEND
0xAE, 0x42, 0x60, 0x82, // Crc
];
_outputPipe.Write(endChunk);
Dispose();
}
public void Dispose()
{
if (!_disposed)
{
if (_memoryHandle != null)
{
_zlibStream!.Dispose();
_memoryStream.Dispose();
}
_disposed = true;
GC.SuppressFinalize(this);
}
}
private unsafe class RawPointerStream : UnmanagedMemoryStream
{
public void Initialize(byte* pointer, int length, int capacity)
{
Initialize(pointer, length, capacity, FileAccess.ReadWrite);
}
}
}

View file

@ -1,4 +1,3 @@
using System.Buffers;
using System.IO.MemoryMappedFiles;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
@ -6,6 +5,7 @@ using SixLabors.ImageSharp.Formats;
using SixLabors.ImageSharp.Formats.Png;
using SixLabors.ImageSharp.PixelFormats;
using StitchATon2.Infra.Buffers;
using StitchATon2.Infra.Synchronization;
namespace StitchATon2.Infra;
@ -18,9 +18,9 @@ public class ImageIntegral : IDisposable
private readonly int _width;
private readonly int _height;
private IMemoryOwner<ManualResetEventSlim>? _rowLocks;
private ManualResetEventSlim[]? _rowLocks;
private MemoryMappedFile? _memoryMappedFile;
private readonly object _lock = new();
private readonly Lock _lock = new();
private readonly ManualResetEventSlim _queueLock = new(true);
private readonly ManualResetEventSlim _initializationLock = new(false);
@ -58,14 +58,14 @@ public class ImageIntegral : IDisposable
{
if (_memoryMappedFile is null)
{
Task.Factory.StartNew(() => Initialize(cancellationToken), cancellationToken);
Task.Run(() => Initialize(cancellationToken), cancellationToken);
_initializationLock.Wait(cancellationToken);
_initializationLock.Dispose();
}
}
}
_rowLocks?.Memory.Span[row].Wait(cancellationToken);
_rowLocks?[row].Wait(cancellationToken);
}
private void Initialize(CancellationToken cancellationToken)
@ -86,8 +86,8 @@ public class ImageIntegral : IDisposable
}
var taskQueue = backedFileStream == null
? Task.CompletedTask
: AllocateBackedFile(backedFileStream, header);
? TaskHelper.SynchronizedTaskFactory.StartNew(() => { }, cancellationToken)
: AllocateBackedFile(backedFileStream, header, cancellationToken);
taskQueue = taskQueue.ContinueWith(
_ =>
@ -100,12 +100,11 @@ public class ImageIntegral : IDisposable
// initialize resource gating, all rows is expected to be locked
// if the backed file require to allocate, it should be safe to do this
// asynchronously
var rowLocks = MemoryAllocator.AllocateManaged<ManualResetEventSlim>(_height);
var rowLocksSpan = rowLocks.Memory.Span;
var rowLocks = new ManualResetEventSlim[_height];
for (int i = 0; i < _height; i++)
{
var isOpen = i < header.ProcessedRows;
rowLocksSpan[i] = new ManualResetEventSlim(isOpen);
rowLocks[i] = new ManualResetEventSlim(isOpen);
}
_rowLocks = rowLocks;
@ -129,7 +128,7 @@ public class ImageIntegral : IDisposable
var imageBuffer = image.Frames.RootFrame.PixelBuffer;
var accumulator = Int32Pixel.Zero;
var buffer = MemoryAllocator.AllocateArray<Int32Pixel>(_width);
var buffer = MemoryAllocator.Allocate<Int32Pixel>(_width);
var processedRows = _processedRows;
Interlocked.Exchange(ref _queueCounter, 0);
@ -143,50 +142,55 @@ public class ImageIntegral : IDisposable
buffer[x] = accumulator;
}
taskQueue = QueueWriterTask(taskQueue, 0, buffer.Clone(_width), cancellationToken);
taskQueue = QueueWriterTask(taskQueue, 0, buffer.Clone(), cancellationToken);
processedRows++;
}
else
{
ReadRow(processedRows - 1, buffer);
ReadRow(processedRows - 1, buffer.Span);
}
if(cancellationToken.IsCancellationRequested)
return;
var prevBuffer = buffer;
buffer = MemoryAllocator.AllocateArray<Int32Pixel>(_width);
for (int y = processedRows; y < image.Height; y++)
buffer = MemoryAllocator.Allocate<Int32Pixel>(_width);
try
{
var sourceRow = imageBuffer.DangerousGetRowSpan(y);
accumulator = (Int32Pixel)sourceRow[0];
buffer[0] = accumulator + prevBuffer[0];
// Process all other columns
for (var x = 1; x < sourceRow.Length; x++)
for (int y = processedRows; y < image.Height; y++)
{
accumulator.Accumulate(sourceRow[x]);
buffer[x] = accumulator + prevBuffer[x];
var sourceRow = imageBuffer.DangerousGetRowSpan(y);
accumulator = (Int32Pixel)sourceRow[0];
buffer[0] = accumulator + prevBuffer[0];
// Process all other columns
for (var x = 1; x < sourceRow.Length; x++)
{
accumulator.Accumulate(sourceRow[x]);
buffer[x] = accumulator + prevBuffer[x];
}
if (_queueCounter >= MaxProcessingQueue)
{
_queueLock.Reset();
_queueLock.Wait(cancellationToken);
}
if(cancellationToken.IsCancellationRequested)
break;
var writeBuffer = prevBuffer;
buffer.Copy(writeBuffer, _width);
taskQueue = QueueWriterTask(taskQueue, y, writeBuffer, cancellationToken);
prevBuffer = buffer;
buffer = MemoryAllocator.Allocate<Int32Pixel>(_width);
}
if (_queueCounter >= MaxProcessingQueue)
{
_queueLock.Reset();
_queueLock.Wait(cancellationToken);
}
if(cancellationToken.IsCancellationRequested)
break;
var writeBuffer = prevBuffer;
Array.Copy(buffer.Array, writeBuffer.Array, image.Width);
taskQueue = QueueWriterTask(taskQueue, y, writeBuffer, cancellationToken);
prevBuffer = buffer;
buffer = MemoryAllocator.AllocateArray<Int32Pixel>(_width);
}
finally
{
buffer.Dispose();
}
buffer.Dispose();
if(cancellationToken.IsCancellationRequested)
return;
@ -205,7 +209,7 @@ public class ImageIntegral : IDisposable
private Task QueueWriterTask(
Task taskQueue,
int row,
ArrayOwner<Int32Pixel> writeBuffer,
IBuffer<Int32Pixel> writeBuffer,
CancellationToken cancellationToken)
{
Interlocked.Increment(ref _queueCounter);
@ -213,10 +217,10 @@ public class ImageIntegral : IDisposable
return taskQueue.ContinueWith(_ =>
{
using (var view = AcquireView(row, MemoryMappedFileAccess.Write))
view.WriteArray(0, writeBuffer.Array, 0, _width);
view.DangerousWriteSpan(0, writeBuffer.Span, 0, _width);
writeBuffer.Dispose();
_rowLocks!.Memory.Span[row].Set();
_rowLocks![row].Set();
Interlocked.Increment(ref _processedRows);
using (var view = AcquireHeaderView(MemoryMappedFileAccess.Write))
@ -249,12 +253,6 @@ public class ImageIntegral : IDisposable
view.DangerousReadSpan(0, buffer, 0, _width);
}
private void ReadRow(int row, IBuffer<Int32Pixel> buffer)
{
using var view = AcquireView(row, MemoryMappedFileAccess.Read);
view.DangerousReadSpan(0, buffer.Span, 0, _width);
}
private FileStream? InitializeBackedFile(string path, out Header header)
{
var expectedHeader = Header.CreateInitial(_width, _height);
@ -311,33 +309,35 @@ public class ImageIntegral : IDisposable
return fs;
}
private static async Task AllocateBackedFile(FileStream fileStream, Header header)
private static Task AllocateBackedFile(FileStream fileStream, Header header, CancellationToken cancellationToken)
{
// The input filestream is expected to be empty with
// initial cursor at the beginning of the file and the content
// is pre-allocated for at least Header.Length bytes
// No other process should be accessed the file while being
// allocated.
// Allocated bytes is not necessary to be zeroed.
return TaskHelper.SynchronizedTaskFactory.StartNew(() =>
{
// The input filestream is expected to be empty with
// initial cursor at the beginning of the file and the content
// is pre-allocated for at least Header.Length bytes
// No other process should be accessed the file while being
// allocated.
// Allocated bytes is not necessary to be zeroed.
// const int writeBufferSize = 4 * 1024;
// using var writeBuffer = MemoryPool<byte>.Shared.Rent(writeBufferSize);
//
// var written = 0;
// while (written + writeBufferSize < header.Length)
// {
// await fileStream.WriteAsync(writeBuffer.Memory, cancellationToken);
// written += writeBufferSize;
// }
//
// if (written < header.Length)
// {
// await fileStream.WriteAsync(writeBuffer.Memory[..(header.Length - written)], cancellationToken);
// }
// const int writeBufferSize = 4 * 1024;
// using var writeBuffer = MemoryPool<byte>.Shared.Rent(writeBufferSize);
//
// var written = 0;
// while (written + writeBufferSize < header.Length)
// {
// await fileStream.WriteAsync(writeBuffer.Memory, cancellationToken);
// written += writeBufferSize;
// }
//
// if (written < header.Length)
// {
// await fileStream.WriteAsync(writeBuffer.Memory[..(header.Length - written)], cancellationToken);
// }
fileStream.SetLength(header.Length + Header.Size);
await fileStream.DisposeAsync();
fileStream.SetLength(header.Length + Header.Size);
fileStream.Dispose();
}, cancellationToken);
}
[StructLayout(LayoutKind.Sequential)]
@ -368,11 +368,8 @@ public class ImageIntegral : IDisposable
if (_rowLocks is { } locks)
{
_rowLocks = null;
var lockSpan = locks.Memory.Span;
for(int i = 0; i < _height; i++)
lockSpan[i].Dispose();
locks.Dispose();
locks[i].Dispose();
}
}

View file

@ -68,6 +68,12 @@ public record struct Int32Pixel
return new Int32Pixel(a.R / b, a.G / b, a.B / b);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Int32Pixel operator /(Int32Pixel a, float b)
{
return new Int32Pixel((byte)(a.R / b), (byte)(a.G / b), (byte)(a.B / b));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static explicit operator Int32Pixel(Rgb24 pixel)
{

View file

@ -1,14 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.10" />
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.11" />
</ItemGroup>
</Project>

View file

@ -2,11 +2,7 @@ namespace StitchATon2.Infra.Synchronization;
public static class TaskHelper
{
public static TaskFactory CreateTaskFactory()
{
return new TaskFactory(
TaskCreationOptions.AttachedToParent,
TaskContinuationOptions.ExecuteSynchronously
);
}
public static readonly TaskFactory SynchronizedTaskFactory = new(
TaskCreationOptions.LongRunning,
TaskContinuationOptions.ExecuteSynchronously);
}

View file

@ -1,25 +1,16 @@
using System.IO.MemoryMappedFiles;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using StitchATon2.Infra.Buffers;
namespace StitchATon2.Infra;
public static class Utils
internal static class Utils
{
[UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_buffer")]
private static extern ref SafeBuffer GetSafeBuffer(this UnmanagedMemoryAccessor view);
[UnsafeAccessor(UnsafeAccessorKind.Field, Name = "_offset")]
private static extern ref long GetOffset(this UnmanagedMemoryAccessor view);
private static unsafe uint AlignedSizeOf<T>() where T : unmanaged
{
uint size = (uint)sizeof(T);
return size is 1 or 2 ? size : (uint)((size + 3) & (~3));
return size is 1 or 2 ? size : (uint)((size + 3) & ~3);
}
public static void DangerousReadSpan<T>(this UnmanagedMemoryAccessor view, long position, Span<T> span, int offset, int count)
internal static void DangerousReadSpan<T>(this MemoryMappedViewAccessor view, long position, Span<T> span, int offset, int count)
where T : unmanaged
{
uint sizeOfT = AlignedSizeOf<T>();
@ -38,19 +29,14 @@ public static class Utils
}
}
var byteOffset = (ulong)(view.GetOffset() + position);
view.GetSafeBuffer().ReadSpan(byteOffset, span.Slice(offset, n));
var byteOffset = (ulong)(view.PointerOffset + position);
view.SafeMemoryMappedViewHandle.ReadSpan(byteOffset, span.Slice(offset, n));
}
public static ArrayOwner<T> Clone<T>(this ArrayOwner<T> arrayOwner, int length) where T : unmanaged
internal static void DangerousWriteSpan<T>(this MemoryMappedViewAccessor view, long position, Span<T> span, int offset, int count)
where T : unmanaged
{
var newArrayOwner = MemoryAllocator.AllocateArray<T>(length);
Array.Copy(arrayOwner.Array, 0, newArrayOwner.Array, 0, length);
return newArrayOwner;
}
public static void CopyTo<T>(this ArrayOwner<T> arrayOwner, ArrayOwner<T> target, int length) where T : unmanaged
{
Array.Copy(arrayOwner.Array, 0, target.Array, 0, length);
var byteOffset = (ulong)(view.PointerOffset + position);
view.SafeMemoryMappedViewHandle.WriteSpan<T>(byteOffset, span.Slice(offset, count));
}
}

View file

@ -1,7 +1,13 @@
# Stitch-a-ton Contest Submission
Repository moved to: https://github.com/denniskematian/GigATon
## Installation
This project is build using .NET 9, with SixLabors.ImageSharp as the only external library.
Please refer to [install dotnet SDK on Debian](https://learn.microsoft.com/en-us/dotnet/core/install/linux-debian?tabs=dotnet9)
for details.
### 1. Build
Install dependencies:
@ -9,7 +15,7 @@ Install dependencies:
wget https://packages.microsoft.com/config/debian/12/packages-microsoft-prod.deb
sudo dpkg -i packages-microsoft-prod.deb
sudo apt update
sudo apt install -y dotnet-sdk-8.0
sudo apt install -y dotnet-sdk-9.0
```
Build the project in `App` folder using command:
@ -24,8 +30,13 @@ After publishing, enter the folder `/publish` then run
./StitchATon2.App
```
## Approach
## Ideas
Since we only serve a static assets and the output is downscale only, I think it's good to pre-process each image to [image integral](https://en.wikipedia.org/wiki/Summed-area_table) first and then save it to another directory (eg. temp).
Later, we can apply Box Filter averaging on the fly to output Stream (HttpResponse).
## Approach
This project is heavily depends on Memory Mapped File to reduce processing overhead and memory pressure to allow HTTP serve more request.
The MMF is used to store the Image Integral of provided asset.

View file

@ -0,0 +1,77 @@
using System.Net.Http.Json;
using BenchmarkDotNet.Attributes;
using StitchATon2.Domain;
namespace StitchATon2.Benchmark;
public class SingleTileBenchmark
{
private const string Url = "http://localhost:5088/api/image/generate";
private readonly TileManager _tileManager = new(Configuration.Default);
private readonly Random _random = new();
private readonly HttpClient _client = new();
private string GetRandomCoordinatePair()
{
var maxId = _tileManager.Configuration.Rows
* _tileManager.Configuration.Columns;
var id = _random.Next(maxId);
var tile = _tileManager.GetTile(id);
return $"{tile.Coordinate}:{tile.Coordinate}";
}
private JsonContent GetRandomDto(float scale, float offsetX = 0, float offsetY = 0)
{
return JsonContent.Create(new
{
canvas_rect = GetRandomCoordinatePair(),
crop_offset = (float[]) [offsetX, offsetY],
crop_size = (float[]) [1f - offsetX, 1f - offsetY],
output_scale = scale,
});
}
private JsonContent GetRandomDtoWithOffset(float scale)
{
var offsetX = _random.NextSingle();
var offsetY = _random.NextSingle();
return GetRandomDto(scale, offsetX, offsetY);
}
[Benchmark]
public async Task NoScaling()
{
await _client.PostAsync(Url, GetRandomDto(1f));
}
[Benchmark]
public async Task ScaleHalf()
{
await _client.PostAsync(Url, GetRandomDto(.5f));
}
[Benchmark]
public async Task ScaleQuarter()
{
await _client.PostAsync(Url, GetRandomDto(.25f));
}
[Benchmark]
public async Task NoScalingWithOffset()
{
await _client.PostAsync(Url, GetRandomDtoWithOffset(1f));
}
[Benchmark]
public async Task ScaleHalfWithOffset()
{
await _client.PostAsync(Url, GetRandomDtoWithOffset(.5f));
}
[Benchmark]
public async Task ScaleQuarterWithOffset()
{
await _client.PostAsync(Url, GetRandomDtoWithOffset(.25f));
}
}

View file

@ -6,6 +6,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StitchATon2.Domain", "Domai
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StitchATon2.Infra", "Infra\StitchATon2.Infra.csproj", "{E602F3FC-6139-4B30-AC5A-75815E6340A4}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StitchATon2.Benchmark", "StitchATon2.Benchmark\StitchATon2.Benchmark.csproj", "{2F9B169C-C799-4489-B864-F912D69C5D3E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConsoleApp", "ConsoleApp\ConsoleApp.csproj", "{D5B2A2E9-974A-43DD-A5D1-E7226CD03AFF}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -24,5 +28,13 @@ Global
{E602F3FC-6139-4B30-AC5A-75815E6340A4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E602F3FC-6139-4B30-AC5A-75815E6340A4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E602F3FC-6139-4B30-AC5A-75815E6340A4}.Release|Any CPU.Build.0 = Release|Any CPU
{2F9B169C-C799-4489-B864-F912D69C5D3E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{2F9B169C-C799-4489-B864-F912D69C5D3E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2F9B169C-C799-4489-B864-F912D69C5D3E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2F9B169C-C799-4489-B864-F912D69C5D3E}.Release|Any CPU.Build.0 = Release|Any CPU
{D5B2A2E9-974A-43DD-A5D1-E7226CD03AFF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D5B2A2E9-974A-43DD-A5D1-E7226CD03AFF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D5B2A2E9-974A-43DD-A5D1-E7226CD03AFF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D5B2A2E9-974A-43DD-A5D1-E7226CD03AFF}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal

View file

@ -1,6 +1,6 @@
{
"sdk": {
"version": "8.0.0",
"version": "9.0.0",
"rollForward": "latestMinor",
"allowPrerelease": false
}