From 57d98efcd0ae6a0a5f26348dd9b2c41f6dbd7fdd Mon Sep 17 00:00:00 2001 From: fboucher-os Date: Sat, 15 Aug 2026 09:11:05 -0400 Subject: [PATCH] feat: implement async background post content extraction to save HTML in blob storage --- .../Endpoints/PostExtractionTests.cs | 74 ++++++++++++++++ .../Fixtures/FakePostParserClient.cs | 13 +++ .../Fixtures/NoteBookmarkApiTestFactory.cs | 3 + src/NoteBookmark.Api/IPostParserClient.cs | 9 ++ src/NoteBookmark.Api/PostEndpoints.cs | 10 ++- .../PostExtractionBackgroundWorker.cs | 87 +++++++++++++++++++ src/NoteBookmark.Api/PostExtractionQueue.cs | 34 ++++++++ src/NoteBookmark.Api/PostParserClient.cs | 56 ++++++++++++ src/NoteBookmark.Api/Program.cs | 5 ++ 9 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 src/NoteBookmark.Api.Tests/Endpoints/PostExtractionTests.cs create mode 100644 src/NoteBookmark.Api.Tests/Fixtures/FakePostParserClient.cs create mode 100644 src/NoteBookmark.Api/IPostParserClient.cs create mode 100644 src/NoteBookmark.Api/PostExtractionBackgroundWorker.cs create mode 100644 src/NoteBookmark.Api/PostExtractionQueue.cs create mode 100644 src/NoteBookmark.Api/PostParserClient.cs diff --git a/src/NoteBookmark.Api.Tests/Endpoints/PostExtractionTests.cs b/src/NoteBookmark.Api.Tests/Endpoints/PostExtractionTests.cs new file mode 100644 index 0000000..4fb5e35 --- /dev/null +++ b/src/NoteBookmark.Api.Tests/Endpoints/PostExtractionTests.cs @@ -0,0 +1,74 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using NoteBookmark.Api.Tests.Fixtures; +using NoteBookmark.Domain; +using System; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading.Tasks; +using Xunit; +using Azure.Storage.Blobs; + +namespace NoteBookmark.Api.Tests.Endpoints; + +public class PostExtractionTests : IClassFixture +{ + private readonly NoteBookmarkApiTestFactory _factory; + private readonly HttpClient _client; + + public PostExtractionTests(NoteBookmarkApiTestFactory factory) + { + _factory = factory; + _client = _factory.CreateClient(); + } + + [Fact] + public async Task ExtractPostDetails_TriggersBackgroundWorkerAndSavesHtmlToBlobStorage() + { + // Arrange + var url = "https://example.com/blog/test-post-" + Guid.NewGuid(); + var extractRequest = new + { + url = url, + tags = "test", + category = "Test" + }; + + // Act - Call the API to extract metadata and save the post + var response = await _client.PostAsJsonAsync("/api/posts/extractPostDetails", extractRequest); + + // Assert API response is OK + response.StatusCode.Should().Be(HttpStatusCode.OK); + + var post = await response.Content.ReadFromJsonAsync(); + post.Should().NotBeNull(); + var postId = post!.Id ?? post.RowKey; + postId.Should().NotBeNullOrEmpty(); + + // Since the extraction happens asynchronously in a BackgroundWorker, + // we poll the blob storage for a short time to verify the file was created. + var blobServiceClient = _factory.Services.GetRequiredService(); + var containerClient = blobServiceClient.GetBlobContainerClient("cleanedposts"); + var blobClient = containerClient.GetBlobClient($"{postId}.html"); + + // Wait up to 5 seconds for the background worker to process + bool blobExists = false; + for (int i = 0; i < 25; i++) + { + if (await blobClient.ExistsAsync()) + { + blobExists = true; + break; + } + await Task.Delay(200); + } + + blobExists.Should().BeTrue("HTML content should be processed by the background worker and saved to blob storage"); + + // Verify the content saved matches the fake content + var downloadResult = await blobClient.DownloadContentAsync(); + var content = downloadResult.Value.Content.ToString(); + content.Should().Contain(url); + } +} diff --git a/src/NoteBookmark.Api.Tests/Fixtures/FakePostParserClient.cs b/src/NoteBookmark.Api.Tests/Fixtures/FakePostParserClient.cs new file mode 100644 index 0000000..5dc2e2e --- /dev/null +++ b/src/NoteBookmark.Api.Tests/Fixtures/FakePostParserClient.cs @@ -0,0 +1,13 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace NoteBookmark.Api.Tests.Fixtures; + +public class FakePostParserClient : IPostParserClient +{ + public Task ExtractContentAsync(string url, CancellationToken cancellationToken = default) + { + // Return a mock HTML snippet for testing + return Task.FromResult($"
Extracted HTML content for {url}
"); + } +} diff --git a/src/NoteBookmark.Api.Tests/Fixtures/NoteBookmarkApiTestFactory.cs b/src/NoteBookmark.Api.Tests/Fixtures/NoteBookmarkApiTestFactory.cs index da53256..cd0686c 100644 --- a/src/NoteBookmark.Api.Tests/Fixtures/NoteBookmarkApiTestFactory.cs +++ b/src/NoteBookmark.Api.Tests/Fixtures/NoteBookmarkApiTestFactory.cs @@ -34,6 +34,9 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) services.AddSingleton(new TableServiceClient(connectionString)); services.AddSingleton(new BlobServiceClient(connectionString)); } + + // Register FakePostParserClient for integration tests + services.AddSingleton(); }); } diff --git a/src/NoteBookmark.Api/IPostParserClient.cs b/src/NoteBookmark.Api/IPostParserClient.cs new file mode 100644 index 0000000..72507ab --- /dev/null +++ b/src/NoteBookmark.Api/IPostParserClient.cs @@ -0,0 +1,9 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace NoteBookmark.Api; + +public interface IPostParserClient +{ + Task ExtractContentAsync(string url, CancellationToken cancellationToken = default); +} diff --git a/src/NoteBookmark.Api/PostEndpoints.cs b/src/NoteBookmark.Api/PostEndpoints.cs index 87c1bb2..66a8a7e 100644 --- a/src/NoteBookmark.Api/PostEndpoints.cs +++ b/src/NoteBookmark.Api/PostEndpoints.cs @@ -94,7 +94,11 @@ static Results SavePost(Post post, TableServiceClient tblClient, } return TypedResults.BadRequest(); } - static async Task, BadRequest>> ExtractPostDetails(ExtractPostRequest request, TableServiceClient tblClient, BlobServiceClient blobClient) + static async Task, BadRequest>> ExtractPostDetails( + ExtractPostRequest request, + TableServiceClient tblClient, + BlobServiceClient blobClient, + PostExtractionQueue queue) { var dataStorageService = new DataStorageService(tblClient, blobClient); @@ -105,6 +109,10 @@ static async Task, BadRequest>> ExtractPostDetails(ExtractPostR if (post != null) { dataStorageService.SavePost(post); + + // Queue background HTML extraction task + queue.QueueBackgroundWorkItem(new ExtractionTask(post.Id ?? post.RowKey, post.Url ?? decodeUrl)); + return TypedResults.Ok(post); } return TypedResults.BadRequest(); diff --git a/src/NoteBookmark.Api/PostExtractionBackgroundWorker.cs b/src/NoteBookmark.Api/PostExtractionBackgroundWorker.cs new file mode 100644 index 0000000..801f507 --- /dev/null +++ b/src/NoteBookmark.Api/PostExtractionBackgroundWorker.cs @@ -0,0 +1,87 @@ +using System; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.Blobs; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace NoteBookmark.Api; + +public class PostExtractionBackgroundWorker : BackgroundService +{ + private readonly PostExtractionQueue _queue; + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + + public PostExtractionBackgroundWorker( + PostExtractionQueue queue, + IServiceProvider serviceProvider, + ILogger logger) + { + _queue = queue; + _serviceProvider = serviceProvider; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("Post Extraction Background Worker started."); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + var task = await _queue.DequeueAsync(stoppingToken); + _logger.LogInformation("Processing extraction for Post: {PostId}, URL: {Url}", task.PostId, task.Url); + + await ProcessExtractionAsync(task, stoppingToken); + } + catch (OperationCanceledException) + { + // Normal shutdown + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error occurred executing background extraction task."); + } + } + + _logger.LogInformation("Post Extraction Background Worker stopped."); + } + + private async Task ProcessExtractionAsync(ExtractionTask task, CancellationToken cancellationToken) + { + using var scope = _serviceProvider.CreateScope(); + var parserClient = scope.ServiceProvider.GetRequiredService(); + var blobServiceClient = scope.ServiceProvider.GetRequiredService(); + + try + { + var content = await parserClient.ExtractContentAsync(task.Url, cancellationToken); + if (string.IsNullOrEmpty(content)) + { + _logger.LogWarning("No content returned for URL: {Url}. Skipping blob upload.", task.Url); + return; + } + + var containerClient = blobServiceClient.GetBlobContainerClient("cleanedposts"); + await containerClient.CreateIfNotExistsAsync(cancellationToken: cancellationToken); + + var blobClient = containerClient.GetBlobClient($"{task.PostId}.html"); + + byte[] contentBytes = Encoding.UTF8.GetBytes(content); + using var stream = new MemoryStream(contentBytes); + + await blobClient.UploadAsync(stream, overwrite: true, cancellationToken: cancellationToken); + _logger.LogInformation("Successfully saved extracted HTML for Post {PostId} to Blob Storage.", task.PostId); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to process extraction for Post {PostId} / URL: {Url}", task.PostId, task.Url); + } + } +} diff --git a/src/NoteBookmark.Api/PostExtractionQueue.cs b/src/NoteBookmark.Api/PostExtractionQueue.cs new file mode 100644 index 0000000..b1698a5 --- /dev/null +++ b/src/NoteBookmark.Api/PostExtractionQueue.cs @@ -0,0 +1,34 @@ +using System; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace NoteBookmark.Api; + +public record ExtractionTask(string PostId, string Url); + +public class PostExtractionQueue +{ + private readonly Channel _queue; + + public PostExtractionQueue() + { + // Unbounded channel is simple and suitable for this task queue. + _queue = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = true, + SingleWriter = false + }); + } + + public void QueueBackgroundWorkItem(ExtractionTask task) + { + ArgumentNullException.ThrowIfNull(task); + _queue.Writer.TryWrite(task); + } + + public async ValueTask DequeueAsync(CancellationToken cancellationToken) + { + return await _queue.Reader.ReadAsync(cancellationToken); + } +} diff --git a/src/NoteBookmark.Api/PostParserClient.cs b/src/NoteBookmark.Api/PostParserClient.cs new file mode 100644 index 0000000..2562eb9 --- /dev/null +++ b/src/NoteBookmark.Api/PostParserClient.cs @@ -0,0 +1,56 @@ +using System; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NoteBookmark.Api; + +public class PostParserClient : IPostParserClient +{ + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + + public PostParserClient(HttpClient httpClient, ILogger logger) + { + _httpClient = httpClient; + _logger = logger; + // Configure base address or default headers if needed, but since URL is fully specified we can just configure it or call it directly. + if (_httpClient.BaseAddress == null) + { + _httpClient.BaseAddress = new Uri("https://azpostlight-parser.azurewebsites.net/"); + } + } + + public async Task ExtractContentAsync(string url, CancellationToken cancellationToken = default) + { + try + { + _logger.LogInformation("Calling parser API for URL: {Url}", url); + var requestBody = new { url = url }; + var response = await _httpClient.PostAsJsonAsync("parser", requestBody, cancellationToken); + + if (!response.IsSuccessStatusCode) + { + _logger.LogWarning("Parser API returned error status: {StatusCode}", response.StatusCode); + return null; + } + + var result = await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken); + return result?.Content; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to extract content for URL: {Url}", url); + return null; + } + } + + private class ParserResponse + { + [JsonPropertyName("content")] + public string? Content { get; set; } + } +} diff --git a/src/NoteBookmark.Api/Program.cs b/src/NoteBookmark.Api/Program.cs index 6def1b2..55fd0f6 100644 --- a/src/NoteBookmark.Api/Program.cs +++ b/src/NoteBookmark.Api/Program.cs @@ -15,6 +15,11 @@ // Register data storage service builder.Services.AddScoped(); +// Register background extraction queue and worker +builder.Services.AddHttpClient(); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(); + // Register AI settings provider builder.Services.AddScoped();