From 860c3327bedd23ed42fb1d7e9e5eff7c7ffbec4c Mon Sep 17 00:00:00 2001 From: Laszlo Takacs Date: Mon, 20 Apr 2026 15:58:54 +0200 Subject: [PATCH 1/2] Add ASP.NET Core middleware --- Prerender.AspNetCore.csproj | 21 +++ PrerenderMiddleware.cs | 106 +++++++++++++++ PrerenderOptions.cs | 7 + PrerenderServiceExtensions.cs | 18 +++ README.md | 69 ++++++++++ tests/Prerender.AspNetCore.Tests.csproj | 17 +++ tests/PrerenderMiddlewareTests.cs | 166 ++++++++++++++++++++++++ 7 files changed, 404 insertions(+) create mode 100644 Prerender.AspNetCore.csproj create mode 100644 PrerenderMiddleware.cs create mode 100644 PrerenderOptions.cs create mode 100644 PrerenderServiceExtensions.cs create mode 100644 README.md create mode 100644 tests/Prerender.AspNetCore.Tests.csproj create mode 100644 tests/PrerenderMiddlewareTests.cs diff --git a/Prerender.AspNetCore.csproj b/Prerender.AspNetCore.csproj new file mode 100644 index 0000000..41af1fd --- /dev/null +++ b/Prerender.AspNetCore.csproj @@ -0,0 +1,21 @@ + + + net8.0 + enable + enable + Prerender.AspNetCore + 1.0.0 + Prerender.io + ASP.NET Core middleware for prerendering JavaScript-rendered pages for SEO via Prerender.io + MIT + README.md + https://github.com/prerender/integrations + git + + + + + + + + diff --git a/PrerenderMiddleware.cs b/PrerenderMiddleware.cs new file mode 100644 index 0000000..c9f05b5 --- /dev/null +++ b/PrerenderMiddleware.cs @@ -0,0 +1,106 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Prerender.AspNetCore; + +public class PrerenderMiddleware : IMiddleware +{ + private static readonly string[] CrawlerUserAgents = + [ + "googlebot", "yahoo", "bingbot", "baiduspider", + "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", + "embedly", "quora link preview", "showyoubot", "outbrain", + "pinterest", "slackbot", "w3c_validator", "perplexity", + "oai-searchbot", "chatgpt-user", "gptbot", "claudebot", "amazonbot", + ]; + + private static readonly string[] ExtensionsToIgnore = + [ + ".js", ".css", ".xml", ".less", ".png", ".jpg", ".jpeg", ".gif", + ".pdf", ".doc", ".txt", ".ico", ".rss", ".zip", ".mp3", ".rar", + ".exe", ".wmv", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", + ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", + ".dmg", ".iso", ".flv", ".m4v", ".torrent", ".ttf", ".woff", ".svg", + ]; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly PrerenderOptions _options; + private readonly ILogger _logger; + + public PrerenderMiddleware( + IHttpClientFactory httpClientFactory, + IOptions options, + ILogger logger) + { + _httpClientFactory = httpClientFactory; + _options = options.Value; + _logger = logger; + } + + public async Task InvokeAsync(HttpContext context, RequestDelegate next) + { + if (!ShouldPrerender(context)) + { + await next(context); + return; + } + + try + { + var client = _httpClientFactory.CreateClient("prerender"); + var apiUrl = BuildApiUrl(context); + + using var request = new HttpRequestMessage(HttpMethod.Get, apiUrl); + request.Headers.TryAddWithoutValidation( + "User-Agent", context.Request.Headers["User-Agent"].ToString()); + if (!string.IsNullOrWhiteSpace(_options.Token)) + request.Headers.TryAddWithoutValidation("X-Prerender-Token", _options.Token); + + using var response = await client.SendAsync(request, context.RequestAborted); + context.Response.StatusCode = (int)response.StatusCode; + var body = await response.Content.ReadAsStringAsync(); + await context.Response.WriteAsync(body); + } + catch (HttpRequestException ex) + { + _logger.LogWarning(ex, "Prerender service unreachable, falling back"); + await next(context); + } + } + + private static bool ShouldPrerender(HttpContext context) + { + if (context.Request.Method != HttpMethods.Get) return false; + + var path = context.Request.Path.Value ?? string.Empty; + if (IsStaticAsset(path)) return false; + + if (context.Request.Query.ContainsKey("_escaped_fragment_")) return true; + if (context.Request.Headers.ContainsKey("X-Bufferbot")) return true; + + var ua = context.Request.Headers["User-Agent"].ToString(); + return !string.IsNullOrEmpty(ua) && IsBot(ua); + } + + private string BuildApiUrl(HttpContext context) + { + var serviceUrl = _options.ServiceUrl.TrimEnd('/') + "/"; + var scheme = context.Request.Scheme; + var host = context.Request.Host.Value; + var pathAndQuery = context.Request.Path + context.Request.QueryString; + return $"{serviceUrl}{scheme}://{host}{pathAndQuery}"; + } + + private static bool IsBot(string userAgent) + { + var ua = userAgent.ToLowerInvariant(); + return CrawlerUserAgents.Any(bot => ua.Contains(bot)); + } + + private static bool IsStaticAsset(string path) + { + var lower = path.ToLowerInvariant(); + return ExtensionsToIgnore.Any(ext => lower.EndsWith(ext)); + } +} diff --git a/PrerenderOptions.cs b/PrerenderOptions.cs new file mode 100644 index 0000000..715f494 --- /dev/null +++ b/PrerenderOptions.cs @@ -0,0 +1,7 @@ +namespace Prerender.AspNetCore; + +public class PrerenderOptions +{ + public string? Token { get; set; } + public string ServiceUrl { get; set; } = "https://service.prerender.io/"; +} diff --git a/PrerenderServiceExtensions.cs b/PrerenderServiceExtensions.cs new file mode 100644 index 0000000..30ac9d8 --- /dev/null +++ b/PrerenderServiceExtensions.cs @@ -0,0 +1,18 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; + +namespace Prerender.AspNetCore; + +public static class PrerenderServiceExtensions +{ + public static IServiceCollection AddPrerender(this IServiceCollection services) + { + services.AddOptions().BindConfiguration("Prerender"); + services.AddHttpClient("prerender"); + services.AddTransient(); + return services; + } + + public static IApplicationBuilder UsePrerender(this IApplicationBuilder app) + => app.UseMiddleware(); +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..b5b6aac --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# prerender-aspnetcore + +ASP.NET Core middleware for [Prerender.io](https://prerender.io). Intercepts requests from bots and crawlers and serves prerendered HTML, so your JavaScript-rendered app is fully indexable by search engines and social media scrapers. + +Compatible with **ASP.NET Core 8+** and **.NET 8+**. + +## Installation + +```bash +dotnet add package Prerender.AspNetCore +``` + +## Setup + +Register the middleware in `Program.cs`: + +```csharp +builder.Services.AddPrerender(); + +var app = builder.Build(); +app.UsePrerender(); // place before routing middleware +``` + +Add your token to `appsettings.json`: + +```json +{ + "Prerender": { + "Token": "YOUR_PRERENDER_TOKEN" + } +} +``` + +The middleware must be placed **before** routing to intercept bot requests early. + +## Settings + +| Setting | Default | Description | +|---------|---------|-------------| +| `Prerender:Token` | `null` | Your Prerender.io token | +| `Prerender:ServiceUrl` | `https://service.prerender.io/` | Prerender service URL (override for self-hosted Prerender) | + +## Self-hosted Prerender + +```json +{ + "Prerender": { + "ServiceUrl": "http://your-prerender-server:3000" + } +} +``` + +## How it works + +Requests are prerendered when **all** of the following are true: + +- The HTTP method is `GET` +- The `User-Agent` matches a known bot/crawler (Googlebot, Bingbot, Twitterbot, GPTBot, ClaudeBot, etc.) + — OR the URL contains `_escaped_fragment_` + — OR the `X-Bufferbot` header is present +- The URL does not end with a static asset extension (`.js`, `.css`, `.png`, etc.) + +Everything else passes through to your normal ASP.NET Core pipeline. + +If the Prerender service is unreachable, the middleware falls back gracefully and serves the normal response. + +## License + +MIT diff --git a/tests/Prerender.AspNetCore.Tests.csproj b/tests/Prerender.AspNetCore.Tests.csproj new file mode 100644 index 0000000..37c1eb9 --- /dev/null +++ b/tests/Prerender.AspNetCore.Tests.csproj @@ -0,0 +1,17 @@ + + + net8.0 + enable + enable + false + + + + + + + + + + + diff --git a/tests/PrerenderMiddlewareTests.cs b/tests/PrerenderMiddlewareTests.cs new file mode 100644 index 0000000..df8ff58 --- /dev/null +++ b/tests/PrerenderMiddlewareTests.cs @@ -0,0 +1,166 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using System.Net; +using Xunit; + +namespace Prerender.AspNetCore.Tests; + +public class PrerenderMiddlewareTests +{ + private const string BotUserAgent = "Mozilla/5.0 (compatible; Googlebot/2.1)"; + private const string BrowserUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"; + private const string PrerenderedHtml = "prerendered"; + + private static TestServer CreateServer( + HttpResponseMessage? fakeResponse = null, + Action? configureOptions = null) + { + var response = fakeResponse ?? new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(PrerenderedHtml) + }; + + var builder = new WebHostBuilder() + .ConfigureServices(services => + { + services.AddPrerender(); + services.AddHttpClient("prerender") + .ConfigurePrimaryHttpMessageHandler(() => new FakeHttpMessageHandler(response)); + if (configureOptions is not null) + services.Configure(configureOptions); + }) + .Configure(app => + { + app.UsePrerender(); + app.Run(ctx => ctx.Response.WriteAsync("normal response")); + }); + + return new TestServer(builder); + } + + [Fact] + public async Task BrowserRequest_PassesThrough() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BrowserUserAgent); + + var response = await client.GetAsync("/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("normal response", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task BotRequest_ReceivesPrerenderedResponse() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + var response = await client.GetAsync("/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains(PrerenderedHtml, await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task BotRequest_StaticAsset_PassesThrough() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + var response = await client.GetAsync("/styles.css"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("normal response", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task EscapedFragment_TriggersPrerender() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BrowserUserAgent); + + var response = await client.GetAsync("/?_escaped_fragment_="); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains(PrerenderedHtml, await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task XBufferbot_TriggersPrerender() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BrowserUserAgent); + client.DefaultRequestHeaders.Add("X-Bufferbot", "true"); + + var response = await client.GetAsync("/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Contains(PrerenderedHtml, await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task PostRequest_BotUa_PassesThrough() + { + using var server = CreateServer(); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + var response = await client.PostAsync("/", new StringContent(string.Empty)); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("normal response", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task NetworkError_FallsBackToNormalResponse() + { + var builder = new WebHostBuilder() + .ConfigureServices(services => + { + services.AddPrerender(); + services.AddHttpClient("prerender") + .ConfigurePrimaryHttpMessageHandler(() => new FailingHttpMessageHandler()); + }) + .Configure(app => + { + app.UsePrerender(); + app.Run(ctx => ctx.Response.WriteAsync("normal response")); + }); + + using var server = new TestServer(builder); + var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", BotUserAgent); + + var response = await client.GetAsync("/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("normal response", await response.Content.ReadAsStringAsync()); + } +} + +internal class FakeHttpMessageHandler : HttpMessageHandler +{ + private readonly HttpResponseMessage _response; + + public FakeHttpMessageHandler(HttpResponseMessage response) => _response = response; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + => Task.FromResult(_response); +} + +internal class FailingHttpMessageHandler : HttpMessageHandler +{ + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + => throw new HttpRequestException("simulated network failure"); +} From 1567069c995605aa54ed307374fdbee7c98c6f8c Mon Sep 17 00:00:00 2001 From: Laszlo Takacs Date: Tue, 21 Apr 2026 09:18:23 +0200 Subject: [PATCH 2/2] Add X-Prerender-Int-Type header --- PrerenderMiddleware.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/PrerenderMiddleware.cs b/PrerenderMiddleware.cs index c9f05b5..5c202a0 100644 --- a/PrerenderMiddleware.cs +++ b/PrerenderMiddleware.cs @@ -56,6 +56,7 @@ public async Task InvokeAsync(HttpContext context, RequestDelegate next) "User-Agent", context.Request.Headers["User-Agent"].ToString()); if (!string.IsNullOrWhiteSpace(_options.Token)) request.Headers.TryAddWithoutValidation("X-Prerender-Token", _options.Token); + request.Headers.TryAddWithoutValidation("X-Prerender-Int-Type", "AspNetCore"); using var response = await client.SendAsync(request, context.RequestAborted); context.Response.StatusCode = (int)response.StatusCode;