Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
PRERENDER_TOKEN=
PRERENDER_SERVICE_URL=
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
node_modules/
.env
67 changes: 67 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# prerender-koa

Koa middleware for [Prerender.io](https://prerender.io). Intercepts requests from bots and crawlers and serves prerendered HTML, so your JavaScript-rendered app is fully indexable by search engines and social media scrapers.

Compatible with **Koa v2+** and **Node.js 18+**.

## Installation

```bash
npm install prerender-koa
```

## Usage

```javascript
const Koa = require('koa');
const prerender = require('prerender-koa');

const app = new Koa();

app.use(prerender({
token: 'YOUR_PRERENDER_TOKEN'
}));

// your other middleware and routes
```

The middleware intercepts bot requests and proxies them to Prerender.io, returning prerendered HTML. Regular browser requests pass through unaffected. All responses include an `X-Prerender: true/false` header.

## Options

| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `token` | `string` | `process.env.PRERENDER_TOKEN` | Your Prerender.io token |
| `serviceUrl` | `string` | `process.env.PRERENDER_SERVICE_URL` or `https://service.prerender.io/` | Prerender service URL (use this for self-hosted Prerender) |
| `protocol` | `string` | `null` | Force a protocol (`http` or `https`). Defaults to the request's protocol |

## Environment variables

```bash
PRERENDER_TOKEN=your_token_here
PRERENDER_SERVICE_URL=https://service.prerender.io/ # optional
```

## Self-hosted Prerender

```javascript
app.use(prerender({
serviceUrl: 'http://your-prerender-server:3000'
}));
```

## How it works

Requests are prerendered when **all** of the following are true:

- The HTTP method is `GET`
- The `User-Agent` matches a known bot/crawler (Googlebot, Bingbot, Twitterbot, GPTBot, ClaudeBot, etc.)
— OR the URL contains `_escaped_fragment_`
— OR the `X-Bufferbot` header is present
- The URL does not end with a static asset extension (`.js`, `.css`, `.png`, etc.)

Everything else passes through to your normal Koa middleware.

## License

MIT
86 changes: 86 additions & 0 deletions index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
'use strict';

const internals = {};

internals.crawlerUserAgents = [
'googlebot', 'yahoo', 'bingbot', 'baiduspider',
'facebookexternalhit', 'twitterbot', 'rogerbot', 'linkedinbot',
'embedly', 'quora link preview', 'showyoubot', 'outbrain',
'pinterest', 'slackbot', 'developers.google.com/+/web/snippet',
'w3c_validator', 'perplexity', 'oai-searchbot', 'chatgpt-user',
'gptbot', 'claudebot', 'amazonbot'
];

internals.extensionsToIgnore = [
'.js', '.css', '.xml', '.less', '.png', '.jpg', '.jpeg', '.gif',
'.pdf', '.doc', '.txt', '.ico', '.rss', '.zip', '.mp3', '.rar',
'.exe', '.wmv', '.avi', '.ppt', '.mpg', '.mpeg', '.tif', '.wav',
'.mov', '.psd', '.ai', '.xls', '.mp4', '.m4a', '.swf', '.dat',
'.dmg', '.iso', '.flv', '.m4v', '.torrent', '.ttf', '.woff', '.svg'
];

internals.defaults = {
serviceUrl: process.env.PRERENDER_SERVICE_URL || 'https://service.prerender.io/',
token: process.env.PRERENDER_TOKEN || null,
protocol: null
};

function isBot(userAgent) {
const ua = userAgent.toLowerCase();
return internals.crawlerUserAgents.some((bot) => ua.includes(bot));
}

function isStaticAsset(path) {
return internals.extensionsToIgnore.some((ext) => path.endsWith(ext));
}

function shouldPrerender(ctx) {
const userAgent = ctx.get('user-agent');
if (!userAgent || ctx.method !== 'GET') return false;
if (isStaticAsset(ctx.path)) return false;

return '_escaped_fragment_' in ctx.query
|| isBot(userAgent)
|| !!ctx.get('x-bufferbot');
}

function buildApiUrl(ctx, settings) {
const protocol = settings.protocol || ctx.protocol;
const base = settings.serviceUrl.endsWith('/')
? settings.serviceUrl
: settings.serviceUrl + '/';
return `${base}${protocol}://${ctx.host}${ctx.url}`;
}

async function fetchPrerendered(apiUrl, ctx, settings) {
const headers = { 'User-Agent': ctx.get('user-agent') };
if (settings.token) {
headers['X-Prerender-Token'] = settings.token;
}
const response = await fetch(apiUrl, { headers, redirect: 'manual' });
const body = await response.text();
return { status: response.status, body };
}

module.exports = function prerenderMiddleware(options = {}) {
const settings = { ...internals.defaults, ...options };

return async function prerender(ctx, next) {
if (!shouldPrerender(ctx)) {
ctx.set('X-Prerender', 'false');
return next();
}

try {
const apiUrl = buildApiUrl(ctx, settings);
const prerendered = await fetchPrerendered(apiUrl, ctx, settings);
ctx.status = prerendered.status;
ctx.body = prerendered.body;
ctx.set('X-Prerender', 'true');
} catch (err) {
console.error('Prerender error, falling back:', err.message);
ctx.set('X-Prerender', 'false');
return next();
}
};
};
Loading