chore: Vendor request-filtering-agent (#12266)

* chore: Vendor request-filtering-agent

* fix: honor fetch timeout and undefined allow list in proxy pre-flight

Default allowIPAddressList to [] so an unset ALLOWED_PRIVATE_IP_ADDRESSES
env var doesn't overwrite the agent's default and crash on .length, and
race the pre-flight DNS lookup against the request's abort signal so the
configured fetch timeout applies to slow DNS resolution.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Tom Moor
2026-05-06 20:26:52 -04:00
committed by GitHub
parent 0f3f7b8da7
commit ad7e6c98ab
6 changed files with 412 additions and 132 deletions
-1
View File
@@ -231,7 +231,6 @@
"redlock": "^5.0.0-beta2",
"reflect-metadata": "^0.2.2",
"refractor": "^3.6.0",
"request-filtering-agent": "^3.2.0",
"resolve-path": "^1.4.0",
"sanitize-filename": "^1.6.4",
"scroll-into-view-if-needed": "^3.1.0",
@@ -1,28 +0,0 @@
const http = require("http");
const https = require("https");
/**
* Mock implementation of request-filtering-agent for Jest testing
* This avoids ESM module issues in the test environment
*/
function useAgent(url, options = {}) {
const parsedUrl = new URL(url);
const isHttps = parsedUrl.protocol === "https:";
// Create a basic agent based on the protocol
const Agent = isHttps ? https.Agent : http.Agent;
// Return a new agent with the provided options
return new Agent({
keepAlive: options.keepAlive,
timeout: options.timeout,
keepAliveMsecs: options.keepAliveMsecs,
maxSockets: options.maxSockets,
maxFreeSockets: options.maxFreeSockets,
maxCachedSessions: options.maxCachedSessions,
});
}
module.exports = {
useAgent,
};
+85 -3
View File
@@ -1,6 +1,8 @@
/* oxlint-disable no-restricted-imports, react/rules-of-hooks */
import { promises as dns } from "node:dns";
import type http from "node:http";
import type https from "node:https";
import * as net from "node:net";
import nodeFetch, {
Headers,
type RequestInit,
@@ -8,11 +10,15 @@ import nodeFetch, {
} from "node-fetch";
import { getProxyForUrl } from "proxy-from-env";
import tunnelAgent, { type TunnelAgent } from "tunnel-agent";
import { useAgent as useFilteringAgent } from "request-filtering-agent";
import env from "@server/env";
import { InternalError } from "@server/errors";
import Logger from "@server/logging/Logger";
import { capitalize } from "lodash";
import {
type RequestFilteringAgentOptions,
useAgent as useFilteringAgent,
validateIPAddress,
} from "./requestFilteringAgent";
interface UrlWithTunnel extends URL {
tunnelMethod?: string;
@@ -44,6 +50,58 @@ export const outlineUserAgent = `Outline-${
export const chromeUserAgent =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36";
/**
* Resolves the URL's hostname and validates every returned address against the
* filtering rules. Used as a pre-flight check when a proxy is configured,
* since both proxy code paths in buildAgent() bypass the per-connection DNS
* hook in the filtering agent.
*
* @param url The target URL to validate.
* @param options Allow/deny rules to apply.
* @param signal Optional abort signal — if it fires (e.g. fetch timeout),
* the validation rejects with an AbortError so the configured timeout
* applies to slow DNS resolution as well.
* @throws An error if any resolved address is disallowed.
*/
const validateTargetURL = async (
url: URL,
options: RequestFilteringAgentOptions,
signal?: AbortSignal | null
): Promise<void> => {
const host = url.hostname;
const lookup =
net.isIP(host) !== 0
? Promise.resolve([{ address: host, family: net.isIP(host) }])
: dns.lookup(host, { all: true });
const addresses = signal
? await Promise.race([
lookup,
new Promise<never>((_, reject) => {
if (signal.aborted) {
reject(Object.assign(new Error("aborted"), { name: "AbortError" }));
return;
}
signal.addEventListener(
"abort",
() =>
reject(
Object.assign(new Error("aborted"), { name: "AbortError" })
),
{ once: true }
);
}),
])
: await lookup;
for (const { address, family } of addresses) {
const err = validateIPAddress({ address, family, host }, options);
if (err) {
throw err;
}
}
};
/**
* Wrapper around fetch that uses the request-filtering-agent in cloud hosted
* environments to filter malicious requests, and the fetch-with-proxy library
@@ -78,6 +136,28 @@ export default async function fetch(
const signal = abortController?.signal || rest.signal;
try {
// When a proxy is configured, the request-filtering-agent's per-connection
// DNS hook does not see the target host (the tunnel-agent path bypasses it
// entirely, and the http-over-http path only filters the proxy's IP).
// Pre-resolve and validate the target ourselves so SSRF protection still
// applies to user-supplied URLs.
//
// We resolve DNS locally, but the proxy performs its own DNS resolution
// when forwarding the request. A determined attacker controlling DNS
// could return a public IP here and a private IP to the proxy. Closing
// that gap would require the proxy itself to enforce egress rules.
const parsedURL = new URL(url);
if (getProxyForUrl(parsedURL.href)) {
await validateTargetURL(
parsedURL,
{
allowPrivateIPAddress,
allowIPAddressList: env.ALLOWED_PRIVATE_IP_ADDRESSES ?? [],
},
signal
);
}
const response = await nodeFetch(url, {
...rest,
headers: {
@@ -195,7 +275,7 @@ function buildAgent(
const filteringOptions = {
...agentOptions,
allowPrivateIPAddress: options.allowPrivateIPAddress,
allowIPAddressList: env.ALLOWED_PRIVATE_IP_ADDRESSES,
allowIPAddressList: env.ALLOWED_PRIVATE_IP_ADDRESSES ?? [],
};
if (proxyURL) {
@@ -221,7 +301,9 @@ function buildAgent(
}
agent = useFilteringAgent(proxyURL.toString(), filteringOptions);
} else {
// Note request filtering agent does not support https tunneling via a proxy
// tunnel-agent bypasses the filtering agent's per-connection DNS hook,
// so SSRF protection for this branch comes from the validateTargetURL
// pre-flight in fetch() above.
agent =
buildTunnel(parsedProxyURL, agentOptions) ||
useFilteringAgent(parsedURL.toString(), filteringOptions);
@@ -0,0 +1,19 @@
Copyright (c) 2019 azu
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+305
View File
@@ -0,0 +1,305 @@
/* oxlint-disable no-restricted-imports */
// Vendored from request-filtering-agent v3.2.0 (MIT, by azu).
// Source: https://github.com/azu/request-filtering-agent/blob/cc0f9fcb9e700cd4246db2ea36245439eede4096/src/request-filtering-agent.ts
// License: see ./LICENSE.md
//
// Vendored to:
// 1. Expose validateIPAddress so the proxy pre-flight check in
// server/utils/fetch.ts can reuse the exact same SSRF rules without
// duplicating them.
// 2. Avoid the upstream package's pure-ESM publish.
//
// When upgrading, diff against upstream and port any changes here.
import * as http from "node:http";
import * as https from "node:https";
import * as net from "node:net";
import * as dns from "node:dns";
import type { TcpNetConnectOpts } from "node:net";
import type { Duplex } from "node:stream";
import ipaddr from "ipaddr.js";
import Logger from "@server/logging/Logger";
export interface RequestFilteringAgentOptions {
/**
* Allow connections to private IP addresses (RFC1918, loopback, link-local,
* etc). Defaults to false.
*/
allowPrivateIPAddress?: boolean;
/**
* Allow connections to the unspecified meta address (0.0.0.0 / ::).
* Defaults to false.
*/
allowMetaIPAddress?: boolean;
/**
* Explicit allow list of IPs or CIDR ranges. Takes precedence over the
* private/meta/deny checks.
*/
allowIPAddressList?: string[];
/**
* Explicit deny list of IPs or CIDR ranges.
*/
denyIPAddressList?: string[];
}
export const DefaultRequestFilteringAgentOptions: Required<RequestFilteringAgentOptions> =
{
allowPrivateIPAddress: false,
allowMetaIPAddress: false,
allowIPAddressList: [],
denyIPAddressList: [],
};
const matchIPAddress = ({
targetAddress,
ipAddressList,
listName,
}: {
targetAddress: {
raw: string;
parsed: ipaddr.IPv4 | ipaddr.IPv6;
};
ipAddressList: string[];
listName: string;
}): boolean => {
for (const ipOrCIDR of ipAddressList) {
if (net.isIP(ipOrCIDR) !== 0) {
if (ipOrCIDR === targetAddress.raw) {
return true;
}
} else {
try {
const cidr = ipaddr.parseCIDR(ipOrCIDR);
if (targetAddress.parsed.match(cidr)) {
return true;
}
} catch (e) {
Logger.warn(
`[request-filtering-agent] Invalid CIDR in ${listName}: ${ipOrCIDR}`,
{ error: e }
);
}
}
}
return false;
};
/**
* Validate a resolved IP address against the configured filtering rules.
*
* @param input The resolved address (and optional host/family for richer error messages).
* @param options Allow/deny rules to apply (undefined values fall back to defaults).
* @returns An Error if the address is disallowed, otherwise undefined.
*/
export const validateIPAddress = (
{
address,
host,
family,
}: { address: string; host?: string; family?: string | number },
options: RequestFilteringAgentOptions = {}
): undefined | Error => {
if (net.isIP(address) === 0) {
return;
}
const resolved: Required<RequestFilteringAgentOptions> = {
...DefaultRequestFilteringAgentOptions,
...options,
};
try {
const parsedAddr = ipaddr.parse(address);
if (resolved.allowIPAddressList.length > 0) {
if (
matchIPAddress({
targetAddress: { raw: address, parsed: parsedAddr },
ipAddressList: resolved.allowIPAddressList,
listName: "allowIPAddressList",
})
) {
return;
}
}
const range = parsedAddr.range();
if (!resolved.allowMetaIPAddress) {
if (range === "unspecified") {
return new Error(
`DNS lookup ${address}(family:${family}, host:${host}) is not allowed. Because, It is meta IP address.`
);
}
}
if (!resolved.allowPrivateIPAddress && range !== "unicast") {
return new Error(
`DNS lookup ${address}(family:${family}, host:${host}) is not allowed. Because, It is private IP address.`
);
}
if (resolved.denyIPAddressList.length > 0) {
if (
matchIPAddress({
targetAddress: { raw: address, parsed: parsedAddr },
ipAddressList: resolved.denyIPAddressList,
listName: "denyIPAddressList",
})
) {
return new Error(
`DNS lookup ${address}(family:${family}, host:${host}) is not allowed. Because It is defined in denyIPAddressList.`
);
}
}
} catch (error) {
return error as Error;
}
return;
};
type LookupOneCallback = (
err: NodeJS.ErrnoException | null,
address?: string,
family?: number
) => void;
type LookupAllCallback = (
err: NodeJS.ErrnoException | null,
addresses?: dns.LookupAddress[]
) => void;
type LookupCallback = LookupOneCallback | LookupAllCallback;
const makeLookup =
(
createConnectionOptions: TcpNetConnectOpts,
requestFilterOptions: Required<RequestFilteringAgentOptions>
): Required<net.TcpSocketConnectOpts>["lookup"] =>
// @ts-expect-error - @types/node has a poor definition of this callback
(hostname, options, cb: LookupCallback) => {
const lookup = createConnectionOptions.lookup || dns.lookup;
let lookupCb: LookupCallback;
if (options.all) {
lookupCb = ((err, addresses) => {
if (err) {
cb(err);
return;
}
for (const { address, family } of addresses!) {
const validationError = validateIPAddress(
{ address, family, host: hostname },
requestFilterOptions
);
if (validationError) {
cb(validationError);
return;
}
}
(cb as LookupAllCallback)(null, addresses);
}) as LookupAllCallback;
} else {
lookupCb = ((err, address, family) => {
if (err) {
cb(err);
return;
}
const validationError = validateIPAddress(
{ address: address!, family: family!, host: hostname },
requestFilterOptions
);
if (validationError) {
cb(validationError);
return;
}
(cb as LookupOneCallback)(null, address!, family!);
}) as LookupOneCallback;
}
// @ts-expect-error - @types/node has a poor definition of this callback
lookup(hostname, options, lookupCb);
};
const resolveOptions = (
options?: RequestFilteringAgentOptions
): Required<RequestFilteringAgentOptions> => ({
...DefaultRequestFilteringAgentOptions,
...options,
});
/**
* An http.Agent that rejects connections to disallowed IP addresses.
*/
export class RequestFilteringHttpAgent extends http.Agent {
private requestFilterOptions: Required<RequestFilteringAgentOptions>;
constructor(options?: http.AgentOptions & RequestFilteringAgentOptions) {
super(options);
this.requestFilterOptions = resolveOptions(options);
}
createConnection(
options: TcpNetConnectOpts,
connectionListener?: (error: Error | null, socket: Duplex) => void
) {
const { host } = options;
if (host !== undefined) {
const validationError = validateIPAddress(
{ address: host },
this.requestFilterOptions
);
if (validationError) {
throw validationError;
}
}
return super.createConnection(
{ ...options, lookup: makeLookup(options, this.requestFilterOptions) },
connectionListener
);
}
}
/**
* An https.Agent that rejects connections to disallowed IP addresses.
*/
export class RequestFilteringHttpsAgent extends https.Agent {
private requestFilterOptions: Required<RequestFilteringAgentOptions>;
constructor(options?: https.AgentOptions & RequestFilteringAgentOptions) {
super(options);
this.requestFilterOptions = resolveOptions(options);
}
createConnection(
options: TcpNetConnectOpts,
connectionListener?: (error: Error | null, socket: Duplex) => void
) {
const { host } = options;
if (host !== undefined) {
const validationError = validateIPAddress(
{ address: host },
this.requestFilterOptions
);
if (validationError) {
throw validationError;
}
}
return super.createConnection(
{ ...options, lookup: makeLookup(options, this.requestFilterOptions) },
connectionListener
);
}
}
export const globalHttpAgent = new RequestFilteringHttpAgent();
export const globalHttpsAgent = new RequestFilteringHttpsAgent();
/**
* Get a filtering agent for the given URL. Returns a process-global agent when
* no options are provided, otherwise constructs a fresh one.
*
* @param url The target URL — used only to pick http vs https.
* @param options Optional filtering and underlying agent options.
* @returns A filtering http or https agent.
*/
export const useAgent = (
url: string,
options?: https.AgentOptions & RequestFilteringAgentOptions
) => {
if (!options) {
return url.startsWith("https") ? globalHttpsAgent : globalHttpAgent;
}
return url.startsWith("https")
? new RequestFilteringHttpsAgent(options)
: new RequestFilteringHttpAgent(options);
};
+3 -100
View File
@@ -812,14 +812,7 @@ __metadata:
languageName: node
linkType: hard
"@babel/compat-data@npm:^7.28.6, @babel/compat-data@npm:^7.29.0":
version: 7.29.0
resolution: "@babel/compat-data@npm:7.29.0"
checksum: 10c0/08f348554989d23aa801bf1405aa34b15e841c0d52d79da7e524285c77a5f9d298e70e11d91cc578d8e2c9542efc586d50c5f5cf8e1915b254a9dcf786913a94
languageName: node
linkType: hard
"@babel/compat-data@npm:^7.29.3":
"@babel/compat-data@npm:^7.28.6, @babel/compat-data@npm:^7.29.3":
version: 7.29.3
resolution: "@babel/compat-data@npm:7.29.3"
checksum: 10c0/81bddd53ce1b1395576fbb7cb739631a976f6b421cd260e6cf2715a9691b9a0ec12ca5c4e1bb88088e60dc87875f6e4ef7fa8674f1dc96ae1bd7c357416605a7
@@ -2069,87 +2062,7 @@ __metadata:
languageName: node
linkType: hard
"@babel/preset-env@npm:^7.11.0":
version: 7.29.2
resolution: "@babel/preset-env@npm:7.29.2"
dependencies:
"@babel/compat-data": "npm:^7.29.0"
"@babel/helper-compilation-targets": "npm:^7.28.6"
"@babel/helper-plugin-utils": "npm:^7.28.6"
"@babel/helper-validator-option": "npm:^7.27.1"
"@babel/plugin-bugfix-firefox-class-in-computed-class-key": "npm:^7.28.5"
"@babel/plugin-bugfix-safari-class-field-initializer-scope": "npm:^7.27.1"
"@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "npm:^7.27.1"
"@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "npm:^7.27.1"
"@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly": "npm:^7.28.6"
"@babel/plugin-proposal-private-property-in-object": "npm:7.21.0-placeholder-for-preset-env.2"
"@babel/plugin-syntax-import-assertions": "npm:^7.28.6"
"@babel/plugin-syntax-import-attributes": "npm:^7.28.6"
"@babel/plugin-syntax-unicode-sets-regex": "npm:^7.18.6"
"@babel/plugin-transform-arrow-functions": "npm:^7.27.1"
"@babel/plugin-transform-async-generator-functions": "npm:^7.29.0"
"@babel/plugin-transform-async-to-generator": "npm:^7.28.6"
"@babel/plugin-transform-block-scoped-functions": "npm:^7.27.1"
"@babel/plugin-transform-block-scoping": "npm:^7.28.6"
"@babel/plugin-transform-class-properties": "npm:^7.28.6"
"@babel/plugin-transform-class-static-block": "npm:^7.28.6"
"@babel/plugin-transform-classes": "npm:^7.28.6"
"@babel/plugin-transform-computed-properties": "npm:^7.28.6"
"@babel/plugin-transform-destructuring": "npm:^7.28.5"
"@babel/plugin-transform-dotall-regex": "npm:^7.28.6"
"@babel/plugin-transform-duplicate-keys": "npm:^7.27.1"
"@babel/plugin-transform-duplicate-named-capturing-groups-regex": "npm:^7.29.0"
"@babel/plugin-transform-dynamic-import": "npm:^7.27.1"
"@babel/plugin-transform-explicit-resource-management": "npm:^7.28.6"
"@babel/plugin-transform-exponentiation-operator": "npm:^7.28.6"
"@babel/plugin-transform-export-namespace-from": "npm:^7.27.1"
"@babel/plugin-transform-for-of": "npm:^7.27.1"
"@babel/plugin-transform-function-name": "npm:^7.27.1"
"@babel/plugin-transform-json-strings": "npm:^7.28.6"
"@babel/plugin-transform-literals": "npm:^7.27.1"
"@babel/plugin-transform-logical-assignment-operators": "npm:^7.28.6"
"@babel/plugin-transform-member-expression-literals": "npm:^7.27.1"
"@babel/plugin-transform-modules-amd": "npm:^7.27.1"
"@babel/plugin-transform-modules-commonjs": "npm:^7.28.6"
"@babel/plugin-transform-modules-systemjs": "npm:^7.29.0"
"@babel/plugin-transform-modules-umd": "npm:^7.27.1"
"@babel/plugin-transform-named-capturing-groups-regex": "npm:^7.29.0"
"@babel/plugin-transform-new-target": "npm:^7.27.1"
"@babel/plugin-transform-nullish-coalescing-operator": "npm:^7.28.6"
"@babel/plugin-transform-numeric-separator": "npm:^7.28.6"
"@babel/plugin-transform-object-rest-spread": "npm:^7.28.6"
"@babel/plugin-transform-object-super": "npm:^7.27.1"
"@babel/plugin-transform-optional-catch-binding": "npm:^7.28.6"
"@babel/plugin-transform-optional-chaining": "npm:^7.28.6"
"@babel/plugin-transform-parameters": "npm:^7.27.7"
"@babel/plugin-transform-private-methods": "npm:^7.28.6"
"@babel/plugin-transform-private-property-in-object": "npm:^7.28.6"
"@babel/plugin-transform-property-literals": "npm:^7.27.1"
"@babel/plugin-transform-regenerator": "npm:^7.29.0"
"@babel/plugin-transform-regexp-modifiers": "npm:^7.28.6"
"@babel/plugin-transform-reserved-words": "npm:^7.27.1"
"@babel/plugin-transform-shorthand-properties": "npm:^7.27.1"
"@babel/plugin-transform-spread": "npm:^7.28.6"
"@babel/plugin-transform-sticky-regex": "npm:^7.27.1"
"@babel/plugin-transform-template-literals": "npm:^7.27.1"
"@babel/plugin-transform-typeof-symbol": "npm:^7.27.1"
"@babel/plugin-transform-unicode-escapes": "npm:^7.27.1"
"@babel/plugin-transform-unicode-property-regex": "npm:^7.28.6"
"@babel/plugin-transform-unicode-regex": "npm:^7.27.1"
"@babel/plugin-transform-unicode-sets-regex": "npm:^7.28.6"
"@babel/preset-modules": "npm:0.1.6-no-external-plugins"
babel-plugin-polyfill-corejs2: "npm:^0.4.15"
babel-plugin-polyfill-corejs3: "npm:^0.14.0"
babel-plugin-polyfill-regenerator: "npm:^0.6.6"
core-js-compat: "npm:^3.48.0"
semver: "npm:^6.3.1"
peerDependencies:
"@babel/core": ^7.0.0-0
checksum: 10c0/d49cb005f2dbc3f2293ab6d80ee8f1380e6215af5518fe26b087c8961c1ea8ebaa554dfce589abe1fbebac25ad7c2515d943dec3859ea2d4981a3f8f4711c580
languageName: node
linkType: hard
"@babel/preset-env@npm:^7.29.3":
"@babel/preset-env@npm:^7.11.0, @babel/preset-env@npm:^7.29.3":
version: 7.29.3
resolution: "@babel/preset-env@npm:7.29.3"
dependencies:
@@ -13589,7 +13502,7 @@ __metadata:
languageName: node
linkType: hard
"ipaddr.js@npm:^2.1.0, ipaddr.js@npm:^2.3.0":
"ipaddr.js@npm:^2.3.0":
version: 2.3.0
resolution: "ipaddr.js@npm:2.3.0"
checksum: 10c0/084bab99e2f6875d7a62adc3325e1c64b038a12c9521e35fb967b5e263a8b3afb1b8884dd77c276092331f5d63298b767491e10997ef147c62da01b143780bbd
@@ -17101,7 +17014,6 @@ __metadata:
redlock: "npm:^5.0.0-beta2"
reflect-metadata: "npm:^0.2.2"
refractor: "npm:^3.6.0"
request-filtering-agent: "npm:^3.2.0"
resolve-path: "npm:^1.4.0"
rimraf: "npm:^6.1.3"
rollup-plugin-webpack-stats: "npm:2.1.11"
@@ -19094,15 +19006,6 @@ __metadata:
languageName: node
linkType: hard
"request-filtering-agent@npm:^3.2.0":
version: 3.2.0
resolution: "request-filtering-agent@npm:3.2.0"
dependencies:
ipaddr.js: "npm:^2.1.0"
checksum: 10c0/f32128c067ca4b700c7e02b41fdd2db430707ec285a2fb44c347f0d9f42326aeb9c32c1986d20c6354630d1368c2fdb1933928a657cec57d637dd895b08cca07
languageName: node
linkType: hard
"require-directory@npm:^2.1.1":
version: 2.1.1
resolution: "require-directory@npm:2.1.1"