From 18dd0b011394c89f7d33739c322e8ad0026e2cf1 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 12 Sep 2026 02:50:47 -0700 Subject: [PATCH 1/2] improvement(network): consolidate validated HTTP transports --- .../self-hosting/environment-variables.mdx | 2 + .../docs/platform/self-hosting/networking.mdx | 35 +- .../docs/platform/self-hosting/security.mdx | 8 +- .../network/environment-proxy.server.test.ts | 390 ++++++++++++++++++ .../environment-proxy-runtime.fixture.ts | 38 ++ apps/sim/lib/core/network/gateway.server.ts | 47 +++ apps/sim/lib/core/network/routing.ts | 15 + apps/sim/lib/core/network/transport.server.ts | 252 +++++++++++ .../security/egress-end-to-end.server.test.ts | 37 +- .../lib/core/security/egress/profiles.test.ts | 108 ++++- apps/sim/lib/core/security/egress/profiles.ts | 60 ++- .../guarded-request-fetch.server.test.ts | 61 ++- .../core/security/input-validation.server.ts | 344 ++++++++++----- .../core/security/pinned-fetch.server.test.ts | 4 +- ...ecure-fetch-request-framing.server.test.ts | 12 + .../secure-fetch-response-cap.server.test.ts | 276 ++++++++++++- apps/sim/lib/core/utils/fetch-deadline.ts | 2 +- apps/sim/lib/mcp/pinned-fetch.ts | 6 +- apps/sim/lib/webhooks/providers/airtable.ts | 13 +- apps/sim/lib/webhooks/providers/ashby.test.ts | 4 +- apps/sim/lib/webhooks/providers/ashby.ts | 9 +- apps/sim/lib/webhooks/providers/attio.ts | 9 +- .../lib/webhooks/providers/bitbucket.test.ts | 3 + apps/sim/lib/webhooks/providers/bitbucket.ts | 7 +- apps/sim/lib/webhooks/providers/calendly.ts | 9 +- .../lib/webhooks/providers/clickup.test.ts | 4 + apps/sim/lib/webhooks/providers/clickup.ts | 9 +- apps/sim/lib/webhooks/providers/fathom.ts | 9 +- apps/sim/lib/webhooks/providers/grain.test.ts | 3 + apps/sim/lib/webhooks/providers/grain.ts | 28 +- .../lib/webhooks/providers/granola.test.ts | 3 + apps/sim/lib/webhooks/providers/granola.ts | 9 +- .../lib/webhooks/providers/instantly.test.ts | 4 +- apps/sim/lib/webhooks/providers/instantly.ts | 9 +- .../lib/webhooks/providers/jotform.test.ts | 10 +- apps/sim/lib/webhooks/providers/jotform.ts | 11 +- apps/sim/lib/webhooks/providers/lemlist.ts | 11 +- .../sim/lib/webhooks/providers/linear.test.ts | 3 + apps/sim/lib/webhooks/providers/linear.ts | 9 +- apps/sim/lib/webhooks/providers/linq.ts | 9 +- .../lib/webhooks/providers/microsoft-teams.ts | 21 +- apps/sim/lib/webhooks/providers/monday.ts | 9 +- apps/sim/lib/webhooks/providers/pagerduty.ts | 13 +- .../sim/lib/webhooks/providers/resend.test.ts | 3 + apps/sim/lib/webhooks/providers/resend.ts | 9 +- .../lib/webhooks/providers/revenuecat.test.ts | 4 +- apps/sim/lib/webhooks/providers/revenuecat.ts | 9 +- .../sim/lib/webhooks/providers/rootly.test.ts | 4 +- apps/sim/lib/webhooks/providers/rootly.ts | 24 +- apps/sim/lib/webhooks/providers/slack.test.ts | 3 + apps/sim/lib/webhooks/providers/slack.ts | 11 +- apps/sim/lib/webhooks/providers/telegram.ts | 9 +- apps/sim/lib/webhooks/providers/typeform.ts | 9 +- .../sim/lib/webhooks/providers/vercel.test.ts | 4 +- apps/sim/lib/webhooks/providers/vercel.ts | 9 +- apps/sim/lib/webhooks/providers/webflow.ts | 9 +- .../lib/webhooks/providers/zendesk.test.ts | 4 + apps/sim/lib/webhooks/providers/zendesk.ts | 13 +- .../lib/webhooks/providers/zoho-desk.test.ts | 3 + apps/sim/lib/webhooks/providers/zoho-desk.ts | 9 +- .../src/mocks/input-validation.mock.ts | 4 + scripts/check-egress-boundary.ts | 5 + ...check-tool-registry-boundary.baseline.json | 14 +- 63 files changed, 1825 insertions(+), 259 deletions(-) create mode 100644 apps/sim/lib/core/network/environment-proxy.server.test.ts create mode 100644 apps/sim/lib/core/network/fixtures/environment-proxy-runtime.fixture.ts create mode 100644 apps/sim/lib/core/network/gateway.server.ts create mode 100644 apps/sim/lib/core/network/routing.ts create mode 100644 apps/sim/lib/core/network/transport.server.ts diff --git a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx index f97b6f246df..819b11fd357 100644 --- a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx +++ b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx @@ -119,6 +119,8 @@ import { Callout } from 'fumadocs-ui/components/callout' | `LITELLM_BASE_URL` | LiteLLM proxy base URL | | `LITELLM_API_KEY` | Optional bearer token for LiteLLM | +On self-hosted deployments, the hosts configured in `OLLAMA_URL`, `VLLM_BASE_URL`, `LITELLM_BASE_URL`, `AZURE_OPENAI_ENDPOINT`, `AZURE_ANTHROPIC_ENDPOINT`, and `OCR_AZURE_ENDPOINT` can be reached on private networks without adding them to `EGRESS_ALLOWED_HOSTS`. Existing allowlists still apply. See [model host permissions](/platform/self-hosting/security#the-ssrf-boundary) for which requests this permits. + ## Login Providers Google, GitHub, and Microsoft sign-in, their callback URLs, and the `DISABLE_*_AUTH` switches are documented in [Authentication](/platform/self-hosting/authentication#social-login). diff --git a/apps/docs/content/docs/platform/self-hosting/networking.mdx b/apps/docs/content/docs/platform/self-hosting/networking.mdx index 4502ea88979..d81fd52e780 100644 --- a/apps/docs/content/docs/platform/self-hosting/networking.mdx +++ b/apps/docs/content/docs/platform/self-hosting/networking.mdx @@ -302,33 +302,28 @@ A proxy body limit of 250 MB accommodates all three defaults. If you lower the a ## Outbound connectivity -The app makes outbound calls to model providers, integration APIs, your email provider, object storage, and your telemetry backend. Whether `HTTP_PROXY` / `HTTPS_PROXY` apply depends on which of those paths a call takes — there is no single answer, and no global setting that covers all of them. +The shared HTTP transport honors `HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY` on both Bun and Node. This covers guarded HTTP requests, pinned model-provider clients, and webhook providers using Sim's fetch adapter. It does not configure a global dispatcher or route non-HTTP protocols. -The server runs on Bun, and Bun's native `fetch` honors `$HTTP_PROXY`, `$HTTPS_PROXY`, and `$NO_PROXY`. Sim installs no global dispatcher, so every call that goes through the default `fetch` is proxied. The rest either build their own HTTP agent or speak a non-HTTP protocol. +Use an `http://` or `https://` proxy URL. For proxy authentication, both the username and password must be nonempty; partial credentials are refused. Lowercase variables take precedence over uppercase ones. `HTTP_PROXY` also applies to HTTPS when `HTTPS_PROXY` is unset. `NO_PROXY` matches the original destination hostname, with optional ports; it accepts exact names, domain suffixes, and `*`, but not CIDR ranges. - - This depends on the runtime. The published images run Bun. If you build the standalone output and run it under Node instead, Node ignores these variables unless started with `NODE_USE_ENV_PROXY=1` (Node 22.21+ / 24.5+), and the `fetch`-based "Yes" rows stop being proxied — model providers, Resend, Gmail sending, the desktop update feed, and the telemetry relay. The Azure Blob, GCS and Azure Communication Services rows still hold: those SDKs read the proxy variables through their own agents rather than through `fetch`. - +The transport resolves and validates destinations locally, then sends the approved numeric IP address in a CONNECT request. The original hostname remains in the HTTP Host header and TLS certificate verification. Both HTTP and HTTPS destinations require a proxy that accepts numeric CONNECT authorities. Proxy tunnels use HTTP/1.1; direct connections, including `NO_PROXY` matches, retain each client's HTTP/2 configuration. A proxy that requires hostname-only CONNECT or resolves names unavailable to the application cannot be used by this transport. Use NAT or transparent egress for those environments. Proxy failures never fall back to a direct connection. + +An environment proxy is trusted deployment configuration, so its own address can be private or loopback. Its DNS answers are checked and pinned before connecting, and cloud metadata addresses are always refused. This trust does not change destination permissions: content URLs remain public-only, while configured private services still need the applicable [egress allowance](/platform/self-hosting/security#the-ssrf-boundary). | Outbound path | Honors `HTTP_PROXY` / `HTTPS_PROXY` | |---|---| -| Model providers reached over the default `fetch` — Anthropic, OpenAI, Google/Gemini, Vertex, Groq, Cerebras, xAI, Mistral, DeepSeek, OpenRouter, Together, Fireworks, Ollama, LiteLLM, and the other OpenAI-compatible providers | Yes | -| Email via Resend, Azure Communication Services, and Gmail sending | Yes | -| The desktop update feed's calls to GitHub | Yes | -| Object storage — Azure Blob and GCS | Yes — their SDK pipelines read the proxy variables | -| Everything through the SSRF guard — the HTTP block, tools, connectors, outbound webhooks, content fetches, MCP servers | No | -| Azure OpenAI, Azure Anthropic, vLLM | Only when the endpoint comes from `AZURE_OPENAI_ENDPOINT`, `AZURE_ANTHROPIC_ENDPOINT`, or `VLLM_BASE_URL`. An endpoint typed into the block is validated and pinned to its resolved IP, which bypasses the proxy | -| Amazon Bedrock, and object storage on S3 | No — the AWS SDK uses its own request handler | -| Email via SMTP | No — Nodemailer opens a raw TCP connection | -| Email via Amazon SES | No — the AWS SDK transport, over HTTPS | +| Shared guarded HTTP transport: tools, connectors, outbound webhooks, content fetches, MCP HTTP connections, and model-provider clients using Sim's fetch adapter | Yes, on Bun and Node, subject to the CONNECT requirements above | +| Amazon Bedrock and AWS integration clients | No, these clients use their own AWS SDK transport | +| Object storage on Azure Blob and GCS; Azure Communication Services email | Yes, through the SDK's own proxy support | +| Native `fetch` paths and SDKs that use it, including Resend email, the desktop update feed, and the `/api/telemetry` relay | Yes on Bun; Node requires [`NODE_USE_ENV_PROXY=1`](https://nodejs.org/api/cli.html#node_use_env_proxy1) (Node 22.21+ / 24.0+) | +| Application object storage on S3; application email via Amazon SES | No, these clients use their own AWS SDK transport | +| Email via SMTP, Postgres, and Redis | No, these use raw TCP | | OTLP export from the server SDK | No | -| The `/api/telemetry` relay that forwards browser events | Yes — it uses the default `fetch` | -| Postgres and Redis | No — raw TCP | -The practical consequence: a mandatory-egress-proxy environment can route most LLM traffic, Resend mail, and Azure/GCS storage through the proxy, but guarded integration calls, S3, Bedrock, SMTP, telemetry, and datastore traffic still need a transparent proxy or NAT-based egress. +An explicit HTTP-block `proxyUrl` takes precedence over environment proxy settings. - Set `NO_PROXY` for every destination that is not on the public internet, not just model endpoints. The app reaches the realtime server (`SOCKET_SERVER_URL`), the Presidio PII service (`PII_URL`), and itself (`INTERNAL_API_BASE_URL`) over the same default `fetch`, alongside self-hosted Ollama, LiteLLM, and vLLM — so a proxy that cannot reach your internal network breaks live updates and PII redaction, not only inference. + Set `NO_PROXY` for every destination that is not on the public internet, not just model endpoints. The app reaches the realtime server (`SOCKET_SERVER_URL`), the Presidio PII service (`PII_URL`), and itself (`INTERNAL_API_BASE_URL`) over HTTP, alongside self-hosted Ollama, LiteLLM, and vLLM — so a proxy that cannot reach your internal network breaks live updates and PII redaction, not only inference. Set it in the application environment, not your shell — under `app.env` on Helm, or the service's `environment:` on Compose. On Helm the suffixes alone are not enough: the chart wires `SOCKET_SERVER_URL`, `PII_URL`, and `OLLAMA_URL` to bare Service names, which no domain suffix matches. Add those names too — `helm template` prints the rendered ones, and the prefix is the release name unless it already contains `sim`, in which case it is the release name alone: @@ -349,9 +344,9 @@ The practical consequence: a mandatory-egress-proxy environment can route most L ``` -### The per-request escape hatch +### Per-request proxies -The HTTP block's `proxyUrl` is honored per request by the SSRF guard, which builds a proxy agent for that call instead of pinning the target IP. It applies only to that path — connectors, content fetches, and MCP calls take a different guarded transport with no per-request proxy option. +The HTTP block's `proxyUrl` builds a proxy agent for that request. It applies only to the HTTP block; connectors, content fetches, and MCP calls use deployment proxy settings. This per-request path validates the destination locally but lets the proxy resolve the destination hostname, unlike the numeric CONNECT used by environment proxies. `proxyUrl` must be an `http://` URL **and** must resolve to a public address. The guard validates the proxy host under a dedicated `proxy` profile that has no operator allowlist, so `EGRESS_ALLOWED_HOSTS` and `EGRESS_ALLOWED_IP_RANGES` do not reach it. A corporate proxy on an RFC 1918 address is refused even when that range is allowlisted for everything else. See [Security](/platform/self-hosting/security#the-ssrf-boundary). diff --git a/apps/docs/content/docs/platform/self-hosting/security.mdx b/apps/docs/content/docs/platform/self-hosting/security.mdx index 1bd8b0fb7a4..189f9a26843 100644 --- a/apps/docs/content/docs/platform/self-hosting/security.mdx +++ b/apps/docs/content/docs/platform/self-hosting/security.mdx @@ -172,7 +172,7 @@ Resource ceilings for the in-process path: ## The SSRF boundary -By default Sim blocks outbound requests to private, reserved, and loopback addresses. This stops a workflow from being used to scan your internal network. Two things soften it on a self-hosted deployment: the provenances marked **Yes** below reach whatever you allowlist, and a configured endpoint, self-hosted service, or request target written as `localhost` or a loopback literal is reachable without any allowlist at all — a local Ollama or Jupyter is the ordinary case. That second carve-out stops short in two places: it does not lift the blocked-port list, and it does not extend to a database, cache, or mail connector on `localhost` — loopback is where Sim's own database and Redis listen, so reaching them has to be asked for. Neither softening applies on Sim Cloud. Every outbound request is classified by where its URL came from: +By default Sim blocks outbound requests to private, reserved, and loopback addresses. The allowances below let a self-hosted deployment reach services its operator has configured. The provenances marked **Yes** reach whatever you allowlist. A configured endpoint, self-hosted service, or request target written as `localhost` or a loopback literal is also reachable without an allowlist. That loopback allowance does not lift the blocked-port list or apply to database, cache, and mail connectors. These allowances do not apply on Sim Cloud. Every outbound request is classified by where its URL came from: | Provenance | Examples | Reaches allowlisted private destinations | |---|---|---| @@ -181,9 +181,11 @@ By default Sim blocks outbound requests to private, reserved, and loopback addre | Request target | The HTTP block's URL, an A2A agent, an RSS feed, a Function block's `fetch` | Yes | | Database host | A database, cache, or mail connector's host | Yes | | Content fetch | An image URL, a file imported by URL, a link from a third-party API response | **No** | -| Proxy | The outbound HTTP proxy itself | **No** | +| Per-request proxy | The HTTP block's `proxyUrl` | **No** | -Content fetches never reach a private destination, allowlist or not — that is the class where SSRF is actually exploited. Nor does the proxy: it is the component deciding where everything else may go, so it is held to public destinations regardless of what the allowlist says. +The hosts in `OLLAMA_URL`, `VLLM_BASE_URL`, and `LITELLM_BASE_URL` are also trusted for self-hosted services. The hosts in `AZURE_OPENAI_ENDPOINT`, `AZURE_ANTHROPIC_ENDPOINT`, and `OCR_AZURE_ENDPOINT` are trusted for configured endpoints. These must be HTTP or HTTPS URLs without embedded credentials or wildcard hostnames. This trust covers the exact host or IP address across ports, without replacing your existing allowlists. It does not grant access through an HTTP block, a Function block's `fetch`, or a URL taken from content. Cloud metadata endpoints remain blocked, and model URL settings grant no private-network access on Sim Cloud. + +Content fetches never reach a private destination, allowlist or not — that is the class where SSRF is actually exploited. The HTTP block's per-request proxy must also resolve to a public address regardless of the allowlist. Deployment proxies configured through `HTTP_PROXY` or `HTTPS_PROXY` can use private or loopback addresses; their DNS answers are checked and pinned, and metadata endpoints remain blocked. They do not expand which destinations a request may reach. See [Outbound connectivity](/platform/self-hosting/networking#outbound-connectivity). Deployments frequently need to reach an internal service by name or address. Name the destinations: diff --git a/apps/sim/lib/core/network/environment-proxy.server.test.ts b/apps/sim/lib/core/network/environment-proxy.server.test.ts new file mode 100644 index 00000000000..2f09c10c241 --- /dev/null +++ b/apps/sim/lib/core/network/environment-proxy.server.test.ts @@ -0,0 +1,390 @@ +/** @vitest-environment node */ +import { execFile } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { createServer as httpServer, type IncomingHttpHeaders } from 'node:http' +import { createSecureServer as http2Server } from 'node:http2' +import { createServer as httpsServer } from 'node:https' +import { type AddressInfo, connect, type Socket } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { getCACertificates, setDefaultCACertificates, type TLSSocket } from 'node:tls' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' +import { resolveHostAddresses } from '@sim/security/dns' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@sim/security/dns', { spy: true }) + +import { + createPinnedFetchWithDispatcher, + createSsrfGuardedFetchWithDispatcher, + secureFetchWithPinnedIP, +} from '@/lib/core/security/input-validation.server' + +const directory = mkdtempSync(join(tmpdir(), 'environment-proxy-tls-')) +const certPath = join(directory, 'cert.pem') +const keyPath = join(directory, 'key.pem') +const sockets = new Set() +const connections: Array<{ authority?: string; authorization?: string; proxySni?: string }> = [] +const requests: Array<{ headers: IncomingHttpHeaders; sni?: string; protocol?: string }> = [] +const origin = httpServer((req, res) => { + requests.push({ headers: req.headers }) + if (req.url === '/wait') return + if (req.url === '/redirect') { + res.writeHead(302, { location: '/done' }) + res.end() + } else { + res.end('reached') + } +}) +const secureOrigin = http2Server({ allowHTTP1: true }, (req, res) => { + requests.push({ + headers: req.headers, + sni: (req.socket as TLSSocket).servername || undefined, + protocol: req.httpVersion, + }) + res.end('tls reached') +}) +const proxy = httpServer() +const secureProxy = httpsServer() +let rejectConnections = false +for (const server of [proxy, secureProxy]) { + server.on('connect', (req, socket, head) => { + socket.on('error', () => {}) + connections.push({ + authority: req.url, + authorization: req.headers['proxy-authorization'], + proxySni: (req.socket as TLSSocket).servername || undefined, + }) + if (rejectConnections) { + socket.end('HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n') + return + } + const target = new URL(`http://${req.url}`) + const upstream = connect(Number(target.port), target.hostname, () => { + socket.write('HTTP/1.1 200 Connection Established\r\n\r\n') + if (head.length) upstream.write(head) + socket.pipe(upstream).pipe(socket) + }) + upstream.on('error', () => socket.destroy()) + socket.on('error', () => upstream.destroy()) + socket.on('close', () => upstream.destroy()) + sockets.add(upstream) + upstream.on('close', () => sockets.delete(upstream)) + }) +} +let originPort = 0 +let securePort = 0 +let proxyPort = 0 +let secureProxyPort = 0 +const trust = getCACertificates('default') +beforeAll(async () => { + await promisify(execFile)('openssl', [ + 'req', + '-x509', + '-newkey', + 'rsa:2048', + '-sha256', + '-nodes', + '-keyout', + keyPath, + '-out', + certPath, + '-days', + '2', + '-subj', + '/CN=localhost', + '-addext', + 'subjectAltName=DNS:localhost', + '-addext', + 'extendedKeyUsage=serverAuth', + ]) + const cert = readFileSync(certPath, 'utf8') + const key = readFileSync(keyPath, 'utf8') + secureOrigin.setSecureContext({ cert, key }) + secureProxy.setSecureContext({ cert, key }) + setDefaultCACertificates([...trust, cert]) + for (const server of [origin, secureOrigin, proxy, secureProxy]) { + server.on('connection', (socket) => { + sockets.add(socket) + socket.on('close', () => sockets.delete(socket)) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + } + originPort = (origin.address() as AddressInfo).port + securePort = (secureOrigin.address() as AddressInfo).port + proxyPort = (proxy.address() as AddressInfo).port + secureProxyPort = (secureProxy.address() as AddressInfo).port +}) +afterAll(async () => { + for (const socket of sockets) socket.destroy() + await Promise.all( + [origin, secureOrigin, proxy, secureProxy].map( + (server) => new Promise((resolve) => server.close(() => resolve())) + ) + ) + setDefaultCACertificates(trust) + rmSync(directory, { recursive: true, force: true }) +}) +beforeEach(() => { + vi.clearAllMocks() + for (const name of [ + 'http_proxy', + 'https_proxy', + 'no_proxy', + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'NO_PROXY', + ]) { + vi.stubEnv(name, '') + } + rejectConnections = false + connections.length = 0 + requests.length = 0 +}) +afterEach(() => vi.unstubAllEnvs()) +const url = () => `http://localhost:${originPort}` +const options = { profile: 'selfHostedService' as const } + +/** Both APIs must select the same transport without requiring caller proxy switches. */ +describe('operator environment proxies', () => { + it.each([ + ['http:', 80], + ['https:', 443], + ])('uses exactly one default port for %s CONNECT authorities', async (protocol, port) => { + vi.stubEnv('http_proxy', `http://localhost:${proxyPort}`) + rejectConnections = true + const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) + try { + await expect(transport.fetch(`${protocol}//localhost`)).rejects.toThrow('GATEWAY_UNAVAILABLE') + expect(connections).toEqual([{ authority: `127.0.0.1:${port}` }]) + expect(requests).toHaveLength(0) + } finally { + await transport.dispatcher.destroy() + } + }) + + it('pins CONNECT numerically, preserves Host, strips proxy credentials, and follows redirects', async () => { + vi.stubEnv('http_proxy', `http://operator:synthetic@localhost:${proxyPort}`) + const transport = createSsrfGuardedFetchWithDispatcher(options) + try { + const response = await transport.fetch(`${url()}/redirect`, { + headers: { 'Proxy-Authorization': 'must-not-reach-origin' }, + }) + expect(await response.text()).toBe('reached') + expect(connections.length).toBeGreaterThan(0) + expect(connections.every(({ authority }) => authority === `127.0.0.1:${originPort}`)).toBe( + true + ) + expect(connections[0].authorization).toBe( + `Basic ${Buffer.from('operator:synthetic').toString('base64')}` + ) + expect(requests).toHaveLength(2) + expect(requests.every(({ headers }) => headers.host === `localhost:${originPort}`)).toBe(true) + expect(requests.every(({ headers }) => headers['proxy-authorization'] === undefined)).toBe( + true + ) + } finally { + await transport.dispatcher.destroy() + } + }) + + it('supports bounded fetch over HTTPS proxy with original origin and proxy TLS identities', async () => { + vi.stubEnv('https_proxy', `https://localhost:${secureProxyPort}`) + const response = await secureFetchWithPinnedIP( + `https://localhost:${securePort}`, + '127.0.0.1', + options + ) + expect(await response.text()).toBe('tls reached') + expect(connections[0]).toMatchObject({ + authority: `127.0.0.1:${securePort}`, + proxySni: 'localhost', + }) + expect(requests[0]).toMatchObject({ + headers: { host: `localhost:${securePort}` }, + sni: 'localhost', + }) + }) + + it('rejects a mismatched origin certificate without a direct fallback', async () => { + vi.stubEnv('https_proxy', `http://localhost:${proxyPort}`) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) + try { + await expect(transport.fetch(`https://127.0.0.1:${securePort}`)).rejects.toThrow() + expect(connections).toHaveLength(1) + expect(requests).toHaveLength(0) + } finally { + await transport.dispatcher.destroy() + } + }) + + it('rejects a mismatched proxy certificate before sending CONNECT', async () => { + vi.stubEnv('https_proxy', `https://127.0.0.1:${secureProxyPort}`) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) + try { + await expect(transport.fetch(`https://localhost:${securePort}`)).rejects.toThrow( + 'GATEWAY_UNAVAILABLE' + ) + expect(connections).toHaveLength(0) + expect(requests).toHaveLength(0) + } finally { + await transport.dispatcher.destroy() + } + }) + + it('uses NO_PROXY against the original hostname while still pinning direct sockets', async () => { + vi.stubEnv('http_proxy', `http://localhost:${proxyPort}`) + vi.stubEnv('no_proxy', `localhost:${originPort}`) + const transport = createSsrfGuardedFetchWithDispatcher(options) + try { + expect(await (await transport.fetch(url())).text()).toBe('reached') + expect(connections).toHaveLength(0) + expect(requests[0].headers.host).toBe(`localhost:${originPort}`) + } finally { + await transport.dispatcher.destroy() + } + }) + + it('retains HTTP/2 on a pinned NO_PROXY connection', async () => { + vi.stubEnv('https_proxy', `http://localhost:${proxyPort}`) + vi.stubEnv('no_proxy', 'localhost') + const transport = createPinnedFetchWithDispatcher('127.0.0.1', { ...options, allowH2: true }) + try { + expect(await (await transport.fetch(`https://localhost:${securePort}`)).text()).toBe( + 'tls reached' + ) + expect(connections).toHaveLength(0) + expect(requests[0]).toMatchObject({ protocol: '2.0', sni: 'localhost' }) + } finally { + await transport.dispatcher.destroy() + } + }) + + it('cancels a proxied request without opening a direct fallback', async () => { + vi.stubEnv('http_proxy', `http://localhost:${proxyPort}`) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) + const controller = new AbortController() + const received = new Promise((resolve) => origin.once('request', () => resolve())) + try { + const pending = transport.fetch(`${url()}/wait`, { signal: controller.signal }) + const rejected = expect(pending).rejects.toThrow() + await received + controller.abort() + await rejected + expect(connections).toHaveLength(1) + expect(requests).toHaveLength(1) + } finally { + await transport.dispatcher.destroy() + } + }) + + it.each(['', '*'])('refuses private content targets even with NO_PROXY=%s', async (noProxy) => { + vi.stubEnv('https_proxy', `http://localhost:${proxyPort}`) + vi.stubEnv('no_proxy', noProxy) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', { profile: 'contentFetch' }) + try { + await expect(transport.fetch(`https://localhost:${securePort}`)).rejects.toThrow() + expect(connections).toHaveLength(0) + expect(requests).toHaveLength(0) + } finally { + await transport.dispatcher.destroy() + } + }) + + it.each(['http://169.254.169.254', 'http://[fd00:ec2::254]'])( + 'refuses metadata proxy %s', + async (proxyUrl) => { + vi.stubEnv('http_proxy', proxyUrl) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) + try { + await expect(transport.fetch(url())).rejects.toThrow() + expect(requests).toHaveLength(0) + } finally { + await transport.dispatcher.destroy() + } + } + ) + + it('checks every resolved proxy address before dialing', async () => { + vi.stubEnv('http_proxy', `http://proxy.invalid:${proxyPort}`) + vi.mocked(resolveHostAddresses).mockResolvedValueOnce({ + addresses: ['127.0.0.1', '169.254.169.254'], + preferred: '127.0.0.1', + }) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) + try { + await expect(transport.fetch(url())).rejects.toThrow() + expect(resolveHostAddresses).toHaveBeenCalledWith('proxy.invalid') + expect(connections).toHaveLength(0) + expect(requests).toHaveLength(0) + } finally { + await transport.dispatcher.destroy() + } + }) + + it.each(['socks5://operator:synthetic@localhost:1080', 'http://operator@localhost:1080'])( + 'fails closed on invalid proxy configuration without exposing credentials: %s', + async (proxyUrl) => { + vi.stubEnv('http_proxy', proxyUrl) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) + try { + await expect(transport.fetch(url())).rejects.toThrow('INVALID_CONFIGURATION') + expect(requests).toHaveLength(0) + } finally { + await transport.dispatcher.destroy() + } + } + ) + + it('does not fall back to a direct socket when the proxy refuses a connection', async () => { + vi.stubEnv('http_proxy', 'http://operator:synthetic@127.0.0.1:1') + const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) + try { + await expect(transport.fetch(url())).rejects.toThrow('GATEWAY_UNAVAILABLE') + expect(requests).toHaveLength(0) + } finally { + await transport.dispatcher.destroy() + } + }) + + it('preserves proxy routing and TLS verification under the actual Bun runtime', async () => { + await promisify(execFile)( + 'bun', + [ + '--no-env-file', + 'run', + fileURLToPath(new URL('./fixtures/environment-proxy-runtime.fixture.ts', import.meta.url)), + String(securePort), + ], + { + /** The child must fail with its diagnostics before Vitest's 10-second deadline. */ + timeout: 8_000, + killSignal: 'SIGKILL', + env: { + ...process.env, + NEXT_PUBLIC_APP_URL: 'http://localhost:3000', + NEXT_PUBLIC_FORCE_HOSTED: 'false', + NODE_EXTRA_CA_CERTS: certPath, + https_proxy: `https://localhost:${secureProxyPort}`, + OUTBOUND_ROUTING_SOURCE: undefined, + OUTBOUND_ROUTING_CONFIG: undefined, + }, + } + ) + expect(connections).toHaveLength(3) + expect(connections.every(({ authority }) => authority === `127.0.0.1:${securePort}`)).toBe(true) + expect(requests).toHaveLength(2) + expect(requests.every(({ sni }) => sni === 'localhost')).toBe(true) + }) + + it('gives an explicit per-request proxy precedence over the environment proxy', async () => { + vi.stubEnv('https_proxy', 'socks5://operator:synthetic@localhost:1080') + const response = await secureFetchWithPinnedIP(`https://localhost:${securePort}`, '127.0.0.1', { + ...options, + proxyUrl: `http://localhost:${proxyPort}`, + }) + expect(await response.text()).toBe('tls reached') + expect(connections[0].authority).toBe(`localhost:${securePort}`) + }) +}) diff --git a/apps/sim/lib/core/network/fixtures/environment-proxy-runtime.fixture.ts b/apps/sim/lib/core/network/fixtures/environment-proxy-runtime.fixture.ts new file mode 100644 index 00000000000..a5bca06661b --- /dev/null +++ b/apps/sim/lib/core/network/fixtures/environment-proxy-runtime.fixture.ts @@ -0,0 +1,38 @@ +/** Real Bun probe invoked against the environment proxy test's local TLS servers. */ +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { + createSsrfGuardedFetchWithDispatcher, + secureFetchWithPinnedIP, +} from '@/lib/core/security/input-validation.server' + +const [originPort] = process.argv.slice(2) +if (!originPort) throw new Error('Local fixture port is required') +const options = { profile: 'selfHostedService' as const } +const signal = AbortSignal.timeout(5_000) +const transport = createSsrfGuardedFetchWithDispatcher(options) +try { + const response = await transport.fetch(`https://localhost:${originPort}`, { + signal, + }) + if ((await response.text()) !== 'tls reached') throw new Error('Guarded response mismatch') + const pinned = await secureFetchWithPinnedIP(`https://localhost:${originPort}`, '127.0.0.1', { + ...options, + timeout: 5_000, + signal, + }) + if ((await pinned.text()) !== 'tls reached') throw new Error('Pinned response mismatch') + let rejected = false + try { + await transport.fetch(`https://127.0.0.1:${originPort}`, { + signal, + }) + } catch (error) { + if (!(error instanceof OutboundRoutingError) || error.code !== 'GATEWAY_UNAVAILABLE') { + throw error + } + rejected = true + } + if (!rejected) throw new Error('Mismatched upstream certificate was accepted') +} finally { + await transport.dispatcher.destroy() +} diff --git a/apps/sim/lib/core/network/gateway.server.ts b/apps/sim/lib/core/network/gateway.server.ts new file mode 100644 index 00000000000..40a012b0235 --- /dev/null +++ b/apps/sim/lib/core/network/gateway.server.ts @@ -0,0 +1,47 @@ +import { isIP, type Socket } from 'node:net' +import { checkServerIdentity, connect as connectTls } from 'node:tls' +import { OutboundRoutingError } from '@/lib/core/network/routing' + +const CONNECT_TIMEOUT_MS = 10_000 + +/** The proxy CA never changes trust for the upstream service. */ +export async function secureOutboundTunnel( + socket: Socket, + hostname: string, + port: number +): Promise { + return new Promise((resolve, reject) => { + const tls = connectTls({ + socket, + host: hostname, + port, + servername: isIP(hostname) ? undefined : hostname, + checkServerIdentity: (_name, certificate) => checkServerIdentity(hostname, certificate), + rejectUnauthorized: true, + ALPNProtocols: ['http/1.1'], + }) + /** The inner TLS stream has no TCP descriptor; QoS must be applied to the outer socket. */ + if ('setTypeOfService' in socket && typeof socket.setTypeOfService === 'function') { + const setTypeOfService = socket.setTypeOfService.bind(socket) + Object.defineProperty(tls, 'setTypeOfService', { + value(tos: number) { + setTypeOfService(tos) + return tls + }, + }) + } + const timer = setTimeout( + () => tls.destroy(new OutboundRoutingError('GATEWAY_UNAVAILABLE')), + CONNECT_TIMEOUT_MS + ) + timer.unref() + tls.once('secureConnect', () => { + clearTimeout(timer) + resolve(tls) + }) + tls.once('error', () => { + clearTimeout(timer) + reject(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + }) + }) +} diff --git a/apps/sim/lib/core/network/routing.ts b/apps/sim/lib/core/network/routing.ts new file mode 100644 index 00000000000..b4caa437859 --- /dev/null +++ b/apps/sim/lib/core/network/routing.ts @@ -0,0 +1,15 @@ +export type OutboundRoutingErrorCode = + | 'CONFIGURATION_UNAVAILABLE' + | 'INVALID_CONFIGURATION' + | 'MISSING_SCOPE' + | 'ROUTE_BLOCKED' + | 'UNSUPPORTED_TRANSPORT' + | 'GATEWAY_UNAVAILABLE' + +/** Public error text never includes routing configuration, credentials or destination details. */ +export class OutboundRoutingError extends Error { + constructor(readonly code: OutboundRoutingErrorCode) { + super(`Outbound routing failed: ${code}`) + this.name = 'OutboundRoutingError' + } +} diff --git a/apps/sim/lib/core/network/transport.server.ts b/apps/sim/lib/core/network/transport.server.ts new file mode 100644 index 00000000000..7be2ec4decc --- /dev/null +++ b/apps/sim/lib/core/network/transport.server.ts @@ -0,0 +1,252 @@ +import { isIP } from 'node:net' +import { checkServerIdentity, type TLSSocket } from 'node:tls' +import { resolveHostAddresses } from '@sim/security/dns' +import { createEgressPolicy, evaluateAddress } from '@sim/security/egress' +import { + type Agent, + buildConnector, + type Dispatcher, + EnvHttpProxyAgent, + Pool, + request, +} from 'undici/index.js' +import { secureOutboundTunnel } from '@/lib/core/network/gateway.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' +import { checkResolvedEgress, validateEgressUrl } from '@/lib/core/security/egress/validate' + +interface OutboundTransportOptions { + profile: EgressProfile + resolvedIP?: string + maxResponseSize?: number + allowH2?: boolean + direct?: Agent + proxyUrl?: string +} + +/** Operator proxies select by original origin; their CONNECT destinations remain locally validated IPs. */ +function createEnvironmentProxyDispatcher(options: OutboundTransportOptions): Dispatcher | null { + const httpProxy = process.env.http_proxy ?? process.env.HTTP_PROXY + const httpsProxy = process.env.https_proxy ?? process.env.HTTPS_PROXY + if (!httpProxy && !httpsProxy) return null + + /** Bun's nested TLS sockets need an explicit identity check, including IP-literal origins. */ + const verifiedConnection = + (target: URL, callback: buildConnector.Callback): buildConnector.Callback => + (...[error, socket]) => { + if (error) { + callback(new OutboundRoutingError('GATEWAY_UNAVAILABLE'), null) + return + } + try { + if (target.protocol === 'https:') { + const tlsSocket = socket as TLSSocket + const hostname = target.hostname.replace(/^\[|\]$/g, '') + if ( + !tlsSocket.authorized || + checkServerIdentity(hostname, tlsSocket.getPeerCertificate()) + ) { + throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } + } + } catch { + socket.destroy() + callback(new OutboundRoutingError('GATEWAY_UNAVAILABLE'), null) + return + } + callback(null, socket) + } + + try { + for (const value of [httpProxy, httpsProxy]) { + if (!value) continue + const proxy = new URL(value) + if ( + !['http:', 'https:'].includes(proxy.protocol) || + proxy.pathname !== '/' || + proxy.search || + proxy.hash || + Boolean(proxy.username) !== Boolean(proxy.password) + ) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + } + + return new EnvHttpProxyAgent({ + httpProxy, + httpsProxy, + proxyTunnel: true, + allowH2: options.allowH2 ?? false, + ...(options.maxResponseSize !== undefined + ? { maxResponseSize: options.maxResponseSize } + : {}), + clientFactory(origin, clientOptions) { + const settings = clientOptions as Pool.Options + const connect = settings.connect + if (typeof connect !== 'function') { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + const hostname = origin.hostname.replace(/^\[|\]$/g, '') + const literal = isIP(hostname) + const policy = createEgressPolicy({ + ...(literal ? { allowedRanges: [hostname] } : { allowedHosts: [hostname] }), + insecureHttp: 'always', + }) + return new Pool(origin, { + ...settings, + connect(connection, callback) { + void (async () => { + const resolved = literal + ? { addresses: [hostname], preferred: hostname } + : await resolveHostAddresses(hostname) + if ( + resolved.addresses.some( + (address) => !evaluateAddress(origin, address, policy).allowed + ) + ) { + throw new OutboundRoutingError('ROUTE_BLOCKED') + } + connect( + { + ...connection, + host: origin.host, + hostname: resolved.preferred, + servername: literal ? undefined : hostname, + }, + verifiedConnection(origin, callback) + ) + })().catch((error) => + callback( + error instanceof OutboundRoutingError + ? error + : new OutboundRoutingError('GATEWAY_UNAVAILABLE'), + null + ) + ) + }, + }) + }, + factory(origin, poolOptions) { + const settings = poolOptions as Pool.Options + const tunneled = typeof settings.connect === 'function' + const connect = + typeof settings.connect === 'function' + ? settings.connect + : buildConnector({ ...settings.connect, allowH2: options.allowH2 ?? false }) + const target = new URL(origin) + const hostname = target.hostname.replace(/^\[|\]$/g, '') + const port = target.port || (target.protocol === 'https:' ? '443' : '80') + return new Pool(origin, { + ...settings, + connect(connection, callback) { + void (async () => { + let address = options.resolvedIP + if (address) { + if ( + !isIP(address) || + !checkResolvedEgress(target, address, options.profile).allowed + ) { + throw new OutboundRoutingError('ROUTE_BLOCKED') + } + } else { + const result = await validateEgressUrl(target.href, 'url', options.profile, { + logDetails: false, + }) + if (!result.isValid) throw new OutboundRoutingError('ROUTE_BLOCKED') + address = result.resolvedIP + } + const authority = `${isIP(address) === 6 ? `[${address}]` : address}:${port}` + const finish = verifiedConnection(target, callback) + connect( + { + ...connection, + protocol: tunneled ? 'http:' : connection.protocol, + port, + host: tunneled ? authority : target.host, + hostname: tunneled ? hostname : address, + servername: isIP(hostname) ? undefined : hostname, + }, + (...[error, socket]) => { + if (error) return finish(error, null) + if (!tunneled || target.protocol !== 'https:') return finish(null, socket) + void secureOutboundTunnel(socket, hostname, Number(port)).then( + (secured) => finish(null, secured), + () => { + socket.destroy() + finish(new OutboundRoutingError('GATEWAY_UNAVAILABLE'), null) + } + ) + } + ) + })().catch((error) => + callback( + error instanceof OutboundRoutingError + ? error + : new OutboundRoutingError('GATEWAY_UNAVAILABLE'), + null + ) + ) + }, + }) + }, + }) + } catch { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } +} + +interface OutboundTransportOwner { + /** Null delegates to the adapter's existing direct transport. */ + selectDispatcher(): Promise + close(): Promise + destroy(): Promise +} + +/** + * Owns direct and environment-proxy connection lifetimes for every HTTP adapter. + * Destination provenance and optional pinning are immutable for this owner. + */ +export function createOutboundTransport(options: OutboundTransportOptions): OutboundTransportOwner { + let closed = false + let environment: Dispatcher | null | undefined + const allPools = () => [ + ...(options.direct ? [options.direct] : []), + ...(environment ? [environment] : []), + ] + return { + async selectDispatcher() { + if (closed) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + if (options.proxyUrl) return options.direct ?? null + if (environment === undefined) environment = createEnvironmentProxyDispatcher(options) + return environment ?? options.direct ?? null + }, + async close() { + closed = true + await Promise.all(allPools().map((pool) => pool.close())) + }, + async destroy() { + closed = true + await Promise.all(allPools().map((pool) => pool.destroy())) + }, + } +} + +type OutboundRequestOptions = Omit< + NonNullable[1]>, + 'headers' | 'dispatcher' +> & { + headers?: Record + dispatcher: Dispatcher +} + +/** + * The shared HTTP wire transport. Import the installed package explicitly: Bun's bare + * undici shim ignores dispatchers. Proxy credentials never become destination headers. + */ +export function requestWithOutboundDispatcher(url: string, options: OutboundRequestOptions) { + const headers = { ...options.headers } + for (const name of Object.keys(headers)) { + if (name.toLowerCase() === 'proxy-authorization') delete headers[name] + } + return request(url, { ...options, headers }) +} diff --git a/apps/sim/lib/core/security/egress-end-to-end.server.test.ts b/apps/sim/lib/core/security/egress-end-to-end.server.test.ts index 87583cd946b..f23e08244fd 100644 --- a/apps/sim/lib/core/security/egress-end-to-end.server.test.ts +++ b/apps/sim/lib/core/security/egress-end-to-end.server.test.ts @@ -11,9 +11,12 @@ import { createServer, type Server } from 'node:http' import type { AddressInfo } from 'node:net' import { networkInterfaces } from 'node:os' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { resetEnvFlagsMock, resetEnvMock, setEnv, setEnvFlags } from '@sim/testing' import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest' -import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' +import { + createSsrfGuardedFetchWithDispatcher, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' /** A non-loopback RFC1918 address on this machine, or null when there is none. */ function privateInterfaceAddress(): string | null { @@ -49,7 +52,10 @@ afterAll(async () => { resetEnvFlagsMock() }) -afterEach(resetEnvFlagsMock) +afterEach(() => { + resetEnvFlagsMock() + resetEnvMock() +}) // Skipped on a host with no private interface (some CI sandboxes); the policy // itself is covered without a socket in packages/security. @@ -81,6 +87,31 @@ describe.skipIf(!host)('reaching a service on a private network', () => { secureFetchWithValidation(`https://${host}:${port}/`, { profile: 'contentFetch' }) ).rejects.toThrow(/private or reserved address/) }) + + it.each([ + ['OLLAMA_URL', 'selfHostedService'], + ['AZURE_OPENAI_ENDPOINT', 'configuredEndpoint'], + ] as const)( + 'reaches the configured %s over both guarded transports', + async (setting, profile) => { + const url = `http://${host}:${port}/` + setEnv({ [setting]: url }) + const bounded = await secureFetchWithValidation(url, { profile }) + expect(await bounded.text()).toBe('reached') + + const transport = createSsrfGuardedFetchWithDispatcher({ profile }) + try { + const streaming = await transport.fetch(url) + expect(await streaming.text()).toBe('reached') + } finally { + await transport.dispatcher.destroy() + } + + await expect( + secureFetchWithValidation(`https://${host}:${port}/`, { profile: 'contentFetch' }) + ).rejects.toThrow(/private or reserved address/) + } + ) }) // Needs no private interface, so it runs everywhere the suite above may not. diff --git a/apps/sim/lib/core/security/egress/profiles.test.ts b/apps/sim/lib/core/security/egress/profiles.test.ts index 0ac86c1f75e..1c8fe34f9ba 100644 --- a/apps/sim/lib/core/security/egress/profiles.test.ts +++ b/apps/sim/lib/core/security/egress/profiles.test.ts @@ -7,7 +7,7 @@ */ import { evaluateAddress, evaluateUrl } from '@sim/security/egress' -import { envFlagsMock, resetEnvFlagsMock } from '@sim/testing' +import { envFlagsMock, resetEnvFlagsMock, resetEnvMock, setEnv } from '@sim/testing' import { afterEach, describe, expect, it } from 'vitest' import { describeEgressDenial, @@ -15,7 +15,10 @@ import { resolveEgressPolicy, } from '@/lib/core/security/egress/profiles' -afterEach(resetEnvFlagsMock) +afterEach(() => { + resetEnvFlagsMock() + resetEnvMock() +}) const ALLOWLIST_PROFILES: EgressProfile[] = [ 'configuredEndpoint', @@ -31,6 +34,107 @@ function decide(profile: EgressProfile, href: string, address?: string) { return address === undefined ? evaluateUrl(url, policy) : evaluateAddress(url, address, policy) } +const OPERATOR_MODEL_ENDPOINTS = [ + ['OLLAMA_URL', 'selfHostedService'], + ['VLLM_BASE_URL', 'selfHostedService'], + ['LITELLM_BASE_URL', 'selfHostedService'], + ['AZURE_OPENAI_ENDPOINT', 'configuredEndpoint'], + ['AZURE_ANTHROPIC_ENDPOINT', 'configuredEndpoint'], + ['OCR_AZURE_ENDPOINT', 'configuredEndpoint'], +] as const + +describe('operator-configured model endpoints', () => { + it.each(OPERATOR_MODEL_ENDPOINTS)('%s trusts its exact host only in %s', (setting, profile) => { + setEnv({ [setting]: 'http://model-service:11434/v1' }) + expect(decide(profile, 'http://model-service:11434/v1/models', '10.4.2.9').allowed).toBe(true) + expect(decide(profile, 'http://model-service:5432/', '10.4.2.9').allowed).toBe(true) + expect(decide(profile, 'https://other.model-service/', '10.4.2.9').allowed).toBe(false) + for (const other of [...ALLOWLIST_PROFILES, ...LOCKED_PROFILES]) { + if (other !== profile) { + expect(decide(other, 'https://model-service/', '10.4.2.9').allowed).toBe(false) + } + } + }) + + it.each([ + ['http://10.4.2.9:11434', '10.4.2.9', '10.4.2.10'], + ['http://[fd12:3456::9]:11434', 'fd12:3456::9', 'fd12:3456::10'], + ])( + 'trusts a configured literal address without widening its range: %s', + (url, address, neighbor) => { + setEnv({ OLLAMA_URL: url }) + expect(decide('selfHostedService', url, address).allowed).toBe(true) + expect(decide('selfHostedService', 'https://other-service/', neighbor).allowed).toBe(false) + expect(decide('requestTarget', url, address).allowed).toBe(false) + expect(decide('contentFetch', url, address).allowed).toBe(false) + } + ) + + it('preserves explicit allowlists while adding model hosts', () => { + envFlagsMock.egressAllowedHosts = 'other-service' + envFlagsMock.egressAllowedIpRanges = '10.9.0.0/24' + setEnv({ OLLAMA_URL: 'http://ollama:11434', AZURE_OPENAI_ENDPOINT: 'https://azure.corp' }) + for (const profile of ALLOWLIST_PROFILES) { + expect(decide(profile, 'https://other-service/', '10.4.2.9').allowed).toBe(true) + expect(decide(profile, 'https://subnet-service/', '10.9.0.5').allowed).toBe(true) + } + expect(decide('selfHostedService', 'http://ollama:11434/', '10.4.2.10').allowed).toBe(true) + expect(decide('configuredEndpoint', 'https://azure.corp/', '10.4.2.11').allowed).toBe(true) + expect(envFlagsMock.egressAllowedHosts).toBe('other-service') + expect(envFlagsMock.egressAllowedIpRanges).toBe('10.9.0.0/24') + }) + + it.each(OPERATOR_MODEL_ENDPOINTS)('%s updates and revokes cached trust', (setting, profile) => { + setEnv({ [setting]: 'https://first-service/' }) + expect(decide(profile, 'https://first-service/', '10.4.2.9').allowed).toBe(true) + setEnv({ [setting]: 'https://second-service/' }) + expect(decide(profile, 'https://first-service/', '10.4.2.9').allowed).toBe(false) + expect(decide(profile, 'https://second-service/', '10.4.2.10').allowed).toBe(true) + setEnv({ [setting]: undefined }) + expect(decide(profile, 'https://second-service/', '10.4.2.10').allowed).toBe(false) + }) + + it.each(OPERATOR_MODEL_ENDPOINTS)( + '%s never grants private access on hosted Sim', + (setting, profile) => { + setEnv({ [setting]: 'https://model-service/' }) + expect(decide(profile, 'https://model-service/', '10.4.2.9').allowed).toBe(true) + envFlagsMock.isHosted = true + expect(decide(profile, 'https://model-service/', '10.4.2.9').allowed).toBe(false) + } + ) + + it.each([ + 'not-a-url', + 'file://model-service/path', + 'ftp://model-service/path', + 'http://user:password@model-service/', + 'http://*.model-service/', + 'http://model-service,other-service/', + 'http://model..service/', + ])('ignores an invalid or unsupported model endpoint: %s', (endpoint) => { + setEnv({ OLLAMA_URL: endpoint }) + expect(decide('selfHostedService', 'https://model-service/', '10.4.2.9').allowed).toBe(false) + expect(decide('selfHostedService', 'https://child.model-service/', '10.4.2.9').allowed).toBe( + false + ) + }) + + it.each([ + ['http://169.254.169.254/', '169.254.169.254'], + ['http://[fd00:ec2::254]/', 'fd00:ec2::254'], + ['https://model-service/', '169.254.169.254'], + ])('still blocks metadata through a configured model URL: %s', (url, address) => { + setEnv({ OLLAMA_URL: url, AZURE_OPENAI_ENDPOINT: url }) + for (const profile of ['selfHostedService', 'configuredEndpoint'] as const) { + expect(decide(profile, url, address)).toMatchObject({ + allowed: false, + reason: 'address-metadata', + }) + } + }) +}) + describe('the operator allowlist reaches exactly the provenances that honor it', () => { it.each(ALLOWLIST_PROFILES)('%s honors an allowlisted range', (profile) => { envFlagsMock.egressAllowedIpRanges = '10.0.0.0/8' diff --git a/apps/sim/lib/core/security/egress/profiles.ts b/apps/sim/lib/core/security/egress/profiles.ts index 9a40f116442..eda108b7ca8 100644 --- a/apps/sim/lib/core/security/egress/profiles.ts +++ b/apps/sim/lib/core/security/egress/profiles.ts @@ -18,6 +18,8 @@ import { type EgressPolicy, type InsecureHttpPolicy, } from '@sim/security/egress' +import { isIpLiteral, unwrapIpv6Brackets } from '@sim/security/ssrf' +import { env } from '@/lib/core/config/env' import { getEgressAllowedHosts, getEgressAllowedIpRanges, @@ -30,7 +32,8 @@ import { * * - `configuredEndpoint` — a base or server URL entered during setup, or a * vendor host built in process: GitHub Enterprise, Grafana, a data-drain - * destination, a connector's host. See `selfHostedService` for the on-prem + * destination, a connector's host. Off-hosted, operator-configured Azure model + * and OCR hosts are trusted alongside the allowlist. See `selfHostedService` for the on-prem * software that expects plain HTTP. * - `requestTarget` — supplied per run by the workflow author: the HTTP block's * `url`, an A2A agent URL, an RSS feed, a Function block's `fetch`. @@ -45,7 +48,8 @@ import { * on-prem: vLLM, Jupyter, 1Password Connect, ClickHouse, an MCP server. Same * reachability as `configuredEndpoint`, but plain HTTP is expected rather than * conditional, because that is how these are ordinarily served inside a - * network. An arbitrary internal port comes with being allowlisted. + * network. Off-hosted, operator-configured Ollama, vLLM and LiteLLM hosts are + * trusted alongside the allowlist. Trust names a host, not a particular port. * - `proxy` — the egress proxy itself. Held to the strictest rule of all, * because it is the component that decides where everything else may go: plain * HTTP by protocol, but public destinations only, and no allowlist. @@ -144,6 +148,8 @@ const SOURCE_NAMES = { interface DeploymentConfig { readonly hosts: string | undefined readonly ranges: string | undefined + readonly selfHostedModelUrls: readonly (string | undefined)[] + readonly configuredModelUrls: readonly (string | undefined)[] readonly legacyPrivate: boolean readonly hosted: boolean } @@ -152,6 +158,12 @@ function readDeploymentConfig(): DeploymentConfig { return { hosts: getEgressAllowedHosts(), ranges: getEgressAllowedIpRanges(), + selfHostedModelUrls: [env.OLLAMA_URL, env.VLLM_BASE_URL, env.LITELLM_BASE_URL], + configuredModelUrls: [ + env.AZURE_OPENAI_ENDPOINT, + env.AZURE_ANTHROPIC_ENDPOINT, + env.OCR_AZURE_ENDPOINT, + ], legacyPrivate: isLegacyPrivateDatabaseAccessAllowed(), hosted: isHosted, } @@ -166,11 +178,49 @@ function buildPolicies(config: DeploymentConfig): Record label.length > 0) + ) { + allowedHosts.push(host) + } + } catch { + /** A malformed model endpoint never grants network access. */ + } + } + } + return [ profile, createEgressPolicy({ - allowedHosts: honorsAllowlist ? config.hosts : undefined, - allowedRanges: honorsAllowlist ? config.ranges : undefined, + allowedHosts: honorsAllowlist ? allowedHosts : undefined, + allowedRanges: honorsAllowlist ? allowedRanges : undefined, insecureHttp: config.hosted && spec.insecureHttp === 'always' && !spec.schemeFixedByProtocol ? 'whenVouched' @@ -188,6 +238,8 @@ function sameConfig(a: DeploymentConfig, b: DeploymentConfig): boolean { return ( a.hosts === b.hosts && a.ranges === b.ranges && + a.selfHostedModelUrls.every((url, index) => url === b.selfHostedModelUrls[index]) && + a.configuredModelUrls.every((url, index) => url === b.configuredModelUrls[index]) && a.legacyPrivate === b.legacyPrivate && a.hosted === b.hosted ) diff --git a/apps/sim/lib/core/security/guarded-request-fetch.server.test.ts b/apps/sim/lib/core/security/guarded-request-fetch.server.test.ts index 45cc68ff51f..f08893e8513 100644 --- a/apps/sim/lib/core/security/guarded-request-fetch.server.test.ts +++ b/apps/sim/lib/core/security/guarded-request-fetch.server.test.ts @@ -54,6 +54,21 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { vi.clearAllMocks() }) + it.each(['manual', 'error'] as const)( + 'checks the initial literal address with redirect mode %s', + async (redirect) => { + const transport = createSsrfGuardedFetchWithDispatcher({ profile: 'contentFetch' }) + try { + await expect(transport.fetch('https://127.0.0.1/', { redirect })).rejects.toThrow( + 'SSRF policy' + ) + expect(mockUndiciRequest).not.toHaveBeenCalled() + } finally { + await transport.dispatcher.destroy() + } + } + ) + it('constructs a Response with the reply status, headers, url, and a streaming body', async () => { mockUndiciRequest.mockResolvedValueOnce( undiciReply( @@ -81,11 +96,28 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { expect(mockUndiciRequest).toHaveBeenCalledTimes(1) const [, options] = mockUndiciRequest.mock.calls[0] expect(options.method).toBe('POST') - expect(options.headers).toEqual({ 'content-type': 'application/json' }) + expect(options.headers).toEqual({ 'content-type': 'application/json', 'user-agent': 'undici' }) expect(options.body).toBe('{"jsonrpc":"2.0"}') expect(options.maxRedirections).toBeUndefined() }) + it('preserves an explicitly supplied User-Agent regardless of casing', async () => { + mockUndiciRequest.mockResolvedValueOnce(undiciReply(200, {}, byteStream('ok'))) + const transport = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) + try { + const response = await transport.fetch('https://api.example.com/data', { + headers: { 'uSeR-aGeNt': 'custom-client/1.0' }, + }) + await response.text() + + expect(mockUndiciRequest.mock.calls[0][1].headers).toEqual({ + 'uSeR-aGeNt': 'custom-client/1.0', + }) + } finally { + await transport.dispatcher.destroy() + } + }) + it('follows a redirect through followRedirectsGuarded and reports the final url', async () => { mockUndiciRequest .mockResolvedValueOnce( @@ -128,7 +160,7 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { }) const [, options] = mockUndiciRequest.mock.calls[0] - expect(options.headers).toEqual({ authorization: 'Bearer t' }) + expect(options.headers).toEqual({ authorization: 'Bearer t', 'user-agent': 'undici' }) expect(Buffer.isBuffer(options.body)).toBe(true) expect(Buffer.from(options.body).toString()).toBe('payload') }) @@ -217,16 +249,21 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { await expect(response.text()).rejects.toThrow() }) - it('rejects the reader when the source is destroyed without an error (abort/reset)', async () => { - const source = new Readable({ read() {} }) // stays open, never pushes - mockUndiciRequest.mockResolvedValueOnce(undiciReply(200, {}, source)) - const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) + it.each([undefined, 'gzip'])( + 'rejects the reader when the source is destroyed without an error (encoding: %s)', + async (encoding) => { + const source = new Readable({ read() {} }) // stays open, never pushes + mockUndiciRequest.mockResolvedValueOnce( + undiciReply(200, encoding ? { 'content-encoding': encoding } : {}, source) + ) + const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) - const response = await fetch('https://mcp.example.com/hang', { method: 'GET' }) - const reader = response.body!.getReader() - const read = reader.read() - source.destroy() // no error argument — mirrors an aborted/reset socket + const response = await fetch('https://mcp.example.com/hang', { method: 'GET' }) + const reader = response.body!.getReader() + const read = reader.read() + source.destroy() // no error argument — mirrors an aborted/reset socket - await expect(read).rejects.toThrow(/closed before completing/) - }) + await expect(read).rejects.toThrow(/closed before completing/) + } + ) }) diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 86891fa1e6e..bcea44d11b7 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -14,9 +14,14 @@ import { HttpsProxyAgent } from 'https-proxy-agent' import { Agent, type Dispatcher, + errors, type RequestInit as UndiciRequestInit, - request as undiciRequest, -} from 'undici' +} from 'undici/index.js' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { + createOutboundTransport, + requestWithOutboundDispatcher, +} from '@/lib/core/network/transport.server' import { describeEgressDenial, type EgressProfile } from '@/lib/core/security/egress/profiles' import { checkEgressUrl, @@ -403,6 +408,7 @@ export interface SecureFetchResponse { } const DEFAULT_MAX_REDIRECTS = 5 +const DEFAULT_USER_AGENT = 'undici' /** * Fail-safe ceiling applied by {@link secureFetchWithPinnedIP} when the caller does not @@ -760,33 +766,29 @@ function contentEncodingDecoder( * `redirect: 'manual'`. */ async function undiciRequestAsResponse( - input: RequestInfo | URL, - init: RequestInit, - dispatcher: Dispatcher + url: string, + effectiveInit: UndiciRequestInit, + dispatcher: Dispatcher, + maxResponseSize?: number ): Promise { - let url: string - let effectiveInit = init as UndiciRequestInit - if (typeof Request !== 'undefined' && input instanceof Request) { - // A Request input carries its own method/headers/body/signal; lift them (explicit - // init fields win, per fetch semantics) so a guarded POST isn't downgraded to GET. - const bodyAllowed = input.method !== 'GET' && input.method !== 'HEAD' - effectiveInit = { - method: input.method, - headers: input.headers, - body: bodyAllowed ? await input.clone().arrayBuffer() : undefined, - signal: input.signal, - ...(init as UndiciRequestInit), - // double-cast-allowed: DOM RequestInit and undici RequestInit differ in TS but match at runtime - } as unknown as UndiciRequestInit - url = input.url - } else { - url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url - } - const method = (effectiveInit.method ?? 'GET').toUpperCase() const canHaveBody = method !== 'GET' && method !== 'HEAD' const requestHeaders = toUndiciRequestHeaders(effectiveInit.headers) ?? {} - const requestBody = canHaveBody ? toUndiciRequestBody(effectiveInit.body) : undefined + if (!Object.keys(requestHeaders).some((name) => name.toLowerCase() === 'user-agent')) { + requestHeaders['user-agent'] = DEFAULT_USER_AGENT + } + let requestBody = canHaveBody ? toUndiciRequestBody(effectiveInit.body) : undefined + if ( + canHaveBody && + (effectiveInit.body instanceof FormData || effectiveInit.body instanceof Blob) + ) { + const encoded = new Request(url, { method, body: effectiveInit.body }) + if (!Object.keys(requestHeaders).some((key) => key.toLowerCase() === 'content-type')) { + const contentType = encoded.headers.get('content-type') + if (contentType) requestHeaders['content-type'] = contentType + } + requestBody = encoded.body ? toUndiciRequestBody(encoded.body) : undefined + } // fetch auto-adds a form content-type for a URLSearchParams body; preserve that parity // when the caller didn't set one (the MCP SDK does set it explicitly, but not every caller). if ( @@ -796,7 +798,7 @@ async function undiciRequestAsResponse( ) { requestHeaders['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8' } - const { statusCode, headers, body } = await undiciRequest(url, { + const { statusCode, headers, body } = await requestWithOutboundDispatcher(url, { method: method as Dispatcher.HttpMethod, headers: requestHeaders, body: requestBody, @@ -815,7 +817,8 @@ async function undiciRequestAsResponse( // Null-body statuses (204/205/304) can't carry a body; drain undici's (empty) stream so its // socket returns to the pool. Attach an error listener first so a socket reset mid-drain // surfaces as a handled event, not an unhandled 'error' that crashes the process. - const isNullBody = statusCode === 204 || statusCode === 205 || statusCode === 304 + const isNullBody = + method === 'HEAD' || statusCode === 204 || statusCode === 205 || statusCode === 304 if (isNullBody) { body.on('error', () => {}) body.resume() @@ -839,11 +842,33 @@ async function undiciRequestAsResponse( // `nodeReadableToWebStream` attaches its `error` listener synchronously, so wiring the pipe // AFTER it means a synchronous zlib error (e.g. a server mislabeling a non-gzip body as gzip) // is caught and rejects the reader instead of taking down the process. - const webBody = nodeReadableToWebStream(decoder ?? body) + let webBody = nodeReadableToWebStream(decoder ?? body) + if (decoder && maxResponseSize !== undefined && maxResponseSize >= 0) { + let decodedBytes = 0 + webBody = webBody.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + decodedBytes += chunk.byteLength + if (decodedBytes > maxResponseSize) throw new errors.ResponseExceededMaxSizeError() + controller.enqueue(chunk) + }, + }) + ) + } if (decoder) { - body.once('error', (err) => decoder.destroy(err)) // forward maxResponseSize / socket reset - decoder.once('close', () => body.destroy()) // tear the source down so the socket can't leak + const signal = effectiveInit.signal + const onAbort = () => decoder.destroy(toError(signal?.reason ?? new Error('Aborted'))) + signal?.addEventListener('abort', onAbort, { once: true }) + body.once('error', (error) => decoder.destroy(error)) + body.once('close', () => { + if (!body.readableEnded) decoder.destroy(new Error('Response body closed before completing')) + }) + decoder.once('close', () => { + signal?.removeEventListener('abort', onAbort) + body.destroy() + }) body.pipe(decoder) + if (signal?.aborted) onAbort() } try { @@ -856,6 +881,7 @@ async function undiciRequestAsResponse( } catch (err) { // `new Response` rejects an out-of-range status (a 1xx undici shouldn't surface, but // defensively): destroy the source so its socket can't leak, then rethrow. + decoder?.destroy() body.destroy() throw err } @@ -867,10 +893,10 @@ async function undiciRequestAsResponse( * fetch semantics) so a manual redirect follower can't silently downgrade a POST Request to a * bare GET or lose its headers. */ -async function liftFetchArgs( +function liftFetchArgs( input: RequestInfo | URL, init?: RequestInit -): Promise<{ target: string; effectiveInit: RequestInit }> { +): { target: string; effectiveInit: RequestInit } { const target = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url if (typeof Request !== 'undefined' && input instanceof Request) { const bodyAllowed = input.method !== 'GET' && input.method !== 'HEAD' @@ -879,7 +905,7 @@ async function liftFetchArgs( effectiveInit: { method: input.method, headers: input.headers, - body: bodyAllowed ? await input.clone().arrayBuffer() : undefined, + body: bodyAllowed ? input.body : undefined, signal: input.signal, // Carry the Request's redirect mode so the pinned fetch honors `manual`/`error` // instead of defaulting a `Request({ redirect: 'manual' })` to `follow`. @@ -891,8 +917,60 @@ async function liftFetchArgs( return { target, effectiveInit: init ?? {} } } +export interface OutboundFetchDispatcher { + close(): Promise + destroy(): Promise +} + +/** Owns routing, redirect validation and connection pools for pinned and DNS-guarded fetches. */ +function createValidatedFetch( + direct: Agent, + options: { + profile: EgressProfile + resolvedIP?: string + maxResponseSize?: number + allowH2?: boolean + } +): { fetch: typeof fetch; dispatcher: OutboundFetchDispatcher } { + const dispatcher = createOutboundTransport({ ...options, direct }) + const rawFetch = async (url: string, init: UndiciRequestInit): Promise => { + const selected = await dispatcher.selectDispatcher() + if (!selected) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + return undiciRequestAsResponse(url, init, selected, options.maxResponseSize) + } + return { + dispatcher, + fetch: async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const { target, effectiveInit } = liftFetchArgs(input, init) + const mode = effectiveInit.redirect ?? 'follow' + // double-cast-allowed: DOM and Undici RequestInit represent the same wire request in this bridge + const undiciInit = effectiveInit as unknown as UndiciRequestInit + if (mode === 'follow') { + return followRedirectsGuarded( + rawFetch, + target, + undiciInit, + options.profile, + options.resolvedIP + ) + } + assertGuardedRedirectTarget(new URL(target), options.profile, options.resolvedIP) + const response = await rawFetch(target, undiciInit) + if ( + mode === 'error' && + isRedirectStatus(response.status) && + response.headers.has('location') + ) { + await response.body?.cancel().catch(() => {}) + throw new TypeError('Outbound fetch received an unexpected redirect') + } + return response + }, + } +} + /** - * SSRF-guarded `fetch` + its `Agent` for outbound requests to user-controlled + * SSRF-guarded `fetch` + its dispatcher for outbound requests to user-controlled * hosts: DNS resolves normally, and every socket connect validates the chosen * addresses via {@link createSsrfGuardedLookup}; redirects are followed manually * with per-hop validation (see {@link followRedirectsGuarded}) so IP-literal @@ -904,7 +982,7 @@ export function createSsrfGuardedFetchWithDispatcher(options: { maxResponseSize?: number }): { fetch: typeof fetch - dispatcher: Agent + dispatcher: OutboundFetchDispatcher } { const dispatcher = new Agent({ allowH2: false, @@ -912,22 +990,7 @@ export function createSsrfGuardedFetchWithDispatcher(options: { ...(options.maxResponseSize !== undefined ? { maxResponseSize: options.maxResponseSize } : {}), }) - const rawFetch = (url: string, init: UndiciRequestInit): Promise => - // double-cast-allowed: DOM RequestInit and undici RequestInit differ in TS but match at runtime - undiciRequestAsResponse(url, init as unknown as RequestInit, dispatcher) - - const guarded = async (input: RequestInfo | URL, init?: RequestInit): Promise => { - const { target, effectiveInit } = await liftFetchArgs(input, init) - return followRedirectsGuarded( - rawFetch, - target, - // double-cast-allowed: DOM RequestInit and undici RequestInit are structurally compatible at runtime but the TS types differ - effectiveInit as unknown as UndiciRequestInit, - options.profile - ) - } - - return { fetch: guarded, dispatcher } + return createValidatedFetch(dispatcher, options) } /** @@ -975,48 +1038,14 @@ export function createPinnedFetch( export function createPinnedFetchWithDispatcher( resolvedIP: string, options: { profile: EgressProfile; allowH2?: boolean; maxResponseSize?: number } -): { fetch: typeof fetch; dispatcher: Agent } { +): { fetch: typeof fetch; dispatcher: OutboundFetchDispatcher } { const dispatcher = new Agent({ allowH2: options.allowH2 ?? false, connect: { lookup: createPinnedLookup(resolvedIP) }, ...(options.maxResponseSize !== undefined ? { maxResponseSize: options.maxResponseSize } : {}), }) - const rawFetch = (url: string, init: UndiciRequestInit): Promise => - // double-cast-allowed: DOM RequestInit and undici RequestInit differ in TS but match at runtime - undiciRequestAsResponse(url, init as unknown as RequestInit, dispatcher) - - // Requests go through `undici.request` (not `undici.fetch`) because fetch's streaming - // `response.body` never delivers under the Bun runtime the server runs on — the same bug - // {@link createSsrfGuardedFetchWithDispatcher} works around. Redirects are handled here (not - // by a caller's wrapper — the pinned fetch is passed straight to provider/A2A SDKs), honoring - // the request's `redirect` mode: `manual`/`error` must NOT transparently follow (e.g. - // `detectMcpAuthType` inspects the 3xx to classify auth). The default `follow` uses - // {@link followRedirectsGuarded}, which drops headers on cross-origin hops (so a redirect - // can't disclose a provider `api-key` to another origin) and stamps the final `response.url`. - // Every hop still dispatches through the pinned `Agent` (its `connect.lookup` forces - // `resolvedIP`), so a redirect can't escape to another address. - const pinned = async (input: RequestInfo | URL, init?: RequestInit): Promise => { - const { target, effectiveInit } = await liftFetchArgs(input, init) - const mode = effectiveInit.redirect ?? 'follow' - // double-cast-allowed: DOM RequestInit and undici RequestInit are structurally compatible at runtime but the TS types differ - const undiciInit = effectiveInit as unknown as UndiciRequestInit - if (mode === 'manual') { - return rawFetch(target, undiciInit) - } - if (mode === 'error') { - const response = await rawFetch(target, undiciInit) - const location = response.headers.get('location') - if (response.status >= 300 && response.status < 400 && location) { - await response.body?.cancel().catch(() => {}) - throw new TypeError('Pinned fetch received an unexpected redirect (redirect: "error")') - } - return response - } - return followRedirectsGuarded(rawFetch, target, undiciInit, options.profile, resolvedIP) - } - - return { fetch: pinned, dispatcher } + return createValidatedFetch(dispatcher, { ...options, resolvedIP }) } /** @@ -1041,14 +1070,23 @@ export async function secureFetchWithPinnedIP( ? requestedMaxResponseBytes : DEFAULT_MAX_RESPONSE_BYTES + const transport = createOutboundTransport({ + profile: options.profile, + resolvedIP, + proxyUrl: options.proxyUrl, + }) + const outboundDispatcher = await transport.selectDispatcher() + return new Promise((resolve, reject) => { const parsed = new URL(url) const isHttps = parsed.protocol === 'https:' const defaultPort = isHttps ? 443 : 80 const port = parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort - let agent: http.Agent - if (options.proxyUrl) { + let agent: http.Agent | undefined + if (outboundDispatcher) { + agent = undefined + } else if (options.proxyUrl) { // Proxy connection is already IP-pinned by validateAndPinProxyUrl; target-IP // pinning is intentionally bypassed (the proxy resolves the target). https // targets tunnel via CONNECT, http targets use absolute-URI forwarding. @@ -1060,6 +1098,9 @@ export async function secureFetchWithPinnedIP( } const { 'accept-encoding': _, ...sanitizedHeaders } = options.headers ?? {} + if (!Object.keys(sanitizedHeaders).some((name) => name.toLowerCase() === 'user-agent')) { + sanitizedHeaders['user-agent'] = DEFAULT_USER_AGENT + } const hasExplicitFraming = Object.keys(sanitizedHeaders).some((name) => { const header = name.toLowerCase() return header === 'content-length' || header === 'transfer-encoding' @@ -1081,8 +1122,14 @@ export async function secureFetchWithPinnedIP( timeout: options.timeout || 300000, } - const protocol = isHttps ? https : http - const req = protocol.request(requestOptions, (res) => { + let destroyRequest: () => void = () => {} + const onResponse = ( + res: Readable & { + statusCode?: number + headers: http.IncomingHttpHeaders + statusMessage?: string + } + ) => { const statusCode = res.statusCode || 0 const location = res.headers.location @@ -1212,6 +1259,7 @@ export async function secureFetchWithPinnedIP( const isBodylessResponse = (requestOptions.method || 'GET').toUpperCase() === 'HEAD' || statusCode === 204 || + statusCode === 205 || statusCode === 304 const contentLength = headersRecord['content-length'] if (contentLength && !isBodylessResponse) { @@ -1219,7 +1267,7 @@ export async function secureFetchWithPinnedIP( if (Number.isFinite(parsedLength) && parsedLength > maxResponseBytes) { cleanupAbort() res.destroy() - req.destroy() + destroyRequest() if (isRetryableHttpStatus(statusCode)) { settledResolve({ ok: false, @@ -1244,38 +1292,69 @@ export async function secureFetchWithPinnedIP( } } + const decoder = isBodylessResponse + ? null + : contentEncodingDecoder((headersRecord['content-encoding'] ?? '').toLowerCase().trim()) + const responseHeaders = decoder + ? stripHeaders(headersRecord, ['content-encoding', 'content-length']) + : headersRecord + let totalBytes = 0 - const nodeRes = res + let bodySettled = false + const nodeRes = decoder ?? res + const destroyTransport = destroyRequest + destroyRequest = () => { + nodeRes.destroy() + if (decoder) res.destroy() + destroyTransport() + } const body = new ReadableStream({ start(controller) { + const fail = (error: Error) => { + if (bodySettled) return + bodySettled = true + cleanupAbort() + controller.error(error) + destroyRequest() + } nodeRes.on('data', (chunk: Buffer) => { + if (bodySettled) return totalBytes += chunk.length if (totalBytes > maxResponseBytes) { - cleanupAbort() - controller.error( + fail( new PayloadSizeLimitError({ label: 'response body', maxBytes: maxResponseBytes, observedBytes: totalBytes, }) ) - nodeRes.destroy() return } controller.enqueue(new Uint8Array(chunk)) }) - nodeRes.on('end', () => { + nodeRes.once('end', () => { + if (bodySettled) return + bodySettled = true cleanupAbort() controller.close() }) - nodeRes.on('error', (err) => { - cleanupAbort() - controller.error(err) + nodeRes.once('error', fail) + nodeRes.once('close', () => { + if (!bodySettled) fail(new Error('Response body closed before completing')) }) + if (decoder) { + res.once('error', (error) => decoder.destroy(error)) + res.once('close', () => { + if (!res.readableEnded) + decoder.destroy(new Error('Response body closed before completing')) + }) + res.pipe(decoder) + } }, cancel() { + bodySettled = true cleanupAbort() - nodeRes.destroy() + destroyRequest() }, }) @@ -1300,7 +1379,7 @@ export async function secureFetchWithPinnedIP( ok: statusCode >= 200 && statusCode < 300, status: statusCode, statusText: res.statusMessage || '', - headers: new SecureFetchHeaders(headersRecord, setCookieArray), + headers: new SecureFetchHeaders(responseHeaders, setCookieArray), body, text: async () => (await readBodyAsBuffer()).toString('utf-8'), json: async () => JSON.parse((await readBodyAsBuffer()).toString('utf-8')), @@ -1309,7 +1388,7 @@ export async function secureFetchWithPinnedIP( return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) as ArrayBuffer }, }) - }) + } let onAbort: (() => void) | null = null const cleanupAbort = () => { @@ -1326,29 +1405,66 @@ export async function secureFetchWithPinnedIP( reject(reason) } - req.on('error', (error) => { - settledReject(error) - }) - - req.on('timeout', () => { - req.destroy() - settledReject(new Error(`Request timed out after ${requestOptions.timeout}ms`)) - }) + let send: () => void + if (outboundDispatcher) { + const dispatcher = outboundDispatcher + const controller = new AbortController() + destroyRequest = () => { + controller.abort() + void transport.destroy() + } + send = () => { + void requestWithOutboundDispatcher(url, { + dispatcher, + method: (options.method || 'GET') as Dispatcher.HttpMethod, + headers: sanitizedHeaders, + body: options.body, + signal: AbortSignal.any([ + controller.signal, + AbortSignal.timeout(options.timeout || 300_000), + ]), + }) + .then(({ statusCode, headers, body }) => { + body.once('close', () => { + void transport.destroy() + }) + onResponse(Object.assign(body, { statusCode, headers })) + }) + .catch((error) => { + void transport.destroy() + settledReject(error) + }) + } + } else { + const protocol = isHttps ? https : http + const req = protocol.request(requestOptions, onResponse) + destroyRequest = () => { + req.destroy() + } + req.on('error', settledReject) + req.on('timeout', () => { + destroyRequest() + settledReject(new Error(`Request timed out after ${requestOptions.timeout}ms`)) + }) + send = () => { + req.end(options.body) + } + } if (options.signal) { if (options.signal.aborted) { - req.destroy() + destroyRequest() settledReject(options.signal.reason ?? new Error('Aborted')) return } onAbort = () => { - req.destroy() + destroyRequest() settledReject(options.signal?.reason ?? new Error('Aborted')) } options.signal.addEventListener('abort', onAbort, { once: true }) } - req.end(options.body) + send() }) } diff --git a/apps/sim/lib/core/security/pinned-fetch.server.test.ts b/apps/sim/lib/core/security/pinned-fetch.server.test.ts index ded5a698a08..a129ad9f999 100644 --- a/apps/sim/lib/core/security/pinned-fetch.server.test.ts +++ b/apps/sim/lib/core/security/pinned-fetch.server.test.ts @@ -109,7 +109,7 @@ describe('createPinnedFetch', () => { expect(url).toBe('https://myresource.openai.azure.com/openai/v1/responses') expect(options.dispatcher).toBeInstanceOf(mockAgent) expect(options.method).toBe('POST') - expect(options.headers).toEqual({ 'api-key': 'secret' }) + expect(options.headers).toEqual({ 'api-key': 'secret', 'user-agent': 'undici' }) expect(options.body).toBe('{}') expect(options.signal).toBe(controller.signal) }) @@ -159,7 +159,7 @@ describe('createPinnedFetch', () => { string > expect(secondHopHeaders['api-key']).toBeUndefined() - expect(Object.keys(secondHopHeaders)).toHaveLength(0) + expect(secondHopHeaders).toEqual({ 'user-agent': 'undici' }) expect(response.status).toBe(200) expect(response.url).toBe('https://other-origin.example/final') expect(response.redirected).toBe(true) diff --git a/apps/sim/lib/core/security/secure-fetch-request-framing.server.test.ts b/apps/sim/lib/core/security/secure-fetch-request-framing.server.test.ts index 3076de3590c..7d977c64e29 100644 --- a/apps/sim/lib/core/security/secure-fetch-request-framing.server.test.ts +++ b/apps/sim/lib/core/security/secure-fetch-request-framing.server.test.ts @@ -83,6 +83,18 @@ async function sendToLengthRequiredEndpoint( } describe('secureFetchWithPinnedIP request framing', () => { + it.each([ + { headers: undefined, expected: 'undici' }, + { headers: { 'uSeR-aGeNt': 'custom-client/1.0' }, expected: 'custom-client/1.0' }, + ])( + 'sends a default User-Agent and preserves an explicit one ($expected)', + async ({ headers, expected }) => { + const received = await sendToLengthRequiredEndpoint({ method: 'POST', headers }) + + expect(received.headers['user-agent']).toBe(expected) + } + ) + it.each(['POST', 'PUT', 'PATCH', 'DELETE'])( 'sends a UTF-8 %s body with its byte length', async (method) => { diff --git a/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts b/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts index ce7e2b7d6d6..ef44693ddc3 100644 --- a/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts +++ b/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts @@ -3,6 +3,9 @@ */ import http from 'node:http' import type { AddressInfo } from 'node:net' +import { Transform } from 'node:stream' +import zlib, { brotliCompressSync, deflateSync, gzipSync } from 'node:zlib' +import { Agent } from 'undici/index.js' import { afterEach, describe, expect, it, vi } from 'vitest' vi.mock('@sim/security/dns', () => ({ @@ -18,7 +21,9 @@ vi.mock('@/lib/core/config/env-flags', () => ({ getProxyUrl: () => undefined, })) +import * as networkTransport from '@/lib/core/network/transport.server' import { + createPinnedFetchWithDispatcher, DEFAULT_MAX_RESPONSE_BYTES, secureFetchWithPinnedIP, } from '@/lib/core/security/input-validation.server' @@ -26,7 +31,11 @@ import { const servers: http.Server[] = [] afterEach(() => { - for (const server of servers.splice(0)) server.close() + vi.restoreAllMocks() + for (const server of servers.splice(0)) { + server.closeAllConnections() + server.close() + } }) /** Starts a throwaway loopback server and returns its origin. */ @@ -38,6 +47,144 @@ async function startServer(handler: http.RequestListener): Promise { } describe('secureFetchWithPinnedIP response cap', () => { + it.each([ + { encoding: 'gzip', compress: gzipSync }, + { encoding: 'deflate', compress: deflateSync }, + { encoding: 'br', compress: brotliCompressSync }, + ])( + 'decodes $encoding JSON and removes encoded framing headers', + async ({ encoding, compress }) => { + const payload = { ok: true, message: 'compressed provider response' } + const encoded = compress(Buffer.from(JSON.stringify(payload))) + const origin = await startServer((_req, res) => { + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Content-Encoding': encoding, + 'Content-Length': String(encoded.length), + }) + res.end(encoded) + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + maxResponseBytes: 1024, + }) + + await expect(response.json()).resolves.toEqual(payload) + expect(response.headers.get('content-type')).toBe('application/json') + expect(response.headers.get('content-encoding')).toBeNull() + expect(response.headers.get('content-length')).toBeNull() + } + ) + + it('limits decoded bytes and closes an upstream still sending compressed content', async () => { + const closed = vi.fn() + const encoded = gzipSync(Buffer.alloc(64 * 1024, 0x41)) + expect(encoded.length).toBeLessThan(1024) + const origin = await startServer((_req, res) => { + res.once('close', closed) + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.write(encoded) + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + maxResponseBytes: 1024, + }) + + await expect(response.text()).rejects.toThrow(/response body/i) + await vi.waitFor(() => expect(closed).toHaveBeenCalledOnce()) + }) + + it('rejects malformed compression and closes the upstream connection', async () => { + const closed = vi.fn() + const origin = await startServer((_req, res) => { + res.once('close', closed) + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.write('this is not gzip') + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + }) + + await expect(response.text()).rejects.toThrow() + await vi.waitFor(() => expect(closed).toHaveBeenCalledOnce()) + }) + + it('rejects the decoded reader when the upstream resets before completing', async () => { + let upstream: http.ServerResponse | undefined + const origin = await startServer((_req, res) => { + upstream = res + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.write(gzipSync(Buffer.from('payload')).subarray(0, 10)) + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + }) + const body = response.text() + upstream!.destroy() + + await expect(body).rejects.toThrow(/aborted|closed before completing/) + }) + + it('rejects a compressed body read when its request is aborted', async () => { + const controller = new AbortController() + const closed = vi.fn() + const origin = await startServer((_req, res) => { + res.once('close', closed) + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.write(gzipSync(Buffer.from('payload')).subarray(0, 10)) + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + signal: controller.signal, + }) + const body = response.text() + controller.abort() + + await expect(body).rejects.toThrow() + await vi.waitFor(() => expect(closed).toHaveBeenCalledOnce()) + }) + + it('destroys the compressed source when the reader cancels', async () => { + const closed = vi.fn() + const origin = await startServer((_req, res) => { + res.once('close', closed) + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.write(gzipSync(Buffer.from('payload'))) + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + }) + await response.body!.cancel() + + await vi.waitFor(() => expect(closed).toHaveBeenCalledOnce()) + }) + + it.each([ + { method: 'HEAD', status: 200 }, + { method: 'GET', status: 204 }, + { method: 'GET', status: 205 }, + { method: 'GET', status: 304 }, + ])('does not decode a bodyless $method $status response', async ({ method, status }) => { + const origin = await startServer((_req, res) => { + res.writeHead(status, { 'Content-Encoding': 'gzip' }) + res.end() + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + method, + }) + + await expect(response.text()).resolves.toBe('') + expect(response.headers.get('content-encoding')).toBe('gzip') + }) + it('rejects a body that exceeds an explicit cap instead of buffering it', async () => { const origin = await startServer((_req, res) => { res.writeHead(200, { 'Content-Type': 'application/octet-stream' }) @@ -112,3 +259,130 @@ describe('secureFetchWithPinnedIP response cap', () => { expect(response.status).toBe(304) }) }) + +describe('pinned fetch response decoding', () => { + it('returns a null body for HEAD even when metadata advertises gzip', async () => { + const origin = await startServer((_req, res) => { + res.writeHead(200, { 'Content-Encoding': 'gzip', 'Content-Length': '10000' }) + res.end() + }) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', { + profile: 'configuredEndpoint', + maxResponseSize: 1024, + }) + try { + const response = await transport.fetch(origin, { method: 'HEAD' }) + + expect(response.body).toBeNull() + await expect(response.text()).resolves.toBe('') + expect(response.headers.get('content-encoding')).toBe('gzip') + expect(response.headers.get('content-length')).toBe('10000') + } finally { + await transport.dispatcher.destroy() + } + }) + + it('enforces maxResponseSize on decoded content and cancels its upstream', async () => { + const encoded = gzipSync(Buffer.alloc(64 * 1024, 0x41)) + const closed = vi.fn() + expect(encoded.length).toBeLessThan(1024) + const origin = await startServer((_req, res) => { + res.once('close', closed) + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.write(encoded) + }) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', { + profile: 'configuredEndpoint', + maxResponseSize: 1024, + }) + try { + const response = await transport.fetch(origin) + + await expect(response.text()).rejects.toMatchObject({ code: 'UND_ERR_RES_EXCEEDED_MAX_SIZE' }) + await vi.waitFor(() => expect(closed).toHaveBeenCalledOnce()) + } finally { + await transport.dispatcher.destroy() + } + }) + + it('still permits decoded responses when maxResponseSize is explicitly unbounded', async () => { + const payload = 'a'.repeat(64 * 1024) + const origin = await startServer((_req, res) => { + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.end(gzipSync(payload)) + }) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', { + profile: 'configuredEndpoint', + maxResponseSize: -1, + }) + try { + const response = await transport.fetch(origin) + await expect(response.text()).resolves.toBe(payload) + } finally { + await transport.dispatcher.destroy() + } + }) + + it.each(['bounded', 'guarded'] as const)( + 'keeps %s cancellation attached until decoding finishes', + async (mode) => { + const decoder = new Transform({ + transform(_chunk, _encoding, done) { + done() + }, + flush() {}, + }) + vi.spyOn(zlib, 'createGunzip').mockReturnValue(decoder as zlib.Gunzip) + const dispatcher = new Agent() + if (mode === 'bounded') { + vi.spyOn(networkTransport, 'createOutboundTransport').mockReturnValueOnce({ + selectDispatcher: async () => dispatcher, + close: () => dispatcher.close(), + destroy: () => dispatcher.destroy(), + }) + } + const pinned = + mode === 'guarded' + ? createPinnedFetchWithDispatcher('127.0.0.1', { profile: 'configuredEndpoint' }) + : undefined + const wireClosed = vi.fn() + const send = networkTransport.requestWithOutboundDispatcher + vi.spyOn(networkTransport, 'requestWithOutboundDispatcher').mockImplementationOnce( + async (...args) => { + const reply = await send(...args) + reply.body.once('close', wireClosed) + return reply + } + ) + const origin = await startServer((_req, res) => { + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.end(gzipSync('payload')) + }) + const controller = new AbortController() + const removeListener = vi.spyOn(controller.signal, 'removeEventListener') + try { + const response = pinned + ? await pinned.fetch(origin, { signal: controller.signal }) + : await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + signal: controller.signal, + }) + await vi.waitFor(() => expect(wireClosed).toHaveBeenCalledOnce()) + if (mode === 'bounded') { + expect(removeListener).not.toHaveBeenCalledWith('abort', expect.any(Function)) + } + const reading = response.text() + const reason = new Error('decoding cancelled') + controller.abort(reason) + + if (mode === 'guarded') await expect(reading).rejects.toBe(reason) + else await expect(reading).rejects.toThrow(/closed before completing/) + expect(decoder.destroyed).toBe(true) + } finally { + decoder.destroy() + await pinned?.dispatcher.destroy() + await dispatcher.destroy() + } + } + ) +}) diff --git a/apps/sim/lib/core/utils/fetch-deadline.ts b/apps/sim/lib/core/utils/fetch-deadline.ts index 006795a427a..f6643784a09 100644 --- a/apps/sim/lib/core/utils/fetch-deadline.ts +++ b/apps/sim/lib/core/utils/fetch-deadline.ts @@ -1,4 +1,4 @@ -import { Agent, type Dispatcher } from 'undici' +import { Agent, type Dispatcher } from 'undici/index.js' /** * Keeps the transport deadline from undercutting the application deadline. diff --git a/apps/sim/lib/mcp/pinned-fetch.ts b/apps/sim/lib/mcp/pinned-fetch.ts index a8360236156..a09d263299f 100644 --- a/apps/sim/lib/mcp/pinned-fetch.ts +++ b/apps/sim/lib/mcp/pinned-fetch.ts @@ -1,10 +1,10 @@ import type { FetchLike } from '@modelcontextprotocol/sdk/shared/transport.js' import { createLogger } from '@sim/logger' import { isPrivateIp } from '@sim/security/ssrf' -import type { Agent } from 'undici' import { createPinnedFetchWithDispatcher, createSsrfGuardedFetchWithDispatcher, + type OutboundFetchDispatcher, } from '@/lib/core/security/input-validation.server' import { MCP_EGRESS_PROFILE, @@ -277,7 +277,7 @@ async function bufferUnderDeadline(response: Response, signal: AbortSignal): Pro */ function releaseStreamOnSettle( response: Response, - dispatcher: Agent | undefined, + dispatcher: OutboundFetchDispatcher | undefined, signal: AbortSignal ): Response { if (!dispatcher || !response.body) { @@ -349,7 +349,7 @@ export function createSsrfGuardedMcpFetch( // Bound every phase — validation, request, body read — by the deadline + caller signal. const signal = init?.signal ? AbortSignal.any([init.signal, timeoutSignal]) : timeoutSignal // Per-request Agent must be torn down (finally): a one-shot leg never reuses its socket. - let dispatcher: Agent | undefined + let dispatcher: OutboundFetchDispatcher | undefined try { logger.info('OAuth guarded fetch: validating', { host }) const resolvedIP = await withDeadline(validateMcpServerSsrf(target, profile), signal) diff --git a/apps/sim/lib/webhooks/providers/airtable.ts b/apps/sim/lib/webhooks/providers/airtable.ts index 99b77074cf6..9326a219e2a 100644 --- a/apps/sim/lib/webhooks/providers/airtable.ts +++ b/apps/sim/lib/webhooks/providers/airtable.ts @@ -3,6 +3,7 @@ import { account, webhook } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { eq } from 'drizzle-orm' import { validateAirtableId } from '@/lib/core/security/input-validation' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getBaseUrl } from '@/lib/core/utils/urls' import { getOAuthToken, @@ -22,6 +23,10 @@ import type { WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Airtable') interface AirtableChange { @@ -194,7 +199,7 @@ async function fetchAndProcessAirtablePayloads( try { const fetchStartTime = Date.now() - const response = await fetch(fullUrl, { + const response = await providerFetch(fullUrl, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, @@ -518,7 +523,7 @@ export const airtableHandler: WebhookProviderHandler = { specification: specification, } - const airtableResponse = await fetch(airtableApiUrl, { + const airtableResponse = await providerFetch(airtableApiUrl, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -639,7 +644,7 @@ export const airtableHandler: WebhookProviderHandler = { const expectedNotificationUrl = getNotificationUrl(webhookRecord) const listUrl = `https://api.airtable.com/v0/bases/${baseId}/webhooks` - const listResp = await fetch(listUrl, { + const listResp = await providerFetch(listUrl, { headers: { Authorization: `Bearer ${accessToken}`, }, @@ -705,7 +710,7 @@ export const airtableHandler: WebhookProviderHandler = { } const airtableDeleteUrl = `https://api.airtable.com/v0/bases/${baseId}/webhooks/${resolvedExternalId}` - const airtableResponse = await fetch(airtableDeleteUrl, { + const airtableResponse = await providerFetch(airtableDeleteUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/webhooks/providers/ashby.test.ts b/apps/sim/lib/webhooks/providers/ashby.test.ts index a636730fcef..80270770e20 100644 --- a/apps/sim/lib/webhooks/providers/ashby.test.ts +++ b/apps/sim/lib/webhooks/providers/ashby.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ import crypto from 'crypto' -import { createMockRequest } from '@sim/testing' +import { createMockRequest, inputValidationMock } from '@sim/testing' import { afterEach, describe, expect, it, vi } from 'vitest' import { ashbyHandler } from '@/lib/webhooks/providers/ashby' import type { @@ -11,6 +11,8 @@ import type { FormatInputContext, } from '@/lib/webhooks/providers/types' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + function authContext( request: AuthContext['request'], rawBody: string, diff --git a/apps/sim/lib/webhooks/providers/ashby.ts b/apps/sim/lib/webhooks/providers/ashby.ts index ac0ab6e1ac5..b8f2a489113 100644 --- a/apps/sim/lib/webhooks/providers/ashby.ts +++ b/apps/sim/lib/webhooks/providers/ashby.ts @@ -4,6 +4,7 @@ import { hmacSha256Hex } from '@sim/security/hmac' import { generateId } from '@sim/utils/id' import { isRecordLike, omit } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { @@ -18,6 +19,10 @@ import type { } from '@/lib/webhooks/providers/types' import { buildFallbackDeliveryFingerprint } from '@/lib/webhooks/providers/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + /** * Kept local rather than imported from `@/tools/ashby/utils`, which has the same * logic. The webhook providers are reachable from workspace page graphs, and the @@ -313,7 +318,7 @@ export const ashbyHandler: WebhookProviderHandler = { secretToken, } - const ashbyResponse = await fetch('https://api.ashbyhq.com/webhook.create', { + const ashbyResponse = await providerFetch('https://api.ashbyhq.com/webhook.create', { method: 'POST', headers: { Authorization: `Basic ${authString}`, @@ -402,7 +407,7 @@ export const ashbyHandler: WebhookProviderHandler = { const authString = Buffer.from(`${apiKey}:`).toString('base64') - const ashbyResponse = await fetch('https://api.ashbyhq.com/webhook.delete', { + const ashbyResponse = await providerFetch('https://api.ashbyhq.com/webhook.delete', { method: 'POST', headers: { Authorization: `Basic ${authString}`, diff --git a/apps/sim/lib/webhooks/providers/attio.ts b/apps/sim/lib/webhooks/providers/attio.ts index 693e7e1f5e6..a232ada9169 100644 --- a/apps/sim/lib/webhooks/providers/attio.ts +++ b/apps/sim/lib/webhooks/providers/attio.ts @@ -3,6 +3,7 @@ import { safeCompare } from '@sim/security/compare' import { hmacSha256Hex } from '@sim/security/hmac' import { toError } from '@sim/utils/errors' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getBaseUrl } from '@/lib/core/utils/urls' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getCredentialOwner, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' @@ -17,6 +18,10 @@ import type { WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Attio') function validateAttioSignature(secret: string, signature: string, body: string): boolean { @@ -169,7 +174,7 @@ export const attioHandler: WebhookProviderHandler = { }, } - const attioResponse = await fetch('https://api.attio.com/v2/webhooks', { + const attioResponse = await providerFetch('https://api.attio.com/v2/webhooks', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -280,7 +285,7 @@ export const attioHandler: WebhookProviderHandler = { return } - const attioResponse = await fetch(`https://api.attio.com/v2/webhooks/${externalId}`, { + const attioResponse = await providerFetch(`https://api.attio.com/v2/webhooks/${externalId}`, { method: 'DELETE', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/webhooks/providers/bitbucket.test.ts b/apps/sim/lib/webhooks/providers/bitbucket.test.ts index 7698e4246ef..874aeffd3e2 100644 --- a/apps/sim/lib/webhooks/providers/bitbucket.test.ts +++ b/apps/sim/lib/webhooks/providers/bitbucket.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { NextRequest } from 'next/server' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -29,6 +30,8 @@ import { buildBitbucketOutputs, } from '@/triggers/bitbucket/utils' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const fetchMock = vi.fn() const CALLBACK_URL = 'https://app.example.com/api/webhooks/trigger/bitbucket-path' const CANDIDATE_DESCRIPTION = 'Sim workflow trigger (bitbucket_push) [sim:webhook-1]' diff --git a/apps/sim/lib/webhooks/providers/bitbucket.ts b/apps/sim/lib/webhooks/providers/bitbucket.ts index 589c81ede5a..666641c997b 100644 --- a/apps/sim/lib/webhooks/providers/bitbucket.ts +++ b/apps/sim/lib/webhooks/providers/bitbucket.ts @@ -5,6 +5,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { toRecord, toRecordOrNull } from '@sim/utils/object' import { truncate } from '@sim/utils/string' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getCredentialOwner, @@ -28,6 +29,10 @@ import { encodeBitbucketSegment, } from '@/tools/bitbucket/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Bitbucket') const BITBUCKET_MANAGEMENT_REQUEST_TIMEOUT_MS = 15_000 @@ -70,7 +75,7 @@ function bitbucketHooksUrl(workspaceSlug: string, repoSlug: string): string { } function fetchBitbucketManagement(url: string, init: RequestInit = {}): Promise { - return fetch(url, { + return providerFetch(url, { ...init, signal: AbortSignal.timeout(BITBUCKET_MANAGEMENT_REQUEST_TIMEOUT_MS), }) diff --git a/apps/sim/lib/webhooks/providers/calendly.ts b/apps/sim/lib/webhooks/providers/calendly.ts index d574d6e9534..0638b460e4c 100644 --- a/apps/sim/lib/webhooks/providers/calendly.ts +++ b/apps/sim/lib/webhooks/providers/calendly.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { DeleteSubscriptionContext, @@ -9,6 +10,10 @@ import type { WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Calendly') export const calendlyHandler: WebhookProviderHandler = { @@ -82,7 +87,7 @@ export const calendlyHandler: WebhookProviderHandler = { scope: 'organization', } - const calendlyResponse = await fetch(calendlyApiUrl, { + const calendlyResponse = await providerFetch(calendlyApiUrl, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -188,7 +193,7 @@ export const calendlyHandler: WebhookProviderHandler = { const calendlyApiUrl = `https://api.calendly.com/webhook_subscriptions/${externalId}` - const calendlyResponse = await fetch(calendlyApiUrl, { + const calendlyResponse = await providerFetch(calendlyApiUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/clickup.test.ts b/apps/sim/lib/webhooks/providers/clickup.test.ts index af6ab66ac90..db280809c3d 100644 --- a/apps/sim/lib/webhooks/providers/clickup.test.ts +++ b/apps/sim/lib/webhooks/providers/clickup.test.ts @@ -1,7 +1,9 @@ /** * @vitest-environment node */ + import { hmacSha256Hex } from '@sim/security/hmac' +import { inputValidationMock } from '@sim/testing' import { NextRequest, NextResponse } from 'next/server' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -23,6 +25,8 @@ vi.mock('@/lib/oauth/credential-service', () => ({ import { clickupHandler } from '@/lib/webhooks/providers/clickup' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const fetchMock = vi.fn() function reqWithHeaders(headers: Record): NextRequest { diff --git a/apps/sim/lib/webhooks/providers/clickup.ts b/apps/sim/lib/webhooks/providers/clickup.ts index 2ca160112a6..e6854a42b6e 100644 --- a/apps/sim/lib/webhooks/providers/clickup.ts +++ b/apps/sim/lib/webhooks/providers/clickup.ts @@ -3,6 +3,7 @@ import { safeCompare } from '@sim/security/compare' import { hmacSha256Hex } from '@sim/security/hmac' import { toError } from '@sim/utils/errors' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getCredentialOwner, @@ -21,6 +22,10 @@ import type { import { createHmacVerifier } from '@/lib/webhooks/providers/utils' import { CLICKUP_API_BASE_URL, clickupAuthorizationHeader } from '@/tools/clickup/shared' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:ClickUp') function validateClickUpSignature(secret: string, signature: string, body: string): boolean { @@ -87,7 +92,7 @@ function parseOptionalStringId(value: unknown): string | undefined { } async function deleteClickUpWebhook(accessToken: string, externalId: string): Promise { - return fetch(`${CLICKUP_API_BASE_URL}/webhook/${externalId}`, { + return providerFetch(`${CLICKUP_API_BASE_URL}/webhook/${externalId}`, { method: 'DELETE', headers: { Authorization: clickupAuthorizationHeader(accessToken) }, }) @@ -188,7 +193,7 @@ export const clickupHandler: WebhookProviderHandler = { if (listId !== undefined) requestBody.list_id = listId if (taskId !== undefined) requestBody.task_id = taskId - const clickupResponse = await fetch( + const clickupResponse = await providerFetch( `${CLICKUP_API_BASE_URL}/team/${encodeURIComponent(workspaceId)}/webhook`, { method: 'POST', diff --git a/apps/sim/lib/webhooks/providers/fathom.ts b/apps/sim/lib/webhooks/providers/fathom.ts index 341f50c4592..451c7842bf4 100644 --- a/apps/sim/lib/webhooks/providers/fathom.ts +++ b/apps/sim/lib/webhooks/providers/fathom.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { validateAlphanumericId } from '@/lib/core/security/input-validation' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { DeleteSubscriptionContext, @@ -8,6 +9,10 @@ import type { WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Fathom') export const fathomHandler: WebhookProviderHandler = { @@ -56,7 +61,7 @@ export const fathomHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const fathomResponse = await fetch('https://api.fathom.ai/external/v1/webhooks', { + const fathomResponse = await providerFetch('https://api.fathom.ai/external/v1/webhooks', { method: 'POST', headers: { 'X-Api-Key': apiKey, @@ -154,7 +159,7 @@ export const fathomHandler: WebhookProviderHandler = { const fathomApiUrl = `https://api.fathom.ai/external/v1/webhooks/${externalId}` - const fathomResponse = await fetch(fathomApiUrl, { + const fathomResponse = await providerFetch(fathomApiUrl, { method: 'DELETE', headers: { 'X-Api-Key': apiKey, diff --git a/apps/sim/lib/webhooks/providers/grain.test.ts b/apps/sim/lib/webhooks/providers/grain.test.ts index b4b2d0336f6..98ebe3eb17a 100644 --- a/apps/sim/lib/webhooks/providers/grain.test.ts +++ b/apps/sim/lib/webhooks/providers/grain.test.ts @@ -1,9 +1,12 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { grainHandler } from '@/lib/webhooks/providers/grain' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const WEBHOOK_ID = 'webhook-uuid-1234' const fetchMock = vi.fn() diff --git a/apps/sim/lib/webhooks/providers/grain.ts b/apps/sim/lib/webhooks/providers/grain.ts index 4270b6babe5..000a30e4ff2 100644 --- a/apps/sim/lib/webhooks/providers/grain.ts +++ b/apps/sim/lib/webhooks/providers/grain.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { DeleteSubscriptionContext, @@ -13,6 +14,10 @@ import type { import { skipByEventTypes } from '@/lib/webhooks/providers/utils' import { GRAIN_V2_TRIGGER_TO_HOOK_TYPES } from '@/triggers/grain/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Grain') const GRAIN_V2_HOOKS_BASE = 'https://api.grain.com/_/public-api/v2/hooks' @@ -62,7 +67,7 @@ async function createGrainV2Hooks(params: { try { for (const hookType of hookTypes) { - const response = await fetch(`${GRAIN_V2_HOOKS_BASE}/create`, { + const response = await providerFetch(`${GRAIN_V2_HOOKS_BASE}/create`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -106,7 +111,7 @@ async function deleteGrainV2Hook(params: { hookId: string requestId: string }): Promise { - const response = await fetch(`${GRAIN_V2_HOOKS_BASE}/${params.hookId}`, { + const response = await providerFetch(`${GRAIN_V2_HOOKS_BASE}/${params.hookId}`, { method: 'DELETE', headers: { Authorization: `Bearer ${params.apiKey}`, @@ -194,7 +199,7 @@ async function createLegacyV1Subscription(params: { requestBody.actions = actions } - const grainResponse = await fetch('https://api.grain.com/_/public-api/hooks', { + const grainResponse = await providerFetch('https://api.grain.com/_/public-api/hooks', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -238,13 +243,16 @@ async function deleteLegacyV1Hook(params: { hookId: string requestId: string }): Promise { - const response = await fetch(`https://api.grain.com/_/public-api/hooks/${params.hookId}`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${params.apiKey}`, - 'Content-Type': 'application/json', - }, - }) + const response = await providerFetch( + `https://api.grain.com/_/public-api/hooks/${params.hookId}`, + { + method: 'DELETE', + headers: { + Authorization: `Bearer ${params.apiKey}`, + 'Content-Type': 'application/json', + }, + } + ) if (!response.ok && response.status !== 404 && response.status !== 410) { throw new Error(`Failed to delete Grain webhook ${params.hookId}: ${response.status}`) } diff --git a/apps/sim/lib/webhooks/providers/granola.test.ts b/apps/sim/lib/webhooks/providers/granola.test.ts index 6999fbe96d4..6b95000b602 100644 --- a/apps/sim/lib/webhooks/providers/granola.test.ts +++ b/apps/sim/lib/webhooks/providers/granola.test.ts @@ -1,8 +1,11 @@ import crypto from 'node:crypto' +import { inputValidationMock } from '@sim/testing' import { NextRequest } from 'next/server' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { granolaHandler } from '@/lib/webhooks/providers/granola' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const SECRET_BYTES = Buffer.from('granola-test-secret-key-padding!!!!!') const SIGNING_SECRET = `whsec_${SECRET_BYTES.toString('base64')}` diff --git a/apps/sim/lib/webhooks/providers/granola.ts b/apps/sim/lib/webhooks/providers/granola.ts index ce3738181df..ef61b2754fc 100644 --- a/apps/sim/lib/webhooks/providers/granola.ts +++ b/apps/sim/lib/webhooks/providers/granola.ts @@ -3,6 +3,7 @@ import { safeCompare } from '@sim/security/compare' import { hmacSha256Base64 } from '@sim/security/hmac' import { toRecordOrNull } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -16,6 +17,10 @@ import type { } from '@/lib/webhooks/providers/types' import { GRANOLA_TRIGGER_TO_EVENT_TYPES } from '@/triggers/granola/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Granola') const GRANOLA_WEBHOOK_ENDPOINTS_URL = 'https://public-api.granola.ai/v1/webhook-endpoints' @@ -114,7 +119,7 @@ function granolaUserFacingError(status: number, body: string): string { /** Delete one Granola webhook endpoint. Treats an already-deleted endpoint as success. */ async function deleteGranolaEndpoint(apiKey: string, endpointId: string): Promise { - const response = await fetch( + const response = await providerFetch( `${GRANOLA_WEBHOOK_ENDPOINTS_URL}/${encodeURIComponent(endpointId)}`, { method: 'DELETE', @@ -289,7 +294,7 @@ export const granolaHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const response = await fetch(GRANOLA_WEBHOOK_ENDPOINTS_URL, { + const response = await providerFetch(GRANOLA_WEBHOOK_ENDPOINTS_URL, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/instantly.test.ts b/apps/sim/lib/webhooks/providers/instantly.test.ts index 72490858e11..5c9c371f437 100644 --- a/apps/sim/lib/webhooks/providers/instantly.test.ts +++ b/apps/sim/lib/webhooks/providers/instantly.test.ts @@ -1,8 +1,10 @@ -import { resetEnvMock, setEnv } from '@sim/testing' +import { inputValidationMock, resetEnvMock, setEnv } from '@sim/testing' import { NextRequest } from 'next/server' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { instantlyHandler } from '@/lib/webhooks/providers/instantly' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + function reqWithHeaders(headers: Record): NextRequest { return new NextRequest('http://localhost/test', { headers }) } diff --git a/apps/sim/lib/webhooks/providers/instantly.ts b/apps/sim/lib/webhooks/providers/instantly.ts index 30acd025ce1..fe476228642 100644 --- a/apps/sim/lib/webhooks/providers/instantly.ts +++ b/apps/sim/lib/webhooks/providers/instantly.ts @@ -3,6 +3,7 @@ import { toError } from '@sim/utils/errors' import { generateShortId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -17,6 +18,10 @@ import type { import { verifyTokenAuth } from '@/lib/webhooks/providers/utils' import { instantlyUrl } from '@/tools/instantly/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Instantly') const SIM_WEBHOOK_TOKEN_HEADER = 'x-sim-webhook-token' @@ -166,7 +171,7 @@ export const instantlyHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const response = await fetch(instantlyUrl('/api/v2/webhooks'), { + const response = await providerFetch(instantlyUrl('/api/v2/webhooks'), { method: 'POST', headers: { Authorization: `Bearer ${apiKey.trim()}`, @@ -228,7 +233,7 @@ export const instantlyHandler: WebhookProviderHandler = { return } - const response = await fetch( + const response = await providerFetch( instantlyUrl(`/api/v2/webhooks/${encodeURIComponent(externalId.trim())}`), { method: 'DELETE', diff --git a/apps/sim/lib/webhooks/providers/jotform.test.ts b/apps/sim/lib/webhooks/providers/jotform.test.ts index 099c68a5ea1..160f88d31c2 100644 --- a/apps/sim/lib/webhooks/providers/jotform.test.ts +++ b/apps/sim/lib/webhooks/providers/jotform.test.ts @@ -1,7 +1,13 @@ /** * @vitest-environment node */ -import { dbChainMock, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { + dbChainMock, + inputValidationMock, + queueTableRows, + resetDbChainMock, + schemaMock, +} from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@sim/db', () => ({ ...dbChainMock, ...schemaMock })) @@ -18,6 +24,8 @@ vi.mock('@/lib/webhooks/provider-subscription-utils', () => ({ import { jotformHandler } from '@/lib/webhooks/providers/jotform' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const fetchMock = vi.fn() function createContext(providerConfig: Record) { diff --git a/apps/sim/lib/webhooks/providers/jotform.ts b/apps/sim/lib/webhooks/providers/jotform.ts index bdad7809d73..181f7652e5b 100644 --- a/apps/sim/lib/webhooks/providers/jotform.ts +++ b/apps/sim/lib/webhooks/providers/jotform.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { and, eq, isNull, ne } from 'drizzle-orm' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { DeleteSubscriptionContext, @@ -22,6 +23,10 @@ import { toStringOrNull, } from '@/tools/jotform/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Jotform') interface JotformSubscriptionCredentials { @@ -67,7 +72,7 @@ async function listWebhookIdForUrl( credentials: JotformSubscriptionCredentials, notificationUrl: string ): Promise { - const response = await fetch( + const response = await providerFetch( buildJotformUrl( credentials, `form/${encodeURIComponent(credentials.formId)}/webhooks` @@ -184,7 +189,7 @@ export const jotformHandler: WebhookProviderHandler = { return {} } - const response = await fetch( + const response = await providerFetch( buildJotformUrl( credentials, `form/${encodeURIComponent(credentials.formId)}/webhooks` @@ -256,7 +261,7 @@ export const jotformHandler: WebhookProviderHandler = { return } - const response = await fetch( + const response = await providerFetch( buildJotformUrl( credentials, `form/${encodeURIComponent(credentials.formId)}/webhooks/${encodeURIComponent(webhookId)}` diff --git a/apps/sim/lib/webhooks/providers/lemlist.ts b/apps/sim/lib/webhooks/providers/lemlist.ts index 24759354a36..f687f030e7b 100644 --- a/apps/sim/lib/webhooks/providers/lemlist.ts +++ b/apps/sim/lib/webhooks/providers/lemlist.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { validateAlphanumericId } from '@/lib/core/security/input-validation' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { DeleteSubscriptionContext, @@ -8,6 +9,10 @@ import type { WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Lemlist') export const lemlistHandler: WebhookProviderHandler = { @@ -65,7 +70,7 @@ export const lemlistHandler: WebhookProviderHandler = { requestBody.campaignId = campaignId } - const lemlistResponse = await fetch(lemlistApiUrl, { + const lemlistResponse = await providerFetch(lemlistApiUrl, { method: 'POST', headers: { Authorization: `Basic ${authString}`, @@ -148,7 +153,7 @@ export const lemlistHandler: WebhookProviderHandler = { } const lemlistApiUrl = `https://api.lemlist.com/api/hooks/${id}` - const lemlistResponse = await fetch(lemlistApiUrl, { + const lemlistResponse = await providerFetch(lemlistApiUrl, { method: 'DELETE', headers: { Authorization: `Basic ${authString}`, @@ -183,7 +188,7 @@ export const lemlistHandler: WebhookProviderHandler = { } const notificationUrl = getNotificationUrl(webhook) - const listResponse = await fetch('https://api.lemlist.com/api/hooks', { + const listResponse = await providerFetch('https://api.lemlist.com/api/hooks', { method: 'GET', headers: { Authorization: `Basic ${authString}`, diff --git a/apps/sim/lib/webhooks/providers/linear.test.ts b/apps/sim/lib/webhooks/providers/linear.test.ts index 9962ba1f790..bff7da94308 100644 --- a/apps/sim/lib/webhooks/providers/linear.test.ts +++ b/apps/sim/lib/webhooks/providers/linear.test.ts @@ -1,8 +1,11 @@ import crypto from 'node:crypto' +import { inputValidationMock } from '@sim/testing' import { NextRequest } from 'next/server' import { describe, expect, it } from 'vitest' import { linearHandler } from '@/lib/webhooks/providers/linear' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + function signLinearBody(secret: string, rawBody: string): string { return crypto.createHmac('sha256', secret).update(rawBody, 'utf8').digest('hex') } diff --git a/apps/sim/lib/webhooks/providers/linear.ts b/apps/sim/lib/webhooks/providers/linear.ts index 0734212bd3b..f5aa12fa702 100644 --- a/apps/sim/lib/webhooks/providers/linear.ts +++ b/apps/sim/lib/webhooks/providers/linear.ts @@ -5,6 +5,7 @@ import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { isRecordLike } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -18,6 +19,10 @@ import type { } from '@/lib/webhooks/providers/types' import { createHmacVerifier } from '@/lib/webhooks/providers/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Linear') function validateLinearSignature(secret: string, signature: string, body: string): boolean { @@ -213,7 +218,7 @@ export const linearHandler: WebhookProviderHandler = { } try { - const response = await fetch('https://api.linear.app/graphql', { + const response = await providerFetch('https://api.linear.app/graphql', { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -292,7 +297,7 @@ export const linearHandler: WebhookProviderHandler = { } try { - const response = await fetch('https://api.linear.app/graphql', { + const response = await providerFetch('https://api.linear.app/graphql', { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/webhooks/providers/linq.ts b/apps/sim/lib/webhooks/providers/linq.ts index 819b87ce0c7..a41c26b1c1c 100644 --- a/apps/sim/lib/webhooks/providers/linq.ts +++ b/apps/sim/lib/webhooks/providers/linq.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { hmacSha256Base64 } from '@sim/security/hmac' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -16,6 +17,10 @@ import type { import { LINQ_API_BASE, linqHeaders } from '@/tools/linq/utils' import { LINQ_ALL_WEBHOOK_EVENT_TYPES, LINQ_TRIGGER_TO_EVENT_TYPE } from '@/triggers/linq/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Linq') /** Max clock skew tolerated between the webhook timestamp and now (seconds). */ @@ -182,7 +187,7 @@ export const linqHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const response = await fetch(`${LINQ_API_BASE}/webhook-subscriptions`, { + const response = await providerFetch(`${LINQ_API_BASE}/webhook-subscriptions`, { method: 'POST', headers: linqHeaders(apiKey), body: JSON.stringify(requestBody), @@ -243,7 +248,7 @@ export const linqHandler: WebhookProviderHandler = { return } - const response = await fetch(`${LINQ_API_BASE}/webhook-subscriptions/${externalId}`, { + const response = await providerFetch(`${LINQ_API_BASE}/webhook-subscriptions/${externalId}`, { method: 'DELETE', headers: linqHeaders(apiKey), }) diff --git a/apps/sim/lib/webhooks/providers/microsoft-teams.ts b/apps/sim/lib/webhooks/providers/microsoft-teams.ts index 98dfe9830a2..afd1200300b 100644 --- a/apps/sim/lib/webhooks/providers/microsoft-teams.ts +++ b/apps/sim/lib/webhooks/providers/microsoft-teams.ts @@ -9,6 +9,7 @@ import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { isMicrosoftContentUrl } from '@/lib/core/security/input-validation' import { + createSsrfGuardedFetchWithDispatcher, type SecureFetchResponse, secureFetchWithPinnedIP, validateUrlWithDNS, @@ -31,6 +32,10 @@ import type { WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:MicrosoftTeams') function validateMicrosoftTeamsSignature( @@ -239,7 +244,9 @@ async function formatTeamsGraphNotification( if (accessToken) { const msgUrl = `https://graph.microsoft.com/v1.0/chats/${encodeURIComponent(resolvedChatId)}/messages/${encodeURIComponent(resolvedMessageId)}` - const res = await fetch(msgUrl, { headers: { Authorization: `Bearer ${accessToken}` } }) + const res = await providerFetch(msgUrl, { + headers: { Authorization: `Bearer ${accessToken}` }, + }) if (res.ok) { message = (await res.json()) as Record @@ -297,7 +304,7 @@ async function formatTeamsGraphNotification( .replace(/=+$/, '') const graphUrl = `https://graph.microsoft.com/v1.0/shares/u!${encodedUrl}/driveItem/content` - const graphRes = await fetch(graphUrl, { + const graphRes = await providerFetch(graphUrl, { headers: { Authorization: `Bearer ${accessToken}` }, redirect: 'follow', }) @@ -344,7 +351,7 @@ async function formatTeamsGraphNotification( } const metadataUrl = `https://graph.microsoft.com/v1.0/shares/${shareToken}/driveItem` - const metadataRes = await fetch(metadataUrl, { + const metadataRes = await providerFetch(metadataUrl, { headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', @@ -353,7 +360,7 @@ async function formatTeamsGraphNotification( if (!metadataRes.ok) { const directUrl = `https://graph.microsoft.com/v1.0/shares/${shareToken}/driveItem/content` - const directRes = await fetch(directUrl, { + const directRes = await providerFetch(directUrl, { headers: { Authorization: `Bearer ${accessToken}` }, redirect: 'follow', }) @@ -639,7 +646,7 @@ export const microsoftTeamsHandler: WebhookProviderHandler = { const existingSubscriptionId = config.externalSubscriptionId as string | undefined if (existingSubscriptionId) { try { - const checkRes = await fetch( + const checkRes = await providerFetch( `https://graph.microsoft.com/v1.0/subscriptions/${existingSubscriptionId}`, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}` } } ) @@ -677,7 +684,7 @@ export const microsoftTeamsHandler: WebhookProviderHandler = { } try { - const res = await fetch('https://graph.microsoft.com/v1.0/subscriptions', { + const res = await providerFetch('https://graph.microsoft.com/v1.0/subscriptions', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -779,7 +786,7 @@ export const microsoftTeamsHandler: WebhookProviderHandler = { return } - const res = await fetch( + const res = await providerFetch( `https://graph.microsoft.com/v1.0/subscriptions/${externalSubscriptionId}`, { method: 'DELETE', diff --git a/apps/sim/lib/webhooks/providers/monday.ts b/apps/sim/lib/webhooks/providers/monday.ts index 64ba5bde7f4..a1d3f83fde1 100644 --- a/apps/sim/lib/webhooks/providers/monday.ts +++ b/apps/sim/lib/webhooks/providers/monday.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { NextResponse } from 'next/server' import { validateMondayNumericId } from '@/lib/core/security/input-validation' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getOAuthToken, refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getCredentialOwner, @@ -18,6 +19,10 @@ import type { } from '@/lib/webhooks/providers/types' import { MONDAY_API_URL, mondayHeaders } from '@/tools/monday/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Monday') /** @@ -106,7 +111,7 @@ export const mondayHandler: WebhookProviderHandler = { const notificationUrl = getNotificationUrl(ctx.webhook) try { - const response = await fetch(MONDAY_API_URL, { + const response = await providerFetch(MONDAY_API_URL, { method: 'POST', headers: mondayHeaders(accessToken), body: JSON.stringify({ @@ -219,7 +224,7 @@ export const mondayHandler: WebhookProviderHandler = { } try { - const response = await fetch(MONDAY_API_URL, { + const response = await providerFetch(MONDAY_API_URL, { method: 'POST', headers: mondayHeaders(accessToken), body: JSON.stringify({ diff --git a/apps/sim/lib/webhooks/providers/pagerduty.ts b/apps/sim/lib/webhooks/providers/pagerduty.ts index fe15d0d475c..70b5917a6fb 100644 --- a/apps/sim/lib/webhooks/providers/pagerduty.ts +++ b/apps/sim/lib/webhooks/providers/pagerduty.ts @@ -2,6 +2,7 @@ import crypto from 'crypto' import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { toRecord } from '@sim/utils/object' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { DeleteSubscriptionContext, @@ -14,6 +15,10 @@ import type { } from '@/lib/webhooks/providers/types' import { createHmacVerifier } from '@/lib/webhooks/providers/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:PagerDuty') const PAGERDUTY_API_BASE = 'https://api.pagerduty.com' @@ -55,7 +60,7 @@ async function cleanupPagerDutySubscription( ): Promise { let id = subscriptionId if (!id) { - const listRes = await fetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions`, { + const listRes = await providerFetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions`, { headers: pagerdutyHeaders(apiKey), }).catch(() => null) if (!listRes || !listRes.ok) return @@ -65,7 +70,7 @@ async function cleanupPagerDutySubscription( id = body?.webhook_subscriptions?.find((sub) => sub.delivery_method?.url === url)?.id } if (!id) return - await fetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions/${id}`, { + await providerFetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions/${id}`, { method: 'DELETE', headers: pagerdutyHeaders(apiKey), }).catch(() => null) @@ -149,7 +154,7 @@ export const pagerdutyHandler: WebhookProviderHandler = { throw new Error('PagerDuty API Key is required to create the webhook subscription.') const { getPagerDutyEvents } = await import('@/triggers/pagerduty/utils') - const res = await fetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions`, { + const res = await providerFetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions`, { method: 'POST', headers: pagerdutyHeaders(apiKey), body: JSON.stringify({ @@ -206,7 +211,7 @@ export const pagerdutyHandler: WebhookProviderHandler = { return } - const res = await fetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions/${externalId}`, { + const res = await providerFetch(`${PAGERDUTY_API_BASE}/webhook_subscriptions/${externalId}`, { method: 'DELETE', headers: pagerdutyHeaders(apiKey), }) diff --git a/apps/sim/lib/webhooks/providers/resend.test.ts b/apps/sim/lib/webhooks/providers/resend.test.ts index 9919ff22930..8558cd056bd 100644 --- a/apps/sim/lib/webhooks/providers/resend.test.ts +++ b/apps/sim/lib/webhooks/providers/resend.test.ts @@ -1,6 +1,9 @@ +import { inputValidationMock } from '@sim/testing' import { describe, expect, it } from 'vitest' import { resendHandler } from '@/lib/webhooks/providers/resend' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + describe('Resend webhook provider', () => { it('formatInput exposes documented email metadata and distinct data.created_at', async () => { const { input } = await resendHandler.formatInput!({ diff --git a/apps/sim/lib/webhooks/providers/resend.ts b/apps/sim/lib/webhooks/providers/resend.ts index c04289e3a00..83adc6b3294 100644 --- a/apps/sim/lib/webhooks/providers/resend.ts +++ b/apps/sim/lib/webhooks/providers/resend.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { hmacSha256Base64 } from '@sim/security/hmac' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -18,6 +19,10 @@ import { RESEND_TRIGGER_TO_EVENT_TYPE, } from '@/triggers/resend/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Resend') /** @@ -185,7 +190,7 @@ export const resendHandler: WebhookProviderHandler = { webhookId: webhook.id, }) - const resendResponse = await fetch('https://api.resend.com/webhooks', { + const resendResponse = await providerFetch('https://api.resend.com/webhooks', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -274,7 +279,7 @@ export const resendHandler: WebhookProviderHandler = { return } - const resendResponse = await fetch(`https://api.resend.com/webhooks/${externalId}`, { + const resendResponse = await providerFetch(`https://api.resend.com/webhooks/${externalId}`, { method: 'DELETE', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/revenuecat.test.ts b/apps/sim/lib/webhooks/providers/revenuecat.test.ts index 0d832733111..2ec52a39654 100644 --- a/apps/sim/lib/webhooks/providers/revenuecat.test.ts +++ b/apps/sim/lib/webhooks/providers/revenuecat.test.ts @@ -1,8 +1,10 @@ -import { resetEnvMock, setEnv } from '@sim/testing' +import { inputValidationMock, resetEnvMock, setEnv } from '@sim/testing' import { NextRequest } from 'next/server' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { revenueCatHandler } from '@/lib/webhooks/providers/revenuecat' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + function requestWithAuth(authValue?: string): NextRequest { return new NextRequest('http://localhost/test', { headers: authValue ? { authorization: authValue } : {}, diff --git a/apps/sim/lib/webhooks/providers/revenuecat.ts b/apps/sim/lib/webhooks/providers/revenuecat.ts index 54ed8c5e499..d04d304f27e 100644 --- a/apps/sim/lib/webhooks/providers/revenuecat.ts +++ b/apps/sim/lib/webhooks/providers/revenuecat.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { generateId } from '@sim/utils/id' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -14,6 +15,10 @@ import type { WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:RevenueCat') /** Base URL for the RevenueCat REST API v2. */ @@ -106,7 +111,7 @@ export const revenueCatHandler: WebhookProviderHandler = { requestBody.environment = environment } - const response = await fetch( + const response = await providerFetch( `${REVENUECAT_API_BASE}/projects/${encodeURIComponent(projectId)}/integrations/webhooks`, { method: 'POST', @@ -182,7 +187,7 @@ export const revenueCatHandler: WebhookProviderHandler = { return } - const response = await fetch( + const response = await providerFetch( `${REVENUECAT_API_BASE}/projects/${encodeURIComponent(projectId)}/integrations/webhooks/${encodeURIComponent(externalId)}`, { method: 'DELETE', diff --git a/apps/sim/lib/webhooks/providers/rootly.test.ts b/apps/sim/lib/webhooks/providers/rootly.test.ts index 47f8114e262..0fe6a4395d9 100644 --- a/apps/sim/lib/webhooks/providers/rootly.test.ts +++ b/apps/sim/lib/webhooks/providers/rootly.test.ts @@ -1,9 +1,11 @@ import crypto from 'node:crypto' -import { resetEnvMock, setEnv } from '@sim/testing' +import { inputValidationMock, resetEnvMock, setEnv } from '@sim/testing' import { NextRequest } from 'next/server' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { rootlyHandler } from '@/lib/webhooks/providers/rootly' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + function signRootlyBody(secret: string, timestamp: string, rawBody: string): string { return crypto.createHmac('sha256', secret).update(`${timestamp}${rawBody}`, 'utf8').digest('hex') } diff --git a/apps/sim/lib/webhooks/providers/rootly.ts b/apps/sim/lib/webhooks/providers/rootly.ts index a0f99798541..f62262efb0f 100644 --- a/apps/sim/lib/webhooks/providers/rootly.ts +++ b/apps/sim/lib/webhooks/providers/rootly.ts @@ -3,6 +3,7 @@ import { safeCompare } from '@sim/security/compare' import { hmacSha256Hex } from '@sim/security/hmac' import { generateId } from '@sim/utils/id' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -15,6 +16,10 @@ import type { WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Rootly') const ROOTLY_WEBHOOK_TIMESTAMP_SKEW_MS = 5 * 60 * 1000 @@ -174,7 +179,7 @@ export const rootlyHandler: WebhookProviderHandler = { }, } - const response = await fetch('https://api.rootly.com/v1/webhooks/endpoints', { + const response = await providerFetch('https://api.rootly.com/v1/webhooks/endpoints', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -255,13 +260,16 @@ export const rootlyHandler: WebhookProviderHandler = { return } - const response = await fetch(`https://api.rootly.com/v1/webhooks/endpoints/${externalId}`, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${apiKey}`, - Accept: 'application/vnd.api+json', - }, - }) + const response = await providerFetch( + `https://api.rootly.com/v1/webhooks/endpoints/${externalId}`, + { + method: 'DELETE', + headers: { + Authorization: `Bearer ${apiKey}`, + Accept: 'application/vnd.api+json', + }, + } + ) if (response.ok || response.status === 404) { await response.body?.cancel() diff --git a/apps/sim/lib/webhooks/providers/slack.test.ts b/apps/sim/lib/webhooks/providers/slack.test.ts index aa071c4c0fd..1984aa66cbd 100644 --- a/apps/sim/lib/webhooks/providers/slack.test.ts +++ b/apps/sim/lib/webhooks/providers/slack.test.ts @@ -1,4 +1,5 @@ import { createHmac } from 'node:crypto' +import { inputValidationMock } from '@sim/testing' import { describe, expect, it } from 'vitest' import { handleSlackChallenge, @@ -7,6 +8,8 @@ import { slackHandler, } from '@/lib/webhooks/providers/slack' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const ctx = (body: unknown) => ({ webhook: {}, workflow: { id: 'wf', userId: 'u' }, diff --git a/apps/sim/lib/webhooks/providers/slack.ts b/apps/sim/lib/webhooks/providers/slack.ts index 196638260f4..edd769e0500 100644 --- a/apps/sim/lib/webhooks/providers/slack.ts +++ b/apps/sim/lib/webhooks/providers/slack.ts @@ -8,6 +8,7 @@ import { isRecordLike } from '@sim/utils/object' import { eq } from 'drizzle-orm' import { NextResponse } from 'next/server' import { + createSsrfGuardedFetchWithDispatcher, secureFetchWithPinnedIP, validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' @@ -25,6 +26,10 @@ import type { } from '@/lib/webhooks/providers/types' import { type SlackEventFilter, slackEventSupportsFilter } from '@/triggers/slack/shared' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Slack') /** 50 MB */ @@ -285,7 +290,7 @@ async function resolveSlackFileInfo( botToken: string ): Promise<{ url_private?: string; name?: string; mimetype?: string; size?: number } | null> { try { - const response = await fetch( + const response = await providerFetch( `https://slack.com/api/files.info?file=${encodeURIComponent(fileId)}`, { headers: { Authorization: `Bearer ${botToken}` } } ) @@ -413,7 +418,7 @@ async function fetchSlackMessageText( ): Promise { try { const params = new URLSearchParams({ channel, timestamp: messageTs }) - const response = await fetch(`https://slack.com/api/reactions.get?${params}`, { + const response = await providerFetch(`https://slack.com/api/reactions.get?${params}`, { headers: { Authorization: `Bearer ${botToken}` }, }) const data = (await response.json()) as { @@ -456,7 +461,7 @@ export async function fetchSlackTeamId(botToken: string): Promise<{ userId: string | undefined teamName: string | undefined }> { - const response = await fetch('https://slack.com/api/auth.test', { + const response = await providerFetch('https://slack.com/api/auth.test', { headers: { Authorization: `Bearer ${botToken}` }, }) const data = (await response.json()) as { diff --git a/apps/sim/lib/webhooks/providers/telegram.ts b/apps/sim/lib/webhooks/providers/telegram.ts index da7303800ed..5ec0d8977fa 100644 --- a/apps/sim/lib/webhooks/providers/telegram.ts +++ b/apps/sim/lib/webhooks/providers/telegram.ts @@ -2,6 +2,7 @@ import { db, webhook, workflowDeploymentVersion } from '@sim/db' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { and, eq, isNull, ne } from 'drizzle-orm' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -13,6 +14,10 @@ import type { WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Telegram') export const telegramHandler: WebhookProviderHandler = { @@ -127,7 +132,7 @@ export const telegramHandler: WebhookProviderHandler = { const telegramApiUrl = `https://api.telegram.org/bot${botToken}/setWebhook` try { - const telegramResponse = await fetch(telegramApiUrl, { + const telegramResponse = await providerFetch(telegramApiUrl, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -198,7 +203,7 @@ export const telegramHandler: WebhookProviderHandler = { } const telegramApiUrl = `https://api.telegram.org/bot${botToken}/deleteWebhook` - const telegramResponse = await fetch(telegramApiUrl, { + const telegramResponse = await providerFetch(telegramApiUrl, { method: 'POST', headers: { 'Content-Type': 'application/json' }, }) diff --git a/apps/sim/lib/webhooks/providers/typeform.ts b/apps/sim/lib/webhooks/providers/typeform.ts index e366366868a..9bc9ca4b878 100644 --- a/apps/sim/lib/webhooks/providers/typeform.ts +++ b/apps/sim/lib/webhooks/providers/typeform.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { hmacSha256Base64 } from '@sim/security/hmac' import { getErrorMessage } from '@sim/utils/errors' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { DeleteSubscriptionContext, @@ -13,6 +14,10 @@ import type { } from '@/lib/webhooks/providers/types' import { createHmacVerifier } from '@/lib/webhooks/providers/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Typeform') function validateTypeformSignature(secret: string, signature: string, body: string): boolean { @@ -104,7 +109,7 @@ export const typeformHandler: WebhookProviderHandler = { requestBody.secret = secret } - const typeformResponse = await fetch(typeformApiUrl, { + const typeformResponse = await providerFetch(typeformApiUrl, { method: 'PUT', headers: { Authorization: `Bearer ${apiKey}`, @@ -192,7 +197,7 @@ export const typeformHandler: WebhookProviderHandler = { const tag = webhookTag || `sim-${(ctx.webhook.id as string).substring(0, 8)}` const typeformApiUrl = `https://api.typeform.com/forms/${formId}/webhooks/${tag}` - const typeformResponse = await fetch(typeformApiUrl, { + const typeformResponse = await providerFetch(typeformApiUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/vercel.test.ts b/apps/sim/lib/webhooks/providers/vercel.test.ts index 9bd56a7bab7..5b5c53ea9e6 100644 --- a/apps/sim/lib/webhooks/providers/vercel.test.ts +++ b/apps/sim/lib/webhooks/providers/vercel.test.ts @@ -2,10 +2,12 @@ * @vitest-environment node */ import crypto from 'crypto' -import { createMockRequest } from '@sim/testing' +import { createMockRequest, inputValidationMock } from '@sim/testing' import { describe, expect, it } from 'vitest' import { vercelHandler } from '@/lib/webhooks/providers/vercel' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + describe('vercelHandler', () => { describe('verifyAuth', () => { const secret = 'test-signing-secret' diff --git a/apps/sim/lib/webhooks/providers/vercel.ts b/apps/sim/lib/webhooks/providers/vercel.ts index 93319493703..4eaf6b83ad3 100644 --- a/apps/sim/lib/webhooks/providers/vercel.ts +++ b/apps/sim/lib/webhooks/providers/vercel.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { isRecordLike } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -15,6 +16,10 @@ import type { WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Vercel') function verifyVercelSignature(secret: string, signature: string, rawBody: string): boolean { @@ -143,7 +148,7 @@ export const vercelHandler: WebhookProviderHandler = { ? `https://api.vercel.com/v1/webhooks?teamId=${encodeURIComponent(teamId)}` : 'https://api.vercel.com/v1/webhooks' - const vercelResponse = await fetch(apiUrl, { + const vercelResponse = await providerFetch(apiUrl, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -228,7 +233,7 @@ export const vercelHandler: WebhookProviderHandler = { ? `https://api.vercel.com/v1/webhooks/${encodeURIComponent(externalId)}?teamId=${encodeURIComponent(teamId)}` : `https://api.vercel.com/v1/webhooks/${encodeURIComponent(externalId)}` - const response = await fetch(apiUrl, { + const response = await providerFetch(apiUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/webhooks/providers/webflow.ts b/apps/sim/lib/webhooks/providers/webflow.ts index 25fabfc36d4..1543adb90c1 100644 --- a/apps/sim/lib/webhooks/providers/webflow.ts +++ b/apps/sim/lib/webhooks/providers/webflow.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { validateAlphanumericId } from '@/lib/core/security/input-validation' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getBaseUrl } from '@/lib/core/utils/urls' import { getOAuthToken, refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getCredentialOwner, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' @@ -13,6 +14,10 @@ import type { WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Webflow') export const webflowHandler: WebhookProviderHandler = { @@ -103,7 +108,7 @@ export const webflowHandler: WebhookProviderHandler = { } } - const webflowResponse = await fetch(webflowApiUrl, { + const webflowResponse = await providerFetch(webflowApiUrl, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -219,7 +224,7 @@ export const webflowHandler: WebhookProviderHandler = { const webflowApiUrl = `https://api.webflow.com/v2/sites/${siteId}/webhooks/${externalId}` - const webflowResponse = await fetch(webflowApiUrl, { + const webflowResponse = await providerFetch(webflowApiUrl, { method: 'DELETE', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/webhooks/providers/zendesk.test.ts b/apps/sim/lib/webhooks/providers/zendesk.test.ts index 831c2ceaeea..763ea9de213 100644 --- a/apps/sim/lib/webhooks/providers/zendesk.test.ts +++ b/apps/sim/lib/webhooks/providers/zendesk.test.ts @@ -1,12 +1,16 @@ /** * @vitest-environment node */ + import crypto from 'crypto' +import { inputValidationMock } from '@sim/testing' import { NextRequest } from 'next/server' import { describe, expect, it } from 'vitest' import { zendeskHandler } from '@/lib/webhooks/providers/zendesk' import { isZendeskEventMatch } from '@/triggers/zendesk/utils' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const SECRET = 'my-signing-secret' function sign(secret: string, timestamp: string, body: string): string { diff --git a/apps/sim/lib/webhooks/providers/zendesk.ts b/apps/sim/lib/webhooks/providers/zendesk.ts index 452eec0ef37..42764021958 100644 --- a/apps/sim/lib/webhooks/providers/zendesk.ts +++ b/apps/sim/lib/webhooks/providers/zendesk.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { toRecord } from '@sim/utils/object' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -15,6 +16,10 @@ import type { WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:Zendesk') /** Zendesk API base for a subdomain. */ @@ -45,7 +50,7 @@ async function deleteZendeskWebhookQuietly( authHeader: string, webhookId: string ): Promise { - await fetch(`${apiBase}/webhooks/${webhookId}`, { + await providerFetch(`${apiBase}/webhooks/${webhookId}`, { method: 'DELETE', headers: { Authorization: authHeader }, }).catch(() => {}) @@ -187,7 +192,7 @@ export const zendeskHandler: WebhookProviderHandler = { const apiBase = zendeskApiBase(subdomain) const authHeader = zendeskAuthHeader(email, apiToken) - const createRes = await fetch(`${apiBase}/webhooks`, { + const createRes = await providerFetch(`${apiBase}/webhooks`, { method: 'POST', headers: { Authorization: authHeader, 'Content-Type': 'application/json' }, body: JSON.stringify({ @@ -219,7 +224,7 @@ export const zendeskHandler: WebhookProviderHandler = { const externalId = toRecord(created.webhook).id as string | undefined if (!externalId) throw new Error('Zendesk webhook created but no webhook ID was returned.') - const secretRes = await fetch(`${apiBase}/webhooks/${externalId}/signing_secret`, { + const secretRes = await providerFetch(`${apiBase}/webhooks/${externalId}/signing_secret`, { headers: { Authorization: authHeader }, }) if (!secretRes.ok) { @@ -264,7 +269,7 @@ export const zendeskHandler: WebhookProviderHandler = { return } - const res = await fetch(`${zendeskApiBase(subdomain)}/webhooks/${externalId}`, { + const res = await providerFetch(`${zendeskApiBase(subdomain)}/webhooks/${externalId}`, { method: 'DELETE', headers: { Authorization: zendeskAuthHeader(email, apiToken) }, }) diff --git a/apps/sim/lib/webhooks/providers/zoho-desk.test.ts b/apps/sim/lib/webhooks/providers/zoho-desk.test.ts index fee16ac6612..940262a251a 100644 --- a/apps/sim/lib/webhooks/providers/zoho-desk.test.ts +++ b/apps/sim/lib/webhooks/providers/zoho-desk.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterEach, describe, expect, it, vi } from 'vitest' vi.mock('@/lib/oauth/credential-service', () => ({ @@ -20,6 +21,8 @@ import { import { getCredentialOwner } from '@/lib/webhooks/provider-subscription-utils' import { mapZohoWebhookError, zohoDeskHandler } from '@/lib/webhooks/providers/zoho-desk' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + function errorStatus(err: unknown): number | undefined { return (err as { status?: number })?.status } diff --git a/apps/sim/lib/webhooks/providers/zoho-desk.ts b/apps/sim/lib/webhooks/providers/zoho-desk.ts index 719038fe31b..1fd1d521a50 100644 --- a/apps/sim/lib/webhooks/providers/zoho-desk.ts +++ b/apps/sim/lib/webhooks/providers/zoho-desk.ts @@ -6,6 +6,7 @@ import { truncate } from '@sim/utils/string' import { eq } from 'drizzle-orm' import * as jose from 'jose' import { NextResponse } from 'next/server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { getCredentialOwner, getNotificationUrl } from '@/lib/webhooks/provider-subscription-utils' import type { @@ -20,6 +21,10 @@ import type { import { isZohoHost } from '@/tools/zoho_desk/host-allowlist' import { withDerivedContentText } from '@/tools/zoho_desk/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WebhookProvider:ZohoDesk') const DEFAULT_ZOHO_DESK_BASE = 'https://desk.zoho.com' @@ -310,7 +315,7 @@ export const zohoDeskHandler: WebhookProviderHandler = { if (direction === 'in' || direction === 'out') filter.direction = direction } - const response = await fetch(`${apiDomain}/api/v1/webhooks`, { + const response = await providerFetch(`${apiDomain}/api/v1/webhooks`, { method: 'POST', headers: { Authorization: `Zoho-oauthtoken ${accessToken}`, @@ -399,7 +404,7 @@ export const zohoDeskHandler: WebhookProviderHandler = { ? safeZohoDeskBase(config.apiDomain) : await resolveZohoDeskApiDomain(owner.accountId) - const response = await fetch(`${apiDomain}/api/v1/webhooks/${externalId}`, { + const response = await providerFetch(`${apiDomain}/api/v1/webhooks/${externalId}`, { method: 'DELETE', headers: { Authorization: `Zoho-oauthtoken ${accessToken}`, diff --git a/packages/testing/src/mocks/input-validation.mock.ts b/packages/testing/src/mocks/input-validation.mock.ts index 48b5a4a92d5..9c22b5a747a 100644 --- a/packages/testing/src/mocks/input-validation.mock.ts +++ b/packages/testing/src/mocks/input-validation.mock.ts @@ -29,6 +29,10 @@ export const inputValidationMockFns = { * ``` */ export const inputValidationMock = { + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + dispatcher: { close: vi.fn(async () => {}), destroy: vi.fn(async () => {}) }, + }), DEFAULT_MAX_RESPONSE_BYTES: 100 * 1024 * 1024, MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, validateUrlWithDNS: inputValidationMockFns.mockValidateUrlWithDNS, diff --git a/scripts/check-egress-boundary.ts b/scripts/check-egress-boundary.ts index 3909ddf245c..537fac33da9 100644 --- a/scripts/check-egress-boundary.ts +++ b/scripts/check-egress-boundary.ts @@ -55,6 +55,7 @@ const TRANSPORTS = new Set([ 'node:https', 'node:http2', 'undici', + 'undici/index.js', 'http-proxy-agent', 'https-proxy-agent', // Present in node_modules as transitive dependencies. Nothing scanned imports @@ -72,6 +73,10 @@ const TRANSPORTS = new Set([ const ALLOWED = new Set([ // The guard itself: resolves, classifies, pins, and follows redirects. 'apps/sim/lib/core/security/input-validation.server.ts', + /** TLS wrapping preserves the validated destination and upstream certificate identity. */ + 'apps/sim/lib/core/network/gateway.server.ts', + /** Owns validated direct and environment-proxy pools for the shared HTTP adapters. */ + 'apps/sim/lib/core/network/transport.server.ts', // Streaming MCP transport, built on the guard's pinned dispatcher. 'apps/sim/lib/mcp/pinned-fetch.ts', // Builds a dispatcher to carry a caller's deadline; issues no request itself. diff --git a/scripts/check-tool-registry-boundary.baseline.json b/scripts/check-tool-registry-boundary.baseline.json index af93cacb593..e4de733eac0 100644 --- a/scripts/check-tool-registry-boundary.baseline.json +++ b/scripts/check-tool-registry-boundary.baseline.json @@ -402,16 +402,16 @@ "gateways": {} }, "app/workspace/[workspaceId]/settings/[section]/page.tsx": { - "modules": 2292, + "modules": 2341, "gateways": { - "apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx": 707, + "apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx": 736, "apps/sim/triggers/registry.ts": 485, - "apps/sim/lib/auth/index.ts": 366, + "apps/sim/lib/auth/index.ts": 378, "apps/sim/blocks/registry.ts": 354, - "apps/sim/lib/webhooks/providers/index.ts": 118, - "apps/sim/lib/webhooks/providers/registry.ts": 116, - "apps/sim/ee/access-control/components/access-control.tsx": 75, - "apps/sim/ee/access-control/components/group-detail.tsx": 73 + "apps/sim/lib/webhooks/providers/index.ts": 117, + "apps/sim/lib/webhooks/providers/registry.ts": 115, + "apps/sim/app/workspace/[workspaceId]/settings/components/recently-deleted/recently-deleted.tsx": 80, + "apps/sim/ee/access-control/components/access-control.tsx": 75 } }, "app/workspace/[workspaceId]/settings/billing/credit-usage/layout.tsx": { From 3082de6efd235ec088940230d1a5f929f463cc22 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 12 Sep 2026 03:11:15 -0700 Subject: [PATCH 2/2] fix(network): replay Request bodies across redirects --- .../guarded-request-fetch.server.test.ts | 87 +++++++++++++++++++ .../core/security/input-validation.server.ts | 36 ++++---- 2 files changed, 108 insertions(+), 15 deletions(-) diff --git a/apps/sim/lib/core/security/guarded-request-fetch.server.test.ts b/apps/sim/lib/core/security/guarded-request-fetch.server.test.ts index f08893e8513..e4ceb5fd380 100644 --- a/apps/sim/lib/core/security/guarded-request-fetch.server.test.ts +++ b/apps/sim/lib/core/security/guarded-request-fetch.server.test.ts @@ -134,6 +134,93 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { expect(await response.text()).toBe('final-body') }) + it.each([ + [307, 'POST'], + [308, 'POST'], + [301, 'PUT'], + [302, 'PUT'], + ] as const)('replays a Request body through same-origin %s redirects', async (status, method) => { + const payloads: string[] = [] + mockUndiciRequest.mockImplementation(async (_url, options: { body: Buffer | Readable }) => { + const chunks: Buffer[] = [] + for await (const chunk of options.body instanceof Readable ? options.body : [options.body]) { + chunks.push(Buffer.from(chunk)) + } + payloads.push(Buffer.concat(chunks).toString()) + return payloads.length < 3 + ? undiciReply(status, { location: `/hop-${payloads.length}` }, byteStream('')) + : undiciReply(200, {}, byteStream('done')) + }) + const transport = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) + try { + const response = await transport.fetch( + new Request('https://api.example.com/start', { + method, + headers: { 'content-type': 'application/json' }, + body: '{"payload":"replay"}', + }) + ) + expect(await response.text()).toBe('done') + expect(payloads).toEqual(Array(3).fill('{"payload":"replay"}')) + expect(response.redirected).toBe(true) + expect(mockUndiciRequest.mock.calls.map(([, options]) => options.method)).toEqual( + Array(3).fill(method) + ) + } finally { + await transport.dispatcher.destroy() + } + }) + + it.each(['manual', 'error'] as const)( + 'keeps Request bodies streaming in %s mode', + async (redirect) => { + const request = new Request('https://api.example.com/upload', { + method: 'POST', + redirect, + body: 'payload', + }) + const transport = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) + mockUndiciRequest.mockImplementationOnce(async (_url, options: { body: Readable }) => { + expect(options.body).toBeInstanceOf(Readable) + const chunks: Buffer[] = [] + for await (const chunk of options.body) chunks.push(Buffer.from(chunk)) + expect(Buffer.concat(chunks).toString()).toBe('payload') + return undiciReply(200, {}, byteStream('done')) + }) + try { + expect(await (await transport.fetch(request)).text()).toBe('done') + } finally { + await transport.dispatcher.destroy() + } + } + ) + + it('does not read the Request body when init supplies a replacement', async () => { + const request = new Request('https://api.example.com/upload', { + method: 'POST', + body: 'original', + }) + const clone = vi.spyOn(request, 'clone') + const bodyOverride = vi.fn().mockReturnValueOnce('replacement').mockReturnValue(undefined) + mockUndiciRequest.mockResolvedValueOnce(undiciReply(200, {}, byteStream('done'))) + const transport = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) + try { + const response = await transport.fetch(request, { + get body() { + return bodyOverride() + }, + }) + expect(await response.text()).toBe('done') + expect(mockUndiciRequest.mock.calls[0][1].body).toBe('replacement') + expect(request.bodyUsed).toBe(false) + expect(clone).not.toHaveBeenCalled() + expect(bodyOverride).toHaveBeenCalledTimes(1) + } finally { + await request.body?.cancel() + await transport.dispatcher.destroy() + } + }) + it('supports buffered reads (.json()) through the constructed body', async () => { mockUndiciRequest.mockResolvedValueOnce( undiciReply( diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index bcea44d11b7..6d29ee1524d 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -893,26 +893,32 @@ async function undiciRequestAsResponse( * fetch semantics) so a manual redirect follower can't silently downgrade a POST Request to a * bare GET or lose its headers. */ -function liftFetchArgs( +async function liftFetchArgs( input: RequestInfo | URL, init?: RequestInit -): { target: string; effectiveInit: RequestInit } { +): Promise<{ target: string; effectiveInit: RequestInit }> { const target = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url if (typeof Request !== 'undefined' && input instanceof Request) { const bodyAllowed = input.method !== 'GET' && input.method !== 'HEAD' - return { - target, - effectiveInit: { - method: input.method, - headers: input.headers, - body: bodyAllowed ? input.body : undefined, - signal: input.signal, - // Carry the Request's redirect mode so the pinned fetch honors `manual`/`error` - // instead of defaulting a `Request({ redirect: 'manual' })` to `follow`. - redirect: input.redirect, - ...init, - }, + const effectiveInit: RequestInit = { + method: input.method, + headers: input.headers, + body: bodyAllowed ? input.body : undefined, + signal: input.signal, + // Carry the Request's redirect mode so the pinned fetch honors `manual`/`error` + // instead of defaulting a `Request({ redirect: 'manual' })` to `follow`. + redirect: input.redirect, + ...init, + } + /** Request hides its original body source, so following redirects requires replayable bytes. */ + if ( + !Object.hasOwn(init ?? {}, 'body') && + effectiveInit.body && + (effectiveInit.redirect ?? 'follow') === 'follow' + ) { + effectiveInit.body = await input.clone().arrayBuffer() } + return { target, effectiveInit } } return { target, effectiveInit: init ?? {} } } @@ -941,7 +947,7 @@ function createValidatedFetch( return { dispatcher, fetch: async (input: RequestInfo | URL, init?: RequestInit): Promise => { - const { target, effectiveInit } = liftFetchArgs(input, init) + const { target, effectiveInit } = await liftFetchArgs(input, init) const mode = effectiveInit.redirect ?? 'follow' // double-cast-allowed: DOM and Undici RequestInit represent the same wire request in this bridge const undiciInit = effectiveInit as unknown as UndiciRequestInit