Firecrawl: scrape any website in 3 lines.

1FIRECRAWL_API_KEY=fc-...

1npm install @mendable/firecrawl-js

1import FirecrawlApp from "@mendable/firecrawl-js";
2 
3const fc = new FirecrawlApp({ apiKey: process.env.FIRECRAWL_API_KEY! });
4 
5export async function scrape(url: string) {
6  const res = await fc.scrapeUrl(url, {
7    formats: ["markdown", "links"],
8    onlyMainContent: true,
9  });
10  if (!res.success) throw new Error(res.error || "scrape failed");
11  return {
12    markdown: res.markdown ?? "",
13    links: res.links ?? [],
14    title: res.metadata?.title,
15  };
16}

1export async function crawl(url: string, opts: { limit?: number } = {}) {
2  const job = await fc.crawlUrl(url, {
3    limit: opts.limit ?? 50,
4    scrapeOptions: { formats: ["markdown"], onlyMainContent: true },
5  });
6  if (!job.success) throw new Error(job.error || "crawl failed");
7  return job.data; // array of pages
8}

1import OpenAI from "openai";
2import { scrape } from "@/lib/firecrawl";
3 
4const ai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
5 
6export async function POST(req: Request) {
7  const { url, question } = await req.json();
8  const page = await scrape(url);
9 
10  const completion = await ai.chat.completions.create({
11    model: "gpt-5-mini",
12    messages: [
13      {
14        role: "system",
15        content:
16          "You answer questions strictly from the provided page content. Cite the source URL.",
17      },
18      {
19        role: "user",
20        content: `URL: ${url}\n\nPAGE:\n${page.markdown.slice(0, 12000)}\n\nQUESTION: ${question}`,
21      },
22    ],
23  });
24 
25  return Response.json({
26    answer: completion.choices[0].message.content,
27    source: url,
28  });
29}