Firecrawl + Next.js: دليل استخراج البيانات من الويب بالذكاء الاصطناعي

npx create-next-app@latest competitor-intel --typescript --tailwind --app
cd competitor-intel

npm install @mendable/firecrawl-js zod

FIRECRAWL_API_KEY=fc-your-api-key-here

import FirecrawlApp from '@mendable/firecrawl-js';
 
if (!process.env.FIRECRAWL_API_KEY) {
  throw new Error('FIRECRAWL_API_KEY is not defined');
}
 
export const firecrawl = new FirecrawlApp({
  apiKey: process.env.FIRECRAWL_API_KEY,
});

import { NextRequest, NextResponse } from 'next/server';
import { firecrawl } from '@/lib/firecrawl';
 
export async function POST(request: NextRequest) {
  const { url } = await request.json();
 
  if (!url || typeof url !== 'string') {
    return NextResponse.json({ error: 'URL is required' }, { status: 400 });
  }
 
  try {
    const result = await firecrawl.scrapeUrl(url, {
      formats: ['markdown', 'html'],
    });
 
    if (!result.success) {
      return NextResponse.json({ error: 'Scrape failed' }, { status: 500 });
    }
 
    return NextResponse.json({
      markdown: result.markdown,
      title: result.metadata?.title,
      description: result.metadata?.description,
    });
  } catch (error) {
    return NextResponse.json(
      { error: 'Failed to scrape URL' },
      { status: 500 }
    );
  }
}

import { z } from 'zod';
 
export const ProductSchema = z.object({
  name: z.string().describe('Product or service name'),
  tagline: z.string().optional().describe('Main marketing tagline'),
  pricing: z
    .array(
      z.object({
        plan: z.string(),
        price: z.string(),
        features: z.array(z.string()),
      })
    )
    .optional()
    .describe('Pricing tiers with features'),
  mainFeatures: z.array(z.string()).describe('Top 5 key features'),
  targetAudience: z.string().optional().describe('Who the product is for'),
  techStack: z.array(z.string()).optional().describe('Technologies mentioned'),
});
 
export type Product = z.infer<typeof ProductSchema>;

import { NextRequest, NextResponse } from 'next/server';
import { firecrawl } from '@/lib/firecrawl';
import { ProductSchema } from '@/lib/schemas';
 
export async function POST(request: NextRequest) {
  const { url } = await request.json();
 
  if (!url || typeof url !== 'string') {
    return NextResponse.json({ error: 'URL is required' }, { status: 400 });
  }
 
  try {
    const result = await firecrawl.scrapeUrl(url, {
      formats: ['extract'],
      extract: {
        schema: ProductSchema,
        prompt:
          'Extract product information, pricing tiers, and key features from this page.',
      },
    });
 
    if (!result.success || !result.extract) {
      return NextResponse.json({ error: 'Extraction failed' }, { status: 500 });
    }
 
    return NextResponse.json({ data: result.extract });
  } catch (error) {
    return NextResponse.json(
      { error: 'Failed to extract data' },
      { status: 500 }
    );
  }
}

import { NextRequest, NextResponse } from 'next/server';
import { firecrawl } from '@/lib/firecrawl';
 
export async function POST(request: NextRequest) {
  const { url, limit = 10 } = await request.json();
 
  if (!url || typeof url !== 'string') {
    return NextResponse.json({ error: 'URL is required' }, { status: 400 });
  }
 
  try {
    const crawlResponse = await firecrawl.asyncCrawlUrl(url, {
      limit,
      scrapeOptions: {
        formats: ['markdown'],
      },
      excludePaths: ['/blog/*', '/news/*'],
    });
 
    if (!crawlResponse.success) {
      return NextResponse.json(
        { error: 'Crawl failed to start' },
        { status: 500 }
      );
    }
 
    return NextResponse.json({ jobId: crawlResponse.id });
  } catch (error) {
    return NextResponse.json(
      { error: 'Failed to start crawl' },
      { status: 500 }
    );
  }
}

// app/api/crawl/status/route.ts
import { NextRequest, NextResponse } from 'next/server';
import { firecrawl } from '@/lib/firecrawl';
 
export async function GET(request: NextRequest) {
  const jobId = request.nextUrl.searchParams.get('jobId');
 
  if (!jobId) {
    return NextResponse.json({ error: 'jobId is required' }, { status: 400 });
  }
 
  const status = await firecrawl.checkCrawlStatus(jobId);
 
  return NextResponse.json({
    status: status.status,
    completed: status.completed,
    total: status.total,
    pages: status.status === 'completed' ? status.data : [],
  });
}

const siteMap = await firecrawl.mapUrl('https://competitor.com', {
  search: 'pricing',
  limit: 50,
});
 
console.log(siteMap.links);
// ['https://competitor.com/pricing', 'https://competitor.com/pricing/enterprise', ...]

'use client';
 
import { useState } from 'react';
import type { Product } from '@/lib/schemas';
 
export default function IntelligenceDashboard() {
  const [url, setUrl] = useState('');
  const [loading, setLoading] = useState(false);
  const [product, setProduct] = useState<Product | null>(null);
  const [error, setError] = useState<string | null>(null);
 
  async function handleExtract() {
    if (!url) return;
    setLoading(true);
    setError(null);
 
    try {
      const res = await fetch('/api/extract', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ url }),
      });
      const json = await res.json();
 
      if (!res.ok) {
        setError(json.error ?? 'Extraction failed');
        return;
      }
      setProduct(json.data);
    } catch {
      setError('Network error — please try again');
    } finally {
      setLoading(false);
    }
  }
 
  return (
    <div className="max-w-4xl mx-auto p-8" dir="rtl">
      <h1 className="text-3xl font-bold mb-2">استخباراتية المنافسين</h1>
      <p className="text-gray-600 mb-8">
        أدخل أي رابط منافس لاستخراج بيانات المنتج البنيوية بالذكاء الاصطناعي.
      </p>
 
      <div className="flex gap-2 mb-8">
        <input
          type="url"
          value={url}
          onChange={(e) => setUrl(e.target.value)}
          placeholder="https://competitor.com/pricing"
          className="flex-1 border rounded-lg px-4 py-2 text-sm"
          dir="ltr"
        />
        <button
          onClick={handleExtract}
          disabled={loading || !url}
          className="bg-orange-500 text-white px-6 py-2 rounded-lg disabled:opacity-50"
        >
          {loading ? 'جاري الاستخراج...' : 'استخراج'}
        </button>
      </div>
 
      {error && (
        <div className="bg-red-50 border border-red-200 rounded-lg p-4 mb-6 text-red-700">
          {error}
        </div>
      )}
 
      {product && <ProductCard product={product} />}
    </div>
  );
}
 
function ProductCard({ product }: { product: Product }) {
  return (
    <div className="border rounded-xl p-6 space-y-4" dir="rtl">
      <div>
        <h2 className="text-2xl font-bold">{product.name}</h2>
        {product.tagline && (
          <p className="text-gray-600 mt-1">{product.tagline}</p>
        )}
        {product.targetAudience && (
          <p className="text-sm text-orange-600 mt-1">
            الجمهور المستهدف: {product.targetAudience}
          </p>
        )}
      </div>
 
      <div>
        <h3 className="font-semibold mb-2">المميزات الرئيسية</h3>
        <ul className="list-disc list-inside space-y-1 text-sm text-gray-700">
          {product.mainFeatures.map((f, i) => (
            <li key={i}>{f}</li>
          ))}
        </ul>
      </div>
 
      {product.pricing && product.pricing.length > 0 && (
        <div>
          <h3 className="font-semibold mb-2">خطط الأسعار</h3>
          <div className="grid grid-cols-1 md:grid-cols-3 gap-3">
            {product.pricing.map((plan, i) => (
              <div key={i} className="border rounded-lg p-3 text-sm">
                <div className="font-medium">{plan.plan}</div>
                <div className="text-orange-600 font-bold">{plan.price}</div>
              </div>
            ))}
          </div>
        </div>
      )}
    </div>
  );
}

// lib/firecrawl-retry.ts
import { firecrawl } from './firecrawl';
 
export async function scrapeWithRetry(
  url: string,
  options = {},
  maxRetries = 3
) {
  let lastError: Error | null = null;
 
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      return await firecrawl.scrapeUrl(url, options);
    } catch (error) {
      lastError = error as Error;
 
      if (attempt < maxRetries) {
        // تراجع أسي: 1 ثانية، 2 ثانية، 4 ثواني
        await new Promise((resolve) =>
          setTimeout(resolve, Math.pow(2, attempt - 1) * 1000)
        );
      }
    }
  }
 
  throw lastError;
}

async function batchScrape(urls: string[]) {
  const results = [];
 
  for (const url of urls) {
    const result = await scrapeWithRetry(url);
    results.push(result);
    // 500 ملي ثانية بين الطلبات تمنع تجاوز حدود المعدل
    await new Promise((resolve) => setTimeout(resolve, 500));
  }
 
  return results;
}

import { unstable_cache } from 'next/cache';
import { firecrawl } from '@/lib/firecrawl';
import { ProductSchema } from '@/lib/schemas';
 
export const getCachedProductData = unstable_cache(
  async (url: string) => {
    const result = await firecrawl.scrapeUrl(url, {
      formats: ['extract'],
      extract: { schema: ProductSchema },
    });
    return result.extract;
  },
  ['product-data'],
  { revalidate: 3600 } // تخزين مؤقت لمدة ساعة
);

const result = await firecrawl.scrapeUrl(url, {
  formats: ['extract'],
  waitFor: 2000,
  extract: { schema: ProductSchema },
});

export const maxDuration = 60;