mkdir ai-scraper && cd ai-scraper
npm init -y

npm install playwright @anthropic-ai/sdk zod commander dotenv
npm install -D typescript @types/node tsx

npx playwright install chromium

npx tsc --init

{
  "compilerOptions": {
    "target": "ES2022",
    "module": "ESNext",
    "moduleResolution": "bundler",
    "strict": true,
    "esModuleInterop": true,
    "outDir": "./dist",
    "rootDir": "./src",
    "declaration": true,
    "resolveJsonModule": true,
    "skipLibCheck": true
  },
  "include": ["src/**/*"]
}

mkdir -p src/{extractors,utils}
touch .env src/index.ts src/scraper.ts src/ai-extractor.ts

ANTHROPIC_API_KEY=sk-ant-your-key-here
MAX_RETRIES=3
RATE_LIMIT_MS=1000

import { z } from "zod";
 
// مخطط العنصر المستخرَج — خصّصه حسب حاجتك
export const ScrapedItemSchema = z.object({
  title: z.string(),
  description: z.string().optional(),
  price: z.string().optional(),
  url: z.string().url().optional(),
  imageUrl: z.string().url().optional(),
  metadata: z.record(z.string()).optional(),
});
 
export type ScrapedItem = z.infer<typeof ScrapedItemSchema>;
 
// مخطط نتيجة الاستخراج الكاملة
export const ExtractionResultSchema = z.object({
  items: z.array(ScrapedItemSchema),
  totalFound: z.number(),
  pageInfo: z.object({
    title: z.string(),
    url: z.string(),
    scrapedAt: z.string(),
  }),
});
 
export type ExtractionResult = z.infer<typeof ExtractionResultSchema>;
 
// إعدادات أداة الاستخراج
export interface ScraperConfig {
  url: string;
  prompt: string;
  schema?: z.ZodSchema;
  waitForSelector?: string;
  maxPages?: number;
  screenshot?: boolean;
  timeout?: number;
}
 
// محتوى صفحة المتصفح
export interface PageContent {
  html: string;
  text: string;
  url: string;
  title: string;
  screenshot?: Buffer;
}

import { chromium, Browser, Page } from "playwright";
import type { PageContent, ScraperConfig } from "./types";
 
export class BrowserScraper {
  private browser: Browser | null = null;
 
  async launch(): Promise<void> {
    this.browser = await chromium.launch({
      headless: true,
      args: [
        "--no-sandbox",
        "--disable-setuid-sandbox",
        "--disable-dev-shm-usage",
      ],
    });
  }
 
  async scrape(config: ScraperConfig): Promise<PageContent> {
    if (!this.browser) {
      throw new Error("Browser not launched. Call launch() first.");
    }
 
    const context = await this.browser.newContext({
      userAgent:
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " +
        "AppleWebKit/537.36 (KHTML, like Gecko) " +
        "Chrome/122.0.0.0 Safari/537.36",
      viewport: { width: 1280, height: 720 },
    });
 
    const page = await context.newPage();
 
    try {
      await page.goto(config.url, {
        waitUntil: "networkidle",
        timeout: config.timeout || 30000,
      });
 
      if (config.waitForSelector) {
        await page.waitForSelector(config.waitForSelector, {
          timeout: 10000,
        });
      }
 
      // التمرير التلقائي لتحميل المحتوى الكسول
      await this.autoScroll(page);
 
      const content = await this.extractContent(page);
 
      let screenshot: Buffer | undefined;
      if (config.screenshot) {
        screenshot = await page.screenshot({
          fullPage: true,
          type: "png",
        });
      }
 
      return { ...content, screenshot };
    } finally {
      await context.close();
    }
  }
 
  private async autoScroll(page: Page): Promise<void> {
    await page.evaluate(async () => {
      await new Promise<void>((resolve) => {
        let totalHeight = 0;
        const distance = 300;
        const timer = setInterval(() => {
          const scrollHeight = document.body.scrollHeight;
          window.scrollBy(0, distance);
          totalHeight += distance;
 
          if (totalHeight >= scrollHeight) {
            clearInterval(timer);
            resolve();
          }
        }, 100);
 
        // مهلة أمان: توقف التمرير بعد 10 ثوانٍ
        setTimeout(() => {
          clearInterval(timer);
          resolve();
        }, 10000);
      });
    });
  }
 
  private async extractContent(
    page: Page
  ): Promise<Omit<PageContent, "screenshot">> {
    const title = await page.title();
    const url = page.url();
 
    const text = await page.evaluate(() => {
      const scripts = document.querySelectorAll(
        "script, style, noscript, iframe"
      );
      scripts.forEach((el) => el.remove());
      return document.body.innerText || "";
    });
 
    const html = await page.evaluate(() => {
      return document.body.innerHTML;
    });
 
    return { html, text, url, title };
  }
 
  async close(): Promise<void> {
    if (this.browser) {
      await this.browser.close();
      this.browser = null;
    }
  }
}

import Anthropic from "@anthropic-ai/sdk";
import { z } from "zod";
import type { PageContent, ExtractionResult } from "./types";
import { ExtractionResultSchema } from "./types";
 
export class AIExtractor {
  private client: Anthropic;
  private model = "claude-sonnet-4-20250514";
 
  constructor(apiKey: string) {
    this.client = new Anthropic({ apiKey });
  }
 
  async extract(
    content: PageContent,
    userPrompt: string,
    schema?: z.ZodSchema
  ): Promise<ExtractionResult> {
    const systemPrompt = `You are a precise data extraction assistant.
Your job is to analyze web page content and extract structured data.
 
Rules:
1. Return ONLY valid JSON — no markdown, no explanations.
2. Extract ALL matching items from the page content.
3. If a field is not found, use null.
4. URLs should be absolute.
5. Clean up text: remove extra whitespace, fix encoding.
6. Be thorough — scan the entire content.`;
 
    const userMessage = `
Page URL: ${content.url}
Page Title: ${content.title}
 
--- PAGE CONTENT START ---
${this.truncateContent(content.text, 80000)}
--- PAGE CONTENT END ---
 
--- HTML STRUCTURE (first 20000 chars) ---
${this.truncateContent(content.html, 20000)}
--- HTML STRUCTURE END ---
 
EXTRACTION REQUEST: ${userPrompt}
 
Return JSON matching this structure:
{
  "items": [
    {
      "title": "string",
      "description": "string or null",
      "price": "string or null",
      "url": "absolute URL or null",
      "imageUrl": "absolute URL or null",
      "metadata": { "key": "value" }
    }
  ],
  "totalFound": number,
  "pageInfo": {
    "title": "page title",
    "url": "page url",
    "scrapedAt": "ISO date string"
  }
}`;
 
    const response = await this.client.messages.create({
      model: this.model,
      max_tokens: 4096,
      system: systemPrompt,
      messages: [{ role: "user", content: userMessage }],
    });
 
    const responseText = response.content
      .filter((block) => block.type === "text")
      .map((block) => block.text)
      .join("");
 
    return this.parseResponse(responseText, schema);
  }
 
  private parseResponse(
    text: string,
    schema?: z.ZodSchema
  ): ExtractionResult {
    let cleaned = text.trim();
 
    if (cleaned.startsWith("\`\`\`")) {
      cleaned = cleaned
        .replace(/^\`\`\`(?:json)?\n?/, "")
        .replace(/\n?\`\`\`$/, "");
    }
 
    let parsed: unknown;
    try {
      parsed = JSON.parse(cleaned);
    } catch (error) {
      throw new Error(
        `فشل تحليل استجابة الذكاء الاصطناعي كـ JSON: ${(error as Error).message}`
      );
    }
 
    const validationSchema = schema || ExtractionResultSchema;
    const result = validationSchema.safeParse(parsed);
 
    if (!result.success) {
      throw new Error(
        `فشل التحقق من الاستجابة:\n${result.error.issues
          .map((i) => `  - ${i.path.join(".")}: ${i.message}`)
          .join("\n")}`
      );
    }
 
    return result.data as ExtractionResult;
  }
 
  private truncateContent(text: string, maxChars: number): string {
    if (text.length <= maxChars) return text;
    return (
      text.substring(0, maxChars) +
      `\n\n[... تم اقتطاع ${text.length - maxChars} حرف ...]`
    );
  }
}

export interface RetryConfig {
  maxRetries: number;
  baseDelayMs: number;
  maxDelayMs: number;
}
 
const DEFAULT_CONFIG: RetryConfig = {
  maxRetries: 3,
  baseDelayMs: 1000,
  maxDelayMs: 10000,
};
 
export async function withRetry<T>(
  fn: () => Promise<T>,
  config: Partial<RetryConfig> = {}
): Promise<T> {
  const { maxRetries, baseDelayMs, maxDelayMs } = {
    ...DEFAULT_CONFIG,
    ...config,
  };
 
  let lastError: Error | undefined;
 
  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    try {
      return await fn();
    } catch (error) {
      lastError = error as Error;
      if (attempt === maxRetries) break;
 
      // تراجع أسي مع تشويش عشوائي
      const delay = Math.min(
        baseDelayMs * Math.pow(2, attempt) + Math.random() * 1000,
        maxDelayMs
      );
 
      console.warn(
        `المحاولة ${attempt + 1} فشلت: ${lastError.message}. ` +
        `إعادة المحاولة خلال ${Math.round(delay)} مللي ثانية...`
      );
 
      await new Promise((resolve) => setTimeout(resolve, delay));
    }
  }
 
  throw lastError;
}

export class RateLimiter {
  private lastCall = 0;
 
  constructor(private minIntervalMs: number) {}
 
  async wait(): Promise<void> {
    const now = Date.now();
    const elapsed = now - this.lastCall;
 
    if (elapsed < this.minIntervalMs) {
      const waitTime = this.minIntervalMs - elapsed;
      await new Promise((resolve) => setTimeout(resolve, waitTime));
    }
 
    this.lastCall = Date.now();
  }
}

import { BrowserScraper } from "./scraper";
import { AIExtractor } from "./ai-extractor";
import { RateLimiter } from "./utils/rate-limiter";
import { withRetry } from "./utils/retry";
import type { ScraperConfig, ExtractionResult, ScrapedItem } from "./types";
 
export class MultiPageScraper {
  private browserScraper: BrowserScraper;
  private aiExtractor: AIExtractor;
  private rateLimiter: RateLimiter;
 
  constructor(apiKey: string, rateLimitMs = 1000) {
    this.browserScraper = new BrowserScraper();
    this.aiExtractor = new AIExtractor(apiKey);
    this.rateLimiter = new RateLimiter(rateLimitMs);
  }
 
  async scrapeMultiplePages(
    configs: ScraperConfig[]
  ): Promise<ExtractionResult> {
    await this.browserScraper.launch();
 
    const allItems: ScrapedItem[] = [];
    let totalFound = 0;
 
    try {
      for (const [index, config] of configs.entries()) {
        console.log(
          `\n📄 استخراج الصفحة ${index + 1}/${configs.length}: ${config.url}`
        );
 
        await this.rateLimiter.wait();
 
        const result = await withRetry(async () => {
          const content = await this.browserScraper.scrape(config);
          console.log(
            `  ✓ تم تحميل الصفحة (${content.text.length} حرف)`
          );
 
          const extraction = await this.aiExtractor.extract(
            content,
            config.prompt,
            config.schema
          );
          console.log(
            `  ✓ تم استخراج ${extraction.items.length} عنصر`
          );
 
          return extraction;
        });
 
        allItems.push(...result.items);
        totalFound += result.totalFound;
      }
    } finally {
      await this.browserScraper.close();
    }
 
    return {
      items: allItems,
      totalFound,
      pageInfo: {
        title: `استخراج متعدد الصفحات (${configs.length} صفحات)`,
        url: configs[0]?.url || "",
        scrapedAt: new Date().toISOString(),
      },
    };
  }
}

import "dotenv/config";
import { Command } from "commander";
import { BrowserScraper } from "./scraper";
import { AIExtractor } from "./ai-extractor";
import { MultiPageScraper } from "./multi-page-scraper";
import { withRetry } from "./utils/retry";
import { writeFileSync } from "fs";
 
const program = new Command();
 
program
  .name("ai-scraper")
  .description("أداة استخراج بيانات ذكية باستخدام Playwright و Claude")
  .version("1.0.0");
 
program
  .command("scrape")
  .description("استخراج بيانات من رابط واحد")
  .requiredOption("-u, --url <url>", "الرابط المطلوب")
  .requiredOption("-p, --prompt <prompt>", "ما الذي تريد استخراجه")
  .option("-o, --output <file>", "ملف JSON للحفظ")
  .option("-s, --screenshot", "أخذ لقطة شاشة كاملة")
  .option("--wait <selector>", "محدد CSS للانتظار")
  .option("--timeout <ms>", "مهلة التنقل بالمللي ثانية", "30000")
  .action(async (options) => {
    const apiKey = process.env.ANTHROPIC_API_KEY;
    if (!apiKey) {
      console.error("❌ لم يتم تعيين ANTHROPIC_API_KEY في .env");
      process.exit(1);
    }
 
    const scraper = new BrowserScraper();
    const extractor = new AIExtractor(apiKey);
 
    try {
      console.log(`🌐 التنقل إلى ${options.url}...`);
      await scraper.launch();
 
      const content = await withRetry(() =>
        scraper.scrape({
          url: options.url,
          prompt: options.prompt,
          waitForSelector: options.wait,
          screenshot: options.screenshot,
          timeout: parseInt(options.timeout),
        })
      );
 
      console.log(`📝 تم تحميل الصفحة: "${content.title}"`);
 
      if (content.screenshot) {
        writeFileSync("screenshot.png", content.screenshot);
        console.log(`📸 تم حفظ لقطة الشاشة`);
      }
 
      console.log(`\n🤖 الإرسال إلى Claude للاستخراج...`);
      const result = await withRetry(() =>
        extractor.extract(content, options.prompt)
      );
 
      console.log(`✅ تم استخراج ${result.items.length} عنصر\n`);
 
      const output = JSON.stringify(result, null, 2);
 
      if (options.output) {
        writeFileSync(options.output, output);
        console.log(`💾 تم الحفظ في ${options.output}`);
      } else {
        console.log(output);
      }
    } catch (error) {
      console.error(`❌ خطأ: ${(error as Error).message}`);
      process.exit(1);
    } finally {
      await scraper.close();
    }
  });
 
program
  .command("multi")
  .description("استخراج من عدة روابط")
  .requiredOption("-u, --urls <urls...>", "الروابط (مفصولة بمسافات)")
  .requiredOption("-p, --prompt <prompt>", "ما الذي تريد استخراجه")
  .option("-o, --output <file>", "ملف JSON للحفظ")
  .action(async (options) => {
    const apiKey = process.env.ANTHROPIC_API_KEY;
    if (!apiKey) {
      console.error("❌ لم يتم تعيين ANTHROPIC_API_KEY في .env");
      process.exit(1);
    }
 
    const multiScraper = new MultiPageScraper(
      apiKey,
      parseInt(process.env.RATE_LIMIT_MS || "1000")
    );
 
    try {
      const configs = options.urls.map((url: string) => ({
        url,
        prompt: options.prompt,
      }));
 
      const result = await multiScraper.scrapeMultiplePages(configs);
      const output = JSON.stringify(result, null, 2);
 
      if (options.output) {
        writeFileSync(options.output, output);
        console.log(`\n💾 تم حفظ ${result.items.length} عنصر في ${options.output}`);
      } else {
        console.log(output);
      }
    } catch (error) {
      console.error(`❌ خطأ: ${(error as Error).message}`);
      process.exit(1);
    }
  });
 
program.parse();

{
  "type": "module",
  "scripts": {
    "scrape": "tsx src/index.ts scrape",
    "multi": "tsx src/index.ts multi",
    "build": "tsc",
    "start": "node dist/index.js"
  }
}

npx tsx src/index.ts scrape \
  -u "https://books.toscrape.com" \
  -p "Extract all book titles, prices, ratings, and availability" \
  -o books.json

{
  "items": [
    {
      "title": "A Light in the Attic",
      "price": "£51.77",
      "metadata": {
        "rating": "Three",
        "availability": "In stock"
      }
    }
  ],
  "totalFound": 20,
  "pageInfo": {
    "title": "All products | Books to Scrape",
    "url": "https://books.toscrape.com",
    "scrapedAt": "2026-03-13T12:00:00.000Z"
  }
}

npx tsx src/index.ts scrape \
  -u "https://news.ycombinator.com/jobs" \
  -p "Extract all job postings: company name, job title, posting date, and link" \
  -o hn-jobs.json

npx tsx src/index.ts multi \
  -u "https://books.toscrape.com/catalogue/page-1.html" \
     "https://books.toscrape.com/catalogue/page-2.html" \
     "https://books.toscrape.com/catalogue/page-3.html" \
  -p "Extract all book titles and prices" \
  -o all-books.json

import { z } from "zod";
 
export const ProductSchema = z.object({
  items: z.array(
    z.object({
      name: z.string(),
      price: z.object({
        amount: z.number(),
        currency: z.string(),
      }),
      rating: z.number().min(0).max(5).optional(),
      reviewCount: z.number().optional(),
      availability: z.enum(["in_stock", "out_of_stock", "preorder"]),
      sku: z.string().optional(),
      brand: z.string().optional(),
      category: z.string().optional(),
      url: z.string().url(),
      imageUrl: z.string().url().optional(),
    })
  ),
  totalFound: z.number(),
  pageInfo: z.object({
    title: z.string(),
    url: z.string(),
    scrapedAt: z.string(),
  }),
});
 
export type ProductResult = z.infer<typeof ProductSchema>;

import { ProductSchema } from "./extractors/product-extractor";
 
const result = await extractor.extract(content, userPrompt, ProductSchema);
// النتيجة الآن محددة الأنواع بالكامل