references/common-patterns.md
# Firecrawl Common Patterns & Best Practices
**Last Updated**: 2025-10-24
---
## Table of Contents
1. [Error Handling](#error-handling)
2. [Retry Strategies](#retry-strategies)
3. [Rate Limit Management](#rate-limit-management)
4. [Batch Processing](#batch-processing)
5. [Caching Strategies](#caching-strategies)
6. [Progress Tracking](#progress-tracking)
7. [Data Storage Patterns](#data-storage-patterns)
8. [Cloudflare Workers Integration](#cloudflare-workers-integration)
---
## Error Handling
### Python: Comprehensive Error Handling
```python
from firecrawl import FirecrawlApp
from firecrawl.exceptions import FirecrawlException
import time
def scrape_with_error_handling(url: str, retries: int = 3) -> dict:
"""
Scrape a URL with comprehensive error handling and retries.
"""
app = FirecrawlApp(api_key=os.environ.get("FIRECRAWL_API_KEY"))
for attempt in range(retries):
try:
result = app.scrape_url(url, params={
"formats": ["markdown"],
"onlyMainContent": True
})
return result
except FirecrawlException as e:
# Firecrawl-specific errors
if "Rate limit" in str(e):
print(f"Rate limited. Waiting before retry {attempt + 1}/{retries}")
time.sleep(60) # Wait 1 minute
continue
elif "Invalid API key" in str(e):
print("Invalid API key. Check your environment variables.")
raise
else:
print(f"Firecrawl error: {e}")
if attempt < retries - 1:
time.sleep(5 * (attempt + 1)) # Exponential backoff
continue
raise
except ConnectionError as e:
print(f"Connection error: {e}")
if attempt < retries - 1:
time.sleep(10)
continue
raise
except Exception as e:
print(f"Unexpected error: {e}")
raise
raise Exception(f"Failed to scrape {url} after {retries} attempts")
```
### TypeScript: Comprehensive Error Handling
```typescript
import FirecrawlApp from 'firecrawl-js';
async function scrapeWithErrorHandling(
url: string,
retries: number = 3
): Promise<any> {
const app = new FirecrawlApp({
apiKey: process.env.FIRECRAWL_API_KEY
});
for (let attempt = 0; attempt < retries; attempt++) {
try {
const result = await app.scrapeUrl(url, {
formats: ['markdown'],
onlyMainContent: true
});
return result;
} catch (error: any) {
// API errors
if (error.response) {
const status = error.response.status;
const message = error.response.data?.error || error.message;
if (status === 429) {
// Rate limited
console.log(`Rate limited. Waiting before retry ${attempt + 1}/${retries}`);
await new Promise(resolve => setTimeout(resolve, 60000));
continue;
} else if (status === 401) {
// Invalid API key
console.error('Invalid API key. Check your environment variables.');
throw error;
} else if (status >= 500) {
// Server error
console.log(`Server error (${status}). Retrying...`);
await new Promise(resolve => setTimeout(resolve, 5000 * (attempt + 1)));
continue;
} else {
console.error(`API error (${status}): ${message}`);
throw error;
}
}
// Network errors
if (error.code === 'ECONNREFUSED' || error.code === 'ETIMEDOUT') {
console.log(`Network error: ${error.message}`);
if (attempt < retries - 1) {
await new Promise(resolve => setTimeout(resolve, 10000));
continue;
}
}
// Unexpected errors
throw error;
}
}
throw new Error(`Failed to scrape ${url} after ${retries} attempts`);
}
```
---
## Retry Strategies
### Exponential Backoff
```python
import time
from typing import Callable, Any
def exponential_backoff(
func: Callable,
max_retries: int = 5,
base_delay: int = 1
) -> Any:
"""
Retry a function with exponential backoff.
"""
for attempt in range(max_retries):
try:
return func()
except Exception as e:
if attempt == max_retries - 1:
raise
delay = base_delay * (2 ** attempt)
print(f"Attempt {attempt + 1} failed. Retrying in {delay}s...")
time.sleep(delay)
# Usage
result = exponential_backoff(
lambda: app.scrape_url("https://example.com"),
max_retries=5,
base_delay=2
)
```
### Circuit Breaker Pattern
```python
from datetime import datetime, timedelta
class CircuitBreaker:
"""
Circuit breaker to prevent cascading failures.
"""
def __init__(self, failure_threshold: int = 5, timeout: int = 60):
self.failure_threshold = failure_threshold
self.timeout = timeout
self.failures = 0
self.last_failure_time = None
self.state = "closed" # closed, open, half-open
def call(self, func: Callable, *args, **kwargs) -> Any:
if self.state == "open":
if datetime.now() - self.last_failure_time > timedelta(seconds=self.timeout):
self.state = "half-open"
else:
raise Exception("Circuit breaker is OPEN")
try:
result = func(*args, **kwargs)
if self.state == "half-open":
self.state = "closed"
self.failures = 0
return result
except Exception as e:
self.failures += 1
self.last_failure_time = datetime.now()
if self.failures >= self.failure_threshold:
self.state = "open"
print(f"Circuit breaker OPEN after {self.failures} failures")
raise
# Usage
breaker = CircuitBreaker(failure_threshold=5, timeout=60)
for url in urls:
try:
result = breaker.call(app.scrape_url, url)
except Exception as e:
print(f"Failed: {e}")
```
---
## Rate Limit Management
### Credit Tracking
```python
class CreditTracker:
"""
Track credit usage and prevent exceeding limits.
"""
def __init__(self, daily_limit: int = 500):
self.daily_limit = daily_limit
self.credits_used = 0
def estimate_credits(self, operation: str, **kwargs) -> int:
"""Estimate credits for an operation."""
if operation == "scrape":
credits = 1
if kwargs.get("screenshot"):
credits += 1
if kwargs.get("actions"):
credits += 1
return credits
elif operation == "crawl":
pages = kwargs.get("limit", 100)
return pages * self.estimate_credits("scrape", **kwargs.get("scrapeOptions", {}))
elif operation == "extract":
return len(kwargs.get("urls", [])) * 5
return 1
def check_and_use(self, credits: int) -> bool:
"""Check if enough credits and mark as used."""
if self.credits_used + credits > self.daily_limit:
print(f"⚠️ Would exceed daily limit ({self.credits_used + credits}/{self.daily_limit})")
return False
self.credits_used += credits
print(f"Credits used: {self.credits_used}/{self.daily_limit}")
return True
# Usage
tracker = CreditTracker(daily_limit=500)
for url in urls:
credits_needed = tracker.estimate_credits("scrape", screenshot=False)
if tracker.check_and_use(credits_needed):
result = app.scrape_url(url)
else:
print("Daily limit reached. Stopping.")
break
```
---
## Batch Processing
### Process URLs in Batches
```python
from typing import List
import time
def batch_scrape(
urls: List[str],
batch_size: int = 10,
delay_between_batches: int = 5
) -> List[dict]:
"""
Scrape URLs in batches with delays.
"""
app = FirecrawlApp(api_key=os.environ.get("FIRECRAWL_API_KEY"))
results = []
for i in range(0, len(urls), batch_size):
batch = urls[i:i + batch_size]
print(f"Processing batch {i // batch_size + 1} ({len(batch)} URLs)")
for url in batch:
try:
result = app.scrape_url(url, params={
"formats": ["markdown"],
"onlyMainContent": True
})
results.append(result)
except Exception as e:
print(f"Failed to scrape {url}: {e}")
results.append({"url": url, "error": str(e)})
# Delay between batches
if i + batch_size < len(urls):
print(f"Waiting {delay_between_batches}s before next batch...")
time.sleep(delay_between_batches)
return results
# Usage
urls = ["https://example.com/page1", "https://example.com/page2", ...]
results = batch_scrape(urls, batch_size=10, delay_between_batches=5)
```
---
## Caching Strategies
### Simple File-Based Cache
```python
import os
import json
import hashlib
from pathlib import Path
class ScrapeCache:
"""
Cache scraped content to avoid re-scraping.
"""
def __init__(self, cache_dir: str = ".cache"):
self.cache_dir = Path(cache_dir)
self.cache_dir.mkdir(exist_ok=True)
def _get_cache_key(self, url: str) -> str:
"""Generate cache key from URL."""
return hashlib.md5(url.encode()).hexdigest()
def get(self, url: str) -> dict | None:
"""Get cached result."""
cache_key = self._get_cache_key(url)
cache_file = self.cache_dir / f"{cache_key}.json"
if cache_file.exists():
with open(cache_file, "r") as f:
return json.load(f)
return None
def set(self, url: str, data: dict):
"""Cache result."""
cache_key = self._get_cache_key(url)
cache_file = self.cache_dir / f"{cache_key}.json"
with open(cache_file, "w") as f:
json.dump(data, f)
# Usage
cache = ScrapeCache()
def cached_scrape(url: str) -> dict:
# Check cache first
cached = cache.get(url)
if cached:
print(f"Using cached result for {url}")
return cached
# Scrape and cache
result = app.scrape_url(url)
cache.set(url, result)
return result
```
---
## Progress Tracking
### Progress Bar for Crawling
```python
from tqdm import tqdm
def crawl_with_progress(url: str, limit: int = 100) -> list:
"""
Crawl with progress bar.
"""
app = FirecrawlApp(api_key=os.environ.get("FIRECRAWL_API_KEY"))
# Start crawl
crawl_id = app.crawl_url(
url=url,
params={"limit": limit},
wait_until_done=False # Don't wait
)
# Poll with progress bar
with tqdm(total=limit, desc="Crawling") as pbar:
while True:
status = app.check_crawl_status(crawl_id)
if status["status"] == "completed":
pbar.n = status["total"]
pbar.refresh()
break
pbar.n = status["completed"]
pbar.refresh()
time.sleep(5)
return status["data"]
```
---
## Data Storage Patterns
### Save to Cloudflare D1
```python
# Assuming D1 binding available in Cloudflare Worker context
async def save_to_d1(pages: list, db):
"""
Save scraped pages to D1 database.
"""
for page in pages:
await db.prepare(
"""
INSERT INTO scraped_pages (url, title, content, scraped_at)
VALUES (?, ?, ?, ?)
"""
).bind(
page["url"],
page["metadata"].get("title"),
page["markdown"],
datetime.now().isoformat()
).run()
```
### Save to Cloudflare R2
```python
# Assuming R2 binding available
async def save_to_r2(pages: list, bucket):
"""
Save scraped pages to R2 storage.
"""
for page in pages:
key = f"scraped/{page['url'].replace('https://', '')}.md"
await bucket.put(
key,
page["markdown"],
{
"httpMetadata": {
"contentType": "text/markdown"
},
"customMetadata": {
"url": page["url"],
"title": page["metadata"].get("title", ""),
"scraped_at": datetime.now().isoformat()
}
}
)
```
---
## Cloudflare Workers Integration
### ⚠️ Important: SDK Compatibility
**The Firecrawl SDK cannot run in Cloudflare Workers** due to Node.js dependencies (`axios`).
**Use direct REST API calls with `fetch` instead** (see example below).
For a complete production-ready example, see `templates/firecrawl-worker-fetch.ts`.
---
### Complete Worker Example (Direct Fetch API)
```typescript
interface Env {
FIRECRAWL_API_KEY: string;
SCRAPED_CONTENT?: KVNamespace;
}
interface FirecrawlScrapeResponse {
success: boolean;
data: {
markdown?: string;
html?: string;
metadata: {
title?: string;
description?: string;
sourceURL: string;
};
};
}
export default {
async fetch(request: Request, env: Env): Promise<Response> {
if (request.method !== 'POST') {
return Response.json({ error: 'Method not allowed' }, { status: 405 });
}
try {
const { url } = await request.json<{ url: string }>();
if (!url) {
return Response.json({ error: 'URL is required' }, { status: 400 });
}
// Check cache (KV)
if (env.SCRAPED_CONTENT) {
const cached = await env.SCRAPED_CONTENT.get(url, 'json');
if (cached) {
return Response.json({ cached: true, data: cached });
}
}
// Call Firecrawl API directly using fetch
const response = await fetch('https://api.firecrawl.dev/v2/scrape', {
method: 'POST',
headers: {
'Authorization': `Bearer ${env.FIRECRAWL_API_KEY}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
url: url,
formats: ['markdown'],
onlyMainContent: true,
removeBase64Images: true
})
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`Firecrawl API error (${response.status}): ${errorText}`);
}
const result = await response.json<FirecrawlScrapeResponse>();
// Cache for 1 hour
if (env.SCRAPED_CONTENT && result.success) {
await env.SCRAPED_CONTENT.put(
url,
JSON.stringify(result.data),
{ expirationTtl: 3600 }
);
}
return Response.json({
cached: false,
data: result.data
});
} catch (error) {
console.error('Worker error:', error);
return Response.json(
{ error: error instanceof Error ? error.message : 'Unknown error' },
{ status: 500 }
);
}
}
};
```
**Setup**:
```bash
# Add API key
npx wrangler secret put FIRECRAWL_API_KEY
# Optional: Add KV binding to wrangler.jsonc
{
"kv_namespaces": [
{ "binding": "SCRAPED_CONTENT", "id": "your-kv-namespace-id" }
]
}
```
### Scheduled Worker (Cron Job)
```typescript
interface Env {
FIRECRAWL_API_KEY: string;
DB: D1Database;
}
export default {
async scheduled(event: ScheduledEvent, env: Env): Promise<void> {
try {
// Call Firecrawl API directly
const response = await fetch('https://api.firecrawl.dev/v2/scrape', {
method: 'POST',
headers: {
'Authorization': `Bearer ${env.FIRECRAWL_API_KEY}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
url: 'https://news.example.com',
formats: ['markdown'],
onlyMainContent: true
})
});
if (!response.ok) {
throw new Error(`Firecrawl API error: ${response.status}`);
}
const result = await response.json<{
success: boolean;
data: { markdown: string };
}>();
if (!result.success) {
throw new Error('Scraping failed');
}
// Store in D1
await env.DB.prepare(
`INSERT INTO daily_scrapes (url, content, scraped_at)
VALUES (?, ?, ?)`
).bind(
'https://news.example.com',
result.data.markdown,
new Date().toISOString()
).run();
console.log('Daily scrape completed successfully');
} catch (error) {
console.error('Scheduled scrape failed:', error);
}
}
};
```
**Add to wrangler.jsonc**:
```jsonc
{
"triggers": {
"crons": ["0 0 * * *"] // Daily at midnight
},
"d1_databases": [
{
"binding": "DB",
"database_name": "my-database",
"database_id": "your-database-id"
}
]
}
```
---
## Best Practices Summary
### Do's ✅
1. **Always use environment variables** for API keys
2. **Implement retry logic** with exponential backoff
3. **Cache results** to avoid re-scraping
4. **Use `onlyMainContent: true`** to save credits
5. **Track credit usage** to avoid unexpected costs
6. **Handle errors gracefully** with specific error types
7. **Use batch processing** for large numbers of URLs
8. **Set reasonable `waitFor` times** for dynamic content
9. **Use `/v2/map` first** to plan efficient crawls
10. **Monitor rate limits** and implement backoff
### Don'ts ❌
1. **Don't hardcode API keys** in source code
2. **Don't scrape without error handling**
3. **Don't ignore rate limits**
4. **Don't scrape the same content repeatedly** without caching
5. **Don't set excessively high `limit` values** on crawls
6. **Don't assume all scrapes succeed** - always check for errors
7. **Don't use synchronous code** in production (use async)
8. **Don't forget to set `removeBase64Images: true`** for large pages
9. **Don't skip `onlyMainContent`** if you want clean data
10. **Don't crawl without setting `excludePaths`** for known problematic routes
---
## Official Documentation
- **API Reference**: https://docs.firecrawl.dev/api-reference
- **Best Practices**: https://docs.firecrawl.dev/best-practices
- **Rate Limits**: https://docs.firecrawl.dev/rate-limits
references/endpoints.md
# Firecrawl API Endpoints Reference
**API Version**: v2
**Base URL**: `https://api.firecrawl.dev`
**Last Updated**: 2025-10-24
---
## Overview
Firecrawl v2 provides four main endpoints for different web scraping needs:
1. **`/v2/scrape`** - Scrape a single page
2. **`/v2/crawl`** - Crawl multiple pages from a starting URL
3. **`/v2/map`** - Discover all URLs on a site
4. **`/v2/extract`** - Extract structured data with AI
---
## 1. `/v2/scrape` - Single Page Scraping
### Purpose
Scrape a single webpage and return its content in various formats.
### Use Cases
- Extract article content
- Get product details from a page
- Convert specific pages to markdown
- Capture screenshots
- Scrape pages with dynamic JavaScript content
### Request
**Endpoint**: `POST https://api.firecrawl.dev/v2/scrape`
**Headers**:
```
Authorization: Bearer YOUR_API_KEY
Content-Type: application/json
```
**Body**:
```json
{
"url": "https://example.com/page",
"formats": ["markdown", "html", "screenshot"],
"onlyMainContent": true,
"waitFor": 3000,
"removeBase64Images": true,
"actions": [
{"type": "click", "selector": "button.load-more"},
{"type": "wait", "milliseconds": 2000}
]
}
```
### Parameters
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `url` | string | **required** | URL to scrape |
| `formats` | array | `["markdown"]` | Output formats: `"markdown"`, `"html"`, `"screenshot"` |
| `onlyMainContent` | boolean | `false` | Remove headers, footers, nav, ads |
| `waitFor` | number | `0` | Milliseconds to wait before scraping |
| `removeBase64Images` | boolean | `false` | Remove base64-encoded images |
| `actions` | array | `[]` | Browser actions to perform before scraping |
| `headers` | object | `{}` | Custom HTTP headers |
### Response
```json
{
"success": true,
"data": {
"markdown": "# Page Title\n\nContent...",
"html": "<html>...</html>",
"screenshot": "data:image/png;base64,...",
"metadata": {
"title": "Page Title",
"description": "Page description",
"language": "en",
"sourceURL": "https://example.com/page"
}
}
}
```
### Python Example
```python
result = app.scrape_url(
url="https://example.com",
params={
"formats": ["markdown", "html"],
"onlyMainContent": True,
"waitFor": 3000
}
)
markdown = result.get("markdown")
```
### TypeScript Example
```typescript
const result = await app.scrapeUrl('https://example.com', {
formats: ['markdown', 'html'],
onlyMainContent: true,
waitFor: 3000
});
const markdown = result.markdown;
```
---
## 2. `/v2/crawl` - Multi-Page Crawling
### Purpose
Crawl multiple pages starting from a URL, following links automatically.
### Use Cases
- Index entire documentation sites
- Archive website content
- Build knowledge bases from multiple pages
- Scrape blog archives
- Collect all product pages
### Request
**Endpoint**: `POST https://api.firecrawl.dev/v2/crawl`
**Headers**:
```
Authorization: Bearer YOUR_API_KEY
Content-Type: application/json
```
**Body**:
```json
{
"url": "https://docs.example.com",
"limit": 100,
"maxDepth": 3,
"scrapeOptions": {
"formats": ["markdown"],
"onlyMainContent": true
},
"allowedDomains": ["docs.example.com"],
"excludePaths": ["/admin/*", "/login"],
"includePaths": ["/docs/*"],
"webhook": "https://your-domain.com/webhook"
}
```
### Parameters
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `url` | string | **required** | Starting URL to crawl |
| `limit` | number | `100` | Maximum pages to crawl |
| `maxDepth` | number | `2` | How many links deep to follow |
| `scrapeOptions` | object | `{}` | Options to pass to scrape endpoint |
| `allowedDomains` | array | `[]` | Only crawl these domains |
| `excludePaths` | array | `[]` | Skip URLs matching these patterns |
| `includePaths` | array | `[]` | Only crawl URLs matching these patterns |
| `webhook` | string | `null` | Webhook URL to receive results |
### Response
```json
{
"success": true,
"id": "crawl_abc123",
"url": "https://api.firecrawl.dev/v2/crawl/abc123"
}
```
**Status Check**: `GET https://api.firecrawl.dev/v2/crawl/abc123`
**Status Response**:
```json
{
"status": "completed",
"total": 47,
"completed": 47,
"creditsUsed": 94,
"data": [
{
"url": "https://docs.example.com/page1",
"markdown": "# Content...",
"metadata": {...}
},
...
]
}
```
### Python Example
```python
crawl_result = app.crawl_url(
url="https://docs.example.com",
params={
"limit": 100,
"maxDepth": 3,
"scrapeOptions": {
"formats": ["markdown"],
"onlyMainContent": True
}
},
poll_interval=5 # Check status every 5 seconds
)
pages = crawl_result.get("data", [])
for page in pages:
print(page["url"])
```
### TypeScript Example
```typescript
const crawlResult = await app.crawlUrl('https://docs.example.com', {
limit: 100,
maxDepth: 3,
scrapeOptions: {
formats: ['markdown'],
onlyMainContent: true
}
});
for (const page of crawlResult.data) {
console.log(page.url);
}
```
---
## 3. `/v2/map` - URL Discovery
### Purpose
Map all URLs on a website without scraping their content.
### Use Cases
- Generate sitemap
- Discover all pages before crawling
- Audit website structure
- Plan crawling strategy
- Find broken links
### Request
**Endpoint**: `POST https://api.firecrawl.dev/v2/map`
**Headers**:
```
Authorization: Bearer YOUR_API_KEY
Content-Type: application/json
```
**Body**:
```json
{
"url": "https://example.com",
"search": "documentation",
"limit": 5000
}
```
### Parameters
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `url` | string | **required** | Starting URL to map |
| `search` | string | `null` | Filter URLs containing this text |
| `limit` | number | `5000` | Maximum URLs to discover |
### Response
```json
{
"success": true,
"links": [
"https://example.com/page1",
"https://example.com/page2",
"https://example.com/docs/intro",
...
]
}
```
### Python Example
```python
map_result = app.map_url("https://example.com")
urls = map_result.get("links", [])
print(f"Found {len(urls)} URLs")
for url in urls:
print(url)
```
### TypeScript Example
```typescript
const mapResult = await app.mapUrl('https://example.com');
const urls = mapResult.links;
console.log(`Found ${urls.length} URLs`);
urls.forEach(url => console.log(url));
```
---
## 4. `/v2/extract` - Structured Data Extraction
### Purpose
Extract structured data from web pages using AI and schemas.
### Use Cases
- Extract product information (price, title, availability)
- Parse contact details from pages
- Build structured datasets from unstructured HTML
- Custom data extraction with schemas
- Scrape directory listings
### Request
**Endpoint**: `POST https://api.firecrawl.dev/v2/extract`
**Headers**:
```
Authorization: Bearer YOUR_API_KEY
Content-Type: application/json
```
**Body**:
```json
{
"urls": [
"https://example.com/product1",
"https://example.com/product2"
],
"schema": {
"type": "object",
"properties": {
"product_name": {"type": "string"},
"price": {"type": "number"},
"in_stock": {"type": "boolean"}
},
"required": ["product_name", "price"]
},
"systemPrompt": "Extract product details including name, price, and availability"
}
```
### Parameters
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `urls` | array | **required** | URLs to extract from |
| `schema` | object | **required** | JSON schema or Zod schema |
| `systemPrompt` | string | `null` | Guide AI extraction behavior |
### Response
```json
{
"success": true,
"data": [
{
"product_name": "Widget Pro",
"price": 29.99,
"in_stock": true
},
{
"product_name": "Gadget Max",
"price": 49.99,
"in_stock": false
}
]
}
```
### Python Example
```python
schema = {
"type": "object",
"properties": {
"title": {"type": "string"},
"price": {"type": "number"},
"availability": {"type": "string"}
},
"required": ["title", "price"]
}
result = app.extract(
urls=["https://example.com/product"],
params={
"schema": schema,
"systemPrompt": "Extract product information"
}
)
print(result)
```
### TypeScript Example (with Zod)
```typescript
import FirecrawlApp from '@mendable/firecrawl-js';
import { z } from 'zod';
const app = new FirecrawlApp({
apiKey: process.env.FIRECRAWL_API_KEY
});
const schema = z.object({
title: z.string(),
price: z.number(),
availability: z.string()
});
const result = await app.extract({
urls: ['https://example.com/product'],
schema: schema,
systemPrompt: 'Extract product information'
});
console.log(result);
```
---
## Credit Usage
Different endpoints consume different amounts of credits:
| Endpoint | Base Credits | Notes |
|----------|--------------|-------|
| `/v2/scrape` | 1-3 | +1 for screenshot, +1 for actions |
| `/v2/crawl` | 1-3 per page | Same as scrape, multiplied by pages |
| `/v2/map` | 1 | Flat rate |
| `/v2/extract` | 5 per page | Uses AI for extraction |
**Credit Optimization**:
- Use `onlyMainContent: true` to reduce credits
- Use `removeBase64Images: true` to reduce response size
- Use `/v2/map` first to plan crawls efficiently
- Batch extract calls when possible
---
## Rate Limits
- **Free tier**: 500 credits/month
- **Paid tiers**: Varies by plan
- **Rate limiting**: Handled automatically by SDK with retries
---
## Error Responses
### 401 Unauthorized
```json
{
"success": false,
"error": "Invalid API key"
}
```
### 429 Rate Limited
```json
{
"success": false,
"error": "Rate limit exceeded. Please upgrade your plan."
}
```
### 500 Server Error
```json
{
"success": false,
"error": "Internal server error. Please try again."
}
```
---
## Official Documentation
- **API Reference**: https://docs.firecrawl.dev/api-reference
- **SDKs**: https://docs.firecrawl.dev/sdks
- **Dashboard**: https://www.firecrawl.dev/app
SKILL.md
---
name: firecrawl-scraper
description: "Firecrawl v2.5 API for web scraping/crawling to LLM-ready markdown. Use for site extraction, dynamic content, or encountering JavaScript rendering, bot detection, content loading errors."
metadata:
keywords:
- firecrawl
- firecrawl api
- web scraping
- web crawler
- scrape website
- crawl website
- extract content
- html to markdown
- site crawler
- content extraction
- web automation
- firecrawl-py
- firecrawl-js
- llm ready data
- structured data extraction
- bot bypass
- javascript rendering
- scraping api
- crawling api
- map urls
- batch scraping
license: MIT
---
# Firecrawl Web Scraper Skill
**Status**: Production Ready ✅
**Last Updated**: 2026-08-03
**Official Docs**: https://docs.firecrawl.dev
**API Version**: v2.5
---
## What is Firecrawl?
Firecrawl is a **Web Data API for AI** that turns entire websites into LLM-ready markdown or structured data. It handles:
- **JavaScript rendering** - Executes client-side JavaScript to capture dynamic content
- **Anti-bot bypass** - Gets past CAPTCHA and bot detection systems
- **Format conversion** - Outputs as markdown, JSON, or structured data
- **Screenshot capture** - Saves visual representations of pages
- **Browser automation** - Full headless browser capabilities
---
## API Endpoints
### 1. `/v2/scrape` - Single Page Scraping
Scrapes a single webpage and returns clean, structured content.
**Use Cases**:
- Extract article content
- Get product details
- Scrape specific pages
- Convert HTML to markdown
**Key Options**:
- `formats`: ["markdown", "html", "screenshot"]
- `onlyMainContent`: true/false (removes nav, footer, ads)
- `waitFor`: milliseconds to wait before scraping
- `actions`: browser automation actions (click, scroll, etc.)
### 2. `/v2/crawl` - Full Site Crawling
Crawls all accessible pages from a starting URL.
**Use Cases**:
- Index entire documentation sites
- Archive website content
- Build knowledge bases
- Scrape multi-page content
**Key Options**:
- `limit`: max pages to crawl
- `maxDepth`: how many links deep to follow
- `allowedDomains`: restrict to specific domains
- `excludePaths`: skip certain URL patterns
### 3. `/v2/map` - URL Discovery
Maps all URLs on a website without scraping content.
**Use Cases**:
- Find sitemap
- Discover all pages
- Plan crawling strategy
- Audit website structure
### 4. `/v2/extract` - Structured Data Extraction
Uses AI to extract specific data fields from pages.
**Use Cases**:
- Extract product prices and names
- Parse contact information
- Build structured datasets
- Custom data schemas
**Key Options**:
- `schema`: Zod or JSON schema defining desired structure
- `systemPrompt`: guide AI extraction behavior
---
## Authentication
Firecrawl requires an API key for all requests.
### Get API Key
1. Sign up at https://www.firecrawl.dev
2. Go to dashboard → API Keys
3. Copy your API key (starts with `fc-`)
### Store Securely
**NEVER hardcode API keys in code!**
```bash
# .env file
FIRECRAWL_API_KEY=fc-your-api-key-here
```
```bash
# .env.local (for local development)
FIRECRAWL_API_KEY=fc-your-api-key-here
```
---
## SDK Quick Start
### Python
```bash
pip install firecrawl-py # v4.5.0+
```
```python
from firecrawl import FirecrawlApp
import os
app = FirecrawlApp(api_key=os.environ.get("FIRECRAWL_API_KEY"))
result = app.scrape_url("https://example.com", params={"formats": ["markdown"], "onlyMainContent": True})
print(result.get("markdown"))
```
### TypeScript/Node.js
```bash
bun add @mendable/firecrawl-js # v4.32.0+
```
```typescript
import FirecrawlApp from '@mendable/firecrawl-js';
const app = new FirecrawlApp({ apiKey: process.env.FIRECRAWL_API_KEY });
const result = await app.scrapeUrl('https://example.com', { formats: ['markdown'], onlyMainContent: true });
console.log(result.markdown);
```
**See**: `templates/` for crawl, extract, and advanced examples
---
## Common Use Cases
| Use Case | Endpoint | Key Options |
|----------|----------|-------------|
| Documentation scraping | `crawl_url()` | `limit: 500`, `allowedDomains` |
| Product data extraction | `extract()` | Zod schema + `systemPrompt` |
| News article scraping | `scrape_url()` | `onlyMainContent: true`, `removeBase64Images` |
| URL discovery | `map()` | Find all pages before crawling |
**See**: `references/common-patterns.md` for complete examples.
---
## Error Handling
```python
# Python
try:
result = app.scrape_url("https://example.com")
except FirecrawlException as e:
print(f"Firecrawl error: {e}")
```
```typescript
// TypeScript
try {
const result = await app.scrapeUrl('https://example.com');
} catch (error) {
console.error('Error:', error.message);
}
```
---
## Rate Limits & Best Practices
| Best Practice | Why |
|---------------|-----|
| Use `onlyMainContent: true` | Reduces credits, cleaner output |
| Set reasonable `limit` | Avoid excessive costs |
| Use `map` endpoint first | Plan crawling strategy |
| Cache results | Avoid re-scraping |
| Batch extract calls | More efficient for multiple URLs |
**Credits**: Free tier = 500/month, paid tiers higher.
---
## Cloudflare Workers Integration
⚠️ **SDK cannot run in Workers** (Node.js dependencies). Use direct REST API:
```typescript
const response = await fetch('https://api.firecrawl.dev/v2/scrape', {
method: 'POST',
headers: {
'Authorization': `Bearer ${env.FIRECRAWL_API_KEY}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({ url, formats: ['markdown'], onlyMainContent: true })
});
```
**See**: `references/common-patterns.md` for complete Workers example with caching.
---
## When to Use This Skill
| ✅ Use Firecrawl | ❌ Don't Use |
|------------------|--------------|
| Modern JS-rendered sites | Simple static HTML (use cheerio) |
| Clean markdown for LLMs | Existing Puppeteer setup works |
| RAG/chatbot content | Direct API available |
| Structured data extraction | Budget constraints |
| Bot protection bypass | |
---
## Common Issues
| Issue | Cause | Fix |
|-------|-------|-----|
| "Invalid API Key" | Key not set | Check `$FIRECRAWL_API_KEY` starts with `fc-` |
| "Rate limit exceeded" | Monthly credits used | Check dashboard, upgrade plan |
| "Timeout error" | Page slow to load | Add `waitFor: 10000` |
| "Content is empty" | JS loads late | Add `actions: [{type: "wait", milliseconds: 3000}]` |
---
## Advanced Features
| Feature | Usage |
|---------|-------|
| **Browser actions** | `actions: [{type: "click", selector: "button"}]` |
| **Custom headers** | `headers: {"User-Agent": "Custom Bot"}` |
| **Webhooks** | `webhook: "https://your-domain.com/webhook"` |
| **Screenshots** | `formats: ["screenshot"]` |
**See**: `references/endpoints.md` for complete API reference.
---
## When to Load References
| Reference | Load When... |
|-----------|--------------|
| `endpoints.md` | Need complete API endpoint documentation |
| `common-patterns.md` | Cloudflare Workers, caching, batch processing, error handling |
---
## Package Versions
| Package | Version |
|---------|---------|
| firecrawl-py | 4.5.0+ |
| @mendable/firecrawl-js | 4.32.0+ |
| API | v2 |
**Note**: Node.js SDK requires Node.js >=22.0.0, cannot run in Workers.
---
**Official Docs**: https://docs.firecrawl.dev | **GitHub**: https://github.com/mendableai/firecrawl
**Token Savings**: ~60% | **Production Ready**: ✅
templates/firecrawl-crawl-example.py
#!/usr/bin/env python3
"""
Firecrawl Full Site Crawling Example (Python)
This template demonstrates how to crawl an entire website and save results.
Requirements:
pip install firecrawl-py python-dotenv
Environment Variables:
FIRECRAWL_API_KEY - Your Firecrawl API key (get from https://www.firecrawl.dev)
Usage:
python firecrawl-crawl-example.py
"""
import os
import json
from pathlib import Path
from dotenv import load_dotenv
from firecrawl import FirecrawlApp
# Load environment variables from .env file
load_dotenv()
def crawl_website(
url: str,
limit: int = 100,
max_depth: int = 3,
output_dir: str = "crawled_data"
) -> list:
"""
Crawl an entire website and save results to disk.
Args:
url: Starting URL to crawl
limit: Maximum number of pages to crawl
max_depth: How many links deep to follow
output_dir: Directory to save scraped content
Returns:
List of crawled page data
"""
# Initialize Firecrawl client
api_key = os.environ.get("FIRECRAWL_API_KEY")
if not api_key:
raise ValueError("FIRECRAWL_API_KEY environment variable not set")
app = FirecrawlApp(api_key=api_key)
print(f"Starting crawl of: {url}")
print(f"Limit: {limit} pages")
print(f"Max depth: {max_depth}")
try:
# Start the crawl
# This will poll the API until the crawl is complete
result = app.crawl_url(
url=url,
params={
# Maximum number of pages to crawl
"limit": limit,
# Maximum depth to follow links
"maxDepth": max_depth,
# Scrape options for each page
"scrapeOptions": {
"formats": ["markdown"],
"onlyMainContent": True,
"removeBase64Images": True,
},
# Only crawl pages within the same domain
# "allowedDomains": ["docs.example.com"],
# Exclude certain paths (e.g., login pages, admin)
# "excludePaths": ["/admin/*", "/login"],
# Include certain paths only
# "includePaths": ["/docs/*"],
},
poll_interval=5 # Check crawl status every 5 seconds
)
# Get crawled pages
pages = result.get("data", [])
print(f"\n✅ Crawled {len(pages)} pages successfully")
# Save results
save_crawled_data(pages, output_dir)
return pages
except Exception as e:
print(f"Error crawling {url}: {e}")
raise
def save_crawled_data(pages: list, output_dir: str):
"""
Save crawled pages to disk as markdown files and JSON metadata.
Args:
pages: List of scraped page data
output_dir: Directory to save files
"""
# Create output directory
output_path = Path(output_dir)
output_path.mkdir(exist_ok=True)
markdown_dir = output_path / "markdown"
markdown_dir.mkdir(exist_ok=True)
metadata_list = []
for i, page in enumerate(pages):
url = page.get("url", "")
markdown = page.get("markdown", "")
metadata = page.get("metadata", {})
# Create safe filename from URL
filename = url.replace("https://", "").replace("http://", "")
filename = filename.replace("/", "_").replace(":", "_")
filename = f"{i:04d}_{filename}.md"
# Save markdown content
markdown_file = markdown_dir / filename
with open(markdown_file, "w", encoding="utf-8") as f:
f.write(f"# {metadata.get('title', 'Untitled')}\n\n")
f.write(f"**Source**: {url}\n\n")
f.write("---\n\n")
f.write(markdown)
print(f"Saved: {markdown_file}")
# Collect metadata
metadata_list.append({
"filename": filename,
"url": url,
"title": metadata.get("title"),
"description": metadata.get("description"),
"language": metadata.get("language"),
})
# Save metadata as JSON
metadata_file = output_path / "metadata.json"
with open(metadata_file, "w", encoding="utf-8") as f:
json.dump(metadata_list, f, indent=2, ensure_ascii=False)
print(f"\n✅ Metadata saved to: {metadata_file}")
def main():
"""Main function demonstrating website crawling."""
# Configuration
url = "https://docs.firecrawl.dev"
limit = 50 # Crawl up to 50 pages
max_depth = 2 # Follow links 2 levels deep
output_dir = "firecrawl_output"
# Crawl the website
pages = crawl_website(
url=url,
limit=limit,
max_depth=max_depth,
output_dir=output_dir
)
# Print summary
print("\n" + "=" * 80)
print("CRAWL SUMMARY")
print("=" * 80)
print(f"Total pages crawled: {len(pages)}")
print(f"Output directory: {output_dir}")
if pages:
print("\nFirst 5 pages:")
for page in pages[:5]:
print(f" - {page.get('url')}")
if __name__ == "__main__":
main()
templates/firecrawl-scrape-python.py
#!/usr/bin/env python3
"""
Firecrawl Basic Scraping Example (Python)
This template demonstrates how to scrape a single webpage using Firecrawl API.
Requirements:
pip install firecrawl-py python-dotenv
Environment Variables:
FIRECRAWL_API_KEY - Your Firecrawl API key (get from https://www.firecrawl.dev)
Usage:
python firecrawl-scrape-python.py
"""
import os
from dotenv import load_dotenv
from firecrawl import FirecrawlApp
# Load environment variables from .env file
load_dotenv()
def scrape_single_page(url: str) -> dict:
"""
Scrape a single webpage and return markdown content.
Args:
url: The URL to scrape
Returns:
dict containing scraped data (markdown, html, metadata)
"""
# Initialize Firecrawl client
# NEVER hardcode API keys! Always use environment variables
api_key = os.environ.get("FIRECRAWL_API_KEY")
if not api_key:
raise ValueError("FIRECRAWL_API_KEY environment variable not set")
app = FirecrawlApp(api_key=api_key)
try:
# Scrape the URL
result = app.scrape_url(
url=url,
params={
# Output formats - can include multiple
"formats": ["markdown", "html"],
# Only extract main content (removes nav, footer, ads)
# This saves credits and improves content quality
"onlyMainContent": True,
# Wait time before scraping (ms) - useful for dynamic content
# "waitFor": 3000,
# Remove base64 images to reduce response size
# "removeBase64Images": True,
# Include screenshot
# "formats": ["markdown", "screenshot"],
}
)
return result
except Exception as e:
print(f"Error scraping {url}: {e}")
raise
def main():
"""Main function demonstrating basic scraping."""
# Example URL to scrape
url = "https://docs.firecrawl.dev"
print(f"Scraping: {url}")
# Scrape the page
result = scrape_single_page(url)
# Access different parts of the result
markdown = result.get("markdown", "")
html = result.get("html", "")
metadata = result.get("metadata", {})
# Print results
print("\n" + "=" * 80)
print("MARKDOWN CONTENT:")
print("=" * 80)
print(markdown[:500]) # First 500 characters
print("...")
print("\n" + "=" * 80)
print("METADATA:")
print("=" * 80)
print(f"Title: {metadata.get('title', 'N/A')}")
print(f"Description: {metadata.get('description', 'N/A')}")
print(f"Language: {metadata.get('language', 'N/A')}")
print(f"Source URL: {metadata.get('sourceURL', 'N/A')}")
# Save to file (optional)
output_file = "scraped_content.md"
with open(output_file, "w", encoding="utf-8") as f:
f.write(markdown)
print(f"\n✅ Full content saved to: {output_file}")
if __name__ == "__main__":
main()
templates/firecrawl-scrape-typescript.ts
/**
* Firecrawl Basic Scraping Example (TypeScript - Node.js)
*
* This template demonstrates how to scrape a single webpage using Firecrawl SDK.
*
* ⚠️ NOTE: This example uses the Firecrawl SDK which requires Node.js runtime.
* For Cloudflare Workers, use firecrawl-worker-fetch.ts instead (direct fetch API).
*
* Requirements:
* npm install @mendable/firecrawl-js
* # or: npm install firecrawl
* npm install -D @types/node
*
* Environment Variables:
* FIRECRAWL_API_KEY - Your Firecrawl API key (get from https://www.firecrawl.dev)
*
* Usage:
* npx tsx firecrawl-scrape-typescript.ts
* # or compile first:
* tsc firecrawl-scrape-typescript.ts && node firecrawl-scrape-typescript.js
*/
import FirecrawlApp from '@mendable/firecrawl-js';
import fs from 'fs/promises';
/**
* Scrape a single webpage and return markdown content
*/
async function scrapeSinglePage(url: string): Promise<{
markdown: string;
html: string;
metadata: Record<string, any>;
}> {
// Initialize Firecrawl client
// NEVER hardcode API keys! Always use environment variables
const apiKey = process.env.FIRECRAWL_API_KEY;
if (!apiKey) {
throw new Error('FIRECRAWL_API_KEY environment variable not set');
}
const app = new FirecrawlApp({ apiKey });
try {
// Scrape the URL
const result = await app.scrapeUrl(url, {
// Output formats - can include multiple
formats: ['markdown', 'html'],
// Only extract main content (removes nav, footer, ads)
// This saves credits and improves content quality
onlyMainContent: true,
// Wait time before scraping (ms) - useful for dynamic content
// waitFor: 3000,
// Remove base64 images to reduce response size
// removeBase64Images: true,
// Include screenshot
// formats: ['markdown', 'screenshot'],
});
return result;
} catch (error) {
console.error(`Error scraping ${url}:`, error);
throw error;
}
}
/**
* Main function demonstrating basic scraping
*/
async function main() {
// Example URL to scrape
const url = 'https://docs.firecrawl.dev';
console.log(`Scraping: ${url}`);
// Scrape the page
const result = await scrapeSinglePage(url);
// Access different parts of the result
const { markdown, html, metadata } = result;
// Print results
console.log('\n' + '='.repeat(80));
console.log('MARKDOWN CONTENT:');
console.log('='.repeat(80));
console.log(markdown.substring(0, 500)); // First 500 characters
console.log('...');
console.log('\n' + '='.repeat(80));
console.log('METADATA:');
console.log('='.repeat(80));
console.log(`Title: ${metadata.title || 'N/A'}`);
console.log(`Description: ${metadata.description || 'N/A'}`);
console.log(`Language: ${metadata.language || 'N/A'}`);
console.log(`Source URL: ${metadata.sourceURL || 'N/A'}`);
// Save to file (optional)
const outputFile = 'scraped_content.md';
await fs.writeFile(outputFile, markdown, 'utf-8');
console.log(`\n✅ Full content saved to: ${outputFile}`);
}
// Run main function
main().catch((error) => {
console.error('Fatal error:', error);
process.exit(1);
});
templates/firecrawl-worker-fetch.ts
/**
* Firecrawl with Cloudflare Workers (Direct Fetch API)
*
* ⚠️ IMPORTANT: The Firecrawl SDK cannot run in Cloudflare Workers due to Node.js
* dependencies (axios uses Node.js http module). This template uses direct REST API
* calls with the fetch API, which works perfectly in Workers.
*
* Features:
* - Direct Firecrawl v2 API integration using fetch
* - Optional KV caching to reduce API calls and credits
* - Proper error handling and TypeScript types
* - Works in Cloudflare Workers runtime
*
* Environment Variables (add via wrangler secrets):
* FIRECRAWL_API_KEY - Your Firecrawl API key (get from https://www.firecrawl.dev)
*
* Setup:
* 1. Add API key: npx wrangler secret put FIRECRAWL_API_KEY
* 2. (Optional) Add KV binding to wrangler.jsonc:
* {
* "kv_namespaces": [
* { "binding": "SCRAPED_CACHE", "id": "your-kv-namespace-id" }
* ]
* }
* 3. Deploy: npx wrangler deploy
*
* Usage:
* POST https://your-worker.workers.dev
* Body: { "url": "https://example.com" }
*/
interface Env {
FIRECRAWL_API_KEY: string;
SCRAPED_CACHE?: KVNamespace; // Optional: for caching results
}
interface FirecrawlScrapeRequest {
url: string;
formats?: ('markdown' | 'html' | 'screenshot')[];
onlyMainContent?: boolean;
waitFor?: number;
removeBase64Images?: boolean;
actions?: Array<{
type: 'click' | 'wait' | 'scroll';
selector?: string;
milliseconds?: number;
direction?: 'up' | 'down';
}>;
headers?: Record<string, string>;
}
interface FirecrawlScrapeResponse {
success: boolean;
data: {
markdown?: string;
html?: string;
screenshot?: string;
metadata: {
title?: string;
description?: string;
language?: string;
sourceURL: string;
statusCode?: number;
};
};
error?: string;
}
interface FirecrawlCrawlRequest {
url: string;
limit?: number;
maxDepth?: number;
scrapeOptions?: Omit<FirecrawlScrapeRequest, 'url'>;
allowedDomains?: string[];
excludePaths?: string[];
includePaths?: string[];
webhook?: string;
}
interface FirecrawlCrawlResponse {
success: boolean;
id: string;
url: string; // Status check URL
}
/**
* Scrape a single URL using Firecrawl v2 API
*/
async function scrapeUrl(
url: string,
apiKey: string,
options: Omit<FirecrawlScrapeRequest, 'url'> = {}
): Promise<FirecrawlScrapeResponse> {
const response = await fetch('https://api.firecrawl.dev/v2/scrape', {
method: 'POST',
headers: {
'Authorization': `Bearer ${apiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
url,
formats: options.formats || ['markdown'],
onlyMainContent: options.onlyMainContent ?? true,
waitFor: options.waitFor,
removeBase64Images: options.removeBase64Images ?? true,
actions: options.actions,
headers: options.headers
})
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`Firecrawl API error (${response.status}): ${errorText}`);
}
return await response.json<FirecrawlScrapeResponse>();
}
/**
* Start a crawl job using Firecrawl v2 API
*/
async function crawlUrl(
apiKey: string,
options: FirecrawlCrawlRequest
): Promise<FirecrawlCrawlResponse> {
const response = await fetch('https://api.firecrawl.dev/v2/crawl', {
method: 'POST',
headers: {
'Authorization': `Bearer ${apiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify(options)
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`Firecrawl API error (${response.status}): ${errorText}`);
}
return await response.json<FirecrawlCrawlResponse>();
}
/**
* Check crawl status
*/
async function checkCrawlStatus(
crawlId: string,
apiKey: string
): Promise<any> {
const response = await fetch(`https://api.firecrawl.dev/v2/crawl/${crawlId}`, {
method: 'GET',
headers: {
'Authorization': `Bearer ${apiKey}`,
}
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(`Firecrawl API error (${response.status}): ${errorText}`);
}
return await response.json();
}
export default {
async fetch(request: Request, env: Env): Promise<Response> {
// CORS headers (if needed for browser access)
const corsHeaders = {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'POST, GET, OPTIONS',
'Access-Control-Allow-Headers': 'Content-Type',
};
// Handle CORS preflight
if (request.method === 'OPTIONS') {
return new Response(null, { headers: corsHeaders });
}
// Only accept POST requests
if (request.method !== 'POST') {
return Response.json(
{ error: 'Method not allowed. Use POST with JSON body: { "url": "..." }' },
{ status: 405, headers: corsHeaders }
);
}
try {
// Parse request body
let body: any;
try {
body = await request.json();
} catch (e) {
return Response.json(
{ error: 'Invalid JSON in request body' },
{ status: 400, headers: corsHeaders }
);
}
const { url, action = 'scrape' } = body;
if (!url) {
return Response.json(
{ error: 'URL is required in request body' },
{ status: 400, headers: corsHeaders }
);
}
// Check if API key is configured
if (!env.FIRECRAWL_API_KEY) {
return Response.json(
{ error: 'FIRECRAWL_API_KEY not configured. Run: npx wrangler secret put FIRECRAWL_API_KEY' },
{ status: 500, headers: corsHeaders }
);
}
// Handle different actions
if (action === 'scrape') {
// Check cache first (if KV is available)
const cacheKey = `scrape:${url}`;
if (env.SCRAPED_CACHE) {
const cached = await env.SCRAPED_CACHE.get(cacheKey, 'json');
if (cached) {
console.log(`Cache hit for ${url}`);
return Response.json(
{ cached: true, data: cached },
{ headers: corsHeaders }
);
}
}
// Scrape the URL
console.log(`Scraping ${url}...`);
const result = await scrapeUrl(url, env.FIRECRAWL_API_KEY, {
formats: body.formats || ['markdown'],
onlyMainContent: body.onlyMainContent ?? true,
waitFor: body.waitFor,
removeBase64Images: body.removeBase64Images ?? true,
actions: body.actions,
headers: body.headers
});
if (!result.success) {
return Response.json(
{ error: result.error || 'Scraping failed' },
{ status: 500, headers: corsHeaders }
);
}
// Cache result for 1 hour (if KV is available)
if (env.SCRAPED_CACHE) {
await env.SCRAPED_CACHE.put(
cacheKey,
JSON.stringify(result.data),
{ expirationTtl: 3600 } // 1 hour
);
console.log(`Cached result for ${url}`);
}
return Response.json(
{ cached: false, data: result.data },
{ headers: corsHeaders }
);
} else if (action === 'crawl') {
// Start a crawl job
console.log(`Starting crawl for ${url}...`);
const crawlResult = await crawlUrl(env.FIRECRAWL_API_KEY, {
url,
limit: body.limit,
maxDepth: body.maxDepth,
scrapeOptions: body.scrapeOptions,
allowedDomains: body.allowedDomains,
excludePaths: body.excludePaths,
includePaths: body.includePaths,
webhook: body.webhook
});
return Response.json(crawlResult, { headers: corsHeaders });
} else if (action === 'crawl-status') {
// Check crawl status
const crawlId = body.crawlId;
if (!crawlId) {
return Response.json(
{ error: 'crawlId is required for crawl-status action' },
{ status: 400, headers: corsHeaders }
);
}
const status = await checkCrawlStatus(crawlId, env.FIRECRAWL_API_KEY);
return Response.json(status, { headers: corsHeaders });
} else {
return Response.json(
{ error: `Unknown action: ${action}. Use 'scrape', 'crawl', or 'crawl-status'` },
{ status: 400, headers: corsHeaders }
);
}
} catch (error) {
console.error('Worker error:', error);
return Response.json(
{
error: error instanceof Error ? error.message : 'Unknown error',
stack: error instanceof Error ? error.stack : undefined
},
{ status: 500, headers: corsHeaders }
);
}
}
};
// Example usage with curl:
/*
# Scrape a single page
curl -X POST https://your-worker.workers.dev \
-H "Content-Type: application/json" \
-d '{"url": "https://docs.firecrawl.dev", "action": "scrape"}'
# Start a crawl
curl -X POST https://your-worker.workers.dev \
-H "Content-Type: application/json" \
-d '{"url": "https://docs.example.com", "action": "crawl", "limit": 10}'
# Check crawl status
curl -X POST https://your-worker.workers.dev \
-H "Content-Type: application/json" \
-d '{"action": "crawl-status", "crawlId": "abc123"}'
*/