firecrawl

import os from firecrawl import FirecrawlApp # Ensure you have your Firecrawl API key set as an environment variable api_key = os.environ.get('FIRECRAWL_API_KEY', '') if not api_key: print("Error: FIRECRAWL_API_KEY environment variable not set.") print("Please set FIRECRAWL_API_KEY to your Firecrawl API key.") exit(1) try: app = FirecrawlApp(api_key=api_key) # Scrape a URL url_to_scrape = "https://www.firecrawl.ai/" print(f"Scraping: {url_to_scrape}") scraped_data = app.scrape(url_to_scrape) print("Scrape successful!") # Access the content from the 'data' field, which is a list of results if scraped_data and isinstance(scraped_data, dict) and 'data' in scraped_data and scraped_data['data']: first_item = scraped_data['data'][0] print(f"Title: {first_item.get('metadata', {}).get('title', 'N/A')}") print(f"Text content length: {len(first_item.get('content', ''))}") else: print("No data found in scrape result.") # Example: Crawl a website (optional, might take longer and consume more credits) # print(f"\nCrawling: {url_to_scrape} up to 1 page") # crawled_data = app.crawl(url_to_scrape, params={'limit': 1}) # if crawled_data and 'data' in crawled_data: # print(f"Crawled {len(crawled_data['data'])} pages.") except Exception as e: print(f"An error occurred: {e}")