"""Bulk web scraping with Scrape.do + LangChain.

Reads a list of URLs from a Google Sheet (via its CSV export link),
scrapes each one through the Scrape.do API using ScrapedoLoader,
and writes the results to results.csv (url, status, content).

Setup:
    1. Put this file in the same folder as scrapedo_loader.py
    2. In Google Sheets: File > Share > Publish to web > select the
       sheet > CSV > Publish, then copy the link.
       (Or use: https://docs.google.com/spreadsheets/d/SHEET_ID/export?format=csv)
    3. Fill in the CONFIG section below.
    4. Run: python3 bulk_scrape.py
"""

import csv
import io
import sys

import requests

from scrapedo_loader import ScrapedoLoader

# ----------------------- CONFIG -----------------------
SCRAPEDO_TOKEN = "1d29ea17366d4b68867fa0b2206265d0c242553231f"   # your Scrape.do API token
SHEET_CSV_URL = "https://docs.google.com/spreadsheets/d/1R8ME8g0LxxDVVU47ZZVG8z-ysGZrYmU82tRrKinHczI/export?format=csv"    # Google Sheet CSV export link
URL_COLUMN = "url"                       # name of the column that holds the URLs
RENDER = False                           # True for JavaScript-heavy pages
OUTPUT_FILE = "results.csv"
# -------------------------------------------------------


def read_urls_from_sheet(sheet_csv_url: str, url_column: str) -> list:
    """Download the sheet as CSV and return the URLs from the given column."""
    response = requests.get(sheet_csv_url, timeout=30)
    response.raise_for_status()
    reader = csv.DictReader(io.StringIO(response.text))
    if reader.fieldnames is None or url_column not in reader.fieldnames:
        sys.exit(
            f"Column '{url_column}' not found in the sheet. "
            f"Available columns: {reader.fieldnames}"
        )
    urls = [row[url_column].strip() for row in reader if row.get(url_column, "").strip()]
    return urls


def main() -> None:
    urls = read_urls_from_sheet(SHEET_CSV_URL, URL_COLUMN)
    if not urls:
        sys.exit("No URLs found in the sheet.")
    print(f"Found {len(urls)} URLs. Scraping...")

    loader = ScrapedoLoader(
        token=SCRAPEDO_TOKEN,
        urls=urls,
        render=RENDER,
        continue_on_failure=True,
    )

    scraped = {}
    for doc in loader.lazy_load():
        scraped[doc.metadata["source"]] = doc
        print(f"  OK  {doc.metadata['source']}")

    with open(OUTPUT_FILE, "w", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow(["url", "status", "content"])
        for url in urls:
            doc = scraped.get(url)
            if doc:
                writer.writerow([url, doc.metadata["status_code"], doc.page_content])
            else:
                writer.writerow([url, "FAILED", ""])

    ok = len(scraped)
    print(f"\nDone. {ok} succeeded, {len(urls) - ok} failed.")
    print(f"Results saved to {OUTPUT_FILE}")


if __name__ == "__main__":
    main()