"""Scrape.do integration for LangChain.

Provides:
  - ScrapedoLoader: a Document Loader that fetches web pages through the
    Scrape.do API (proxy rotation, anti-bot bypass, optional JS rendering)
    and returns LangChain Document objects.
  - scrapedo_tool: a ready-to-use agent Tool for live URL fetching.

Requirements:
    pip install langchain-core requests

Usage:
    from scrapedo_loader import ScrapedoLoader

    loader = ScrapedoLoader(
        token="1d29ea17366d4b68867fa0b2206265d0c242553231f",
        urls=["https://example.com"],
        render=False,   # set True for JavaScript-heavy pages
    )
    docs = loader.load()
    print(docs[0].page_content[:500])
    print(docs[0].metadata)
"""

from typing import Any, Dict, Iterator, List, Optional, Union

import requests
from langchain_core.document_loaders import BaseLoader
from langchain_core.documents import Document

SCRAPEDO_API_URL = "https://api.scrape.do/"


class ScrapedoLoader(BaseLoader):
    """Load web pages via the Scrape.do API into LangChain Documents.

    Unlike WebBaseLoader, this handles JavaScript rendering, proxy
    rotation, and anti-bot protection server-side through Scrape.do.

    Args:
        token: Your Scrape.do API token.
        urls: A single URL string or a list of URLs to scrape.
        render: If True, renders the page with a headless browser
            (JavaScript-heavy sites). Uses more credits.
        super_proxy: If True, uses residential/mobile proxies for
            hard-to-scrape targets (sends super=true).
        extra_params: Optional dict of additional Scrape.do query
            parameters, e.g. {"geoCode": "us"}.
        timeout: Request timeout in seconds (default 90).
        continue_on_failure: If True (default), a failed URL is skipped
            with a warning instead of raising an exception.
    """

    def __init__(
        self,
        token: str,
        urls: Union[str, List[str]],
        render: bool = False,
        super_proxy: bool = False,
        extra_params: Optional[Dict[str, Any]] = None,
        timeout: int = 90,
        continue_on_failure: bool = True,
    ) -> None:
        if not token:
            raise ValueError("A Scrape.do API token is required.")
        self.token = token
        self.urls = [urls] if isinstance(urls, str) else list(urls)
        self.render = render
        self.super_proxy = super_proxy
        self.extra_params = extra_params or {}
        self.timeout = timeout
        self.continue_on_failure = continue_on_failure

    def _build_params(self, url: str) -> Dict[str, Any]:
        # requests URL-encodes parameter values automatically,
        # which satisfies Scrape.do's encoded-URL requirement.
        params: Dict[str, Any] = {"token": self.token, "url": url}
        if self.render:
            params["render"] = "true"
        if self.super_proxy:
            params["super"] = "true"
        params.update(self.extra_params)
        return params

    def lazy_load(self) -> Iterator[Document]:
        """Fetch each URL through Scrape.do and yield Documents one by one."""
        for url in self.urls:
            try:
                response = requests.get(
                    SCRAPEDO_API_URL,
                    params=self._build_params(url),
                    timeout=self.timeout,
                )
                response.raise_for_status()
            except requests.RequestException as exc:
                # Never leak the API token in error output.
                safe_error = str(exc).replace(self.token, "***TOKEN***")
                if self.continue_on_failure:
                    print(f"[ScrapedoLoader] Skipping {url}: {safe_error}")
                    continue
                raise requests.RequestException(safe_error) from None

            yield Document(
                page_content=response.text,
                metadata={
                    "source": url,
                    "status_code": response.status_code,
                    "render": self.render,
                    "super_proxy": self.super_proxy,
                },
            )

    def load(self) -> List[Document]:
        """Fetch all URLs and return the Documents as a list."""
        return list(self.lazy_load())


# ---------------------------------------------------------------------------
# Agent Tool (bonus): lets a LangChain agent fetch any URL live.
# ---------------------------------------------------------------------------

def make_scrapedo_tool(token: str, render: bool = False):
    """Create a LangChain tool bound to your Scrape.do token.

    Usage:
        from scrapedo_loader import make_scrapedo_tool
        scrape = make_scrapedo_tool(token="YOUR_SCRAPEDO_TOKEN")
        # pass [scrape] into your agent's tools list
    """
    from langchain_core.tools import tool

    @tool
    def scrapedo_scrape(url: str) -> str:
        """Fetch the full content of a web page. Handles JavaScript
        rendering, proxies, and anti-bot protection. Input: a URL."""
        loader = ScrapedoLoader(token=token, urls=url, render=render)
        docs = loader.load()
        if not docs:
            return f"Failed to fetch {url}."
        return docs[0].page_content

    return scrapedo_scrape