feat(lambda): add AWS Lambda integration in examples/integrations/aws_lambda/ (#177)

feat(lambda): add AWS Lambda one-shot scrape integration

Self-contained example in examples/integrations/aws_lambda/ — Dockerfile,
entrypoint, handler, and docs for running CloakBrowser stealth scrapes in
AWS Lambda (container image). Includes smart_wait DOM-stability polling,
Xvfb headed mode, and Lambda-specific Chromium flags.

Contributed by @AlexTech314.

examples/integrations/aws_lambda/lambda_handler.py

@@ -0,0 +1,219 @@
+"""AWS Lambda handler for one-off stealth-browser invocations.
+
+Always runs **headed** via the Xvfb display started by `lambda-entrypoint.sh`.
+
+Event schema (all fields except `url` are optional):
+
+    Launch options (passed to cloakbrowser.launch_context_async):
+        url                 str              required, the page to scrape
+        proxy               str|dict         http://user:pass@host:port  or  Playwright proxy dict
+        humanize            bool             False — enable human-like mouse/keyboard/scroll
+        human_preset        str              "default" | "careful"
+        geoip               bool             False — auto timezone+locale from proxy IP
+        timezone            str              IANA tz, e.g. "America/New_York"
+        locale              str              BCP-47, e.g. "en-US"
+        viewport            {width,height}   defaults to 1920x947 (cloakbrowser DEFAULT_VIEWPORT)
+        user_agent          str              custom UA (rare — cloakbrowser sets one already)
+        extra_args          list[str]        additional Chromium CLI flags
+
+    Navigation options (passed to page.goto):
+        wait_until          str              "load"|"domcontentloaded"|"networkidle"|"commit"
+                                              default "domcontentloaded"
+        goto_timeout_ms     int              30000
+
+    Post-navigation waits (run in this order if specified):
+        smart_wait                       bool ON by default if no other wait is set.
+                                              Polls document.outerHTML.length and bails when it
+                                              hasn't changed for `dom_stable_ms`. Handles lazy
+                                              hydration, async chunks, and lazy images, and is
+                                              immune to analytics beacons / long-poll that keep
+                                              the network busy without mutating the DOM.
+        dom_stable_ms                    int  1500   — how long DOM must be quiet
+        max_settle_ms                    int  15000  — hard cap on smart_wait
+        wait_for_load_state              str  "load"|"domcontentloaded"|"networkidle"
+        wait_for_load_state_timeout_ms   int  30000
+        wait_for_selector                str  CSS or XPath selector
+        wait_for_selector_state          str  "attached"|"detached"|"visible"|"hidden", default "visible"
+        wait_for_selector_timeout_ms     int  30000
+        wait_for_function                str  JS expression that returns truthy when ready
+        wait_for_function_timeout_ms     int  30000
+        wait_ms                          int  fixed pause in ms (page.wait_for_timeout)
+
+    Capture options:
+        screenshot              bool         True
+        full_page_screenshot    bool         False — capture entire scrollable page
+
+Returns:
+    {"title": ..., "url": ..., "html": ..., "screenshot_b64"?: ...}
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import logging
+import subprocess
+from pathlib import Path
+from typing import Any
+
+from cloakbrowser import launch_context_async
+
+logger = logging.getLogger("cloakbrowser.lambda")
+logger.setLevel(logging.INFO)
+
+
+def _diag_snapshot() -> str:
+    """Capture Xvfb status, Xvfb log, X11 socket state, and env for error reports."""
+    import os
+    parts = []
+    try:
+        r = subprocess.run(["pgrep", "-fa", "Xvfb"], capture_output=True, text=True)
+        parts.append(f"pgrep Xvfb: rc={r.returncode} stdout={r.stdout.strip()!r}")
+    except Exception as e:
+        parts.append(f"pgrep failed: {e}")
+    try:
+        r = subprocess.run(["ls", "-la", "/tmp/.X11-unix"], capture_output=True, text=True)
+        parts.append(f"ls /tmp/.X11-unix:\n{r.stdout}{r.stderr}")
+    except Exception as e:
+        parts.append(f"ls /tmp/.X11-unix failed: {e}")
+    try:
+        log = Path("/tmp/Xvfb.log").read_text()
+        parts.append(f"/tmp/Xvfb.log:\n{log}")
+    except Exception as e:
+        parts.append(f"Xvfb log unreadable: {e}")
+    parts.append(f"env: DISPLAY={os.environ.get('DISPLAY')!r} HOME={os.environ.get('HOME')!r}")
+    return "\n".join(parts)
+
+
+def handler(event: dict, context: Any) -> dict:
+    return asyncio.run(_run(event))
+
+
+def _build_launch_kwargs(event: dict) -> dict:
+    """Translate the event dict into kwargs for launch_context_async.
+
+    Only includes keys explicitly set in the event so cloakbrowser's defaults
+    (DEFAULT_VIEWPORT etc.) kick in when fields are absent — passing
+    viewport=None would *disable* viewport emulation, which we don't want.
+    """
+    kwargs: dict = {
+        "headless": False,  # always headed via Xvfb
+        "args": [
+            # Lambda /dev/shm is ~64 MB — Chromium crashes mid-render without this.
+            "--disable-dev-shm-usage",
+            # Lambda's restricted process model can't fork from Chromium's zygote
+            # — without this, child renderer processes fail to spawn.
+            "--no-zygote",
+            *event.get("extra_args", []),
+        ],
+    }
+    for key in ("proxy", "humanize", "human_preset", "geoip",
+                "timezone", "locale", "viewport", "user_agent"):
+        if key in event:
+            kwargs[key] = event[key]
+    return kwargs
+
+
+async def _smart_wait(page, dom_stable_ms: int = 1500, max_settle_ms: int = 15000) -> None:
+    """Wait until the document HTML hasn't changed for `dom_stable_ms`.
+
+    Generic stopping condition for at-scale scraping when you can't tune
+    selectors per site. More robust than `networkidle` because it ignores
+    network activity that doesn't mutate the DOM (analytics beacons,
+    long-poll, websockets, web vitals streams).
+    """
+    js = f"""
+    (() => {{
+        if (!window.__cb_settle) {{
+            window.__cb_settle = {{ len: -1, since: Date.now() }};
+        }}
+        const cur = document.documentElement.outerHTML.length;
+        const s = window.__cb_settle;
+        if (cur !== s.len) {{
+            s.len = cur;
+            s.since = Date.now();
+            return false;
+        }}
+        return (Date.now() - s.since) >= {int(dom_stable_ms)};
+    }})()
+    """
+    try:
+        await page.wait_for_function(js, timeout=max_settle_ms, polling=200)
+    except Exception:
+        # Hit max_settle_ms cap — return what we have rather than fail the whole invoke
+        logger.warning("smart_wait hit max_settle_ms=%d cap", max_settle_ms)
+
+
+_EXPLICIT_WAIT_KEYS = (
+    "wait_for_load_state", "wait_for_selector", "wait_for_function", "wait_ms",
+)
+
+
+async def _post_nav_waits(page, event: dict) -> None:
+    """Run waits in priority order. smart_wait is the default unless the
+    caller asked for a more specific stopping condition."""
+    explicit = any(k in event for k in _EXPLICIT_WAIT_KEYS)
+    if event.get("smart_wait", not explicit):
+        await _smart_wait(
+            page,
+            dom_stable_ms=event.get("dom_stable_ms", 1500),
+            max_settle_ms=event.get("max_settle_ms", 15000),
+        )
+    if "wait_for_load_state" in event:
+        await page.wait_for_load_state(
+            event["wait_for_load_state"],
+            timeout=event.get("wait_for_load_state_timeout_ms", 30000),
+        )
+    if "wait_for_selector" in event:
+        await page.wait_for_selector(
+            event["wait_for_selector"],
+            state=event.get("wait_for_selector_state", "visible"),
+            timeout=event.get("wait_for_selector_timeout_ms", 30000),
+        )
+    if "wait_for_function" in event:
+        await page.wait_for_function(
+            event["wait_for_function"],
+            timeout=event.get("wait_for_function_timeout_ms", 30000),
+        )
+    if "wait_ms" in event:
+        await page.wait_for_timeout(event["wait_ms"])
+
+
+async def _run(event: dict) -> dict:
+    url = event["url"]
+
+    try:
+        ctx = await launch_context_async(**_build_launch_kwargs(event))
+    except Exception as e:
+        diag = _diag_snapshot()
+        logger.error("launch_context_async failed: %s\nDIAG:\n%s", e, diag)
+        raise RuntimeError(f"launch failed: {e}\n--- DIAG ---\n{diag}") from e
+
+    try:
+        page = await ctx.new_page()
+        await page.goto(
+            url,
+            wait_until=event.get("wait_until", "domcontentloaded"),
+            timeout=event.get("goto_timeout_ms", 30000),
+        )
+
+        await _post_nav_waits(page, event)
+
+        result: dict = {
+            "title": await page.title(),
+            "url": page.url,
+            "html": await page.content(),
+        }
+
+        if event.get("screenshot", True):
+            png = await page.screenshot(
+                full_page=event.get("full_page_screenshot", False),
+            )
+            result["screenshot_b64"] = base64.b64encode(png).decode()
+
+        return result
+    finally:
+        try:
+            await ctx.close()
+        except Exception:
+            pass