diff --git a/CHANGELOG.md b/CHANGELOG.md index cd1836d..b8e05a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Bright Data Python SDK Changelog +## Version 3.0.0 - Discover API removed + +### ๐Ÿšจ Breaking Changes + +The Bright Data Discover API integration is removed from the SDK. Everything below is gone: + +- **Async client**: `BrightDataClient.discover()` and `BrightDataClient.discover_trigger()` +- **Sync client**: `SyncBrightDataClient.discover()`, `discover_trigger()`, `discover_status()`, `discover_wait()`, `discover_fetch()` and `discover_to_result()` +- **Exported models**: `DiscoverResult`, `DiscoverJob` and `DiscoverSnapshot` are no longer exported from `brightdata` +- **Module**: the `brightdata.discover` package (`DiscoverService` and its models) is deleted + +**Migration**: use `client.search.google()` / `bing()` / `yandex()` to find sources and `client.scrape_url()` to read them. + +Unaffected: the dataset *discovery* endpoints (`client.search.instagram.posts()`, `discover_by_category()`, the `InstagramPostsDiscoverPayload` / `InstagramReelsDiscoverPayload` payloads, and every `discover_by` / `discover_new` dataset trigger) are a different feature and are untouched. + ## Version 2.5.1 - Lint cleanup - **Chore**: pinned an explicit `ruff` lint rule scope (`select`) and sorted imports repo-wide. No behavior changes. diff --git a/README.md b/README.md index cbee752..46d5477 100644 --- a/README.md +++ b/README.md @@ -184,31 +184,6 @@ async with async_playwright() as pw: **When to use:** sites that require full browser automation โ€” JS rendering, login flows, interactive clicks. For plain HTML fetches, prefer `client.scrape_url()`. -### Discover API - -AI-ranked web search. Unlike SERP (which returns engine-ordered results), Discover takes a `query` plus an `intent` phrase and re-ranks by relevance. Optionally extracts full page content as markdown. - -```python -async with BrightDataClient() as client: - result = await client.discover( - query="artificial intelligence trends 2026", - intent="latest AI technology developments", - country="us", - num_results=10, - ) - for item in result.data: - print(f"[{item['relevance_score']:.2f}] {item['title']} {item['link']}") -``` - -For long-running discoveries, trigger and poll separately: - -```python -job = await client.discover_trigger(query="...", intent="...") -result = await job.wait_and_fetch(timeout=60) -``` - -**When to use Discover vs SERP:** Discover when you want *entity-level* relevance ranking driven by a natural-language intent (e.g. "find sustainability-focused AI companies"). SERP when you want raw search engine results. - ### Scraper Studio Run custom collectors built in [Bright Data's Scraper Studio](https://brightdata.com/cp/scrapers). One call triggers the job, polls until ready, and returns the records: diff --git a/notebooks/07_discover_api.ipynb b/notebooks/07_discover_api.ipynb deleted file mode 100644 index 12a1a0e..0000000 --- a/notebooks/07_discover_api.ipynb +++ /dev/null @@ -1,646 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# ๐Ÿ”Ž Discover API - AI-Powered Web Search\n", - "\n", - "Test the Discover API โ€” web search with AI-powered relevance ranking:\n", - "- Basic search\n", - "- Intent-based relevance ranking\n", - "- Filtering with keywords, country, city\n", - "- Full-page content extraction\n", - "- Manual trigger/poll/fetch control\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "API Token: 859a9772-4...4362\n", - "Setup complete!\n" - ] - } - ], - "source": [ - "import os\n", - "from dotenv import load_dotenv\n", - "load_dotenv()\n", - "\n", - "API_TOKEN = os.getenv(\"BRIGHTDATA_API_TOKEN\")\n", - "if not API_TOKEN:\n", - " raise ValueError(\"Set BRIGHTDATA_API_TOKEN in .env file\")\n", - "\n", - "print(f\"API Token: {API_TOKEN[:10]}...{API_TOKEN[-4:]}\")\n", - "print(\"Setup complete!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Initialize Client" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Client initialized\n" - ] - } - ], - "source": [ - "from brightdata import BrightDataClient\n", - "\n", - "client = BrightDataClient(token=API_TOKEN)\n", - "\n", - "print(\"Client initialized\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "## Test 1: Basic Discover Search\n", - "\n", - "Simple search query โ€” returns AI-ranked results with relevance scores." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Discover Search: 'artificial intelligence trends 2026'\n", - "\n", - "Success: True\n", - "Total Results: 20\n", - "Task ID: 214ecd4f-ab5b-4db5-8ca6-5d0b145cfcc0\n", - "Duration: 6s\n", - "\n", - "--- Top 10 Results ---\n", - "\n", - "1. [0.94] 10 Artificial Intelligence (AI) Trends That Will Define 2026\n", - " URL: https://sisuadigital.com/blog/artificial-intelligence-trends-2026-sisua-digital/\n", - " Dec 26, 2025 โ€” Discover the 10 Artificial Intelligence (AI) Trends That Will Def...\n", - "\n", - "2. [0.93] The Future of AI in 2026: Major Trends and Predictions\n", - " URL: https://medium.com/predict/the-future-of-ai-in-2026-major-trends-and-predictions-fad3b6f9ecbe\n", - " The future of AI in 2026 includes agentic AI, autonomous systems, AI-native appl...\n", - "\n", - "3. [0.93] Top 10 AI Trends to Watch in 2026\n", - " URL: https://www.usaii.org/ai-insights/top-10-ai-trends-to-watch-in-2026\n", - " Oct 1, 2025 โ€” Top 10 AI Trends to Watch in 2026 ยท 1. Agentic AI ยท 2. AI Prompt E...\n", - "\n", - "4. [0.92] Top AI Trends in 2026: How Ready Are You?\n", - " URL: https://fptsoftware.com/resource-center/blogs/top-ai-trends-in-2026\n", - " Jan 14, 2026 โ€” Top AI Trends in 2026: How Ready Are You? ยท AI agents will increa...\n", - "\n", - "5. [0.91] The 8 Biggest AI Trends For 2026 That Everyone Must Be ...\n", - " URL: https://bernardmarr.com/the-8-biggest-ai-trends-for-2026-that-everyone-must-be-ready-for-now/\n", - " Oct 7, 2025 โ€” The 8 Biggest AI Trends For 2026 That Everyone Must Be Ready For N...\n", - "\n", - "6. [0.91] What's next in AI: 7 trends to watch in 2026\n", - " URL: https://news.microsoft.com/source/features/ai/whats-next-in-ai-7-trends-to-watch-in-2026/\n", - " Dec 8, 2025 โ€” Seven AI trends to watch in 2026 will make AI a true partner โ€” boo...\n", - "\n", - "7. [0.91] What's next in AI: 7 trends to watch in 2026\n", - " URL: https://news.microsoft.com/source/features/ai/whats-next-in-ai-7-trends-to-watch-in-2026/\n", - " Dec 8, 2025 โ€” Seven AI trends to watch in 2026 will make AI a true partner โ€” boo...\n", - "\n", - "8. [0.91] AI Technology Trends 2026 โ€“ The Future Of Innovation\n", - " URL: https://prolifics.com/usa/resource-center/blog/ai-technology-trends-2026\n", - " Jan 12, 2026 โ€” The Trends That Will Shape AI and Tech in 2026: A Strategic Roadm...\n", - "\n", - "9. [0.91] Top AI Development Trends for 2026-2030: What's Next?\n", - " URL: https://tech-stack.com/blog/top-ai-dev-trends/\n", - " Jan 12, 2026 โ€” By 2026, 75% of businesses will use generative AI to create synth...\n", - "\n", - "10. [0.90] Generative AI Trends for 2026: What Enterprises Must ...\n", - " URL: https://vassardigital.ai/blog/generative-ai-trends-for-2026-what-enterprises-must-prepare-for-next/\n", - " Jan 6, 2026 โ€” Generative AI Trends for 2026: What Enterprises Must Prepare for N...\n" - ] - } - ], - "source": [ - "QUERY = \"artificial intelligence trends 2026\"\n", - "\n", - "print(f\"Discover Search: '{QUERY}'\\n\")\n", - "\n", - "async with client:\n", - " result = await client.discover(\n", - " query=QUERY\n", - " )\n", - "\n", - "print(f\"Success: {result.success}\")\n", - "print(f\"Total Results: {result.total_results}\")\n", - "print(f\"Task ID: {result.task_id}\")\n", - "print(f\"Duration: {result.duration_seconds}s\" if result.duration_seconds else \"Duration: N/A\")\n", - "\n", - "if result.success and result.data:\n", - " print(\"\\n--- Top 10 Results ---\")\n", - " for i, item in enumerate(result.data[:10]):\n", - " score = item.get('relevance_score', 0)\n", - " title = item.get('title', 'N/A')\n", - " link = item.get('link', 'N/A')\n", - " desc = item.get('description', 'N/A')\n", - " print(f\"\\n{i+1}. [{score:.2f}] {title}\")\n", - " print(f\" URL: {link}\")\n", - " print(f\" {desc[:80]}...\" if len(str(desc)) > 80 else f\" {desc}\")\n", - "else:\n", - " print(f\"\\nError: {result.error}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "## Test 2: Search with Intent\n", - "\n", - "The `intent` parameter separates *what* you're searching from *why*.\n", - "The API uses this to rank results by semantic relevance, not just keyword matching." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Query: 'Tesla battery technology'\n", - "Intent: 'recent breakthroughs in EV battery chemistry'\n", - "\n", - "Success: True\n", - "Total Results: 20\n", - "\n", - "--- Results ranked by relevance to intent ---\n", - "\n", - "1. [0.8438] Elon Musk Hails 'Major' Battery Chemistry Breakthrough By ...\n", - " https://finance.yahoo.com/news/elon-musk-hails-major-battery-133103578.html\n", - "\n", - "2. [0.8320] A New Solid-State Battery Could Take Your EV 800 Miles ...\n", - " https://www.popularmechanics.com/cars/hybrid-electric/a69110918/a-new-solid-state-battery-could-take-your-ev-an-astounding-800-miles-on-a-single-charge/\n", - "\n", - "3. [0.8203] Tesla's Quantum Chemistry Breakthrough: Dry Electrode ...\n", - " https://www.linkedin.com/posts/waheedbrown_new-tesla-4680-battery-cells-patent-achieves-activity-7400947804782804992-5q8_\n", - "\n", - "4. [0.8164] The future of electric vehicles & battery chemistry\n", - " https://www.mckinsey.com/industries/automotive-and-assembly/our-insights/the-battery-chemistries-powering-the-future-of-electric-vehicles\n", - "\n", - "5. [0.8086] Chinese researchers achieved a lithium battery with an ... - IO+\n", - " https://ioplus.nl/en/posts/chinese-researchers-achieved-a-lithium-battery-with-an-unprecedented-energy-density-in-2023-this-is-where-the-promising-technology-stands-now\n", - "\n", - "6. [0.8008] Forget solid-state batteries โ€“ researchers have made a ...\n", - " https://www.techradar.com/vehicle-tech/hybrid-electric-vehicles/forget-solid-state-batteries-researchers-have-made-a-lithium-ion-breakthrough-that-could-boost-range-and-drastically-lower-costs\n", - "\n", - "7. [0.8008] Tesla Researchers Tout 100 Year Battery Design\n", - " https://www.silicon.co.uk/e-innovation/research/tesla-researchers-tout-100-year-battery-design-459676\n", - "\n", - "8. [0.7969] Tesla Researchers Announce a 4 Million-Mile Battery But ...\n", - " https://www.torquenews.com/11826/tesla-researchers-announce-4-million-mile-battery-its-only-3rd-bets-cell-chemistry-they\n", - "\n", - "9. [0.7969] Tesla Researchers Announce a 4 Million-Mile Battery But ...\n", - " https://www.torquenews.com/11826/tesla-researchers-announce-4-million-mile-battery-its-only-3rd-bets-cell-chemistry-they\n", - "\n", - "10. [0.7969] Electric Vehicle Innovations Shaping the Future of Mobility\n", - " https://ev-lectron.com/blogs/blog/electric-vehicle-innovations-shaping-the-future-of-mobility?srsltid=AfmBOor4y6tUvkNneh1lU6dtMfskihmx690zBLzPPhy-BFGvRD3JSK74\n" - ] - } - ], - "source": [ - "QUERY = \"Tesla battery technology\"\n", - "INTENT = \"recent breakthroughs in EV battery chemistry\"\n", - "\n", - "print(f\"Query: '{QUERY}'\")\n", - "print(f\"Intent: '{INTENT}'\\n\")\n", - "\n", - "async with client:\n", - " result = await client.discover(\n", - " query=QUERY,\n", - " intent=INTENT\n", - " )\n", - "\n", - "print(f\"Success: {result.success}\")\n", - "print(f\"Total Results: {result.total_results}\")\n", - "\n", - "if result.success and result.data:\n", - " print(\"\\n--- Results ranked by relevance to intent ---\")\n", - " for i, item in enumerate(result.data[:10]):\n", - " score = item.get('relevance_score', 0)\n", - " title = item.get('title', 'N/A')\n", - " link = item.get('link', 'N/A')\n", - " print(f\"\\n{i+1}. [{score:.4f}] {title}\")\n", - " print(f\" {link}\")\n", - "else:\n", - " print(f\"\\nError: {result.error}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "## Test 3: Filter Keywords & Localization\n", - "\n", - "Use `filter_keywords` to narrow results, and `country`/`city` for localization." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Query: 'sustainable fashion brands'\n", - "Intent: 'eco-friendly clothing companies'\n", - "Filter Keywords: ['sustainability', 'eco-friendly', 'organic']\n", - "Country: us\n", - "\n", - "Success: True\n", - "Total Results: 10\n", - "\n", - "--- Filtered Results ---\n", - "\n", - "1. [0.91] 5 Eco-Friendly Clothing Brands Changing The World\n", - " https://www.realthread.com/blog/5-eco-friendly-clothing-brands-changing-the-world\n", - "\n", - "2. [0.91] 5 Eco-Friendly Clothing Brands Changing The World\n", - " https://www.realthread.com/blog/5-eco-friendly-clothing-brands-changing-the-world\n", - "\n", - "3. [0.90] 5 Eco-Friendly Clothing Brands Changing The World\n", - " https://www.realthread.com/blog/5-eco-friendly-clothing-brands-changing-the-world\n", - "\n", - "4. [0.88] 15 Most Eco-Conscious Clothing Brands In 2026\n", - " https://www.thegoodtrade.com/features/eco-friendly-clothing-brands/\n", - "\n", - "5. [0.88] Best Ethical and Sustainable Fashion Brands To Love - BLANC\n", - " https://blancliving.co/blogs/news/best-ethical-sustainable-fashion-brands-to-love\n", - "\n", - "6. [0.88] 24 Sustainable Clothing Brands Leading the Way in 2026\n", - " https://www.sumissura.com/en-us/blog/best-sustainable-clothing-brands\n", - "\n", - "7. [0.88] The Top 5 of the Best Eco-Friendly Fashion Brands\n", - " https://www.weavabel.com/blog/best-eco-friendly-fashion-brands-and-why-theyre-winning-the-ethical-race\n", - "\n", - "8. [0.88] Toad&Co | Sustainable, Organic & Eco Friendly Clothing\n", - " https://www.toadandco.com/\n", - "\n", - "9. [0.88] Toad&Co | Sustainable, Organic & Eco Friendly Clothing\n", - " https://www.toadandco.com/\n", - "\n", - "10. [0.87] The 7 Sustainable Clothing Manufacturers\n", - " https://taianbowinsgarment.com/the-7-sustainable-clothing-manufacturers/\n" - ] - } - ], - "source": [ - "QUERY = \"sustainable fashion brands\"\n", - "INTENT = \"eco-friendly clothing companies\"\n", - "FILTER_KEYWORDS = [\"sustainability\", \"eco-friendly\", \"organic\"]\n", - "\n", - "print(f\"Query: '{QUERY}'\")\n", - "print(f\"Intent: '{INTENT}'\")\n", - "print(f\"Filter Keywords: {FILTER_KEYWORDS}\")\n", - "print(f\"Country: us\\n\")\n", - "\n", - "async with client:\n", - " result = await client.discover(\n", - " query=QUERY,\n", - " intent=INTENT,\n", - " filter_keywords=FILTER_KEYWORDS,\n", - " country=\"us\",\n", - " num_results=10\n", - " )\n", - "\n", - "print(f\"Success: {result.success}\")\n", - "print(f\"Total Results: {result.total_results}\")\n", - "\n", - "if result.success and result.data:\n", - " print(\"\\n--- Filtered Results ---\")\n", - " for i, item in enumerate(result.data[:10]):\n", - " score = item.get('relevance_score', 0)\n", - " title = item.get('title', 'N/A')\n", - " link = item.get('link', 'N/A')\n", - " print(f\"\\n{i+1}. [{score:.2f}] {title}\")\n", - " print(f\" {link}\")\n", - "else:\n", - " print(f\"\\nError: {result.error}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "## Test 4: Include Page Content\n", - "\n", - "Set `include_content=True` to get full page content as markdown alongside each result." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Query: 'python asyncio tutorial'\n", - "Intent: 'beginner-friendly guide to async programming in Python'\n", - "Include Content: True\n", - "\n", - "Success: True\n", - "Total Results: 3\n", - "\n", - "============================================================\n", - "1. [0.88] Python asyncio Tutorial: A Step-by-Step Walkthrough With ...\n", - " URL: https://python.plainenglish.io/python-asyncio-tutorial-a-step-by-step-walkthrough-with-code-6cdb9bc8059e\n", - " Content preview: Python asyncio Tutorial: A Step-by-Step Walkthrough With Code | by Vijay | Python in Plain EnglishA quick intro guide to asynchronous programming in Python (using asyncio) | by Nicolas Galer | Wix Engineering | Medium ...\n" - ] - } - ], - "source": [ - "QUERY = \"python asyncio tutorial\"\n", - "INTENT = \"beginner-friendly guide to async programming in Python\"\n", - "\n", - "print(f\"Query: '{QUERY}'\")\n", - "print(f\"Intent: '{INTENT}'\")\n", - "print(\"Include Content: True\\n\")\n", - "\n", - "async with client:\n", - " result = await client.discover(\n", - " query=QUERY,\n", - " intent=INTENT,\n", - " include_content=True,\n", - " num_results=3\n", - " )\n", - "\n", - "print(f\"Success: {result.success}\")\n", - "print(f\"Total Results: {result.total_results}\")\n", - "\n", - "if result.success and result.data:\n", - " for i, item in enumerate(result.data[:3]):\n", - " score = item.get('relevance_score', 0)\n", - " title = item.get('title', 'N/A')\n", - " content = item.get('content')\n", - " print(f\"\\n{'='*60}\")\n", - " print(f\"{i+1}. [{score:.2f}] {title}\")\n", - " print(f\" URL: {item.get('link', 'N/A')}\")\n", - " if content:\n", - " preview = content[:200].replace('\\n', ' ')\n", - " print(f\" Content preview: {preview}...\")\n", - " else:\n", - " print(\" Content: None\")\n", - "else:\n", - " print(f\"\\nError: {result.error}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "## Test 5: Manual Trigger / Poll / Fetch\n", - "\n", - "Use `discover_trigger()` for manual control โ€” trigger the search, do other work, then poll and fetch." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "QUERY = \"market research SaaS pricing 2026\"\n", - "INTENT = \"competitor pricing strategies for B2B SaaS\"\n", - "\n", - "print(f\"Query: '{QUERY}'\")\n", - "print(f\"Intent: '{INTENT}'\\n\")\n", - "\n", - "async with client:\n", - " # Step 1: Trigger the search\n", - " job = await client.discover_trigger(\n", - " query=QUERY,\n", - " intent=INTENT,\n", - " num_results=5\n", - " )\n", - " print(f\"Step 1 - Triggered: {job}\")\n", - " print(f\" Task ID: {job.task_id}\")\n", - "\n", - " # Step 2: Check status\n", - " status = await job.status()\n", - " print(f\"\\nStep 2 - Status: {status}\")\n", - "\n", - " # Step 3: Wait for completion\n", - " print(\"\\nStep 3 - Waiting for results...\")\n", - " final_status = await job.wait(timeout=60, poll_interval=2)\n", - " print(f\" Final status: {final_status}\")\n", - "\n", - " # Step 4: Fetch results\n", - " data = await job.fetch()\n", - " print(f\"\\nStep 4 - Fetched {len(data)} results\")\n", - "\n", - " for i, item in enumerate(data):\n", - " score = item.get('relevance_score', 0)\n", - " title = item.get('title', 'N/A')\n", - " link = item.get('link', 'N/A')\n", - " print(f\"\\n {i+1}. [{score:.2f}] {title}\")\n", - " print(f\" {link}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "## Test 6: Timing Metadata" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Use result from Test 2\n", - "print(\"=== Result Metadata ===\")\n", - "print(f\"success: {result.success}\")\n", - "print(f\"query: {result.query}\")\n", - "print(f\"intent: {result.intent}\")\n", - "print(f\"total_results: {result.total_results}\")\n", - "print(f\"duration_seconds: {result.duration_seconds}\")\n", - "print(f\"task_id: {result.task_id}\")\n", - "\n", - "print(\"\\n=== Timing ===\")\n", - "print(f\"trigger_sent_at: {result.trigger_sent_at}\")\n", - "print(f\"data_fetched_at: {result.data_fetched_at}\")\n", - "\n", - "elapsed = result.elapsed_ms()\n", - "if elapsed:\n", - " print(f\"\\nTotal time: {elapsed / 1000:.2f} seconds\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "## Test 7: Export Results to JSON" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "from pathlib import Path\n", - "\n", - "if result.success and result.data:\n", - " output_file = Path.cwd() / \"discover_results.json\"\n", - "\n", - " export_data = {\n", - " \"success\": result.success,\n", - " \"query\": result.query,\n", - " \"intent\": result.intent,\n", - " \"total_results\": result.total_results,\n", - " \"duration_seconds\": result.duration_seconds,\n", - " \"results\": result.data,\n", - " }\n", - "\n", - " with open(output_file, \"w\") as f:\n", - " json.dump(export_data, f, indent=2, default=str)\n", - "\n", - " print(f\"Exported to: {output_file}\")\n", - " print(f\"Results count: {len(result.data)}\")\n", - "else:\n", - " print(\"No successful results to export\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "## Summary\n", - "\n", - "### Methods\n", - "\n", - "| Method | Description |\n", - "|--------|-------------|\n", - "| `client.discover(query, ...)` | Search + poll + return results |\n", - "| `client.discover_trigger(query, ...)` | Trigger search, return `DiscoverJob` for manual control |\n", - "\n", - "### Parameters\n", - "\n", - "| Parameter | Type | Description | Default |\n", - "|-----------|------|-------------|--------|\n", - "| `query` | str | Search query | Required |\n", - "| `intent` | str | Why you're searching โ€” guides AI relevance ranking | `None` |\n", - "| `include_content` | bool | Return page content as markdown | `False` |\n", - "| `country` | str | Country code for localized results | `None` |\n", - "| `city` | str | City for localized results | `None` |\n", - "| `language` | str | Language code | `None` |\n", - "| `filter_keywords` | list[str] | Filter results by keywords | `None` |\n", - "| `num_results` | int | Number of results to return | `None` |\n", - "| `format` | str | Response format | `\"json\"` |\n", - "| `timeout` | int | Max seconds to wait | `60` |\n", - "| `poll_interval` | int | Seconds between status checks | `2` |\n", - "\n", - "### DiscoverJob Methods (Manual Control)\n", - "\n", - "| Method | Description |\n", - "|--------|-------------|\n", - "| `job.status()` | Check task status (`'processing'` or `'done'`) |\n", - "| `job.wait(timeout, poll_interval)` | Poll until done or timeout |\n", - "| `job.fetch()` | Fetch results after completion |\n", - "| `job.to_result()` | Wait + fetch + wrap as `DiscoverResult` |\n", - "\n", - "### Result Item Fields\n", - "\n", - "| Field | Type | Description |\n", - "|-------|------|-------------|\n", - "| `link` | str | URL of the result |\n", - "| `title` | str | Page title |\n", - "| `description` | str | Snippet / meta description |\n", - "| `relevance_score` | float | 0.0โ€“1.0, AI-computed relevance to stated intent |\n", - "| `content` | str/null | Page content in markdown (only if `include_content=True`) |\n", - "\n", - "### Discover vs SERP\n", - "\n", - "| | SERP (`client.search.google`) | Discover (`client.discover`) |\n", - "|---|---|---|\n", - "| Results | Raw search engine results | AI-ranked by intent |\n", - "| Relevance | Keyword-based | Semantic (via `intent`) |\n", - "| Content | Snippets only | Optional full-page markdown |\n", - "| Use case | Known search queries | Open-ended research |" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv (3.11.10)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.10" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/pyproject.toml b/pyproject.toml index 2cbdc1f..326f878 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ where = ["src"] [project] name = "brightdata-sdk" -version = "2.5.1" +version = "3.0.0" description = "Modern async-first Python SDK for Bright Data APIs" authors = [{name = "Bright Data", email = "support@brightdata.com"}] license = {text = "MIT"} diff --git a/src/brightdata/__init__.py b/src/brightdata/__init__.py index 0f5f7c7..3807dc4 100644 --- a/src/brightdata/__init__.py +++ b/src/brightdata/__init__.py @@ -13,9 +13,6 @@ from .client import BrightDataClient from .core.zone_manager import ZoneManager -# Export Discover API models -from .discover.models import DiscoverJob, DiscoverResult, DiscoverSnapshot - # Export exceptions from .exceptions import ( APIError, @@ -134,10 +131,6 @@ "ScraperStudioJob", "JobStatus", "ScraperStudioService", - # Discover API - "DiscoverResult", - "DiscoverJob", - "DiscoverSnapshot", # Services "WebUnlockerService", "BrowserService", diff --git a/src/brightdata/client.py b/src/brightdata/client.py index b559ba7..9e0bd61 100644 --- a/src/brightdata/client.py +++ b/src/brightdata/client.py @@ -29,8 +29,6 @@ from .core.zone_manager import ZoneManager from .crawler.service import CrawlerService from .datasets import DatasetsClient -from .discover.models import DiscoverJob, DiscoverResult -from .discover.service import DiscoverService from .exceptions import APIError, AuthenticationError, ValidationError from .models import ScrapeResult from .scraper_studio.service import ScraperStudioService @@ -161,7 +159,6 @@ def __init__( self._datasets_client: Optional[DatasetsClient] = None self._scraper_studio_service: Optional[ScraperStudioService] = None self._browser_service: Optional[BrowserService] = None - self._discover_service: Optional[DiscoverService] = None self._zone_manager: Optional[ZoneManager] = None self._is_connected = False self._account_info: Optional[Dict[str, Any]] = None @@ -642,127 +639,6 @@ async def scrape_url( poll_timeout=poll_timeout, ) - async def discover( - self, - query: str, - intent: Optional[str] = None, - include_content: bool = False, - country: Optional[str] = None, - city: Optional[str] = None, - language: Optional[str] = None, - filter_keywords: Optional[List[str]] = None, - num_results: Optional[int] = None, - format: str = "json", - timeout: int = 60, - poll_interval: int = 2, - ) -> DiscoverResult: - """ - Search the web with AI-powered relevance ranking. - - Triggers a search, polls until complete, and returns ranked results. - Uses the Discover API which adds AI relevance ranking via `intent` - and optional full-page content extraction. - - Args: - query: Search query string. - intent: Why you're searching โ€” guides AI relevance ranking. - include_content: If True, returns page content as markdown. - country: Country code for localized results (e.g., "us"). - city: City for localized results (e.g., "new york"). - language: Language code for localized results. - filter_keywords: Filter results by keywords (e.g., ["sustainability"]). - num_results: Number of results to return. - format: Response format (default: "json"). - timeout: Max seconds to wait for results (default: 60). - poll_interval: Seconds between status checks (default: 2). - - Returns: - DiscoverResult with AI-ranked search results. - - Example: - >>> async with BrightDataClient() as client: - ... result = await client.discover( - ... query="artificial intelligence trends 2026", - ... intent="latest AI technology developments", - ... ) - ... for item in result.data: - ... print(f"[{item['relevance_score']:.2f}] {item['title']}") - """ - self._ensure_initialized() - if self._discover_service is None: - self._discover_service = DiscoverService(self.engine) - - return await self._discover_service.search( - query=query, - intent=intent, - include_content=include_content, - country=country, - city=city, - language=language, - filter_keywords=filter_keywords, - num_results=num_results, - format=format, - timeout=timeout, - poll_interval=poll_interval, - ) - - async def discover_trigger( - self, - query: str, - intent: Optional[str] = None, - include_content: bool = False, - country: Optional[str] = None, - city: Optional[str] = None, - language: Optional[str] = None, - filter_keywords: Optional[List[str]] = None, - num_results: Optional[int] = None, - format: str = "json", - ) -> DiscoverJob: - """ - Trigger a discover search and return a job for manual polling. - - Use this when you want to do other work while waiting for results. - - Args: - query: Search query string. - intent: Why you're searching โ€” guides AI relevance ranking. - include_content: If True, returns page content as markdown. - country: Country code for localized results. - city: City for localized results. - language: Language code for localized results. - filter_keywords: Filter results by keywords. - num_results: Number of results to return. - format: Response format (default: "json"). - - Returns: - DiscoverJob for manual polling and fetching. - - Example: - >>> async with BrightDataClient() as client: - ... job = await client.discover_trigger( - ... query="market research SaaS pricing", - ... intent="competitor pricing strategies", - ... ) - ... # Do other work... - ... await job.wait(timeout=60) - ... data = await job.fetch() - """ - self._ensure_initialized() - if self._discover_service is None: - self._discover_service = DiscoverService(self.engine) - - return await self._discover_service.trigger( - query=query, - intent=intent, - include_content=include_content, - country=country, - city=city, - language=language, - filter_keywords=filter_keywords, - num_results=num_results, - format=format, - ) - async def __aenter__(self): """Async context manager entry.""" await self.engine.__aenter__() diff --git a/src/brightdata/discover/__init__.py b/src/brightdata/discover/__init__.py deleted file mode 100644 index 045df99..0000000 --- a/src/brightdata/discover/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Discover API โ€” AI-powered web search with relevance ranking.""" - -from .models import DiscoverJob, DiscoverResult -from .service import DiscoverService - -__all__ = ["DiscoverService", "DiscoverResult", "DiscoverJob"] diff --git a/src/brightdata/discover/models.py b/src/brightdata/discover/models.py deleted file mode 100644 index f9c324f..0000000 --- a/src/brightdata/discover/models.py +++ /dev/null @@ -1,193 +0,0 @@ -"""Models for Discover API results and jobs.""" - -from __future__ import annotations - -import asyncio -import time -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Dict, List, Optional - -from ..models import BaseResult - -if TYPE_CHECKING: - from .service import DiscoverService - - -@dataclass -class DiscoverResult(BaseResult): - """ - Result from a Discover API search. - - Contains AI-ranked web search results with relevance scores - and optional full-page content. - - Attributes: - query: Original search query. - intent: Intent description used for relevance ranking. - data: List of result dicts with link, title, description, relevance_score, content. - duration_seconds: Server-side processing duration. - total_results: Number of results returned. - task_id: Discover API task identifier. - """ - - query: str = "" - intent: Optional[str] = None - data: Optional[List[Dict[str, Any]]] = None - duration_seconds: Optional[float] = None - total_results: Optional[int] = None - task_id: Optional[str] = None - - def __repr__(self) -> str: - """String representation with query info.""" - base_repr = super().__repr__() - query_str = self.query[:50] + "..." if len(self.query) > 50 else self.query - total_str = f" results={self.total_results}" if self.total_results else "" - return f"" - - -@dataclass -class DiscoverSnapshot: - """ - Colorless handle for a triggered Discover search โ€” data only, no I/O methods. - - Mirrors the crawler's CrawlJob shape: the verbs live on the service, not the - handle. Returned by the sync client's ``discover_trigger()``; poll/fetch via - ``discover_status`` / ``discover_wait`` / ``discover_fetch`` / ``discover_to_result`` - (by ``task_id``), or the async ``DiscoverService`` verbs. - - Attributes: - task_id: Discover API task identifier. - query: Original search query (echo). - intent: Intent used for ranking (echo). - """ - - task_id: str - query: str = "" - intent: Optional[str] = None - - def __repr__(self) -> str: - return f"" - - -@dataclass -class DiscoverJob: - """ - Handle for a pending Discover API search. - - Created by discover_trigger(), allows manual polling and fetching. - - Example: - >>> job = await client.discover_trigger( - ... query="market research SaaS pricing", - ... intent="competitor pricing strategies", - ... ) - >>> print(f"Task ID: {job.task_id}") - >>> await job.wait(timeout=60) - >>> data = await job.fetch() - """ - - task_id: str - _service: "DiscoverService" = field(repr=False) - query: str = "" - intent: Optional[str] = None - - def __repr__(self) -> str: - return f"" - - async def status(self) -> str: - """ - Check task status. - - Returns: - 'processing' or 'done' - """ - response_data = await self._service._poll_once(self.task_id) - return response_data.get("status", "processing") - - async def wait(self, timeout: int = 60, poll_interval: int = 2) -> str: - """ - Poll until done or timeout. - - Args: - timeout: Maximum seconds to wait. - poll_interval: Seconds between status checks. - - Returns: - Final status string. - - Raises: - TimeoutError: If timeout is reached. - APIError: If task fails. - """ - from ..exceptions import APIError - - start = time.time() - while True: - elapsed = time.time() - start - if elapsed > timeout: - raise TimeoutError(f"Discover task {self.task_id} timed out after {timeout}s") - - response_data = await self._service._poll_once(self.task_id) - status = response_data.get("status", "processing") - - if status == "done": - self._last_response = response_data - return status - elif status in ("error", "failed"): - raise APIError(f"Discover task {self.task_id} failed with status: {status}") - - await asyncio.sleep(poll_interval) - - async def fetch(self) -> List[Dict[str, Any]]: - """ - Fetch results. Call after wait() returns 'done'. - - Returns: - List of result dicts. - """ - if hasattr(self, "_last_response") and self._last_response.get("status") == "done": - return self._last_response.get("results", []) - - response_data = await self._service._poll_once(self.task_id) - return response_data.get("results", []) - - async def to_result(self, timeout: int = 60, poll_interval: int = 2) -> DiscoverResult: - """ - Wait + fetch + wrap in DiscoverResult. - - Args: - timeout: Maximum seconds to wait. - poll_interval: Seconds between status checks. - - Returns: - DiscoverResult with full data. - """ - from datetime import datetime, timezone - - start_time = datetime.now(timezone.utc) - - try: - await self.wait(timeout=timeout, poll_interval=poll_interval) - results = await self.fetch() - end_time = datetime.now(timezone.utc) - - return DiscoverResult( - success=True, - query=self.query, - intent=self.intent, - data=results, - total_results=len(results), - task_id=self.task_id, - trigger_sent_at=start_time, - data_fetched_at=end_time, - ) - except Exception as e: - return DiscoverResult( - success=False, - query=self.query, - intent=self.intent, - error=str(e), - task_id=self.task_id, - trigger_sent_at=start_time, - data_fetched_at=datetime.now(timezone.utc), - ) diff --git a/src/brightdata/discover/service.py b/src/brightdata/discover/service.py deleted file mode 100644 index b84cd66..0000000 --- a/src/brightdata/discover/service.py +++ /dev/null @@ -1,296 +0,0 @@ -""" -Discover API service โ€” AI-powered web search with relevance ranking. - -Provides access to Bright Data's Discover API, which performs web searches -with AI-powered relevance ranking based on stated intent. -""" - -import asyncio -import time -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional - -from ..core.engine import AsyncEngine -from ..exceptions import APIError -from .models import DiscoverJob, DiscoverResult - - -class DiscoverService: - """ - Bright Data Discover API โ€” AI-powered web search with relevance ranking. - - The Discover API differs from SERP scraping: it uses an `intent` parameter - to separate what you're searching for from why, enabling AI-powered - relevance ranking and optional full-page content extraction. - - Example: - >>> async with BrightDataClient() as client: - ... result = await client.discover( - ... query="artificial intelligence trends 2026", - ... intent="latest AI technology developments", - ... ) - ... for item in result.data: - ... print(f"[{item['relevance_score']:.2f}] {item['title']}") - """ - - def __init__(self, engine: AsyncEngine): - self._engine = engine - - async def search( - self, - query: str, - intent: Optional[str] = None, - include_content: bool = False, - country: Optional[str] = None, - city: Optional[str] = None, - language: Optional[str] = None, - filter_keywords: Optional[List[str]] = None, - num_results: Optional[int] = None, - format: str = "json", - timeout: int = 60, - poll_interval: int = 2, - ) -> DiscoverResult: - """ - Search the web with AI-powered relevance ranking. - - Triggers a search, polls until complete, and returns results. - - Args: - query: Search query string. - intent: Why you're searching โ€” guides AI relevance ranking. - include_content: If True, returns page content as markdown. - country: Country code for localized results (e.g., "us"). - city: City for localized results (e.g., "new york"). - language: Language code for localized results. - filter_keywords: Filter results by keywords. - num_results: Number of results to return. - format: Response format (default: "json"). - timeout: Max seconds to wait for results (default: 60). - poll_interval: Seconds between status checks (default: 2). - - Returns: - DiscoverResult with ranked search results. - - Raises: - APIError: If the API request fails. - TimeoutError: If polling exceeds timeout. - """ - trigger_time = datetime.now(timezone.utc) - - task_id = await self._trigger( - query=query, - intent=intent, - include_content=include_content, - country=country, - city=city, - language=language, - filter_keywords=filter_keywords, - num_results=num_results, - format=format, - ) - - response_data = await self._poll_until_done(task_id, timeout, poll_interval) - fetch_time = datetime.now(timezone.utc) - - results = response_data.get("results", []) - duration = response_data.get("duration_seconds") - - return DiscoverResult( - success=True, - query=query, - intent=intent, - data=results, - duration_seconds=duration, - total_results=len(results), - task_id=task_id, - trigger_sent_at=trigger_time, - data_fetched_at=fetch_time, - ) - - async def trigger( - self, - query: str, - intent: Optional[str] = None, - include_content: bool = False, - country: Optional[str] = None, - city: Optional[str] = None, - language: Optional[str] = None, - filter_keywords: Optional[List[str]] = None, - num_results: Optional[int] = None, - format: str = "json", - ) -> DiscoverJob: - """ - Trigger a search and return a job for manual polling. - - Use this when you want to do other work while waiting for results. - - Args: - query: Search query string. - intent: Why you're searching โ€” guides AI relevance ranking. - include_content: If True, returns page content as markdown. - country: Country code for localized results. - city: City for localized results. - language: Language code for localized results. - filter_keywords: Filter results by keywords. - num_results: Number of results to return. - format: Response format (default: "json"). - - Returns: - DiscoverJob for manual polling and fetching. - - Raises: - APIError: If the trigger request fails. - """ - task_id = await self._trigger( - query=query, - intent=intent, - include_content=include_content, - country=country, - city=city, - language=language, - filter_keywords=filter_keywords, - num_results=num_results, - format=format, - ) - - return DiscoverJob( - task_id=task_id, - _service=self, - query=query, - intent=intent, - ) - - # ------------------------------------------------------------------ - # PUBLIC SERVICE VERBS (drive a discover task by task_id) - # ------------------------------------------------------------------ - # Discover was the one subsystem lacking id-based status/fetch on the - # service (it had only the DiscoverJob methods). These are additive and - # mirror DiscoverJob.status/fetch/wait/to_result, keyed by task_id, so a - # caller can poll/fetch a triggered search with only its task_id. - - async def status(self, task_id: str) -> str: - """Check a discover task's status by task_id ('processing' or 'done').""" - response_data = await self._poll_once(task_id) - return response_data.get("status", "processing") - - async def fetch(self, task_id: str) -> List[Dict[str, Any]]: - """Fetch a discover task's results by task_id. Call after status == 'done'.""" - response_data = await self._poll_once(task_id) - return response_data.get("results", []) - - async def wait(self, task_id: str, timeout: int = 60, poll_interval: int = 2) -> str: - """Poll a discover task until done (or fail/timeout), by task_id.""" - await self._poll_until_done(task_id, timeout, poll_interval) - return "done" - - async def to_result( - self, task_id: str, timeout: int = 60, poll_interval: int = 2 - ) -> DiscoverResult: - """ - Wait + fetch + wrap a discover task (by task_id) as DiscoverResult. - - Note: query/intent are not recoverable from a bare task_id, so they are - left empty here; use DiscoverJob.to_result() (or the service's search()) - when you need them populated. - """ - trigger_time = datetime.now(timezone.utc) - try: - response_data = await self._poll_until_done(task_id, timeout, poll_interval) - fetch_time = datetime.now(timezone.utc) - results = response_data.get("results", []) - return DiscoverResult( - success=True, - data=results, - duration_seconds=response_data.get("duration_seconds"), - total_results=len(results), - task_id=task_id, - trigger_sent_at=trigger_time, - data_fetched_at=fetch_time, - ) - except Exception as e: - return DiscoverResult( - success=False, - error=str(e), - task_id=task_id, - trigger_sent_at=trigger_time, - data_fetched_at=datetime.now(timezone.utc), - ) - - async def _trigger( - self, - query: str, - intent: Optional[str] = None, - include_content: bool = False, - country: Optional[str] = None, - city: Optional[str] = None, - language: Optional[str] = None, - filter_keywords: Optional[List[str]] = None, - num_results: Optional[int] = None, - format: str = "json", - ) -> str: - """POST /discover, return task_id.""" - payload: Dict[str, Any] = {"query": query} - - if intent: - payload["intent"] = intent - if include_content: - payload["include_content"] = True - if country: - payload["country"] = country - if city: - payload["city"] = city - if language: - payload["language"] = language - if filter_keywords: - payload["filter_keywords"] = filter_keywords - if num_results is not None: - payload["num_results"] = num_results - if format: - payload["format"] = format - - async with self._engine.post("/discover", json_data=payload) as response: - if response.status >= 400: - text = await response.text() - raise APIError( - f"Discover trigger failed (HTTP {response.status}): {text}", - status_code=response.status, - ) - - data = await response.json() - task_id = data.get("task_id") - if not task_id: - raise APIError("No task_id in discover response") - return task_id - - async def _poll_once(self, task_id: str) -> Dict[str, Any]: - """GET /discover?task_id=, return response data.""" - async with self._engine.get("/discover", params={"task_id": task_id}) as response: - if response.status >= 400: - text = await response.text() - raise APIError( - f"Discover poll failed (HTTP {response.status}): {text}", - status_code=response.status, - ) - return await response.json() - - async def _poll_until_done( - self, task_id: str, timeout: int, poll_interval: int - ) -> Dict[str, Any]: - """Poll GET /discover?task_id= until done or timeout.""" - start = time.time() - - while True: - elapsed = time.time() - start - if elapsed > timeout: - raise TimeoutError(f"Discover task {task_id} timed out after {timeout}s") - - response_data = await self._poll_once(task_id) - status = response_data.get("status", "processing") - - if status == "done": - return response_data - elif status in ("error", "failed"): - error_msg = response_data.get("error", "Unknown error") - raise APIError(f"Discover task failed: {error_msg}") - - await asyncio.sleep(poll_interval) diff --git a/src/brightdata/models.py b/src/brightdata/models.py index 86773aa..0108123 100644 --- a/src/brightdata/models.py +++ b/src/brightdata/models.py @@ -348,6 +348,3 @@ def __repr__(self) -> str: Result = Union[BaseResult, ScrapeResult, SearchResult, CrawlResult] - -# DiscoverResult is defined in discover/models.py to avoid circular imports -# It extends BaseResult with query, intent, and relevance-scored results diff --git a/src/brightdata/sync_client.py b/src/brightdata/sync_client.py index 447ee1b..a95fd7c 100644 --- a/src/brightdata/sync_client.py +++ b/src/brightdata/sync_client.py @@ -12,7 +12,6 @@ from .browser.service import BrowserService from .client import BrightDataClient -from .discover.models import DiscoverResult, DiscoverSnapshot from .models import ScrapeResult, SearchResult from .types import AccountInfo @@ -193,50 +192,6 @@ def scrape_url(self, url, **kwargs): """Scrape URL using Web Unlocker.""" return self._run(self._async_client.scrape_url(url, **kwargs)) - def discover(self, query: str, **kwargs) -> DiscoverResult: - """Search the web with AI-powered relevance ranking.""" - return self._run(self._async_client.discover(query, **kwargs)) - - def discover_trigger(self, query: str, **kwargs) -> DiscoverSnapshot: - """Trigger a discover search; returns a colorless DiscoverSnapshot. - - Poll/fetch with discover_status / discover_wait / discover_fetch / - discover_to_result (by task_id). (Previously returned the async-only - DiscoverJob, which could not be driven from sync.) - """ - job = self._run(self._async_client.discover_trigger(query, **kwargs)) - return DiscoverSnapshot( - task_id=job.task_id, - query=getattr(job, "query", "") or "", - intent=getattr(job, "intent", None), - ) - - def _discover_service(self): - """The async DiscoverService, ensured to exist (lazy, same as the async client).""" - svc = self._async_client._discover_service - if svc is None: - from .discover.service import DiscoverService - - svc = DiscoverService(self._async_client.engine) - self._async_client._discover_service = svc - return svc - - def discover_status(self, task_id: str) -> str: - """Check a triggered discover search's status by task_id ('processing'/'done').""" - return self._run(self._discover_service().status(task_id)) - - def discover_wait(self, task_id: str, **kwargs) -> str: - """Poll a triggered discover search until done, by task_id.""" - return self._run(self._discover_service().wait(task_id, **kwargs)) - - def discover_fetch(self, task_id: str): - """Fetch a triggered discover search's results by task_id.""" - return self._run(self._discover_service().fetch(task_id)) - - def discover_to_result(self, task_id: str, **kwargs) -> DiscoverResult: - """Wait + fetch + wrap a triggered discover search as DiscoverResult, by task_id.""" - return self._run(self._discover_service().to_result(task_id, **kwargs)) - # ======================================== # Service Properties # ======================================== diff --git a/tests/unit/test_colorless_service_verbs.py b/tests/unit/test_colorless_service_verbs.py index 85a1f7f..549c735 100644 --- a/tests/unit/test_colorless_service_verbs.py +++ b/tests/unit/test_colorless_service_verbs.py @@ -4,7 +4,6 @@ Covers the additive service-level verbs that let a triggered job be driven by its id alone: - BaseWebScraper.status / wait / fetch / to_result(snapshot_id) - - DiscoverService.status / wait / fetch / to_result(task_id) Also guards that the existing colored ScrapeJob methods still work (no regression) and that the relocated logic matches the job's behavior. @@ -12,12 +11,10 @@ Mocked at the api_client / _poll_once seam (not raw aiohttp), per the plan. """ -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock import pytest -from brightdata.discover.models import DiscoverResult -from brightdata.discover.service import DiscoverService from brightdata.exceptions import APIError from brightdata.models import ScrapeResult from brightdata.scrapers.amazon import AmazonScraper @@ -122,54 +119,3 @@ async def test_service_verb_matches_job(self): job = ScrapeJob(snapshot_id="snap_1", api_client=api) assert await s.fetch("snap_1") == await job.fetch() assert await s.status("snap_1") == await job.status() - - -# --------------------------------------------------------------------------- -# DiscoverService service verbs (the previously-missing id-based path) -# --------------------------------------------------------------------------- - - -class TestDiscoverServiceVerbs: - def _svc(self): - return DiscoverService(engine=MagicMock()) - - @pytest.mark.asyncio - async def test_status_by_task_id(self): - svc = self._svc() - svc._poll_once = AsyncMock(return_value={"status": "done", "results": []}) - assert await svc.status("t1") == "done" - svc._poll_once.assert_awaited_once_with("t1") - - @pytest.mark.asyncio - async def test_fetch_by_task_id(self): - svc = self._svc() - svc._poll_once = AsyncMock(return_value={"status": "done", "results": [{"r": 1}]}) - assert await svc.fetch("t1") == [{"r": 1}] - - @pytest.mark.asyncio - async def test_wait_by_task_id(self): - svc = self._svc() - svc._poll_until_done = AsyncMock(return_value={"status": "done", "results": []}) - assert await svc.wait("t1") == "done" - svc._poll_until_done.assert_awaited_once() - - @pytest.mark.asyncio - async def test_to_result_by_task_id(self): - svc = self._svc() - svc._poll_until_done = AsyncMock( - return_value={"status": "done", "results": [{"r": 1}], "duration_seconds": 1.2} - ) - res = await svc.to_result("t1") - assert isinstance(res, DiscoverResult) - assert res.success is True - assert res.total_results == 1 - assert res.task_id == "t1" - - @pytest.mark.asyncio - async def test_to_result_failure_is_caught(self): - svc = self._svc() - svc._poll_until_done = AsyncMock(side_effect=APIError("Discover task failed: boom")) - res = await svc.to_result("t1") - assert res.success is False - assert res.task_id == "t1" - assert res.error and "boom" in res.error diff --git a/tests/unit/test_sync_client_coverage.py b/tests/unit/test_sync_client_coverage.py index 4eede2e..08b2c92 100644 --- a/tests/unit/test_sync_client_coverage.py +++ b/tests/unit/test_sync_client_coverage.py @@ -16,7 +16,6 @@ import pytest from brightdata import SyncBrightDataClient -from brightdata.discover.models import DiscoverSnapshot from brightdata.models import ScrapeResult from brightdata.scrapers.service import ScrapeService from brightdata.serp.service import SearchService @@ -170,39 +169,3 @@ def test_instagram_search_new_methods(self, loop): s = SyncInstagramSearchScraper(api, loop) assert s.profiles("u") == "P" assert s.reels_all("u") == "R" - - -class TestDiscoverSyncPath: - """Discover's sync manual path โ€” the one subsystem that had none before.""" - - def _client(self, loop): - c = SyncBrightDataClient(token="x" * 12) - c._loop = loop # inject the loop; do NOT enter the context (no network) - return c - - def test_discover_trigger_returns_colorless_snapshot(self, loop): - c = self._client(loop) - fake_job = MagicMock(task_id="t1", query="q", intent="i") - c._async_client.discover_trigger = AsyncMock(return_value=fake_job) - snap = c.discover_trigger("q", intent="i") - assert isinstance(snap, DiscoverSnapshot) - assert (snap.task_id, snap.query, snap.intent) == ("t1", "q", "i") - # colorless: no I/O methods on the handle - assert not hasattr(snap, "fetch") - - def test_discover_status_wait_fetch_by_task_id(self, loop): - c = self._client(loop) - svc = MagicMock() - svc.status = AsyncMock(return_value="done") - svc.wait = AsyncMock(return_value="done") - svc.fetch = AsyncMock(return_value=[{"r": 1}]) - c._async_client._discover_service = svc - assert c.discover_status("t1") == "done" - assert c.discover_wait("t1") == "done" - assert c.discover_fetch("t1") == [{"r": 1}] - - def test_discover_service_is_ensured_when_missing(self, loop): - c = self._client(loop) - c._async_client._discover_service = None # not yet created - svc = c._discover_service() - assert svc is not None and c._async_client._discover_service is svc