From 45957fb0c377663d3fbba3d07fade25057a97f0e Mon Sep 17 00:00:00 2001 From: Tarek Etman Date: Wed, 11 Nov 2026 05:12:23 +0000 Subject: [PATCH 1/2] feat(integrations): let Google Custom Search use another API address An optional API address field on the google_cse integration. Left empty, searches and the connect check go to https://www.googleapis.com/customsearch/v1 as before. --- changes/+google-cse-address.added.md | 2 + sentient/integrations/plugins/alternatives.py | 14 ++++- sentient/integrations/plugins/search.py | 8 ++- tests/integrations/test_google_cse_address.py | 60 +++++++++++++++++++ 4 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 changes/+google-cse-address.added.md create mode 100644 tests/integrations/test_google_cse_address.py diff --git a/changes/+google-cse-address.added.md b/changes/+google-cse-address.added.md new file mode 100644 index 000000000..e8779a214 --- /dev/null +++ b/changes/+google-cse-address.added.md @@ -0,0 +1,2 @@ +**Google Custom Search can use another API address.** An optional field when you connect it sends searches to a +service that answers in the same format; left empty, searches go to Google as before. diff --git a/sentient/integrations/plugins/alternatives.py b/sentient/integrations/plugins/alternatives.py index 57ab45781..8561b2e61 100644 --- a/sentient/integrations/plugins/alternatives.py +++ b/sentient/integrations/plugins/alternatives.py @@ -8,6 +8,7 @@ from sentient.integrations.base import IntegrationError, IntegrationPlugin, SetupField from sentient.integrations.common import http_client +from sentient.integrations.plugins.search import GOOGLE_CSE_URL if TYPE_CHECKING: # pragma: no cover from sentient.integrations.service import IntegrationManager @@ -126,6 +127,9 @@ class GoogleCSEPlugin(IntegrationPlugin): setup_fields = [ SetupField("api_key", "API key", secret=True, help="Google Cloud Console → APIs & Services → Credentials."), SetupField("cx", "Search engine ID", secret=False, help="From programmablesearchengine.google.com (the 'cx' value)."), + SetupField("base_url", "API address", required=False, placeholder="https://www.googleapis.com/customsearch/v1", + help="Optional. Leave empty to use Google. Set it only for a service that answers in the " + "Custom Search JSON format."), ] docs_url = "https://developers.google.com/custom-search/v1/overview" instructions_md = ( @@ -140,12 +144,18 @@ class GoogleCSEPlugin(IntegrationPlugin): async def validate(self, fields: dict[str, str], mgr: IntegrationManager) -> tuple[dict, str | None]: key, cx = _require(fields, "api_key", "cx") + base_url = str(fields.get("base_url", "")).strip() + if base_url and not base_url.lower().startswith(("http://", "https://")): + raise IntegrationError("The API address has to start with http:// or https://.") async with http_client() as http: - r = await http.get("https://www.googleapis.com/customsearch/v1", params={"key": key, "cx": cx, "q": "test", "num": 1}) + r = await http.get(base_url or GOOGLE_CSE_URL, params={"key": key, "cx": cx, "q": "test", "num": 1}) if r.status_code == 400: raise IntegrationError("Google didn't accept the search engine ID. Check the 'cx' value.") _rejected(r, "Google Custom Search") - return {"api_key": key, "cx": cx}, None + creds = {"api_key": key, "cx": cx} + if base_url: + creds["base_url"] = base_url + return creds, None class GoogleMapsPlugin(IntegrationPlugin): diff --git a/sentient/integrations/plugins/search.py b/sentient/integrations/plugins/search.py index 2db620aa2..5c1e356e1 100644 --- a/sentient/integrations/plugins/search.py +++ b/sentient/integrations/plugins/search.py @@ -11,6 +11,8 @@ log = logging.getLogger(__name__) +GOOGLE_CSE_URL = "https://www.googleapis.com/customsearch/v1" + def _ddg_sync(query: str, n: int, region: str) -> list[dict]: from ddgs import DDGS @@ -36,9 +38,9 @@ async def brave_search(key: str, query: str, n: int) -> list[dict]: return [{"title": x.get("title"), "url": x.get("url"), "snippet": x.get("description")} for x in rows] -async def google_cse_search(key: str, cx: str, query: str, n: int) -> list[dict]: +async def google_cse_search(key: str, cx: str, query: str, n: int, base_url: str = "") -> list[dict]: async with http_client() as http: - r = await http.get("https://www.googleapis.com/customsearch/v1", + r = await http.get(base_url or GOOGLE_CSE_URL, params={"key": key, "cx": cx, "q": query, "num": min(n, 10)}) r.raise_for_status() rows = r.json().get("items") or [] @@ -72,7 +74,7 @@ async def web_search(ctx: ToolContext, query: str, max_results: int = 8) -> dict elif provider == "google_cse": c = await mgr.get_credentials("google_cse") if await mgr.is_connected("google_cse") else None if c and c.get("api_key") and c.get("cx"): - results = await google_cse_search(c["api_key"], c["cx"], query, n) + results = await google_cse_search(c["api_key"], c["cx"], query, n, c.get("base_url", "")) else: note = "Google Custom Search isn't connected; used DuckDuckGo instead." elif provider == "searxng": diff --git a/tests/integrations/test_google_cse_address.py b/tests/integrations/test_google_cse_address.py new file mode 100644 index 000000000..436b8b8bc --- /dev/null +++ b/tests/integrations/test_google_cse_address.py @@ -0,0 +1,60 @@ +"""Which address Google Custom Search calls: Google by default, the connected base_url when one is set.""" + +from __future__ import annotations + +import httpx +import pytest +import respx + +from sentient.integrations.base import IntegrationError +from sentient.integrations.plugins.alternatives import GoogleCSEPlugin +from sentient.integrations.plugins.search import GOOGLE_CSE_URL, google_cse_search + +OTHER = "http://search.example.test/customsearch/v1" +ITEMS = {"items": [{"title": "T", "link": "https://example.com/a", "snippet": "S"}]} + + +def test_default_address_is_google(): + assert GOOGLE_CSE_URL == "https://www.googleapis.com/customsearch/v1" + + +@respx.mock +async def test_search_calls_google_without_base_url(): + route = respx.get(GOOGLE_CSE_URL).mock(return_value=httpx.Response(200, json=ITEMS)) + rows = await google_cse_search("k", "cx1", "sentient", 3) + assert route.called + assert route.calls.last.request.url.params["cx"] == "cx1" + assert rows == [{"title": "T", "url": "https://example.com/a", "snippet": "S"}] + + +@respx.mock +async def test_search_calls_base_url_with_the_same_params(): + google = respx.get(GOOGLE_CSE_URL) + route = respx.get(OTHER).mock(return_value=httpx.Response(200, json=ITEMS)) + rows = await google_cse_search("k", "cx1", "sentient", 3, OTHER) + params = route.calls.last.request.url.params + assert (params["key"], params["cx"], params["q"], params["num"]) == ("k", "cx1", "sentient", "3") + assert not google.called + assert rows[0]["url"] == "https://example.com/a" + + +@respx.mock +async def test_connect_without_base_url_checks_google_and_stores_no_address(): + route = respx.get(GOOGLE_CSE_URL).mock(return_value=httpx.Response(200, json=ITEMS)) + creds, _ = await GoogleCSEPlugin().validate({"api_key": "k", "cx": "cx1", "base_url": " "}, None) + assert route.called + assert creds == {"api_key": "k", "cx": "cx1"} + + +@respx.mock +async def test_connect_with_base_url_checks_it_and_stores_it(): + google = respx.get(GOOGLE_CSE_URL) + route = respx.get(OTHER).mock(return_value=httpx.Response(200, json=ITEMS)) + creds, _ = await GoogleCSEPlugin().validate({"api_key": "k", "cx": "cx1", "base_url": OTHER}, None) + assert route.called and not google.called + assert creds == {"api_key": "k", "cx": "cx1", "base_url": OTHER} + + +async def test_connect_rejects_a_non_http_address(): + with pytest.raises(IntegrationError): + await GoogleCSEPlugin().validate({"api_key": "k", "cx": "cx1", "base_url": "file:///etc/passwd"}, None) From 3419b08328b410c052c76ffd1b57d1fdf7788352 Mon Sep 17 00:00:00 2001 From: Tarek Etman Date: Wed, 11 Nov 2026 12:41:32 +0000 Subject: [PATCH 2/2] fix(integrations): check a custom Google Custom Search address before connecting A malformed or unreachable address now gives an IntegrationError, as does an answer that is not a JSON object. A query already in the address, such as ?tenant=abc, is kept in the connect check and in searches; httpx params= replaced it. With no address set, searches and the connect check go to Google as before. --- sentient/integrations/plugins/alternatives.py | 19 +++++++-- sentient/integrations/plugins/search.py | 11 ++++- tests/integrations/test_google_cse_address.py | 41 +++++++++++++++++++ 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/sentient/integrations/plugins/alternatives.py b/sentient/integrations/plugins/alternatives.py index 8561b2e61..634d072ea 100644 --- a/sentient/integrations/plugins/alternatives.py +++ b/sentient/integrations/plugins/alternatives.py @@ -8,7 +8,7 @@ from sentient.integrations.base import IntegrationError, IntegrationPlugin, SetupField from sentient.integrations.common import http_client -from sentient.integrations.plugins.search import GOOGLE_CSE_URL +from sentient.integrations.plugins.search import google_cse_url if TYPE_CHECKING: # pragma: no cover from sentient.integrations.service import IntegrationManager @@ -147,11 +147,24 @@ async def validate(self, fields: dict[str, str], mgr: IntegrationManager) -> tup base_url = str(fields.get("base_url", "")).strip() if base_url and not base_url.lower().startswith(("http://", "https://")): raise IntegrationError("The API address has to start with http:// or https://.") - async with http_client() as http: - r = await http.get(base_url or GOOGLE_CSE_URL, params={"key": key, "cx": cx, "q": "test", "num": 1}) + try: + async with http_client() as http: + r = await http.get(google_cse_url(base_url, {"key": key, "cx": cx, "q": "test", "num": 1})) + except (httpx.HTTPError, httpx.InvalidURL) as exc: + if not base_url: + raise + raise IntegrationError("Couldn't reach the API address. Check it and try again.") from exc if r.status_code == 400: raise IntegrationError("Google didn't accept the search engine ID. Check the 'cx' value.") _rejected(r, "Google Custom Search") + if base_url: + try: + body = r.json() + except ValueError: + body = None + if not isinstance(body, dict): + raise IntegrationError("The API address didn't answer in the Custom Search JSON format. " + "Check the address.") creds = {"api_key": key, "cx": cx} if base_url: creds["base_url"] = base_url diff --git a/sentient/integrations/plugins/search.py b/sentient/integrations/plugins/search.py index 5c1e356e1..f3dc0b80b 100644 --- a/sentient/integrations/plugins/search.py +++ b/sentient/integrations/plugins/search.py @@ -5,6 +5,8 @@ import asyncio import logging +import httpx + from sentient.integrations.base import IntegrationError, IntegrationPlugin, itool, manager_from from sentient.integrations.common import http_client from sentient.tools.base import ToolContext @@ -14,6 +16,12 @@ GOOGLE_CSE_URL = "https://www.googleapis.com/customsearch/v1" +def google_cse_url(base_url: str, params: dict) -> httpx.URL: + """The address to call with the search parameters added to any it already has (httpx's ``params=`` would + replace a query such as ``?tenant=abc`` in a custom address).""" + return httpx.URL(base_url or GOOGLE_CSE_URL).copy_merge_params(params) + + def _ddg_sync(query: str, n: int, region: str) -> list[dict]: from ddgs import DDGS @@ -40,8 +48,7 @@ async def brave_search(key: str, query: str, n: int) -> list[dict]: async def google_cse_search(key: str, cx: str, query: str, n: int, base_url: str = "") -> list[dict]: async with http_client() as http: - r = await http.get(base_url or GOOGLE_CSE_URL, - params={"key": key, "cx": cx, "q": query, "num": min(n, 10)}) + r = await http.get(google_cse_url(base_url, {"key": key, "cx": cx, "q": query, "num": min(n, 10)})) r.raise_for_status() rows = r.json().get("items") or [] return [{"title": x.get("title"), "url": x.get("link"), "snippet": x.get("snippet")} for x in rows] diff --git a/tests/integrations/test_google_cse_address.py b/tests/integrations/test_google_cse_address.py index 436b8b8bc..20ed0badb 100644 --- a/tests/integrations/test_google_cse_address.py +++ b/tests/integrations/test_google_cse_address.py @@ -55,6 +55,47 @@ async def test_connect_with_base_url_checks_it_and_stores_it(): assert creds == {"api_key": "k", "cx": "cx1", "base_url": OTHER} +@respx.mock +async def test_search_keeps_the_query_already_in_base_url(): + route = respx.get(OTHER).mock(return_value=httpx.Response(200, json=ITEMS)) + await google_cse_search("k", "cx1", "sentient", 3, OTHER + "?tenant=abc") + params = route.calls.last.request.url.params + assert (params["tenant"], params["cx"], params["q"]) == ("abc", "cx1", "sentient") + + +@respx.mock +async def test_connect_keeps_the_query_already_in_base_url(): + route = respx.get(OTHER).mock(return_value=httpx.Response(200, json=ITEMS)) + creds, _ = await GoogleCSEPlugin().validate({"api_key": "k", "cx": "cx1", "base_url": OTHER + "?tenant=abc"}, None) + params = route.calls.last.request.url.params + assert (params["tenant"], params["cx"]) == ("abc", "cx1") + assert creds["base_url"] == OTHER + "?tenant=abc" + + +@respx.mock +async def test_connect_rejects_a_malformed_address(): + with pytest.raises(IntegrationError): + await GoogleCSEPlugin().validate({"api_key": "k", "cx": "cx1", "base_url": "https://example.test:notaport"}, None) + + +@respx.mock +async def test_connect_reports_an_unreachable_address_as_an_integration_error(): + respx.get(OTHER).mock(side_effect=httpx.ConnectError("refused")) + with pytest.raises(IntegrationError): + await GoogleCSEPlugin().validate({"api_key": "k", "cx": "cx1", "base_url": OTHER}, None) + + +@pytest.mark.parametrize("answer", [ + httpx.Response(200, text="Sign in", headers={"Content-Type": "text/html"}), + httpx.Response(200, json=[1, 2]), +]) +@respx.mock +async def test_connect_rejects_an_address_that_does_not_answer_with_a_json_object(answer): + respx.get(OTHER).mock(return_value=answer) + with pytest.raises(IntegrationError): + await GoogleCSEPlugin().validate({"api_key": "k", "cx": "cx1", "base_url": OTHER}, None) + + async def test_connect_rejects_a_non_http_address(): with pytest.raises(IntegrationError): await GoogleCSEPlugin().validate({"api_key": "k", "cx": "cx1", "base_url": "file:///etc/passwd"}, None)