Karlostavitch1 nsarrazin HF staff commited on
Commit
7c22da3
1 Parent(s): 881070b

Added support for SearXNG as a websearch source (#805)

Browse files

* Created searchSearxng.ts for querying and parsing searxng instance.
Added searxng to the searchWeb.ts to ensure inclusion in the search initialisation script.
Added searxng to +layout.server.ts to ensure the search button is enabled if url provided in .env.local assignment
Updated .env to include searxng configuration
Updated README.md under Web Search Config to mention addition of searxng URL use in .env

* Changes requested
consolidated env variable to SEARXNG_QUERY_URL (felt considering the content the word QUERY was more relevant)
refactored the searchSearxng.ts to generate the query with the new variable
updated related references to variable (.env, searchWeb.ts and +layout.server.ts)
Updated readme and variable comment on .env to reflect the same

* urefectored to display SearXNG in search dialoge when used.

* lint

* types

---------

Co-authored-by: Nathan Sarrazin <[email protected]>

.env CHANGED
@@ -18,6 +18,7 @@ SERPER_API_KEY=#your serper.dev api key here
18
  SERPAPI_KEY=#your serpapi key here
19
  SERPSTACK_API_KEY=#your serpstack api key here
20
  USE_LOCAL_WEBSEARCH=#set to true to parse google results yourself, overrides other API keys
 
21
 
22
  WEBSEARCH_ALLOWLIST=`[]` # if it's defined, allow websites from only this list.
23
  WEBSEARCH_BLOCKLIST=`[]` # if it's defined, block websites from this list.
 
18
  SERPAPI_KEY=#your serpapi key here
19
  SERPSTACK_API_KEY=#your serpstack api key here
20
  USE_LOCAL_WEBSEARCH=#set to true to parse google results yourself, overrides other API keys
21
+ SEARXNG_QUERY_URL=# where '<query>' will be replaced with query keywords see https://docs.searxng.org/dev/search_api.html eg https://searxng.yourdomain.com/search?q=<query>&engines=duckduckgo,google&format=json
22
 
23
  WEBSEARCH_ALLOWLIST=`[]` # if it's defined, allow websites from only this list.
24
  WEBSEARCH_BLOCKLIST=`[]` # if it's defined, block websites from this list.
README.md CHANGED
@@ -166,7 +166,7 @@ PUBLIC_APP_DISCLAIMER=
166
 
167
  You can enable the web search through an API by adding `YDC_API_KEY` ([docs.you.com](https://docs.you.com)) or `SERPER_API_KEY` ([serper.dev](https://serper.dev/)) or `SERPAPI_KEY` ([serpapi.com](https://serpapi.com/)) or `SERPSTACK_API_KEY` ([serpstack.com](https://serpstack.com/)) to your `.env.local`.
168
 
169
- You can also simply enable the local websearch by setting `USE_LOCAL_WEBSEARCH=true` in your `.env.local`.
170
 
171
  ### Custom models
172
 
 
166
 
167
  You can enable the web search through an API by adding `YDC_API_KEY` ([docs.you.com](https://docs.you.com)) or `SERPER_API_KEY` ([serper.dev](https://serper.dev/)) or `SERPAPI_KEY` ([serpapi.com](https://serpapi.com/)) or `SERPSTACK_API_KEY` ([serpstack.com](https://serpstack.com/)) to your `.env.local`.
168
 
169
+ You can also simply enable the local google websearch by setting `USE_LOCAL_WEBSEARCH=true` in your `.env.local` or specify a SearXNG instance by adding the query URL to `SEARXNG_QUERY_URL`.
170
 
171
  ### Custom models
172
 
src/lib/server/websearch/searchSearxng.ts ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { SEARXNG_QUERY_URL } from "$env/static/private";
2
+
3
+ export async function searchSearxng(query: string) {
4
+ const abortController = new AbortController();
5
+ setTimeout(() => abortController.abort(), 10000);
6
+
7
+ // Insert the query into the URL template
8
+ let url = SEARXNG_QUERY_URL.replace("<query>", query);
9
+
10
+ // Check if "&format=json" already exists in the URL
11
+ if (!url.includes("&format=json")) {
12
+ url += "&format=json";
13
+ }
14
+
15
+ // Call the URL to return JSON data
16
+ const jsonResponse = await fetch(url, {
17
+ signal: abortController.signal,
18
+ })
19
+ .then((response) => response.json() as Promise<{ results: { url: string }[] }>)
20
+ .catch((error) => {
21
+ console.error("Failed to fetch or parse JSON", error);
22
+ throw new Error("Failed to fetch or parse JSON");
23
+ });
24
+
25
+ // Extract 'url' elements from the JSON response and trim to the top 5 URLs
26
+ const urls = jsonResponse.results.slice(0, 5).map((item) => item.url);
27
+
28
+ if (!urls.length) {
29
+ throw new Error(`Response doesn't contain any "url" elements`);
30
+ }
31
+
32
+ // Map URLs to the correct object shape
33
+ return { organic_results: urls.map((link) => ({ link })) };
34
+ }
src/lib/server/websearch/searchWeb.ts CHANGED
@@ -5,15 +5,23 @@ import {
5
  SERPER_API_KEY,
6
  SERPSTACK_API_KEY,
7
  USE_LOCAL_WEBSEARCH,
 
8
  YDC_API_KEY,
9
  } from "$env/static/private";
10
  import { getJson } from "serpapi";
11
  import type { GoogleParameters } from "serpapi";
12
  import { searchWebLocal } from "./searchWebLocal";
 
13
 
14
  // get which SERP api is providing web results
15
  export function getWebSearchProvider() {
16
- return YDC_API_KEY ? WebSearchProvider.YOU : WebSearchProvider.GOOGLE;
 
 
 
 
 
 
17
  }
18
 
19
  // Show result as JSON
@@ -21,6 +29,9 @@ export async function searchWeb(query: string) {
21
  if (USE_LOCAL_WEBSEARCH) {
22
  return await searchWebLocal(query);
23
  }
 
 
 
24
  if (SERPER_API_KEY) {
25
  return await searchWebSerper(query);
26
  }
 
5
  SERPER_API_KEY,
6
  SERPSTACK_API_KEY,
7
  USE_LOCAL_WEBSEARCH,
8
+ SEARXNG_QUERY_URL,
9
  YDC_API_KEY,
10
  } from "$env/static/private";
11
  import { getJson } from "serpapi";
12
  import type { GoogleParameters } from "serpapi";
13
  import { searchWebLocal } from "./searchWebLocal";
14
+ import { searchSearxng } from "./searchSearxng";
15
 
16
  // get which SERP api is providing web results
17
  export function getWebSearchProvider() {
18
+ if (YDC_API_KEY) {
19
+ return WebSearchProvider.YOU;
20
+ } else if (SEARXNG_QUERY_URL) {
21
+ return WebSearchProvider.SEARXNG;
22
+ } else {
23
+ return WebSearchProvider.GOOGLE;
24
+ }
25
  }
26
 
27
  // Show result as JSON
 
29
  if (USE_LOCAL_WEBSEARCH) {
30
  return await searchWebLocal(query);
31
  }
32
+ if (SEARXNG_QUERY_URL) {
33
+ return await searchSearxng(query);
34
+ }
35
  if (SERPER_API_KEY) {
36
  return await searchWebSerper(query);
37
  }
src/lib/types/WebSearch.ts CHANGED
@@ -42,4 +42,5 @@ interface YouSearchHit {
42
  export enum WebSearchProvider {
43
  GOOGLE = "Google",
44
  YOU = "You.com",
 
45
  }
 
42
  export enum WebSearchProvider {
43
  GOOGLE = "Google",
44
  YOU = "You.com",
45
+ SEARXNG = "SearXNG",
46
  }
src/routes/+layout.server.ts CHANGED
@@ -12,6 +12,7 @@ import {
12
  MESSAGES_BEFORE_LOGIN,
13
  YDC_API_KEY,
14
  USE_LOCAL_WEBSEARCH,
 
15
  ENABLE_ASSISTANTS,
16
  } from "$env/static/private";
17
  import { ObjectId } from "mongodb";
@@ -126,7 +127,8 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => {
126
  SERPER_API_KEY ||
127
  SERPSTACK_API_KEY ||
128
  YDC_API_KEY ||
129
- USE_LOCAL_WEBSEARCH
 
130
  ),
131
  ethicsModalAccepted: !!settings?.ethicsModalAcceptedAt,
132
  ethicsModalAcceptedAt: settings?.ethicsModalAcceptedAt ?? null,
 
12
  MESSAGES_BEFORE_LOGIN,
13
  YDC_API_KEY,
14
  USE_LOCAL_WEBSEARCH,
15
+ SEARXNG_QUERY_URL,
16
  ENABLE_ASSISTANTS,
17
  } from "$env/static/private";
18
  import { ObjectId } from "mongodb";
 
127
  SERPER_API_KEY ||
128
  SERPSTACK_API_KEY ||
129
  YDC_API_KEY ||
130
+ USE_LOCAL_WEBSEARCH ||
131
+ SEARXNG_QUERY_URL
132
  ),
133
  ethicsModalAccepted: !!settings?.ethicsModalAcceptedAt,
134
  ethicsModalAcceptedAt: settings?.ethicsModalAcceptedAt ?? null,