@@ -1,5 +1,63 @@
11#!/usr/bin/env python
2+"""Script to run searx from terminal.
23
4+Getting categories without initiate the engine will only return `['general']`
5+
6+>>> import searx.engines
7+... list(searx.engines.categories.keys())
8+['general']
9+>>> import searx
10+... searx.engines.initialize_engines(searx.settings['engines'])
11+... list(searx.engines.categories.keys())
12+['general', 'it', 'science', 'images', 'news', 'videos', 'music', 'files', 'social media', 'map']
13+
14+Example to use this script:
15+
16+.. code:: bash
17+
18+ $ SEARX_DEBUG=1 python3 utils/standalone_searx.py rain
19+
20+Example to run it from python:
21+
22+>>> import importlib
23+... import json
24+... import sys
25+... import searx
26+... import searx.engines
27+... search_query = 'rain'
28+... # initialize engines
29+... searx.engines.initialize_engines(searx.settings['engines'])
30+... # load engines categories once instead of each time the function called
31+... engine_cs = list(searx.engines.categories.keys())
32+... # load module
33+... spec = importlib.util.spec_from_file_location(
34+... 'utils.standalone_searx', 'utils/standalone_searx.py')
35+... sas = importlib.util.module_from_spec(spec)
36+... spec.loader.exec_module(sas)
37+... # use function from module
38+... prog_args = sas.parse_argument([search_query], category_choices=engine_cs)
39+... search_q = sas.get_search_query(prog_args, engine_categories=engine_cs)
40+... res_dict = sas.to_dict(search_q)
41+... sys.stdout.write(json.dumps(
42+... res_dict, sort_keys=True, indent=4, ensure_ascii=False,
43+... default=sas.json_serial))
44+{
45+ "answers": [],
46+ "infoboxes": [ {...} ],
47+ "paging": true,
48+ "results": [... ],
49+ "results_number": 820000000.0,
50+ "search": {
51+ "lang": "all",
52+ "pageno": 1,
53+ "q": "rain",
54+ "safesearch": 0,
55+ "timerange": null
56+ },
57+ "suggestions": [...]
58+}
59+""" # noqa: E501
60+# pylint: disable=pointless-string-statement
361'''
462searx is free software: you can redistribute it and/or modify
563it under the terms of the GNU Affero General Public License as published by
@@ -16,90 +74,145 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
1674
1775(C) 2016- by Alexandre Flament, <alex@al-f.net>
1876'''
19−
20−# set path
21−from sys import path
22−from os.path import realpath, dirname
23−path.append(realpath(dirname(realpath(__file__)) + '/../'))
24−
25−# initialization
26−from json import dumps
27−from searx import settings
77+# pylint: disable=wrong-import-position
78+import argparse
2879import sys
29−import codecs
30−import searx.query
31−import searx.search
80+from datetime import datetime
81+from json import dumps
82+from typing import Any, Dict, List, Optional
83+
84+import searx
3285import searx.engines
33−import searx.webapdater
3486import searx.preferences
87+import searx.query
88+import searx.search
3589import searx.webadapter
36−import argparse
3790
38−searx.engines.initialize_engines(settings['engines'])
39−
40−# command line parsing
41−parser = argparse.ArgumentParser(description='Standalone searx.')
42−parser.add_argument('query', type=str,
43− help='Text query')
44−parser.add_argument('--category', type=str, nargs='?',
45− choices=searx.engines.categories.keys(),
46− default='general',
47− help='Search category')
48−parser.add_argument('--lang', type=str, nargs='?',default='all',
49− help='Search language')
50−parser.add_argument('--pageno', type=int, nargs='?', default=1,
51− help='Page number starting from 1')
52−parser.add_argument('--safesearch', type=str, nargs='?', choices=['0', '1', '2'], default='0',
53− help='Safe content filter from none to strict')
54−parser.add_argument('--timerange', type=str, nargs='?', choices=['day', 'week', 'month', 'year'],
55− help='Filter by time range')
56−args = parser.parse_args()
57−
58−# search results for the query
59−form = {
60− "q":args.query,
61− "categories":args.category.decode(),
62− "pageno":str(args.pageno),
63− "language":args.lang,
64− "time_range":args.timerange
65−}
66−preferences = searx.preferences.Preferences(['oscar'], searx.engines.categories.keys(), searx.engines.engines, [])
67−preferences.key_value_settings['safesearch'].parse(args.safesearch)
91+EngineCategoriesVar = Optional[List[str]]
6892
69−search_query, raw_text_query, _, _ = searx.webadapter.get_search_query_from_webapp(preferences, form)
70−search = searx.search.Search(search_query)
71−result_container = search.search()
7293
73−# output
74−from datetime import datetime
94+def get_search_query(
95+ args: argparse.Namespace, engine_categories: EngineCategoriesVar = None
96+) -> searx.search.SearchQuery:
97+ """Get search results for the query"""
98+ if engine_categories is None:
99+ engine_categories = list(searx.engines.categories.keys())
100+ try:
101+ category = args.category.decode('utf-8')
102+ except AttributeError:
103+ category = args.category
104+ form = {
105+ "q": args.query,
106+ "categories": category,
107+ "pageno": str(args.pageno),
108+ "language": args.lang,
109+ "time_range": args.timerange
110+ }
111+ preferences = searx.preferences.Preferences(
112+ ['oscar'], engine_categories, searx.engines.engines, [])
113+ preferences.key_value_settings['safesearch'].parse(args.safesearch)
114+
115+ search_query = searx.webadapter.get_search_query_from_webapp(
116+ preferences, form)[0]
117+ return search_query
75118
76−def no_parsed_url(results):
119+
120+def no_parsed_url(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
121+ """Remove parsed url from dict."""
77122 for result in results:
78123 del result['parsed_url']
79124 return results
80125
81−def json_serial(obj):
82− """JSON serializer for objects not serializable by default json code"""
126+
127+def json_serial(obj: Any) -> Any:
128+ """JSON serializer for objects not serializable by default json code.
129+
130+ :raise TypeError: raised when **obj** is not serializable
131+ """
83132 if isinstance(obj, datetime):
84133 serial = obj.isoformat()
85134 return serial
86− raise TypeError ("Type not serializable")
135+ if isinstance(obj, bytes):
136+ return obj.decode('utf8')
137+ if isinstance(obj, set):
138+ return list(obj)
139+ raise TypeError("Type ({}) not serializable".format(type(obj)))
87140
88−result_container_json = {
89− "search": {
90− "q": search_query.query,
91− "pageno": search_query.pageno,
92− "lang": search_query.lang,
93− "safesearch": search_query.safesearch,
94− "timerange": search_query.time_range,
95− "engines": search_query.engines
96− },
97− "results": no_parsed_url(result_container.get_ordered_results()),
98− "infoboxes": result_container.infoboxes,
99− "suggestions": list(result_container.suggestions),
100− "answers": list(result_container.answers),
101− "paging": result_container.paging,
102− "results_number": result_container.results_number()
103−}
104−sys.stdout = codecs.getwriter("UTF-8")(sys.stdout)
105−sys.stdout.write(dumps(result_container_json, sort_keys=True, indent=4, ensure_ascii=False, encoding="utf-8", default=json_serial))
141+
142+def to_dict(search_query: searx.search.SearchQuery) -> Dict[str, Any]:
143+ """Get result from parsed arguments."""
144+ result_container = searx.search.Search(search_query).search()
145+ result_container_json = {
146+ "search": {
147+ "q": search_query.query,
148+ "pageno": search_query.pageno,
149+ "lang": search_query.lang,
150+ "safesearch": search_query.safesearch,
151+ "timerange": search_query.time_range,
152+ },
153+ "results": no_parsed_url(result_container.get_ordered_results()),
154+ "infoboxes": result_container.infoboxes,
155+ "suggestions": list(result_container.suggestions),
156+ "answers": list(result_container.answers),
157+ "paging": result_container.paging,
158+ "results_number": result_container.results_number()
159+ }
160+ return result_container_json
161+
162+
163+def parse_argument(
164+ args: Optional[List[str]]=None,
165+ category_choices: EngineCategoriesVar=None
166+) -> argparse.Namespace:
167+ """Parse command line.
168+
169+ :raise SystemExit: Query argument required on `args`
170+
171+ Examples:
172+
173+ >>> import importlib
174+ ... # load module
175+ ... spec = importlib.util.spec_from_file_location(
176+ ... 'utils.standalone_searx', 'utils/standalone_searx.py')
177+ ... sas = importlib.util.module_from_spec(spec)
178+ ... spec.loader.exec_module(sas)
179+ ... sas.parse_argument()
180+ usage: ptipython [-h] [--category [{general}]] [--lang [LANG]] [--pageno [PAGENO]] [--safesearch [{0,1,2}]] [--timerange [{day,week,month,year}]]
181+ query
182+ SystemExit: 2
183+ >>> sas.parse_argument(['rain'])
184+ Namespace(category='general', lang='all', pageno=1, query='rain', safesearch='0', timerange=None)
185+ """ # noqa: E501
186+ if not category_choices:
187+ category_choices = list(searx.engines.categories.keys())
188+ parser = argparse.ArgumentParser(description='Standalone searx.')
189+ parser.add_argument('query', type=str,
190+ help='Text query')
191+ parser.add_argument('--category', type=str, nargs='?',
192+ choices=category_choices,
193+ default='general',
194+ help='Search category')
195+ parser.add_argument('--lang', type=str, nargs='?', default='all',
196+ help='Search language')
197+ parser.add_argument('--pageno', type=int, nargs='?', default=1,
198+ help='Page number starting from 1')
199+ parser.add_argument(
200+ '--safesearch', type=str, nargs='?',
201+ choices=['0', '1', '2'], default='0',
202+ help='Safe content filter from none to strict')
203+ parser.add_argument(
204+ '--timerange', type=str,
205+ nargs='?', choices=['day', 'week', 'month', 'year'],
206+ help='Filter by time range')
207+ return parser.parse_args(args)
208+
209+
210+if __name__ == '__main__':
211+ searx.engines.initialize_engines(searx.settings['engines'])
212+ engine_cs = list(searx.engines.categories.keys())
213+ prog_args = parse_argument(category_choices=engine_cs)
214+ search_q = get_search_query(prog_args, engine_categories=engine_cs)
215+ res_dict = to_dict(search_q)
216+ sys.stdout.write(dumps(
217+ res_dict, sort_keys=True, indent=4, ensure_ascii=False,
218+ default=json_serial))