git.iko.soy/searx
on master help sign in
[enh] settings unification - new dependency: pyyaml
3afdd1dasciimoo
.gitignore+2
compare with master
unifiedsplit
@@ -1,6 +1,8 @@
1env
1env
2engines.cfg
2engines.cfg
3.installed.cfg
3.installed.cfg
4+.coverage
5+covearge/
4setup.cfg
6setup.cfg
5
7
6*.pyc
8*.pyc
README.md+1−2
@@ -25,8 +25,7 @@ List of [running instances](https://github.com/asciimoo/searx/wiki/Searx-instanc
25
25
26* clone source: `git clone git@github.com:asciimoo/searx.git && cd searx`
26* clone source: `git clone git@github.com:asciimoo/searx.git && cd searx`
27* install dependencies: `pip install -r requirements.txt`
27* install dependencies: `pip install -r requirements.txt`
28−* edit your [searx/settings.py](https://github.com/asciimoo/searx/blob/master/searx/settings.py) (set your `secret_key`!)
28+* edit your [settings.yml](https://github.com/asciimoo/searx/blob/master/settings.yml) (set your `secret_key`!)
29−* rename `engines.cfg_sample` to `engines.cfg`
30* run `python searx/webapp.py` to start the application
29* run `python searx/webapp.py` to start the application
31
30
32For all the details, follow this [step by step installation](https://github.com/asciimoo/searx/wiki/Installation)
31For all the details, follow this [step by step installation](https://github.com/asciimoo/searx/wiki/Installation)
engines.cfg_sampledeleted−99
@@ -1,99 +0,0 @@
1−[wikipedia]
2−engine = mediawiki
3−url = https://en.wikipedia.org/
4−number_of_results = 1
5−
6−[bing]
7−engine = bing
8−locale = en-US
9−
10−[currency]
11−engine=currency_convert
12−categories = general
13−
14−[deviantart]
15−engine = deviantart
16−categories = images
17−
18−[ddg definitions]
19−engine = duckduckgo_definitions
20−
21−[duckduckgo]
22−engine = duckduckgo
23−locale = en-us
24−
25−[filecrop]
26−engine = filecrop
27−categories = files
28−
29−[flickr]
30−engine = flickr
31−categories = images
32−
33−[github]
34−engine = github
35−categories = it
36−
37−[google]
38−engine = json_engine
39−search_url = https://ajax.googleapis.com/ajax/services/search/web?v=2.0&start=0&rsz=large&safe=off&filter=off&q={query}
40−categories = general
41−url_query = /responseData/results/unescapedUrl
42−content_query = /responseData/results/content
43−title_query = /responseData/results/titleNoFormatting
44−
45−[google images]
46−engine = google_images
47−categories = images
48−
49−[piratebay]
50−engine = piratebay
51−categories = videos, music, files
52−
53−[soundcloud]
54−engine = soundcloud
55−categories = music
56−
57−[stackoverflow]
58−engine = stackoverflow
59−categories = it
60−
61−[startpage]
62−engine = startpage
63−
64−[twitter]
65−engine = twitter
66−categories = social media
67−
68−[urbandictionary]
69−engine = xpath
70−search_url = http://www.urbandictionary.com/define.php?term={query}
71−url_xpath = //div[@class="word"]//a/@href
72−title_xpath = //div[@class="word"]//a
73−content_xpath = //div[@class="definition"]
74−
75−[yahoo]
76−engine = xpath
77−search_url = http://search.yahoo.com/search?p={query}
78−results_xpath = //div[@class="res"]
79−url_xpath = .//h3/a/@href
80−title_xpath = .//h3/a
81−content_xpath = .//div[@class="abstr"]
82−suggestion_xpath = //div[@id="satat"]//a
83−
84−[youtube]
85−engine = youtube
86−categories = videos
87−
88−[dailymotion]
89−engine = dailymotion
90−locale = en_US
91−categories = videos
92−
93−[vimeo]
94−engine = vimeo
95−categories = videos
96−results_xpath = //div[@id="browse_content"]/ol/li
97−url_xpath=./a/@href
98−title_xpath=./a/div[@class="data"]/p[@class="title"]/text()
99−content_xpath=./a/img/@src
requirements.txt+1
@@ -1,3 +1,4 @@
1flask
1flask
2grequests
2grequests
3lxml
3lxml
4+pyyaml
searx/__init__.py+22
@@ -0,0 +1,22 @@
1+from os import environ
2+from os.path import realpath, dirname, join
3+try:
4+ from yaml import load
5+except:
6+ from sys import exit, stderr
7+ stderr.write('[E] install pyyaml\n')
8+ exit(2)
9+
10+
11+searx_dir = realpath(dirname(realpath(__file__))+'/../')
12+engine_dir = dirname(realpath(__file__))
13+
14+if 'SEARX_SETTINGS_PATH' in environ:
15+ settings_path = environ['SEARX_SETTINGS_PATH']
16+else:
17+ settings_path = join(searx_dir, 'settings.yml')
18+
19+
20+with open(settings_path) as settings_yaml:
21+ settings = load(settings_yaml)
22+
searx/engines/__init__.py+9−14
@@ -24,15 +24,11 @@ from operator import itemgetter
24from urlparse import urlparse
24from urlparse import urlparse
25from searx import settings
25from searx import settings
26from searx.utils import gen_useragent
26from searx.utils import gen_useragent
27−import ConfigParser
28import sys
27import sys
29from datetime import datetime
28from datetime import datetime
30
29
31engine_dir = dirname(realpath(__file__))
30engine_dir = dirname(realpath(__file__))
32−searx_dir = join(engine_dir, '../../')
33
31
34−engines_config = ConfigParser.SafeConfigParser()
35−engines_config.read(join(searx_dir, 'engines.cfg'))
36number_of_searches = 0
32number_of_searches = 0
37
33
38engines = {}
34engines = {}
@@ -48,24 +44,23 @@ def load_module(filename):
48 module.name = modname
44 module.name = modname
49 return module
45 return module
50
46
51−if not engines_config.sections():
47+if not 'engines' in settings or not settings['engines']:
52− print '[E] Error no engines found. Edit your engines.cfg'
48+ print '[E] Error no engines found. Edit your settings.yml'
53 exit(2)
49 exit(2)
54
50
55−for engine_config_name in engines_config.sections():
51+for engine_data in settings['engines']:
56− engine_data = engines_config.options(engine_config_name)
52+ engine_name = engine_data['engine']
57− engine = load_module(engines_config.get(engine_config_name, 'engine')+'.py')
53+ engine = load_module(engine_name+'.py')
58− engine.name = engine_config_name
59 for param_name in engine_data:
54 for param_name in engine_data:
60 if param_name == 'engine':
55 if param_name == 'engine':
61 continue
56 continue
62 if param_name == 'categories':
57 if param_name == 'categories':
63− if engines_config.get(engine_config_name, param_name) == 'none':
58+ if engine_data['categories'] == 'none':
64 engine.categories = []
59 engine.categories = []
65 else:
60 else:
66− engine.categories = map(str.strip, engines_config.get(engine_config_name, param_name).split(','))
61+ engine.categories = map(str.strip, engine_data['categories'].split(','))
67 continue
62 continue
68− setattr(engine, param_name, engines_config.get(engine_config_name, param_name))
63+ setattr(engine, param_name, engine_data[param_name])
69 for engine_attr in dir(engine):
64 for engine_attr in dir(engine):
70 if engine_attr.startswith('_'):
65 if engine_attr.startswith('_'):
71 continue
66 continue
@@ -170,7 +165,7 @@ def search(query, request, selected_engines):
170 request_args = dict(headers = request_params['headers']
165 request_args = dict(headers = request_params['headers']
171 ,hooks = dict(response=callback)
166 ,hooks = dict(response=callback)
172 ,cookies = request_params['cookies']
167 ,cookies = request_params['cookies']
173− ,timeout = settings.request_timeout
168+ ,timeout = settings['server']['request_timeout']
174 )
169 )
175
170
176 if request_params['method'] == 'GET':
171 if request_params['method'] == 'GET':
searx/webapp.py+6−6
@@ -41,7 +41,7 @@ from searx.utils import highlight_content, html_to_text
41
41
42
42
43app = Flask(__name__)
43app = Flask(__name__)
44−app.secret_key = settings.secret_key
44+app.secret_key = settings['server']['secret_key']
45
45
46
46
47opensearch_xml = '''<?xml version="1.0" encoding="utf-8"?>
47opensearch_xml = '''<?xml version="1.0" encoding="utf-8"?>
@@ -58,8 +58,8 @@ opensearch_xml = '''<?xml version="1.0" encoding="utf-8"?>
58
58
59
59
60def get_base_url():
60def get_base_url():
61− if settings.base_url:
61+ if settings['server']['base_url']:
62− hostname = settings.base_url
62+ hostname = settings['server']['base_url']
63 else:
63 else:
64 scheme = 'http'
64 scheme = 'http'
65 if request.is_secure:
65 if request.is_secure:
@@ -243,9 +243,9 @@ def run():
243 from gevent import monkey
243 from gevent import monkey
244 monkey.patch_all()
244 monkey.patch_all()
245
245
246− app.run(debug = settings.debug
246+ app.run(debug = settings['server']['debug']
247− ,use_debugger = settings.debug
247+ ,use_debugger = settings['server']['debug']
248− ,port = settings.port
248+ ,port = settings['server']['port']
249 )
249 )
250
250
251
251
settings.ymlnew file+107
@@ -0,0 +1,107 @@
1+server:
2+ port : 8888
3+ secret_key : "ultrasecretkey" # change this!
4+ debug : True
5+ request_timeout : 3.0 # seconds
6+ base_url: False
7+
8+engines:
9+ - name : wikipedia
10+ engine : mediawiki
11+ url : https://en.wikipedia.org/
12+ number_of_results : 1
13+
14+ - name : bing
15+ engine : bing
16+ locale : en-US
17+
18+ - name : currency
19+ engine : currency_convert
20+ categories : general
21+
22+ - name : deviantart
23+ engine : deviantart
24+ categories : images
25+
26+ - name : ddg definitions
27+ engine : duckduckgo_definitions
28+
29+ - name : duckduckgo
30+ engine : duckduckgo
31+ locale : en-us
32+
33+ - name : filecrop
34+ engine : filecrop
35+ categories : files
36+
37+ - name : flickr
38+ engine : flickr
39+ categories : images
40+
41+ - name : github
42+ engine : github
43+ categories : it
44+
45+ - name : google
46+ engine : json_engine
47+ search_url : https://ajax.googleapis.com/ajax/services/search/web?v=2.0&start=0&rsz=large&safe=off&filter=off&q={query}
48+ categories : general
49+ url_query : /responseData/results/unescapedUrl
50+ content_query : /responseData/results/content
51+ title_query : /responseData/results/titleNoFormatting
52+
53+ - name : google images
54+ engine : google_images
55+ categories : images
56+
57+ - name : piratebay
58+ engine : piratebay
59+ categories : videos, music, files
60+
61+ - name : soundcloud
62+ engine : soundcloud
63+ categories : music
64+
65+ - name : stackoverflow
66+ engine : stackoverflow
67+ categories : it
68+
69+ - name : startpage
70+ engine : startpage
71+
72+ - name : twitter
73+ engine : twitter
74+ categories : social media
75+
76+ - name : urbandictionary
77+ engine : xpath
78+ search_url : http://www.urbandictionary.com/define.php?term={query}
79+ url_xpath : //div[@class="word"]//a/@href
80+ title_xpath : //div[@class="word"]//a
81+ content_xpath : //div[@class="definition"]
82+
83+ - name : yahoo
84+ engine : xpath
85+ search_url : http://search.yahoo.com/search?p={query}
86+ results_xpath : //div[@class="res"]
87+ url_xpath : .//h3/a/@href
88+ title_xpath : .//h3/a
89+ content_xpath : .//div[@class="abstr"]
90+ suggestion_xpath : //div[@id="satat"]//a
91+
92+ - name : youtube
93+ engine : youtube
94+ categories : videos
95+
96+ - name : dailymotion
97+ engine : dailymotion
98+ locale : en_US
99+ categories : videos
100+
101+ - name : vimeo
102+ engine : vimeo
103+ categories : videos
104+ results_xpath : //div[@id="browse_content"]/ol/li
105+ url_xpath : ./a/@href
106+ title_xpath : ./a/div[@class="data"]/p[@class="title"]/text()
107+ content_xpath : ./a/img/@src
setup.py+1
@@ -32,6 +32,7 @@ setup(
32 'flask',
32 'flask',
33 'grequests',
33 'grequests',
34 'lxml',
34 'lxml',
35+ 'pyyaml',
35 'setuptools',
36 'setuptools',
36 ],
37 ],
37 extras_require={
38 extras_require={
versions.cfg+1
@@ -16,6 +16,7 @@ mccabe = 0.2.1
16pep8 = 1.4.6
16pep8 = 1.4.6
17plone.testing = 4.0.8
17plone.testing = 4.0.8
18pyflakes = 0.7.3
18pyflakes = 0.7.3
19+pyyaml = 3.10
19requests = 2.2.0
20requests = 2.2.0
20robotframework-debuglibrary = 0.3
21robotframework-debuglibrary = 0.3
21robotframework-httplibrary = 0.4.2
22robotframework-httplibrary = 0.4.2