git.iko.soy/searx
on master help sign in
[enh] settings unification - new dependency: pyyaml
commit3afdd1d9941527e23cd7c05d2c15dd24a32de834 authorasciimoo <asciimoo@gmail.com> dateSun 19 Jan 2014, 00:17 (13 years ago) parent39ebe1d
10 files changed+150−121
compare with master
unifiedsplit
.gitignore+2 README.md+1−2 engines.cfg_sample−99 requirements.txt+1 searx/__init__.py+22 searx/engines/__init__.py+9−14 searx/webapp.py+6−6 settings.yml+107 setup.py+1 versions.cfg+1
.gitignore+2
@@ -1,6 +1,8 @@
11env
22engines.cfg
33.installed.cfg
4+.coverage
5+covearge/
46setup.cfg
57
68*.pyc
README.md+1−2
@@ -25,8 +25,7 @@ List of [running instances](https://github.com/asciimoo/searx/wiki/Searx-instanc
2525
2626* clone source: `git clone git@github.com:asciimoo/searx.git && cd searx`
2727* install dependencies: `pip install -r requirements.txt`
28−* edit your [searx/settings.py](https://github.com/asciimoo/searx/blob/master/searx/settings.py) (set your `secret_key`!)
29−* rename `engines.cfg_sample` to `engines.cfg`
28+* edit your [settings.yml](https://github.com/asciimoo/searx/blob/master/settings.yml) (set your `secret_key`!)
3029* run `python searx/webapp.py` to start the application
3130
3231For all the details, follow this [step by step installation](https://github.com/asciimoo/searx/wiki/Installation)
engines.cfg_sampledeleted−99
@@ -1,99 +0,0 @@
1−[wikipedia]
2−engine = mediawiki
3−url = https://en.wikipedia.org/
4−number_of_results = 1
5−
6−[bing]
7−engine = bing
8−locale = en-US
9−
10−[currency]
11−engine=currency_convert
12−categories = general
13−
14−[deviantart]
15−engine = deviantart
16−categories = images
17−
18−[ddg definitions]
19−engine = duckduckgo_definitions
20−
21−[duckduckgo]
22−engine = duckduckgo
23−locale = en-us
24−
25−[filecrop]
26−engine = filecrop
27−categories = files
28−
29−[flickr]
30−engine = flickr
31−categories = images
32−
33−[github]
34−engine = github
35−categories = it
36−
37−[google]
38−engine = json_engine
39−search_url = https://ajax.googleapis.com/ajax/services/search/web?v=2.0&start=0&rsz=large&safe=off&filter=off&q={query}
40−categories = general
41−url_query = /responseData/results/unescapedUrl
42−content_query = /responseData/results/content
43−title_query = /responseData/results/titleNoFormatting
44−
45−[google images]
46−engine = google_images
47−categories = images
48−
49−[piratebay]
50−engine = piratebay
51−categories = videos, music, files
52−
53−[soundcloud]
54−engine = soundcloud
55−categories = music
56−
57−[stackoverflow]
58−engine = stackoverflow
59−categories = it
60−
61−[startpage]
62−engine = startpage
63−
64−[twitter]
65−engine = twitter
66−categories = social media
67−
68−[urbandictionary]
69−engine = xpath
70−search_url = http://www.urbandictionary.com/define.php?term={query}
71−url_xpath = //div[@class="word"]//a/@href
72−title_xpath = //div[@class="word"]//a
73−content_xpath = //div[@class="definition"]
74−
75−[yahoo]
76−engine = xpath
77−search_url = http://search.yahoo.com/search?p={query}
78−results_xpath = //div[@class="res"]
79−url_xpath = .//h3/a/@href
80−title_xpath = .//h3/a
81−content_xpath = .//div[@class="abstr"]
82−suggestion_xpath = //div[@id="satat"]//a
83−
84−[youtube]
85−engine = youtube
86−categories = videos
87−
88−[dailymotion]
89−engine = dailymotion
90−locale = en_US
91−categories = videos
92−
93−[vimeo]
94−engine = vimeo
95−categories = videos
96−results_xpath = //div[@id="browse_content"]/ol/li
97−url_xpath=./a/@href
98−title_xpath=./a/div[@class="data"]/p[@class="title"]/text()
99−content_xpath=./a/img/@src
requirements.txt+1
@@ -1,3 +1,4 @@
11flask
22grequests
33lxml
4+pyyaml
searx/__init__.py+22
@@ -0,0 +1,22 @@
1+from os import environ
2+from os.path import realpath, dirname, join
3+try:
4+ from yaml import load
5+except:
6+ from sys import exit, stderr
7+ stderr.write('[E] install pyyaml\n')
8+ exit(2)
9+
10+
11+searx_dir = realpath(dirname(realpath(__file__))+'/../')
12+engine_dir = dirname(realpath(__file__))
13+
14+if 'SEARX_SETTINGS_PATH' in environ:
15+ settings_path = environ['SEARX_SETTINGS_PATH']
16+else:
17+ settings_path = join(searx_dir, 'settings.yml')
18+
19+
20+with open(settings_path) as settings_yaml:
21+ settings = load(settings_yaml)
22+
searx/engines/__init__.py+9−14
@@ -24,15 +24,11 @@ from operator import itemgetter
2424from urlparse import urlparse
2525from searx import settings
2626from searx.utils import gen_useragent
27−import ConfigParser
2827import sys
2928from datetime import datetime
3029
3130engine_dir = dirname(realpath(__file__))
32−searx_dir = join(engine_dir, '../../')
3331
34−engines_config = ConfigParser.SafeConfigParser()
35−engines_config.read(join(searx_dir, 'engines.cfg'))
3632number_of_searches = 0
3733
3834engines = {}
@@ -48,24 +44,23 @@ def load_module(filename):
4844 module.name = modname
4945 return module
5046
51−if not engines_config.sections():
52− print '[E] Error no engines found. Edit your engines.cfg'
47+if not 'engines' in settings or not settings['engines']:
48+ print '[E] Error no engines found. Edit your settings.yml'
5349 exit(2)
5450
55−for engine_config_name in engines_config.sections():
56− engine_data = engines_config.options(engine_config_name)
57− engine = load_module(engines_config.get(engine_config_name, 'engine')+'.py')
58− engine.name = engine_config_name
51+for engine_data in settings['engines']:
52+ engine_name = engine_data['engine']
53+ engine = load_module(engine_name+'.py')
5954 for param_name in engine_data:
6055 if param_name == 'engine':
6156 continue
6257 if param_name == 'categories':
63− if engines_config.get(engine_config_name, param_name) == 'none':
58+ if engine_data['categories'] == 'none':
6459 engine.categories = []
6560 else:
66− engine.categories = map(str.strip, engines_config.get(engine_config_name, param_name).split(','))
61+ engine.categories = map(str.strip, engine_data['categories'].split(','))
6762 continue
68− setattr(engine, param_name, engines_config.get(engine_config_name, param_name))
63+ setattr(engine, param_name, engine_data[param_name])
6964 for engine_attr in dir(engine):
7065 if engine_attr.startswith('_'):
7166 continue
@@ -170,7 +165,7 @@ def search(query, request, selected_engines):
170165 request_args = dict(headers = request_params['headers']
171166 ,hooks = dict(response=callback)
172167 ,cookies = request_params['cookies']
173− ,timeout = settings.request_timeout
168+ ,timeout = settings['server']['request_timeout']
174169 )
175170
176171 if request_params['method'] == 'GET':
searx/webapp.py+6−6
@@ -41,7 +41,7 @@ from searx.utils import highlight_content, html_to_text
4141
4242
4343app = Flask(__name__)
44−app.secret_key = settings.secret_key
44+app.secret_key = settings['server']['secret_key']
4545
4646
4747opensearch_xml = '''<?xml version="1.0" encoding="utf-8"?>
@@ -58,8 +58,8 @@ opensearch_xml = '''<?xml version="1.0" encoding="utf-8"?>
5858
5959
6060def get_base_url():
61− if settings.base_url:
62− hostname = settings.base_url
61+ if settings['server']['base_url']:
62+ hostname = settings['server']['base_url']
6363 else:
6464 scheme = 'http'
6565 if request.is_secure:
@@ -243,9 +243,9 @@ def run():
243243 from gevent import monkey
244244 monkey.patch_all()
245245
246− app.run(debug = settings.debug
247− ,use_debugger = settings.debug
248− ,port = settings.port
246+ app.run(debug = settings['server']['debug']
247+ ,use_debugger = settings['server']['debug']
248+ ,port = settings['server']['port']
249249 )
250250
251251
settings.ymlnew file+107
@@ -0,0 +1,107 @@
1+server:
2+ port : 8888
3+ secret_key : "ultrasecretkey" # change this!
4+ debug : True
5+ request_timeout : 3.0 # seconds
6+ base_url: False
7+
8+engines:
9+ - name : wikipedia
10+ engine : mediawiki
11+ url : https://en.wikipedia.org/
12+ number_of_results : 1
13+
14+ - name : bing
15+ engine : bing
16+ locale : en-US
17+
18+ - name : currency
19+ engine : currency_convert
20+ categories : general
21+
22+ - name : deviantart
23+ engine : deviantart
24+ categories : images
25+
26+ - name : ddg definitions
27+ engine : duckduckgo_definitions
28+
29+ - name : duckduckgo
30+ engine : duckduckgo
31+ locale : en-us
32+
33+ - name : filecrop
34+ engine : filecrop
35+ categories : files
36+
37+ - name : flickr
38+ engine : flickr
39+ categories : images
40+
41+ - name : github
42+ engine : github
43+ categories : it
44+
45+ - name : google
46+ engine : json_engine
47+ search_url : https://ajax.googleapis.com/ajax/services/search/web?v=2.0&start=0&rsz=large&safe=off&filter=off&q={query}
48+ categories : general
49+ url_query : /responseData/results/unescapedUrl
50+ content_query : /responseData/results/content
51+ title_query : /responseData/results/titleNoFormatting
52+
53+ - name : google images
54+ engine : google_images
55+ categories : images
56+
57+ - name : piratebay
58+ engine : piratebay
59+ categories : videos, music, files
60+
61+ - name : soundcloud
62+ engine : soundcloud
63+ categories : music
64+
65+ - name : stackoverflow
66+ engine : stackoverflow
67+ categories : it
68+
69+ - name : startpage
70+ engine : startpage
71+
72+ - name : twitter
73+ engine : twitter
74+ categories : social media
75+
76+ - name : urbandictionary
77+ engine : xpath
78+ search_url : http://www.urbandictionary.com/define.php?term={query}
79+ url_xpath : //div[@class="word"]//a/@href
80+ title_xpath : //div[@class="word"]//a
81+ content_xpath : //div[@class="definition"]
82+
83+ - name : yahoo
84+ engine : xpath
85+ search_url : http://search.yahoo.com/search?p={query}
86+ results_xpath : //div[@class="res"]
87+ url_xpath : .//h3/a/@href
88+ title_xpath : .//h3/a
89+ content_xpath : .//div[@class="abstr"]
90+ suggestion_xpath : //div[@id="satat"]//a
91+
92+ - name : youtube
93+ engine : youtube
94+ categories : videos
95+
96+ - name : dailymotion
97+ engine : dailymotion
98+ locale : en_US
99+ categories : videos
100+
101+ - name : vimeo
102+ engine : vimeo
103+ categories : videos
104+ results_xpath : //div[@id="browse_content"]/ol/li
105+ url_xpath : ./a/@href
106+ title_xpath : ./a/div[@class="data"]/p[@class="title"]/text()
107+ content_xpath : ./a/img/@src
setup.py+1
@@ -32,6 +32,7 @@ setup(
3232 'flask',
3333 'grequests',
3434 'lxml',
35+ 'pyyaml',
3536 'setuptools',
3637 ],
3738 extras_require={
versions.cfg+1
@@ -16,6 +16,7 @@ mccabe = 0.2.1
1616pep8 = 1.4.6
1717plone.testing = 4.0.8
1818pyflakes = 0.7.3
19+pyyaml = 3.10
1920requests = 2.2.0
2021robotframework-debuglibrary = 0.3
2122robotframework-httplibrary = 0.4.2