d793c27 · searx · git.iko.soy

git.iko.soy/searx

[enh] engine types

commitd793c2733c7aac3aacf40f3f5cf9fc0919305e76 authorasciimoo <asciimoo@gmail.com> dateTue 15 Oct 2013, 19:11 (13 years ago) parentc3b7ed8

7 files changed+34−29

compare with master

unifiedsplit

examples/basic_engine.py+1−1 searx/__init__.py−7 searx/engines/__init__.py+8−6 searx/engines/duckduckgo.py+12−7 searx/engines/duckduckgo_definitions.py+6−6 searx/static/css/style.css+3−1 searx/templates/results.html+4−1

examples/basic_engine.py+1−1

@@ -17,5 +17,5 @@ def response(resp):

1717 '''post-response callback

1818 resp: requests response object

1919 '''

20− return [resp.text]

20+ return [{'url': '', 'title': '', 'content': ''}]

2121

searx/__init__.py−7

@@ -1,7 +0,0 @@

1−

2−base_result_template = """

3−<div class="result">

4− <h3 class="result_title"><a href="{url}">{title}</a></h3>

5− <p class="content">{content}<br />{url}</p>

6−</div>

7−"""

searx/engines/__init__.py+8−6

@@ -6,7 +6,7 @@ import grequests

77engine_dir = dirname(realpath(__file__))

9−engines = []

9+engines = {}

1010

1111for filename in listdir(engine_dir):

1212 modname = splitext(filename)[0]

@@ -16,14 +16,16 @@ for filename in listdir(engine_dir):

1616 engine = load_source(modname, filepath)

1717 if not hasattr(engine, 'request') or not hasattr(engine, 'response'):

1818 continue

19− engines.append(engine)

19+ engines[modname] = engine

2020

2121def default_request_params():

2222 return {'method': 'GET', 'headers': {}, 'data': {}, 'url': ''}

2323

24−def make_callback(results, callback):

24+def make_callback(engine_name, results, callback):

2525 def process_callback(response, **kwargs):

26− results.extend(callback(response))

26+ for result in callback(response):

27+ result['engine'] = engine_name

28+ results.append(result)

2729 return process_callback

2830

2931def search(query, request):

@@ -31,11 +33,11 @@ def search(query, request):

3133 requests = []

3234 results = []

3335 user_agent = request.headers.get('User-Agent', '')

34− for engine in engines:

36+ for ename, engine in engines.items():

3537 headers = default_request_params()

3638 headers['User-Agent'] = user_agent

3739 request_params = engine.request(query, headers)

38− callback = make_callback(results, engine.response)

40+ callback = make_callback(ename, results, engine.response)

3941 if request_params['method'] == 'GET':

4042 req = grequests.get(request_params['url']

4143 ,headers=headers

searx/engines/duckduckgo.py+12−7

@@ -1,14 +1,19 @@

1−from lxml import html

1+from json import loads

44def request(query, params):

5− params['method'] = 'POST'

6− params['url'] = 'https://duckduckgo.com/html'

7− params['data']['q'] = query

5+ params['url'] = 'https://duckduckgo.com/d.js?q=%s&l=us-en&p=1&s=0' % query

86 return params

108

119def response(resp):

12− dom = html.fromstring(resp.text)

13− results = dom.xpath('//div[@class="results_links results_links_deep web-result"]')

14− return [html.tostring(x) for x in results]

10+ results = []

11+ search_res = loads(resp.text[resp.text.find('[{'):-2])[:-1]

12+ for r in search_res:

13+ if not r.get('t'):

14+ continue

15+ results.append({'title': r['t']

16+ ,'content': r['a']

17+ ,'url': r['u']

18+ })

19+ return results

searx/engines/duckduckgo_definitions.py+6−6

@@ -1,5 +1,4 @@

11import json

2−from searx import base_result_template

43def request(query, params):

54 params['url'] = 'http://api.duckduckgo.com/?q=%s&format=json&pretty=0' % query

@@ -10,10 +9,11 @@ def response(resp):

109 search_res = json.loads(resp.text)

1110 results = []

1211 if 'Definition' in search_res:

13− res = {'title' : search_res.get('Heading', '')

14− ,'content' : search_res.get('Definition', '')

15− ,'url' : search_res.get('AbstractURL', '')

16− }

17− results.append(base_result_template.format(**res))

12+ if search_res.get('AbstractURL'):

13+ res = {'title' : search_res.get('Heading', '')

14+ ,'content' : search_res.get('Definition', '')

15+ ,'url' : search_res.get('AbstractURL', '')

16+ }

17+ results.append(res)

1818

1919 return results

searx/static/css/style.css+3−1

@@ -8,7 +8,9 @@ html {

99h1 { font-size: 5em; }

1010

11−input { border: 2px solid #8888FF; padding: 8px; background-color: #FFFFFF; font-size: 1.6em; }

11+input { border: 2px solid #8888FF; padding: 8px; background-color: #FFFFFF; font-size: 1.3em; }

12+

13+a { text-decoration: none; }

1214

1315.result_title { margin-bottom: 0; }

1416

searx/templates/results.html+4−1

@@ -5,6 +5,9 @@

55 <input type="submit" value="search" />

66</form>

77{% for result in results %}

8− <p>{{ result|safe }}</p>

8+ <div class="result">

9+ <h3 class="result_title"><a href="{{ result.url }}">{{ result.title|safe }}</a></h3>

10+ <p class="content"><span class="engine">{{ result.engine }}</span><br />{% if result.content %}{{ result.content|safe }}<br />{% endif %}<span class="url">{{ result.url }}</span></p>

11+ </div>

912{% endfor %}

1013{% endblock %}