d5ec0f4 · searx · git.iko.soy

git.iko.soy/searx

[mod] search refactor

commitd5ec0f43e465e1bd016f1c841122fc3c774fc718 authorasciimoo <asciimoo@gmail.com> dateTue 15 Oct 2013, 18:19 (13 years ago) parent2440e74

2 files changed+40−34

compare with master

unifiedsplit

searx/engines/__init__.py+38−1 searx/webapp.py+2−33

searx/engines/__init__.py+38−1

@@ -2,6 +2,7 @@

22from os.path import realpath, dirname, splitext, join

33from os import listdir

44from imp import load_source

5+import grequests

67engine_dir = dirname(realpath(__file__))

@@ -12,4 +13,40 @@ for filename in listdir(engine_dir):

1213 if filename.startswith('_') or not filename.endswith('.py'):

1314 continue

1415 filepath = join(engine_dir, filename)

15− engines.append(load_source(modname, filepath))

16+ engine = load_source(modname, filepath)

17+ if not hasattr(engine, 'request') or not hasattr(engine, 'response'):

18+ continue

19+ engines.append(engine)

20+

21+def default_request_params():

22+ return {'method': 'GET', 'headers': {}, 'data': {}, 'url': ''}

23+

24+def make_callback(results, callback):

25+ def process_callback(response, **kwargs):

26+ results.extend(callback(response))

27+ return process_callback

28+

29+def search(query, request):

30+ global engines

31+ requests = []

32+ results = []

33+ user_agent = request.headers.get('User-Agent', '')

34+ for engine in engines:

35+ headers = default_request_params()

36+ headers['User-Agent'] = user_agent

37+ request_params = engine.request(query, headers)

38+ callback = make_callback(results, engine.response)

39+ if request_params['method'] == 'GET':

40+ req = grequests.get(request_params['url']

41+ ,headers=headers

42+ ,hooks=dict(response=callback)

43+ )

44+ else:

45+ req = grequests.post(request_params['url']

46+ ,data=request_params['data']

47+ ,headers=headers

48+ ,hooks=dict(response=callback)

49+ )

50+ requests.append(req)

51+ grequests.map(requests)

52+ return results

searx/webapp.py+2−33

@@ -25,8 +25,7 @@ if __name__ == "__main__":

2525from flask import Flask, request, flash, render_template

2626import ConfigParser

2727from os import getenv

28−from searx.engines import engines

29−import grequests

28+from searx.engines import search

3029

3130cfg = ConfigParser.SafeConfigParser()

3231cfg.read('/etc/searx.conf')

@@ -38,14 +37,6 @@ cfg.read('searx.conf')

3837app = Flask(__name__)

3938app.secret_key = cfg.get('app', 'secret_key')

4039

41−def default_request_params():

42− return {'method': 'GET', 'headers': {}, 'data': {}, 'url': ''}

43−

44−def make_callback(results, callback):

45− def process_callback(response, **kwargs):

46− results.extend(callback(response))

47− return process_callback

48−

4940@app.route('/', methods=['GET', 'POST'])

5041def index():

5142 if request.method=='POST':

@@ -53,30 +44,8 @@ def index():

5344 flash('Wrong post data')

5445 return render_template('index.html')

5546 query = request.form['q']

56− requests = []

57− results = []

58− user_agent = request.headers.get('User-Agent', '')

59− for engine in engines:

60− headers = default_request_params()

61− headers['User-Agent'] = user_agent

62− request_params = engine.request(query, headers)

63− callback = make_callback(results, engine.response)

64− if request_params['method'] == 'GET':

65− req = grequests.get(request_params['url']

66− ,headers=headers

67− ,hooks=dict(response=callback)

68− )

69− else:

70− req = grequests.post(request_params['url']

71− ,data=request_params['data']

72− ,headers=headers

73− ,hooks=dict(response=callback)

74− )

75− requests.append(req)

76− grequests.map(requests)

47+ results = search(query, request)

7748 return render_template('results.html', results=results, q=query)

78−

79−

8049 return render_template('index.html')

8150

8251if __name__ == "__main__":