dd46629 · searx · git.iko.soy

git.iko.soy/searx

fix: robot fw, entry points, some flake8, package searx egg

commitdd4662978dd74c0dce089790689fe0a8a4f9bb16 authorMatej Cotman <cotman.matej@gmail.com> dateSun 19 Jan 2014, 22:59 (13 years ago) parentb7fa790

17 files changed+293−130

compare with master

unifiedsplit

.gitignore+11−7 Makefile+4−8 buildout.cfg−2 minimal.cfg−2 production.cfg−2 searx/__init__.py+2−4 searx/engines/__init__.py+72−38 searx/engines/bing.py+3−2 searx/engines/currency_convert.py+15−12 searx/engines/dailymotion.py+4−1 settings.yml → searx/settings.yml searx/settings_robot.py−16 searx/settings_robot.yml+107 searx/testing.py+16−3 searx/utils.py+5−1 searx/webapp.py+39−32 setup.py+15

.gitignore+11−7

@@ -1,20 +1,24 @@

1−env

2−engines.cfg

3−.installed.cfg

41.coverage

5−coverage/

2+.installed.cfg

3+engines.cfg

4+env

5+robot_log.html

6+robot_output.xml

7+robot_report.html

68setup.cfg

810*.pyc

911*/*.pyc

1012

1113bin/

12−include/

13−lib/

1414build/

15+covearge/

1516develop-eggs/

17+dist/

1618eggs/

19+include/

20+lib/

1721local/

18−searx.egg-info/

1922parts/

23+searx.egg-info/

2024var/

Makefile+4−8

@@ -21,11 +21,7 @@ $(python):

2121tests: .installed.cfg

2222 @bin/test

2323

24−enginescfg:

25− @test -f ./engines.cfg || echo "Copying engines.cfg ..."

26− @cp --no-clobber engines.cfg_sample engines.cfg

27−

28−robot: .installed.cfg enginescfg

24+robot: .installed.cfg

2925 @bin/robot

3026

3127flake8: .installed.cfg

@@ -37,18 +33,18 @@ coverage: .installed.cfg

3733 @bin/coverage report --show-missing

3834 @bin/coverage html --directory ./coverage

3935

40−production: bin/buildout production.cfg setup.py enginescfg

36+production: bin/buildout production.cfg setup.py

4137 bin/buildout -c production.cfg $(options)

4238 @echo "* Please modify `readlink --canonicalize-missing ./searx/settings.py`"

4339 @echo "* Hint 1: on production, disable debug mode and change secret_key"

4440 @echo "* Hint 2: searx will be executed at server startup by crontab"

4541 @echo "* Hint 3: to run immediatley, execute 'bin/supervisord'"

4642

47−minimal: bin/buildout minimal.cfg setup.py enginescfg

43+minimal: bin/buildout minimal.cfg setup.py

4844 bin/buildout -c minimal.cfg $(options)

4945

5046clean:

5147 @rm -rf .installed.cfg .mr.developer.cfg bin parts develop-eggs \

5248 searx.egg-info lib include .coverage coverage

5349

54−.PHONY: all tests enginescfg robot flake8 coverage production minimal clean

50+.PHONY: all tests robot flake8 coverage production minimal clean

buildout.cfg−2

@@ -16,8 +16,6 @@ recipe = zc.recipe.egg:script

1616eggs = ${buildout:eggs}

1717interpreter = py

1818dependent-scripts = true

19−entry-points =

20− searx-run=searx.webapp:run

2119

2220

2321[robot]

minimal.cfg−2

@@ -13,5 +13,3 @@ parts +=

1313recipe = zc.recipe.egg:script

1414eggs = ${buildout:eggs}

1515interpreter = py

16−entry-points =

17− searx-run=searx.webapp:run

production.cfg−2

@@ -15,8 +15,6 @@ parts +=

1515recipe = zc.recipe.egg:script

1616eggs = ${buildout:eggs}

1717interpreter = py

18−entry-points =

19− searx-run=searx.webapp:run

2018

2119

2220[supervisor]

searx/__init__.py+2−4

@@ -1,5 +1,5 @@

11from os import environ

2−from os.path import realpath, dirname, join

2+from os.path import realpath, dirname, join, abspath

33try:

44 from yaml import load

55except:

@@ -7,8 +7,7 @@ except:

77 stderr.write('[E] install pyyaml\n')

88 exit(2)

10−

11−searx_dir = realpath(dirname(realpath(__file__))+'/../')

10+searx_dir = abspath(dirname(__file__))

1211engine_dir = dirname(realpath(__file__))

1312

1413if 'SEARX_SETTINGS_PATH' in environ:

@@ -19,4 +18,3 @@ else:

1918

2019with open(settings_path) as settings_yaml:

2120 settings = load(settings_yaml)

22−

searx/engines/__init__.py+72−38

@@ -35,6 +35,7 @@ engines = {}

3535

3636categories = {'general': []}

3737

38+

3839def load_module(filename):

3940 modname = splitext(filename)[0]

4041 if modname in sys.modules:

@@ -50,7 +51,7 @@ if not 'engines' in settings or not settings['engines']:

5051

5152for engine_data in settings['engines']:

5253 engine_name = engine_data['engine']

53− engine = load_module(engine_name+'.py')

54+ engine = load_module(engine_name + '.py')

5455 for param_name in engine_data:

5556 if param_name == 'engine':

5657 continue

@@ -58,38 +59,50 @@ for engine_data in settings['engines']:

5859 if engine_data['categories'] == 'none':

5960 engine.categories = []

6061 else:

61− engine.categories = map(str.strip, engine_data['categories'].split(','))

62+ engine.categories = map(

63+ str.strip, engine_data['categories'].split(','))

6264 continue

6365 setattr(engine, param_name, engine_data[param_name])

6466 for engine_attr in dir(engine):

6567 if engine_attr.startswith('_'):

6668 continue

6769 if getattr(engine, engine_attr) == None:

68− print '[E] Engine config error: Missing attribute "{0}.{1}"'.format(engine.name, engine_attr)

70+ print '[E] Engine config error: Missing attribute "{0}.{1}"'.format(engine.name, engine_attr) # noqa

6971 sys.exit(1)

7072 engines[engine.name] = engine

71− engine.stats = {'result_count': 0, 'search_count': 0, 'page_load_time': 0, 'score_count': 0, 'errors': 0}

73+ engine.stats = {

74+ 'result_count': 0,

75+ 'search_count': 0,

76+ 'page_load_time': 0,

77+ 'score_count': 0,

78+ 'errors': 0

79+ }

7280 if hasattr(engine, 'categories'):

7381 for category_name in engine.categories:

7482 categories.setdefault(category_name, []).append(engine)

7583 else:

7684 categories['general'].append(engine)

7785

86+

7887def default_request_params():

79− return {'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}}

88+ return {

89+ 'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}}

90+

8091

8192def make_callback(engine_name, results, suggestions, callback, params):

8293 # creating a callback wrapper for the search engine results

8394 def process_callback(response, **kwargs):

8495 cb_res = []

8596 response.search_params = params

86− engines[engine_name].stats['page_load_time'] += (datetime.now() - params['started']).total_seconds()

97+ engines[engine_name].stats['page_load_time'] += \

98+ (datetime.now() - params['started']).total_seconds()

8799 try:

88100 search_results = callback(response)

89101 except Exception, e:

90102 engines[engine_name].stats['errors'] += 1

91103 results[engine_name] = cb_res

92− print '[E] Error with engine "{0}":\n\t{1}'.format(engine_name, str(e))

104+ print '[E] Error with engine "{0}":\n\t{1}'.format(

105+ engine_name, str(e))

93106 return

94107 for result in search_results:

95108 result['engine'] = engine_name

@@ -101,23 +114,25 @@ def make_callback(engine_name, results, suggestions, callback, params):

101114 results[engine_name] = cb_res

102115 return process_callback

103116

117+

104118def score_results(results):

105− flat_res = filter(None, chain.from_iterable(izip_longest(*results.values())))

119+ flat_res = filter(

120+ None, chain.from_iterable(izip_longest(*results.values())))

106121 flat_len = len(flat_res)

107122 engines_len = len(results)

108123 results = []

109124 # deduplication + scoring

110− for i,res in enumerate(flat_res):

125+ for i, res in enumerate(flat_res):

111126 res['parsed_url'] = urlparse(res['url'])

112127 res['engines'] = [res['engine']]

113128 weight = 1.0

114129 if hasattr(engines[res['engine']], 'weight'):

115130 weight = float(engines[res['engine']].weight)

116− score = int((flat_len - i)/engines_len)*weight+1

131+ score = int((flat_len - i) / engines_len) * weight + 1

117132 duplicated = False

118133 for new_res in results:

119− p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path

120− p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path

134+ p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa

135+ p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path # noqa

121136 if res['parsed_url'].netloc == new_res['parsed_url'].netloc and\

122137 p1 == p2 and\

123138 res['parsed_url'].query == new_res['parsed_url'].query and\

@@ -125,7 +140,7 @@ def score_results(results):

125140 duplicated = new_res

126141 break

127142 if duplicated:

128− if len(res.get('content', '')) > len(duplicated.get('content', '')):

143+ if len(res.get('content', '')) > len(duplicated.get('content', '')): # noqa

129144 duplicated['content'] = res['content']

130145 duplicated['score'] += score

131146 duplicated['engines'].append(res['engine'])

@@ -139,6 +154,7 @@ def score_results(results):

139154 results.append(res)

140155 return sorted(results, key=itemgetter('score'), reverse=True)

141156

157+

142158def search(query, request, selected_engines):

143159 global engines, categories, number_of_searches

144160 requests = []

@@ -160,13 +176,20 @@ def search(query, request, selected_engines):

160176 request_params['started'] = datetime.now()

161177 request_params = engine.request(query, request_params)

162178

163− callback = make_callback(selected_engine['name'], results, suggestions, engine.response, request_params)

164−

165− request_args = dict(headers = request_params['headers']

166− ,hooks = dict(response=callback)

167− ,cookies = request_params['cookies']

168− ,timeout = settings['server']['request_timeout']

169− )

179+ callback = make_callback(

180+ selected_engine['name'],

181+ results,

182+ suggestions,

183+ engine.response,

184+ request_params

185+ )

186+

187+ request_args = dict(

188+ headers=request_params['headers'],

189+ hooks=dict(response=callback),

190+ cookies=request_params['cookies'],

191+ timeout=settings['server']['request_timeout']

192+ )

170193

171194 if request_params['method'] == 'GET':

172195 req = grequests.get

@@ -180,7 +203,7 @@ def search(query, request, selected_engines):

180203

181204 requests.append(req(request_params['url'], **request_args))

182205 grequests.map(requests)

183− for engine_name,engine_results in results.items():

206+ for engine_name, engine_results in results.items():

184207 engines[engine_name].stats['search_count'] += 1

185208 engines[engine_name].stats['result_count'] += len(engine_results)

186209

@@ -192,6 +215,7 @@ def search(query, request, selected_engines):

192215

193216 return results, suggestions

194217

218+

195219def get_engines_stats():

196220 # TODO refactor

197221 pageloads = []

@@ -200,14 +224,15 @@ def get_engines_stats():

200224 errors = []

201225 scores_per_result = []

202226

203− max_pageload = max_results = max_score = max_errors = max_score_per_result = 0

227+ max_pageload = max_results = max_score = max_errors = max_score_per_result = 0 # noqa

204228 for engine in engines.values():

205229 if engine.stats['search_count'] == 0:

206230 continue

207− results_num = engine.stats['result_count']/float(engine.stats['search_count'])

208− load_times = engine.stats['page_load_time']/float(engine.stats['search_count'])

231+ results_num = \

232+ engine.stats['result_count'] / float(engine.stats['search_count'])

233+ load_times = engine.stats['page_load_time'] / float(engine.stats['search_count']) # noqa

209234 if results_num:

210− score = engine.stats['score_count'] / float(engine.stats['search_count'])

235+ score = engine.stats['score_count'] / float(engine.stats['search_count']) # noqa

211236 score_per_result = score / results_num

212237 else:

213238 score = score_per_result = 0.0

@@ -220,30 +245,39 @@ def get_engines_stats():

220245 results.append({'avg': results_num, 'name': engine.name})

221246 scores.append({'avg': score, 'name': engine.name})

222247 errors.append({'avg': engine.stats['errors'], 'name': engine.name})

223− scores_per_result.append({'avg': score_per_result, 'name': engine.name})

248+ scores_per_result.append({

249+ 'avg': score_per_result,

250+ 'name': engine.name

251+ })

224252

225253 for engine in pageloads:

226− engine['percentage'] = int(engine['avg']/max_pageload*100)

254+ engine['percentage'] = int(engine['avg'] / max_pageload * 100)

227255

228256 for engine in results:

229− engine['percentage'] = int(engine['avg']/max_results*100)

257+ engine['percentage'] = int(engine['avg'] / max_results * 100)

230258

231259 for engine in scores:

232− engine['percentage'] = int(engine['avg']/max_score*100)

260+ engine['percentage'] = int(engine['avg'] / max_score * 100)

233261

234262 for engine in scores_per_result:

235− engine['percentage'] = int(engine['avg']/max_score_per_result*100)

263+ engine['percentage'] = int(engine['avg'] / max_score_per_result * 100)

236264

237265 for engine in errors:

238266 if max_errors:

239− engine['percentage'] = int(float(engine['avg'])/max_errors*100)

267+ engine['percentage'] = int(float(engine['avg']) / max_errors * 100)

240268 else:

241269 engine['percentage'] = 0

242270

243−

244− return [('Page loads (sec)', sorted(pageloads, key=itemgetter('avg')))

245− ,('Number of results', sorted(results, key=itemgetter('avg'), reverse=True))

246− ,('Scores', sorted(scores, key=itemgetter('avg'), reverse=True))

247− ,('Scores per result', sorted(scores_per_result, key=itemgetter('avg'), reverse=True))

248− ,('Errors', sorted(errors, key=itemgetter('avg'), reverse=True))

249− ]

271+ return [

272+ ('Page loads (sec)', sorted(pageloads, key=itemgetter('avg'))),

273+ (

274+ 'Number of results',

275+ sorted(results, key=itemgetter('avg'), reverse=True)

276+ ),

277+ ('Scores', sorted(scores, key=itemgetter('avg'), reverse=True)),

278+ (

279+ 'Scores per result',

280+ sorted(scores_per_result, key=itemgetter('avg'), reverse=True)

281+ ),

282+ ('Errors', sorted(errors, key=itemgetter('avg'), reverse=True)),

283+ ]

searx/engines/bing.py+3−2

@@ -4,11 +4,12 @@ from cgi import escape

55base_url = 'http://www.bing.com/'

66search_string = 'search?{query}'

7−locale = 'en-US' # see http://msdn.microsoft.com/en-us/library/dd251064.aspx

7+locale = 'en-US' # see http://msdn.microsoft.com/en-us/library/dd251064.aspx

1010def request(query, params):

11− search_path = search_string.format(query=urlencode({'q': query, 'setmkt': locale}))

11+ search_path = search_string.format(

12+ query=urlencode({'q': query, 'setmkt': locale}))

1213 #if params['category'] == 'images':

1314 # params['url'] = base_url + 'images/' + search_path

1415 params['url'] = base_url + search_path

searx/engines/currency_convert.py+15−12

@@ -7,6 +7,7 @@ weight = 100

88parser_re = re.compile(r'^\W*(\d+(?:\.\d+)?)\W*([a-z]{3})\W*(?:in)?\W*([a-z]{3})\W*$', re.I)

10+

1011def request(query, params):

1112 m = parser_re.match(query)

1213 if not m:

@@ -19,7 +20,7 @@ def request(query, params):

1920 # wrong params

2021 return params

2122

22− q = (from_currency+to_currency).upper()

23+ q = (from_currency + to_currency).upper()

2324

2425 params['url'] = url.format(query=q)

2526 params['ammount'] = ammount

@@ -33,25 +34,27 @@ def response(resp):

3334 global base_url

3435 results = []

3536 try:

36− _,conversion_rate,_ = resp.text.split(',', 2)

37+ _, conversion_rate, _ = resp.text.split(',', 2)

3738 conversion_rate = float(conversion_rate)

3839 except:

3940 return results

4041

41− title = '{0} {1} in {2} is {3}'.format(resp.search_params['ammount']

42− ,resp.search_params['from']

43− ,resp.search_params['to']

44− ,resp.search_params['ammount']*conversion_rate

45− )

42+ title = '{0} {1} in {2} is {3}'.format(

43+ resp.search_params['ammount'],

44+ resp.search_params['from'],

45+ resp.search_params['to'],

46+ resp.search_params['ammount'] * conversion_rate

47+ )

4648

4749 content = '1 {0} is {1} {2}'.format(resp.search_params['from'], conversion_rate, resp.search_params['to'])

4850 now_date = datetime.now().strftime('%Y%m%d')

4951 url = 'http://finance.yahoo.com/currency/converter-results/{0}/{1}-{2}-to-{3}.html'

50− url = url.format(now_date

51− ,resp.search_params['ammount']

52− ,resp.search_params['from'].lower()

53− ,resp.search_params['to'].lower()

54− )

52+ url = url.format(

53+ now_date,

54+ resp.search_params['ammount'],

55+ resp.search_params['from'].lower(),

56+ resp.search_params['to'].lower()

57+ )

5558 results.append({'title': title, 'content': content, 'url': url})

5659

5760 return results

searx/engines/dailymotion.py+4−1

@@ -8,9 +8,11 @@ locale = 'en_US'

88# see http://www.dailymotion.com/doc/api/obj-video.html

99search_url = 'https://api.dailymotion.com/videos?fields=title,description,duration,url,thumbnail_360_url&sort=relevance&limit=25&page=1&{query}'

1010

11+

1112def request(query, params):

1213 global search_url

13− params['url'] = search_url.format(query=urlencode({'search': query, 'localization': locale }))

14+ params['url'] = search_url.format(

15+ query=urlencode({'search': query, 'localization': locale}))

1416 return params

1517

1618

@@ -32,6 +34,7 @@ def response(resp):

3234 results.append({'url': url, 'title': title, 'content': content})

3335 return results

3436

37+

3538def text_content_from_html(html_string):

3639 desc_html = html.fragment_fromstring(html_string, create_parent=True)

3740 return desc_html.text_content()

settings.yml → searx/settings.ymlrenamed

The content is the same.

searx/settings_robot.pydeleted−16

@@ -1,16 +0,0 @@

1−

2−port = 11111

3−

4−secret_key = "ultrasecretkey" # change this!

5−

6−debug = False

7−

8−request_timeout = 5.0 # seconds

9−

10−weights = {} # 'search_engine_name': float(weight) | default is 1.0

11−

12−blacklist = [] # search engine blacklist

13−

14−categories = {} # custom search engine categories

15−

16−base_url = None # "https://your.domain.tld/" or None (to use request parameters)

searx/settings_robot.ymlnew file+107

@@ -0,0 +1,107 @@

1+server:

2+ port : 11111

3+ secret_key : "ultrasecretkey" # change this!

4+ debug : False

5+ request_timeout : 3.0 # seconds

6+ base_url: False

8+engines:

9+ - name : wikipedia

10+ engine : mediawiki

11+ url : https://en.wikipedia.org/

12+ number_of_results : 1

13+

14+ - name : bing

15+ engine : bing

16+ locale : en-US

17+

18+ - name : currency

19+ engine : currency_convert

20+ categories : general

21+

22+ - name : deviantart

23+ engine : deviantart

24+ categories : images

25+

26+ - name : ddg definitions

27+ engine : duckduckgo_definitions

28+

29+ - name : duckduckgo

30+ engine : duckduckgo

31+ locale : en-us

32+

33+ - name : filecrop

34+ engine : filecrop

35+ categories : files

36+

37+ - name : flickr

38+ engine : flickr

39+ categories : images

40+

41+ - name : github

42+ engine : github

43+ categories : it

44+

45+ - name : google

46+ engine : json_engine

47+ search_url : https://ajax.googleapis.com/ajax/services/search/web?v=2.0&start=0&rsz=large&safe=off&filter=off&q={query}

48+ categories : general

49+ url_query : /responseData/results/unescapedUrl

50+ content_query : /responseData/results/content

51+ title_query : /responseData/results/titleNoFormatting

52+

53+ - name : google images

54+ engine : google_images

55+ categories : images

56+

57+ - name : piratebay

58+ engine : piratebay

59+ categories : videos, music, files

60+

61+ - name : soundcloud

62+ engine : soundcloud

63+ categories : music

64+

65+ - name : stackoverflow

66+ engine : stackoverflow

67+ categories : it

68+

69+ - name : startpage

70+ engine : startpage

71+

72+ - name : twitter

73+ engine : twitter

74+ categories : social media

75+

76+ - name : urbandictionary

77+ engine : xpath

78+ search_url : http://www.urbandictionary.com/define.php?term={query}

79+ url_xpath : //div[@class="word"]//a/@href

80+ title_xpath : //div[@class="word"]//a

81+ content_xpath : //div[@class="definition"]

82+

83+ - name : yahoo

84+ engine : xpath

85+ search_url : http://search.yahoo.com/search?p={query}

86+ results_xpath : //div[@class="res"]

87+ url_xpath : .//h3/a/@href

88+ title_xpath : .//h3/a

89+ content_xpath : .//div[@class="abstr"]

90+ suggestion_xpath : //div[@id="satat"]//a

91+

92+ - name : youtube

93+ engine : youtube

94+ categories : videos

95+

96+ - name : dailymotion

97+ engine : dailymotion

98+ locale : en_US

99+ categories : videos

100+

101+ - name : vimeo

102+ engine : vimeo

103+ categories : videos

104+ results_xpath : //div[@id="browse_content"]/ol/li

105+ url_xpath : ./a/@href

106+ title_xpath : ./a/div[@class="data"]/p[@class="title"]/text()

107+ content_xpath : ./a/img/@src

searx/testing.py+16−3

@@ -7,10 +7,10 @@ from unittest2 import TestCase

88import os

99import subprocess

10−import sys

1110

1211

1312class SearxTestLayer:

13+ """Base layer for non-robot tests."""

1414

1515 __name__ = u'SearxTestLayer'

1616

@@ -36,24 +36,37 @@ class SearxRobotLayer(Layer):

3636

3737 def setUp(self):

3838 os.setpgrp() # create new process group, become its leader

39+

40+ # get program paths

3941 webapp = os.path.join(

4042 os.path.abspath(os.path.dirname(os.path.realpath(__file__))),

4143 'webapp.py'

4244 )

4345 exe = os.path.abspath(os.path.dirname(__file__) + '/../bin/py')

46+

47+ # set robot settings path

48+ os.environ['SEARX_SETTINGS_PATH'] = os.path.abspath(

49+ os.path.dirname(__file__) + '/settings_robot.yml')

50+

51+ # run the server

4452 self.server = subprocess.Popen(

45− [exe, webapp, 'settings_robot'],

53+ [exe, webapp],

4654 stdout=subprocess.PIPE,

4755 stderr=subprocess.STDOUT

4856 )

4957

5058 def tearDown(self):

51− # TERM all processes in my group

59+ # send TERM signal to all processes in my group, to stop subprocesses

5260 os.killpg(os.getpgid(self.server.pid), 15)

5361

62+ # remove previously set environment variable

63+ del os.environ['SEARX_SETTINGS_PATH']

64+

5465

5566SEARXROBOTLAYER = SearxRobotLayer()

5667

5768

5869class SearxTestCase(TestCase):

70+ """Base test case for non-robot tests."""

71+

5972 layer = SearxTestLayer

searx/utils.py+5−1

@@ -5,10 +5,12 @@ import codecs

55import cStringIO

66import re

89def gen_useragent():

910 # TODO

1011 return "Mozilla/5.0 (X11; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0"

1112

13+

1214def highlight_content(content, query):

1315

1416 if not content:

@@ -34,10 +36,11 @@ def highlight_content(content, query):

3436

3537 return content

3638

39+

3740class HTMLTextExtractor(HTMLParser):

3841 def __init__(self):

3942 HTMLParser.__init__(self)

40− self.result = [ ]

43+ self.result = []

4144

4245 def handle_data(self, d):

4346 self.result.append(d)

@@ -54,6 +57,7 @@ class HTMLTextExtractor(HTMLParser):

5457 def get_text(self):

5558 return u''.join(self.result)

5659

60+

5761def html_to_text(html):

5862 s = HTMLTextExtractor()

5963 s.feed(html)

searx/webapp.py+39−32

@@ -17,13 +17,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1818'''

1919

20−import os

21−import sys

22−if __name__ == "__main__":

23− sys.path.append(os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/../'))

24−

2520from searx import settings

26−

2721from flask import Flask, request, render_template, url_for, Response, make_response, redirect

2822from searx.engines import search, categories, engines, get_engines_stats

2923import json

@@ -33,10 +27,16 @@ from flask import send_from_directory

3327from searx.utils import highlight_content, html_to_text

3428

3529

30+import os

3631

37−app = Flask(__name__)

38−app.secret_key = settings['server']['secret_key']

3932

33+app = Flask(

34+ __name__,

35+ static_folder=os.path.join(os.path.dirname(__file__), 'static'),

36+ template_folder=os.path.join(os.path.dirname(__file__), 'templates')

37+)

38+

39+app.secret_key = settings['server']['secret_key']

4040

4141#TODO configurable via settings.yml

4242favicons = ['wikipedia', 'youtube', 'vimeo', 'soundcloud',

@@ -81,6 +81,7 @@ def render(template_name, **kwargs):

8181 kwargs['selected_categories'] = ['general']

8282 return render_template(template_name, **kwargs)

8383

84+

8485def parse_query(query):

8586 query_engines = []

8687 query_parts = query.split()

@@ -94,7 +95,7 @@ def parse_query(query):

9495def index():

9596 global categories

9697

97− if request.method=='POST':

98+ if request.method == 'POST':

9899 request_data = request.form

99100 else:

100101 request_data = request.args

@@ -106,7 +107,7 @@ def index():

106107 query, selected_engines = parse_query(request_data['q'].encode('utf-8'))

107108

108109 if not len(selected_engines):

109− for pd_name,pd in request_data.items():

110+ for pd_name, pd in request_data.items():

110111 if pd_name.startswith('category_'):

111112 category = pd_name[9:]

112113 if not category in categories:

@@ -159,23 +160,24 @@ def index():

159160 response.headers.add('Content-Disposition', 'attachment;Filename=searx_-_{0}.csv'.format('_'.join(query.split())))

160161 return response

161162 elif request_data.get('format') == 'rss':

162− response_rss = render('opensearch_response_rss.xml'

163− ,results=results

164− ,q=request_data['q']

165− ,number_of_results=len(results)

166− ,base_url=get_base_url()

167− )

163+ response_rss = render(

164+ 'opensearch_response_rss.xml',

165+ results=results,

166+ q=request_data['q'],

167+ number_of_results=len(results),

168+ base_url=get_base_url()

169+ )

168170 return Response(response_rss, mimetype='text/xml')

169171

170−

171− return render('results.html'

172− ,results=results

173− ,q=request_data['q']

174− ,selected_categories=selected_categories

175− ,number_of_results=len(results)+len(featured_results)

176− ,featured_results=featured_results

177− ,suggestions=suggestions

178− )

172+ return render(

173+ 'results.html',

174+ results=results,

175+ q=request_data['q'],

176+ selected_categories=selected_categories,

177+ number_of_results=len(results) + len(featured_results),

178+ featured_results=featured_results,

179+ suggestions=suggestions

180+ )

179181

180182

181183@app.route('/about', methods=['GET'])

@@ -192,9 +194,9 @@ def list_engines():

192194@app.route('/preferences', methods=['GET', 'POST'])

193195def preferences():

194196

195− if request.method=='POST':

197+ if request.method == 'POST':

196198 selected_categories = []

197− for pd_name,pd in request.form.items():

199+ for pd_name, pd in request.form.items():

198200 if pd_name.startswith('category_'):

199201 category = pd_name[9:]

200202 if not category in categories:

@@ -203,7 +205,10 @@ def preferences():

203205 if selected_categories:

204206 resp = make_response(redirect('/'))

205207 # cookie max age: 4 weeks

206− resp.set_cookie('categories', ','.join(selected_categories), max_age=60*60*24*7*4)

208+ resp.set_cookie(

209+ 'categories', ','.join(selected_categories),

210+ max_age=60 * 60 * 24 * 7 * 4

211+ )

207212 return resp

208213 return render('preferences.html')

209214

@@ -238,6 +243,7 @@ def opensearch():

238243 mimetype="application/xml")

239244 return resp

240245

246+

241247@app.route('/favicon.ico')

242248def favicon():

243249 return send_from_directory(os.path.join(app.root_path, 'static/img'),

@@ -248,10 +254,11 @@ def run():

248254 from gevent import monkey

249255 monkey.patch_all()

250256

251− app.run(debug = settings['server']['debug']

252− ,use_debugger = settings['server']['debug']

253− ,port = settings['server']['port']

254− )

257+ app.run(

258+ debug=settings['server']['debug'],

259+ use_debugger=settings['server']['debug'],

260+ port=settings['server']['port']

261+ )

255262

256263

257264if __name__ == "__main__":

setup.py+15

@@ -49,4 +49,19 @@ setup(

4949 'zope.testrunner',

5050 ]

5151 },

52+ entry_points={

53+ 'console_scripts': [

54+ 'searx-run = searx.webapp:run'

55+ ]

56+ },

57+ package_data={

58+ 'searx': [

59+ 'settings.yml',

60+ '../README.md',

61+ 'static/*/*',

62+ 'templates/*.html',

63+ 'templates/result_templates/*.html',

64+ ],

65+ },

66+

5267)