0d93ad2 · searx · git.iko.soy

engines.cfg_sample+9−7

compare with master

unifiedsplit

@@ -5,7 +5,7 @@ number_of_results = 1

5

6[bing]

7engine = bing

8−language = en-us

8+locale = en-US

9

10[cc]

11engine=currency_convert

@@ -20,6 +20,7 @@ engine = duckduckgo_definitions

20

21[duckduckgo]

22engine = duckduckgo

23+locale = en-us

23

24

24[flickr]

25[flickr]

25engine = flickr

26engine = flickr

@@ -63,17 +64,17 @@ categories = social media

63[urbandictionary]

64[urbandictionary]

64engine = xpath

65engine = xpath

65search_url = http://www.urbandictionary.com/define.php?term={query}

66search_url = http://www.urbandictionary.com/define.php?term={query}

66−url_xpath = //div[@id="entries"]//div[@class="word"]//a

67+url_xpath = //div[@id="entries"]//div[@class="word"]/a/@href

67−title_xpath = //div[@id="entries"]//div[@class="word"]//span//text()

68+title_xpath = //div[@id="entries"]//div[@class="word"]/span

68−content_xpath = //div[@id="entries"]//div[@class="text"]//div[@class="definition"]//text()

69+content_xpath = //div[@id="entries"]//div[@class="text"]/div[@class="definition"]

69

70

70[yahoo]

71[yahoo]

71engine = xpath

72engine = xpath

72search_url = http://search.yahoo.com/search?p={query}

73search_url = http://search.yahoo.com/search?p={query}

73results_xpath = //div[@class="res"]

74results_xpath = //div[@class="res"]

74−url_xpath = .//span[@class="url"]//text()

75+url_xpath = .//h3/a/@href

76+title_xpath = .//h3/a

75−content_xpath = .//div[@class="abstr"]//text()

77+content_xpath = .//div[@class="abstr"]

76−title_xpath = .//h3/a//text()

77suggestion_xpath = //div[@id="satat"]//a

78suggestion_xpath = //div[@id="satat"]//a

78

79

79[youtube]

80[youtube]

@@ -82,5 +83,6 @@ categories = videos

82

83

83[dailymotion]

84[dailymotion]

84engine = dailymotion

85engine = dailymotion

86+locale = en_US

85categories = videos

87categories = videos

86

88

searx/engines/__init__.py+1−1

@@ -261,7 +261,7 @@ def get_engines_stats():

261

262 for engine in errors:

263 if max_errors:

264− engine['percentage'] = int(engine['avg']/max_errors*100)

264+ engine['percentage'] = int(float(engine['avg'])/max_errors*100)

265 else:

266 engine['percentage'] = 0

267

searx/engines/bing.py+2−2

@@ -4,11 +4,11 @@ from cgi import escape

4

5base_url = 'http://www.bing.com/'

6search_string = 'search?{query}'

7−language = 'en-us' # see http://msdn.microsoft.com/en-us/library/dd251064.aspx

7+locale = 'en-US' # see http://msdn.microsoft.com/en-us/library/dd251064.aspx

8

9

10def request(query, params):

11− search_path = search_string.format(query=urlencode({'q': query, 'setmkt': language}))

11+ search_path = search_string.format(query=urlencode({'q': query, 'setmkt': locale}))

12 #if params['category'] == 'images':

13 # params['url'] = base_url + 'images/' + search_path

14 params['url'] = base_url + search_path

searx/engines/dailymotion.py+9−3

@@ -1,16 +1,17 @@

1from urllib import urlencode

2+from lxml import html

2from json import loads

3from json import loads

3from cgi import escape

4from cgi import escape

4

5

5categories = ['videos']

6categories = ['videos']

6−localization = 'en'

7+locale = 'en_US'

7

8

8# see http://www.dailymotion.com/doc/api/obj-video.html

9# see http://www.dailymotion.com/doc/api/obj-video.html

9search_url = 'https://api.dailymotion.com/videos?fields=title,description,duration,url,thumbnail_360_url&sort=relevance&limit=25&page=1&{query}'

10search_url = 'https://api.dailymotion.com/videos?fields=title,description,duration,url,thumbnail_360_url&sort=relevance&limit=25&page=1&{query}'

10

11

11def request(query, params):

12def request(query, params):

12 global search_url

13 global search_url

13− params['url'] = search_url.format(query=urlencode({'search': query, 'localization': localization }))

14+ params['url'] = search_url.format(query=urlencode({'search': query, 'localization': locale }))

14 return params

15 return params

15

16

17

@@ -27,6 +28,11 @@ def response(resp):

27 else:

28 else:

28 content = ''

29 content = ''

29 if res['description']:

30 if res['description']:

31+ description = text_content_from_html(res['description'])

30− content += escape(res['description'][:500])

32+ content += description[:500]

31 results.append({'url': url, 'title': title, 'content': content})

33 results.append({'url': url, 'title': title, 'content': content})

32 return results

34 return results

35+

36+def text_content_from_html(html_string):

37+ desc_html = html.fragment_fromstring(html_string, create_parent=True)

38+ return desc_html.text_content()

searx/engines/duckduckgo.py+3−2

@@ -3,10 +3,11 @@ from urllib import urlencode

3from searx.utils import html_to_text

4

5url = 'https://duckduckgo.com/'

6−search_url = url + 'd.js?{query}&l=us-en&p=1&s=0'

6+search_url = url + 'd.js?{query}&p=1&s=0'

7+locale = 'us-en'

7

8

8def request(query, params):

9def request(query, params):

9− params['url'] = search_url.format(query=urlencode({'q': query}))

10+ params['url'] = search_url.format(query=urlencode({'q': query, 'l': locale}))

10 return params

11 return params

11

12

13

searx/engines/duckduckgo_definitions.py+1−1

@@ -1,7 +1,7 @@

1import json

2from urllib import urlencode

3

4−url = 'http://api.duckduckgo.com/?{query}&format=json&pretty=0'

4+url = 'http://api.duckduckgo.com/?{query}&format=json&pretty=0&no_redirect=1'

5

6def request(query, params):

7 params['url'] = url.format(query=urlencode({'q': query}))

searx/engines/filecrop.py+4−6

@@ -1,6 +1,4 @@

1−from json import loads

2from urllib import urlencode

1from urllib import urlencode

3−from searx.utils import html_to_text

4from HTMLParser import HTMLParser

2from HTMLParser import HTMLParser

5

3

6url = 'http://www.filecrop.com/'

4url = 'http://www.filecrop.com/'

@@ -10,7 +8,7 @@ class FilecropResultParser(HTMLParser):

10 def __init__(self):

8 def __init__(self):

11 HTMLParser.__init__(self)

9 HTMLParser.__init__(self)

12 self.__start_processing = False

10 self.__start_processing = False

13−

11+

14 self.results = []

12 self.results = []

15 self.result = {}

13 self.result = {}

16

14

@@ -22,7 +20,7 @@ class FilecropResultParser(HTMLParser):

22 if tag == 'tr':

20 if tag == 'tr':

23 if ('bgcolor', '#edeff5') in attrs or ('bgcolor', '#ffffff') in attrs:

21 if ('bgcolor', '#edeff5') in attrs or ('bgcolor', '#ffffff') in attrs:

24 self.__start_processing = True

22 self.__start_processing = True

25−

23+

26 if not self.__start_processing:

24 if not self.__start_processing:

27 return

25 return

28

26

@@ -50,7 +48,7 @@ class FilecropResultParser(HTMLParser):

50 self.data_counter = 0

48 self.data_counter = 0

51 self.results.append(self.result)

49 self.results.append(self.result)

52 self.result = {}

50 self.result = {}

53−

51+

54 def handle_data(self, data):

52 def handle_data(self, data):

55 if not self.__start_processing:

53 if not self.__start_processing:

56 return

54 return

@@ -59,7 +57,7 @@ class FilecropResultParser(HTMLParser):

59 self.result['content'] += data + ' '

57 self.result['content'] += data + ' '

60 else:

58 else:

61 self.result['content'] = data + ' '

59 self.result['content'] = data + ' '

62−

60+

63 self.data_counter += 1

61 self.data_counter += 1

64

62

65def request(query, params):

63def request(query, params):

searx/engines/flickr.pymode 100755 → 100644

Only the mode changed.

searx/engines/google_images.pymode 100755 → 100644

Only the mode changed.

searx/engines/startpage.py+5−6

@@ -19,14 +19,13 @@ def response(resp):

19 global base_url

20 results = []

21 dom = html.fromstring(resp.content)

22+ # ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]

23+ # not ads : div[@class="result"] are the direct childs of div[@id="results"]

22− for result in dom.xpath('//div[@class="result"]'):

24+ for result in dom.xpath('//div[@id="results"]/div[@class="result"]'):

23 link = result.xpath('.//h3/a')[0]

25 link = result.xpath('.//h3/a')[0]

24 url = link.attrib.get('href')

26 url = link.attrib.get('href')

25 parsed_url = urlparse(url)

27 parsed_url = urlparse(url)

26− # TODO better google link detection

27− if parsed_url.netloc.find('www.google.com') >= 0:

28− continue

29− title = ' '.join(link.xpath('.//text()'))

28+ title = link.text_content()

30− content = escape(' '.join(result.xpath('.//p[@class="desc"]//text()')))

29+ content = result.xpath('./p[@class="desc"]')[0].text_content()

31 results.append({'url': url, 'title': title, 'content': content})

30 results.append({'url': url, 'title': title, 'content': content})

32 return results

31 return results

searx/engines/xpath.py+56−20

@@ -1,5 +1,5 @@

1from lxml import html

2−from urllib import urlencode

2+from urllib import urlencode, unquote

3from urlparse import urlparse, urljoin

4from cgi import escape

5from lxml.etree import _ElementStringResult

@@ -11,32 +11,64 @@ title_xpath = None

11suggestion_xpath = ''

12results_xpath = ''

13

14+'''

15+if xpath_results is list, extract the text from each result and concat the list

16+if xpath_results is a xml element, extract all the text node from it ( text_content() method from lxml )

17+if xpath_results is a string element, then it's already done

18+'''

14−def extract_url(xpath_results):

19+def extract_text(xpath_results):

15− url = ''

16− parsed_search_url = urlparse(search_url)

17 if type(xpath_results) == list:

20 if type(xpath_results) == list:

21+ # it's list of result : concat everything using recursive call

18 if not len(xpath_results):

22 if not len(xpath_results):

19 raise Exception('Empty url resultset')

23 raise Exception('Empty url resultset')

24+ result = ''

25+ for e in xpath_results:

26+ result = result + extract_text(e)

27+ return result

20− if type(xpath_results[0]) == _ElementStringResult:

28+ elif type(xpath_results) == _ElementStringResult:

29+ # it's a string

21− url = ''.join(xpath_results)

30+ return ''.join(xpath_results)

22− if url.startswith('//'):

23− url = parsed_search_url.scheme+url

24− elif url.startswith('/'):

25− url = urljoin(search_url, url)

26− #TODO

27− else:

28− url = xpath_results[0].attrib.get('href')

29 else:

31 else:

32+ # it's a element

30− url = xpath_results.attrib.get('href')

33+ return xpath_results.text_content()

34+

35+

36+def extract_url(xpath_results):

37+ url = extract_text(xpath_results)

38+

31− if not url.startswith('http://') and not url.startswith('https://'):

39+ if url.startswith('//'):

32− url = 'http://'+url

40+ # add http or https to this kind of url //example.com/

41+ parsed_search_url = urlparse(search_url)

42+ url = parsed_search_url.scheme+url

43+ elif url.startswith('/'):

44+ # fix relative url to the search engine

45+ url = urljoin(search_url, url)

46+

47+ # normalize url

48+ url = normalize_url(url)

49+

50+ return url

51+

52+

53+def normalize_url(url):

33 parsed_url = urlparse(url)

54 parsed_url = urlparse(url)

55+

56+ # add a / at this end of the url if there is no path

34 if not parsed_url.netloc:

57 if not parsed_url.netloc:

35 raise Exception('Cannot parse url')

58 raise Exception('Cannot parse url')

36 if not parsed_url.path:

59 if not parsed_url.path:

37 url += '/'

60 url += '/'

61+

62+ # FIXME : hack for yahoo

63+ if parsed_url.hostname == 'search.yahoo.com' and parsed_url.path.startswith('/r'):

64+ p = parsed_url.path

65+ mark = p.find('/**')

66+ if mark != -1:

67+ return unquote(p[mark+3:]).decode('utf-8')

68+

38 return url

69 return url

39

70

71+

40def request(query, params):

72def request(query, params):

41 query = urlencode({'q': query})[2:]

73 query = urlencode({'q': query})[2:]

42 params['url'] = search_url.format(query=query)

74 params['url'] = search_url.format(query=query)

@@ -50,15 +82,19 @@ def response(resp):

50 if results_xpath:

82 if results_xpath:

51 for result in dom.xpath(results_xpath):

83 for result in dom.xpath(results_xpath):

52 url = extract_url(result.xpath(url_xpath))

84 url = extract_url(result.xpath(url_xpath))

53− title = ' '.join(result.xpath(title_xpath))

85+ title = extract_text(result.xpath(title_xpath)[0 ])

54− content = escape(' '.join(result.xpath(content_xpath)))

86+ content = extract_text(result.xpath(content_xpath)[0])

55 results.append({'url': url, 'title': title, 'content': content})

87 results.append({'url': url, 'title': title, 'content': content})

56 else:

88 else:

89+ for url, title, content in zip(

57− for content, url, title in zip(dom.xpath(content_xpath), map(extract_url, dom.xpath(url_xpath)), dom.xpath(title_xpath)):

90+ map(extract_url, dom.xpath(url_xpath)), \

91+ map(extract_text, dom.xpath(title_xpath)), \

92+ map(extract_text, dom.xpath(content_xpath)), \

93+ ):

58 results.append({'url': url, 'title': title, 'content': content})

94 results.append({'url': url, 'title': title, 'content': content})

59

95

60 if not suggestion_xpath:

96 if not suggestion_xpath:

61 return results

97 return results

62 for suggestion in dom.xpath(suggestion_xpath):

98 for suggestion in dom.xpath(suggestion_xpath):

63− results.append({'suggestion': escape(''.join(suggestion.xpath('.//text()')))})

99+ results.append({'suggestion': extract_text(suggestion)})

64 return results

100 return results

searx/engines/yacy.py+5−5

@@ -1,5 +1,5 @@

1from json import loads

2−from urllib import urlencode, quote

2+from urllib import urlencode

3

4url = 'http://localhost:8090'

5search_url = '/yacysearch.json?{query}&maximumRecords=10'

@@ -10,7 +10,7 @@ def request(query, params):

10

11def response(resp):

12 raw_search_results = loads(resp.text)

13−

13+

14 if not len(raw_search_results):

15 return []

16

@@ -22,10 +22,10 @@ def response(resp):

22 tmp_result = {}

23 tmp_result['title'] = result['title']

24 tmp_result['url'] = result['link']

25− tmp_result['content'] = ''

25+ tmp_result['content'] = ''

26−

26+

27 if len(result['description']):

28− tmp_result['content'] += result['description'] +"<br/>"

28+ tmp_result['content'] += result['description'] +"<br/>"

29

30 if len(result['pubDate']):

31 tmp_result['content'] += result['pubDate'] + "<br/>"

searx/templates/about.html+1−1

@@ -37,7 +37,7 @@

37<p>It's ok if you don't trust us regarding the logs, <a href="https://github.com/asciimoo/searx">take the code</a> and run it yourself! decentralize!</p>

38<h3>How to add to firefox?</h3>

39<p><a href="#" onclick="window.external.AddSearchProvider(window.location.protocol + '//' + window.location.host + '/opensearch.xml')">Install</a> searx as a search engine on any version of Firefox! (javascript required)</p>

40−<h2 id="faq">Developer FAQ</h2>

40+<h2 id="dev_faq">Developer FAQ</h2>

41<h3>New engines?</h3>

42<p><ul>

43 <li>Edit your engines.cfg, see <a href="https://raw.github.com/asciimoo/searx/master/engines.cfg_sample">sample config</a></li>

searx/webapp.py+2−1

@@ -152,7 +152,8 @@ def preferences():

152 selected_categories.append(category)

153 if selected_categories:

154 resp = make_response(redirect('/'))

155+ # cookie max age: 4 weeks

155− resp.set_cookie('categories', ','.join(selected_categories))

156+ resp.set_cookie('categories', ','.join(selected_categories), max_age=60*60*24*7*4)

156 return resp

157 return resp

157 return render('preferences.html')

158 return render('preferences.html')

158

159