f6db77d · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

Merge pull request #210 from Cqoicebordel/unit-tests

f6db77dAdam Tauber12 years ago

searx/engines/currency_convert.py+3−6

compare with master

unifiedsplit

@@ -13,12 +13,9 @@ def request(query, params):

13 if not m:

14 # wrong query

15 return params

16− try:

16+

17− ammount, from_currency, to_currency = m.groups()

17+ ammount, from_currency, to_currency = m.groups()

18− ammount = float(ammount)

18+ ammount = float(ammount)

19− except:

20− # wrong params

21− return params

23 q = (from_currency + to_currency).upper()

20 q = (from_currency + to_currency).upper()

searx/engines/duckduckgo.py+5−5

@@ -15,7 +15,7 @@

16from urllib import urlencode

17from lxml.html import fromstring

18−from searx.utils import html_to_text

18+from searx.engines.xpath import extract_text

20# engine dependent config

21categories = ['general']

@@ -28,8 +28,8 @@ url = 'https://duckduckgo.com/html?{query}&s={offset}'

28# specific xpath variables

29result_xpath = '//div[@class="results_links results_links_deep web-result"]' # noqa

30url_xpath = './/a[@class="large"]/@href'

31−title_xpath = './/a[@class="large"]//text()'

31+title_xpath = './/a[@class="large"]'

32−content_xpath = './/div[@class="snippet"]//text()'

32+content_xpath = './/div[@class="snippet"]'

35# do search-request

@@ -64,8 +64,8 @@ def response(resp):

64 if not res_url:

65 continue

67− title = html_to_text(''.join(r.xpath(title_xpath)))

67+ title = extract_text(r.xpath(title_xpath))

68− content = html_to_text(''.join(r.xpath(content_xpath)))

68+ content = extract_text(r.xpath(content_xpath))

70 # append result

71 results.append({'title': title,

searx/engines/duckduckgo_definitions.py+3−2

@@ -25,9 +25,10 @@ def request(query, params):

27def response(resp):

28− search_res = json.loads(resp.text)

29 results = []

28 results = []

30+ search_res = json.loads(resp.text)

31+

31 content = ''

32 content = ''

32 heading = search_res.get('Heading', '')

33 heading = search_res.get('Heading', '')

33 attributes = []

34 attributes = []

@@ -68,7 +69,7 @@ def response(resp):

68 results.append({'title': heading, 'url': firstURL})

69 results.append({'title': heading, 'url': firstURL})

70 # related topics

71 # related topics

71− for ddg_result in search_res.get('RelatedTopics', None):

72+ for ddg_result in search_res.get('RelatedTopics', []):

72 if 'FirstURL' in ddg_result:

73 if 'FirstURL' in ddg_result:

73 suggestion = result_to_text(ddg_result.get('FirstURL', None),

74 suggestion = result_to_text(ddg_result.get('FirstURL', None),

74 ddg_result.get('Text', None),

75 ddg_result.get('Text', None),

searx/engines/faroo.py+3−5

@@ -37,7 +37,7 @@ search_category = {'general': 'web',

38# do search-request

39def request(query, params):

40− offset = (params['pageno']-1) * number_of_results + 1

40+ offset = (params['pageno'] - 1) * number_of_results + 1

41 categorie = search_category.get(params['category'], 'web')

43 if params['language'] == 'all':

@@ -45,11 +45,11 @@ def request(query, params):

45 else:

46 language = params['language'].split('_')[0]

48− # skip, if language is not supported

48+ # if language is not supported, put it in english

49 if language != 'en' and\

50 language != 'de' and\

51 language != 'zh':

52− return params

52+ language = 'en'

54 params['url'] = search_url.format(offset=offset,

55 number_of_results=number_of_results,

@@ -69,12 +69,10 @@ def response(resp):

69 # HTTP-Code 401: api-key is not valide

70 if resp.status_code == 401:

71 raise Exception("API key is not valide")

72− return []

74 # HTTP-Code 429: rate limit exceeded

73 # HTTP-Code 429: rate limit exceeded

75 if resp.status_code == 429:

74 if resp.status_code == 429:

76 raise Exception("rate limit has been exceeded!")

75 raise Exception("rate limit has been exceeded!")

77− return []

79 results = []

77 results = []

searx/engines/openstreetmap.py+5−4

@@ -38,6 +38,9 @@ def response(resp):

39 # parse results

40 for r in json:

41+ if 'display_name' not in r:

42+ continue

43+

41 title = r['display_name']

44 title = r['display_name']

42 osm_type = r.get('osm_type', r.get('type'))

45 osm_type = r.get('osm_type', r.get('type'))

43 url = result_base_url.format(osm_type=osm_type,

46 url = result_base_url.format(osm_type=osm_type,

@@ -49,10 +52,8 @@ def response(resp):

49 geojson = r.get('geojson')

52 geojson = r.get('geojson')

51 # if no geojson is found and osm_type is a node, add geojson Point

54 # if no geojson is found and osm_type is a node, add geojson Point

52− if not geojson and\

55+ if not geojson and osm_type == 'node':

53− osm_type == 'node':

54− geojson = {u'type': u'Point',

55− u'coordinates': [r['lon'], r['lat']]}

56+ geojson = {u'type': u'Point', u'coordinates': [r['lon'], r['lat']]}

57 address_raw = r.get('address')

58 address_raw = r.get('address')

58 address = {}

59 address = {}

searx/engines/photon.py+1−1

@@ -61,7 +61,7 @@ def response(resp):

61 continue

63 # get title

64− title = properties['name']

64+ title = properties.get('name')

66 # get osm-type

67 if properties.get('osm_type') == 'N':

searx/engines/startpage.py+5−8

@@ -13,6 +13,7 @@

13from lxml import html

14from cgi import escape

15import re

16+from searx.engines.xpath import extract_text

17# engine dependent config

18# engine dependent config

18categories = ['general']

19categories = ['general']

@@ -45,8 +46,7 @@ def request(query, params):

46 # set language if specified

47 # set language if specified

47 if params['language'] != 'all':

48 if params['language'] != 'all':

48− params['data']['with_language'] = ('lang_' +

49+ params['data']['with_language'] = ('lang_' + params['language'].split('_')[0])

49− params['language'].split('_')[0])

51 return params

@@ -64,18 +64,15 @@ def response(resp):

64 continue

65 link = links[0]

66 url = link.attrib.get('href')

67− try:

68− title = escape(link.text_content())

69− except UnicodeDecodeError:

70− continue

72 # block google-ad url's

68 # block google-ad url's

73 if re.match("^http(s|)://www.google.[a-z]+/aclk.*$", url):

69 if re.match("^http(s|)://www.google.[a-z]+/aclk.*$", url):

74 continue

70 continue

72+ title = escape(extract_text(link))

73+

76 if result.xpath('./p[@class="desc"]'):

74 if result.xpath('./p[@class="desc"]'):

77− content = escape(result.xpath('./p[@class="desc"]')[0]

75+ content = escape(extract_text(result.xpath('./p[@class="desc"]')))

78− .text_content())

79 else:

76 else:

80 content = ''

77 content = ''

searx/engines/subtitleseeker.py+8−7

@@ -12,6 +12,7 @@ from cgi import escape

12from urllib import quote_plus

13from lxml import html

14from searx.languages import language_codes

15+from searx.engines.xpath import extract_text

16# engine dependent config

17# engine dependent config

17categories = ['videos']

18categories = ['videos']

@@ -20,7 +21,7 @@ language = ""

21# search-url

22# search-url

22url = 'http://www.subtitleseeker.com/'

23url = 'http://www.subtitleseeker.com/'

23−search_url = url+'search/TITLES/{query}&p={pageno}'

24+search_url = url + 'search/TITLES/{query}&p={pageno}'

25# specific xpath variables

26# specific xpath variables

26results_xpath = '//div[@class="boxRows"]'

27results_xpath = '//div[@class="boxRows"]'

@@ -44,7 +45,7 @@ def response(resp):

44 if resp.search_params['language'] != 'all':

45 if resp.search_params['language'] != 'all':

45 search_lang = [lc[1]

46 search_lang = [lc[1]

46 for lc in language_codes

47 for lc in language_codes

47− if lc[0][:2] == resp.search_params['language']][0]

48+ if lc[0][:2] == resp.search_params['language'].split('_')[0]][0]

49 # parse results

50 # parse results

50 for result in dom.xpath(results_xpath):

51 for result in dom.xpath(results_xpath):

@@ -56,17 +57,17 @@ def response(resp):

56 elif search_lang:

57 elif search_lang:

57 href = href + search_lang + '/'

58 href = href + search_lang + '/'

59− title = escape(link.xpath(".//text()")[0])

60+ title = escape(extract_text(link))

61− content = result.xpath('.//div[contains(@class,"red")]//text()')[0]

62+ content = extract_text(result.xpath('.//div[contains(@class,"red")]'))

62 content = content + " - "

63 content = content + " - "

63− text = result.xpath('.//div[contains(@class,"grey-web")]')[0]

64+ text = extract_text(result.xpath('.//div[contains(@class,"grey-web")]')[0])

64− content = content + html.tostring(text, method='text')

65+ content = content + text

66 if result.xpath(".//span") != []:

67 if result.xpath(".//span") != []:

67 content = content +\

68 content = content +\

68 " - (" +\

69 " - (" +\

69− result.xpath(".//span//text()")[0].strip() +\

70+ extract_text(result.xpath(".//span")) +\

70 ")"

71 ")"

72 # append result

73 # append result

searx/engines/twitter.py+8−5

@@ -13,8 +13,8 @@

13from urlparse import urljoin

14from urllib import urlencode

15from lxml import html

16−from cgi import escape

17from datetime import datetime

16from datetime import datetime

17+from searx.engines.xpath import extract_text

19# engine dependent config

20categories = ['social media']

@@ -22,12 +22,12 @@ language_support = True

23# search-url

24base_url = 'https://twitter.com/'

25−search_url = base_url+'search?'

25+search_url = base_url + 'search?'

27# specific xpath variables

28results_xpath = '//li[@data-item-type="tweet"]'

29link_xpath = './/small[@class="time"]//a'

30−title_xpath = './/span[@class="username js-action-profile-name"]//text()'

30+title_xpath = './/span[@class="username js-action-profile-name"]'

31content_xpath = './/p[@class="js-tweet-text tweet-text"]'

32timestamp_xpath = './/span[contains(@class,"_timestamp")]'

@@ -39,6 +39,8 @@ def request(query, params):

39 # set language if specified

40 if params['language'] != 'all':

41 params['cookies']['lang'] = params['language'].split('_')[0]

42+ else:

43+ params['cookies']['lang'] = 'en'

43 return params

45 return params

@@ -53,8 +55,9 @@ def response(resp):

53 for tweet in dom.xpath(results_xpath):

55 for tweet in dom.xpath(results_xpath):

54 link = tweet.xpath(link_xpath)[0]

56 link = tweet.xpath(link_xpath)[0]

55 url = urljoin(base_url, link.attrib.get('href'))

57 url = urljoin(base_url, link.attrib.get('href'))

56− title = ''.join(tweet.xpath(title_xpath))

58+ title = extract_text(tweet.xpath(title_xpath))

57− content = escape(html.tostring(tweet.xpath(content_xpath)[0], method='text', encoding='UTF-8').decode("utf-8"))

59+ content = extract_text(tweet.xpath(content_xpath)[0])

60+

58 pubdate = tweet.xpath(timestamp_xpath)

61 pubdate = tweet.xpath(timestamp_xpath)

59 if len(pubdate) > 0:

62 if len(pubdate) > 0:

60 timestamp = float(pubdate[0].attrib.get('data-time'))

63 timestamp = float(pubdate[0].attrib.get('data-time'))

searx/engines/yacy.py+11−8

@@ -25,10 +25,10 @@ number_of_results = 5

25# search-url

26base_url = 'http://localhost:8090'

27search_url = '/yacysearch.json?{query}'\

28− '&startRecord={offset}'\

28+ '&startRecord={offset}'\

29− '&maximumRecords={limit}'\

29+ '&maximumRecords={limit}'\

30− '&contentdom={search_type}'\

30+ '&contentdom={search_type}'\

31− '&resource=global' # noqa

31+ '&resource=global'

33# yacy specific type-definitions

34search_types = {'general': 'text',

@@ -41,7 +41,7 @@ search_types = {'general': 'text',

41# do search-request

42def request(query, params):

43 offset = (params['pageno'] - 1) * number_of_results

44− search_type = search_types.get(params['category'], '0')

44+ search_type = search_types.get(params.get('category'), '0')

46 params['url'] = base_url +\

47 search_url.format(query=urlencode({'query': query}),

@@ -66,9 +66,12 @@ def response(resp):

66 if not raw_search_results:

67 return []

69− search_results = raw_search_results.get('channels', {})[0].get('items', [])

69+ search_results = raw_search_results.get('channels', [])

71+ if len(search_results) == 0:

72+ return []

73+

71− for result in search_results:

74+ for result in search_results[0].get('items', []):

72 # parse image results

75 # parse image results

73 if result.get('image'):

76 if result.get('image'):

74 # append result

77 # append result

@@ -88,7 +91,7 @@ def response(resp):

88 'content': result['description'],

91 'content': result['description'],

89 'publishedDate': publishedDate})

92 'publishedDate': publishedDate})

91− #TODO parse video, audio and file results

94+ # TODO parse video, audio and file results

93 # return results

96 # return results

94 return results

97 return results

searx/engines/yahoo.py+2−2

@@ -35,7 +35,7 @@ suggestion_xpath = '//div[@id="satat"]//a'

35def parse_url(url_string):

36 endings = ['/RS', '/RK']

37 endpositions = []

38− start = url_string.find('http', url_string.find('/RU=')+1)

38+ start = url_string.find('http', url_string.find('/RU=') + 1)

40 for ending in endings:

41 endpos = url_string.rfind(ending)

@@ -91,7 +91,7 @@ def response(resp):

91 'content': content})

93 # if no suggestion found, return results

94− if not suggestion_xpath:

94+ if not dom.xpath(suggestion_xpath):

95 return results

97 # parse suggestion

searx/tests/engines/test_currency_convert.pynew file+44

@@ -0,0 +1,44 @@

1+from collections import defaultdict

2+from datetime import datetime

3+import mock

4+from searx.engines import currency_convert

5+from searx.testing import SearxTestCase

8+class TestCurrencyConvertEngine(SearxTestCase):

10+ def test_request(self):

11+ query = 'test_query'

12+ dicto = defaultdict(dict)

13+ dicto['pageno'] = 1

14+ params = currency_convert.request(query, dicto)

15+ self.assertNotIn('url', params)

16+

17+ query = '1.1.1 EUR in USD'

18+ params = currency_convert.request(query, dicto)

19+ self.assertNotIn('url', params)

20+

21+ query = '10 eur in usd'

22+ params = currency_convert.request(query, dicto)

23+ self.assertIn('url', params)

24+ self.assertIn('finance.yahoo.com', params['url'])

25+ self.assertIn('EUR', params['url'])

26+ self.assertIn('USD', params['url'])

27+

28+ def test_response(self):

29+ dicto = defaultdict(dict)

30+ dicto['ammount'] = 10

31+ dicto['from'] = "EUR"

32+ dicto['to'] = "USD"

33+ response = mock.Mock(text='a,b,c,d', search_params=dicto)

34+ self.assertEqual(currency_convert.response(response), [])

35+

36+ csv = "2,0.5,1"

37+ response = mock.Mock(text=csv, search_params=dicto)

38+ results = currency_convert.response(response)

39+ self.assertEqual(type(results), list)

40+ self.assertEqual(len(results), 1)

41+ self.assertEqual(results[0]['answer'], '10 EUR = 5.0 USD (1 EUR = 0.5 USD)')

42+ now_date = datetime.now().strftime('%Y%m%d')

43+ self.assertEqual(results[0]['url'], 'http://finance.yahoo.com/currency/converter-results/' +

44+ now_date + '/10-eur-to-usd.html')

searx/tests/engines/test_duckduckgo.pynew file+90

@@ -0,0 +1,90 @@

1+from collections import defaultdict

2+import mock

3+from searx.engines import duckduckgo

4+from searx.testing import SearxTestCase

7+class TestDuckduckgoEngine(SearxTestCase):

9+ def test_request(self):

10+ query = 'test_query'

11+ dicto = defaultdict(dict)

12+ dicto['pageno'] = 1

13+ dicto['language'] = 'fr_FR'

14+ params = duckduckgo.request(query, dicto)

15+ self.assertIn('url', params)

16+ self.assertIn(query, params['url'])

17+ self.assertIn('duckduckgo.com', params['url'])

18+ self.assertIn('fr-fr', params['url'])

19+

20+ dicto['language'] = 'all'

21+ params = duckduckgo.request(query, dicto)

22+ self.assertIn('en-us', params['url'])

23+

24+ def test_response(self):

25+ self.assertRaises(AttributeError, duckduckgo.response, None)

26+ self.assertRaises(AttributeError, duckduckgo.response, [])

27+ self.assertRaises(AttributeError, duckduckgo.response, '')

28+ self.assertRaises(AttributeError, duckduckgo.response, '[]')

29+

30+ response = mock.Mock(text='<html></html>')

31+ self.assertEqual(duckduckgo.response(response), [])

32+

33+ html = """

34+ <div class="results_links results_links_deep web-result">

35+ <div class="icon_fav" style="display: block;">

36+ <a rel="nofollow" href="https://www.test.com/">

37+ <img width="16" height="16" alt=""

38+ src="/i/www.test.com.ico" style="visibility: visible;" name="i15" />

39+ </a>

40+ </div>

41+ <div class="links_main links_deep">

42+ <a rel="nofollow" class="large" href="http://this.should.be.the.link/">

43+ This <b>is</b> <b>the</b> title

44+ </a>

45+ <div class="snippet"><b>This</b> should be the content.</div>

46+ <div class="url">

47+ http://this.should.be.the.link/

48+ </div>

49+ </div>

50+ </div>

51+ """

52+ response = mock.Mock(text=html)

53+ results = duckduckgo.response(response)

54+ self.assertEqual(type(results), list)

55+ self.assertEqual(len(results), 1)

56+ self.assertEqual(results[0]['title'], 'This is the title')

57+ self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')

58+ self.assertEqual(results[0]['content'], 'This should be the content.')

59+

60+ html = """

61+ <div class="results_links results_links_deep web-result">

62+ <div class="icon_fav" style="display: block;">

63+ </div>

64+ <div class="links_main links_deep">

65+ <div class="snippet"><b>This</b> should be the content.</div>

66+ <div class="url">

67+ http://this.should.be.the.link/

68+ </div>

69+ </div>

70+ </div>

71+ <div class="results_links results_links_deep web-result">

72+ <div class="icon_fav" style="display: block;">

73+ <img width="16" height="16" alt=""

74+ src="/i/www.test.com.ico" style="visibility: visible;" name="i15" />

75+ </div>

76+ <div class="links_main links_deep">

77+ <a rel="nofollow" class="large" href="">

78+ This <b>is</b> <b>the</b> title

79+ </a>

80+ <div class="snippet"><b>This</b> should be the content.</div>

81+ <div class="url">

82+ http://this.should.be.the.link/

83+ </div>

84+ </div>

85+ </div>

86+ """

87+ response = mock.Mock(text=html)

88+ results = duckduckgo.response(response)

89+ self.assertEqual(type(results), list)

90+ self.assertEqual(len(results), 0)

searx/tests/engines/test_duckduckgo_definitions.pynew file+250

@@ -0,0 +1,250 @@

1+from collections import defaultdict

2+import mock

3+from searx.engines import duckduckgo_definitions

4+from searx.testing import SearxTestCase

7+class TestDDGDefinitionsEngine(SearxTestCase):

9+ def test_result_to_text(self):

10+ url = ''

11+ text = 'Text'

12+ html_result = 'Html'

13+ result = duckduckgo_definitions.result_to_text(url, text, html_result)

14+ self.assertEqual(result, text)

15+

16+ html_result = '<a href="url">Text in link</a>'

17+ result = duckduckgo_definitions.result_to_text(url, text, html_result)

18+ self.assertEqual(result, 'Text in link')

19+

20+ def test_request(self):

21+ query = 'test_query'

22+ dicto = defaultdict(dict)

23+ dicto['pageno'] = 1

24+ params = duckduckgo_definitions.request(query, dicto)

25+ self.assertIn('url', params)

26+ self.assertIn(query, params['url'])

27+ self.assertIn('duckduckgo.com', params['url'])

28+

29+ def test_response(self):

30+ self.assertRaises(AttributeError, duckduckgo_definitions.response, None)

31+ self.assertRaises(AttributeError, duckduckgo_definitions.response, [])

32+ self.assertRaises(AttributeError, duckduckgo_definitions.response, '')

33+ self.assertRaises(AttributeError, duckduckgo_definitions.response, '[]')

34+

35+ response = mock.Mock(text='{}')

36+ self.assertEqual(duckduckgo_definitions.response(response), [])

37+

38+ response = mock.Mock(text='{"data": []}')

39+ self.assertEqual(duckduckgo_definitions.response(response), [])

40+

41+ json = """

42+ {

43+ "DefinitionSource": "definition source",

44+ "Heading": "heading",

45+ "ImageWidth": 0,

46+ "RelatedTopics": [

47+ {

48+ "Result": "Top-level domains",

49+ "Icon": {

50+ "URL": "",

51+ "Height": "",

52+ "Width": ""

53+ },

54+ "FirstURL": "https://first.url",

55+ "Text": "text"

56+ },

57+ {

58+ "Topics": [

59+ {

60+ "Result": "result topic",

61+ "Icon": {

62+ "URL": "",

63+ "Height": "",

64+ "Width": ""

65+ },

66+ "FirstURL": "https://duckduckgo.com/?q=2%2F2",

67+ "Text": "result topic text"

68+ }

69+ ],

70+ "Name": "name"

71+ }

72+ ],

73+ "Entity": "Entity",

74+ "Type": "A",

75+ "Redirect": "",

76+ "DefinitionURL": "http://definition.url",

77+ "AbstractURL": "https://abstract.url",

78+ "Definition": "this is the definition",

79+ "AbstractSource": "abstract source",

80+ "Infobox": {

81+ "content": [

82+ {

83+ "data_type": "string",

84+ "value": "1999",

85+ "label": "Introduced",

86+ "wiki_order": 0

87+ }

88+ ],

89+ "meta": [

90+ {

91+ "data_type": "string",

92+ "value": ".test",

93+ "label": "article_title"

94+ }

95+ ]

96+ },

97+ "Image": "image.png",

98+ "ImageIsLogo": 0,

99+ "Abstract": "abstract",

100+ "AbstractText": "abstract text",

101+ "AnswerType": "",

102+ "ImageHeight": 0,

103+ "Results": [{

104+ "Result" : "result title",

105+ "Icon" : {

106+ "URL" : "result url",

107+ "Height" : 16,

108+ "Width" : 16

109+ },

110+ "FirstURL" : "result first url",

111+ "Text" : "result text"

112+ }

113+ ],

114+ "Answer": "answer"

115+ }

116+ """

117+ response = mock.Mock(text=json)

118+ results = duckduckgo_definitions.response(response)

119+ self.assertEqual(type(results), list)

120+ self.assertEqual(len(results), 4)

121+ self.assertEqual(results[0]['answer'], 'answer')

122+ self.assertEqual(results[1]['title'], 'heading')

123+ self.assertEqual(results[1]['url'], 'result first url')

124+ self.assertEqual(results[2]['suggestion'], 'text')

125+ self.assertEqual(results[3]['infobox'], 'heading')

126+ self.assertEqual(results[3]['id'], 'http://definition.url')

127+ self.assertEqual(results[3]['entity'], 'Entity')

128+ self.assertIn('abstract', results[3]['content'])

129+ self.assertIn('this is the definition', results[3]['content'])

130+ self.assertEqual(results[3]['img_src'], 'image.png')

131+ self.assertIn('Introduced', results[3]['attributes'][0]['label'])

132+ self.assertIn('1999', results[3]['attributes'][0]['value'])

133+ self.assertIn({'url': 'https://abstract.url', 'title': 'abstract source'}, results[3]['urls'])

134+ self.assertIn({'url': 'http://definition.url', 'title': 'definition source'}, results[3]['urls'])

135+ self.assertIn({'name': 'name', 'suggestions': ['result topic text']}, results[3]['relatedTopics'])

136+

137+ json = """

138+ {

139+ "DefinitionSource": "definition source",

140+ "Heading": "heading",

141+ "ImageWidth": 0,

142+ "RelatedTopics": [],

143+ "Entity": "Entity",

144+ "Type": "A",

145+ "Redirect": "",

146+ "DefinitionURL": "",

147+ "AbstractURL": "https://abstract.url",

148+ "Definition": "",

149+ "AbstractSource": "abstract source",

150+ "Image": "",

151+ "ImageIsLogo": 0,

152+ "Abstract": "",

153+ "AbstractText": "abstract text",

154+ "AnswerType": "",

155+ "ImageHeight": 0,

156+ "Results": [],

157+ "Answer": ""

158+ }

159+ """

160+ response = mock.Mock(text=json)

161+ results = duckduckgo_definitions.response(response)

162+ self.assertEqual(type(results), list)

163+ self.assertEqual(len(results), 1)

164+ self.assertEqual(results[0]['url'], 'https://abstract.url')

165+ self.assertEqual(results[0]['title'], 'heading')

166+ self.assertEqual(results[0]['content'], '')

167+

168+ json = """

169+ {

170+ "DefinitionSource": "definition source",

171+ "Heading": "heading",

172+ "ImageWidth": 0,

173+ "RelatedTopics": [

174+ {

175+ "Result": "Top-level domains",

176+ "Icon": {

177+ "URL": "",

178+ "Height": "",

179+ "Width": ""

180+ },

181+ "FirstURL": "https://first.url",

182+ "Text": "heading"

183+ },

184+ {

185+ "Name": "name"

186+ },

187+ {

188+ "Topics": [

189+ {

190+ "Result": "result topic",

191+ "Icon": {

192+ "URL": "",

193+ "Height": "",

194+ "Width": ""

195+ },

196+ "FirstURL": "https://duckduckgo.com/?q=2%2F2",

197+ "Text": "heading"

198+ }

199+ ],

200+ "Name": "name"

201+ }

202+ ],

203+ "Entity": "Entity",

204+ "Type": "A",

205+ "Redirect": "",

206+ "DefinitionURL": "http://definition.url",

207+ "AbstractURL": "https://abstract.url",

208+ "Definition": "this is the definition",

209+ "AbstractSource": "abstract source",

210+ "Infobox": {

211+ "meta": [

212+ {

213+ "data_type": "string",

214+ "value": ".test",

215+ "label": "article_title"

216+ }

217+ ]

218+ },

219+ "Image": "image.png",

220+ "ImageIsLogo": 0,

221+ "Abstract": "abstract",

222+ "AbstractText": "abstract text",

223+ "AnswerType": "",

224+ "ImageHeight": 0,

225+ "Results": [{

226+ "Result" : "result title",

227+ "Icon" : {

228+ "URL" : "result url",

229+ "Height" : 16,

230+ "Width" : 16

231+ },

232+ "Text" : "result text"

233+ }

234+ ],

235+ "Answer": ""

236+ }

237+ """

238+ response = mock.Mock(text=json)

239+ results = duckduckgo_definitions.response(response)

240+ self.assertEqual(type(results), list)

241+ self.assertEqual(len(results), 1)

242+ self.assertEqual(results[0]['infobox'], 'heading')

243+ self.assertEqual(results[0]['id'], 'http://definition.url')

244+ self.assertEqual(results[0]['entity'], 'Entity')

245+ self.assertIn('abstract', results[0]['content'])

246+ self.assertIn('this is the definition', results[0]['content'])

247+ self.assertEqual(results[0]['img_src'], 'image.png')

248+ self.assertIn({'url': 'https://abstract.url', 'title': 'abstract source'}, results[0]['urls'])

249+ self.assertIn({'url': 'http://definition.url', 'title': 'definition source'}, results[0]['urls'])

250+ self.assertIn({'name': 'name', 'suggestions': []}, results[0]['relatedTopics'])

searx/tests/engines/test_faroo.pynew file+116

@@ -0,0 +1,116 @@

1+# -*- coding: utf-8 -*-

2+from collections import defaultdict

3+import mock

4+from searx.engines import faroo

5+from searx.testing import SearxTestCase

8+class TestFarooEngine(SearxTestCase):

10+ def test_request(self):

11+ query = 'test_query'

12+ dicto = defaultdict(dict)

13+ dicto['pageno'] = 1

14+ dicto['language'] = 'fr_FR'

15+ dicto['category'] = 'general'

16+ params = faroo.request(query, dicto)

17+ self.assertIn('url', params)

18+ self.assertIn(query, params['url'])

19+ self.assertIn('faroo.com', params['url'])

20+ self.assertIn('en', params['url'])

21+ self.assertIn('web', params['url'])

22+

23+ dicto['language'] = 'all'

24+ params = faroo.request(query, dicto)

25+ self.assertIn('en', params['url'])

26+

27+ dicto['language'] = 'de_DE'

28+ params = faroo.request(query, dicto)

29+ self.assertIn('de', params['url'])

30+

31+ def test_response(self):

32+ self.assertRaises(AttributeError, faroo.response, None)

33+ self.assertRaises(AttributeError, faroo.response, [])

34+ self.assertRaises(AttributeError, faroo.response, '')

35+ self.assertRaises(AttributeError, faroo.response, '[]')

36+

37+ response = mock.Mock(text='{}')

38+ self.assertEqual(faroo.response(response), [])

39+

40+ response = mock.Mock(text='{"data": []}')

41+ self.assertEqual(faroo.response(response), [])

42+

43+ response = mock.Mock(text='{"data": []}', status_code=401)

44+ self.assertRaises(Exception, faroo.response, response)

45+

46+ response = mock.Mock(text='{"data": []}', status_code=429)

47+ self.assertRaises(Exception, faroo.response, response)

48+

49+ json = """

50+ {

51+ "results": [

52+ {

53+ "title": "This is the title",

54+ "kwic": "This is the content",

55+ "content": "",

56+ "url": "http://this.is.the.url/",

57+ "iurl": "",

58+ "domain": "css3test.com",

59+ "author": "Jim Dalrymple",

60+ "news": true,

61+ "votes": "10",

62+ "date": 1360622563000,

63+ "related": []

64+ },

65+ {

66+ "title": "This is the title2",

67+ "kwic": "This is the content2",

68+ "content": "",

69+ "url": "http://this.is.the.url2/",

70+ "iurl": "",

71+ "domain": "css3test.com",

72+ "author": "Jim Dalrymple",

73+ "news": false,

74+ "votes": "10",

75+ "related": []

76+ },

77+ {

78+ "title": "This is the title3",

79+ "kwic": "This is the content3",

80+ "content": "",

81+ "url": "http://this.is.the.url3/",

82+ "iurl": "http://upload.wikimedia.org/optimized.jpg",

83+ "domain": "css3test.com",

84+ "author": "Jim Dalrymple",

85+ "news": false,

86+ "votes": "10",

87+ "related": []

88+ }

89+ ],

90+ "query": "test",

91+ "suggestions": [],

92+ "count": 100,

93+ "start": 1,

94+ "length": 10,

95+ "time": "15"

96+ }

97+ """

98+ response = mock.Mock(text=json)

99+ results = faroo.response(response)

100+ self.assertEqual(type(results), list)

101+ self.assertEqual(len(results), 4)

102+ self.assertEqual(results[0]['title'], 'This is the title')

103+ self.assertEqual(results[0]['url'], 'http://this.is.the.url/')

104+ self.assertEqual(results[0]['content'], 'This is the content')

105+ self.assertEqual(results[1]['title'], 'This is the title2')

106+ self.assertEqual(results[1]['url'], 'http://this.is.the.url2/')

107+ self.assertEqual(results[1]['content'], 'This is the content2')

108+ self.assertEqual(results[3]['img_src'], 'http://upload.wikimedia.org/optimized.jpg')

109+

110+ json = """

111+ {}

112+ """

113+ response = mock.Mock(text=json)

114+ results = faroo.response(response)

115+ self.assertEqual(type(results), list)

116+ self.assertEqual(len(results), 0)

searx/tests/engines/test_google.pynew file+162

@@ -0,0 +1,162 @@

1+# -*- coding: utf-8 -*-

2+from collections import defaultdict

3+import mock

4+import lxml

5+from searx.engines import google

6+from searx.testing import SearxTestCase

9+class TestGoogleEngine(SearxTestCase):

10+

11+ def test_request(self):

12+ query = 'test_query'

13+ dicto = defaultdict(dict)

14+ dicto['pageno'] = 1

15+ dicto['language'] = 'fr_FR'

16+ params = google.request(query, dicto)

17+ self.assertIn('url', params)

18+ self.assertIn(query, params['url'])

19+ self.assertIn('google.com', params['url'])

20+ self.assertIn('PREF', params['cookies'])

21+ self.assertIn('fr', params['headers']['Accept-Language'])

22+

23+ dicto['language'] = 'all'

24+ params = google.request(query, dicto)

25+ self.assertIn('en', params['headers']['Accept-Language'])

26+

27+ def test_response(self):

28+ self.assertRaises(AttributeError, google.response, None)

29+ self.assertRaises(AttributeError, google.response, [])

30+ self.assertRaises(AttributeError, google.response, '')

31+ self.assertRaises(AttributeError, google.response, '[]')

32+

33+ response = mock.Mock(text='<html></html>')

34+ self.assertEqual(google.response(response), [])

35+

36+ html = """

37+ <li class="g">

38+ <h3 class="r">

39+ <a href="http://this.should.be.the.link/">

40+ <b>This</b> is <b>the</b> title

41+ </a>

42+ </h3>

43+ <div class="s">

44+ <div class="kv" style="margin-bottom:2px">

45+ <cite>

46+ <b>test</b>.psychologies.com/

47+ </cite>

48+ <div class="_nBb"><U+200E>

49+ <div style="display:inline" onclick="google.sham(this);" aria-expanded="false"

50+ aria-haspopup="true" tabindex="0" data-ved="0CBUQ7B0wAA">

51+ <span class="_O0">

52+ </span>

53+ </div>

54+ <div style="display:none" class="am-dropdown-menu" role="menu" tabindex="-1">

55+ <ul>

56+ <li class="_Ykb">

57+ <a class="_Zkb" href="http://www.google.fr/url?url=http://webcache.googleusercontent

58+ .com/search%3Fcache:R1Z_4pGXjuIJ:http://test.psychologies.com/">

59+ En cache

60+ </a>

61+ </li>

62+ <li class="_Ykb">

63+ <a class="_Zkb" href="/search?safe=off&q=related:test.psy.com/">

64+ Pages similaires

65+ </a>

66+ </li>

67+ </ul>

68+ </div>

69+ </div>

70+ </div>

71+ <span class="st">

72+ This should be the content.

73+ </span>

74+ <br>

75+ <div class="osl"><U+200E>

76+ <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/">

77+ Test Personnalité

78+ </a> - <U+200E>

79+ <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/">

80+ Tests - Moi

81+ </a> - <U+200E>

82+ <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/tests-couple">

83+ Test Couple

84+ </a>

85+ - <U+200E>

86+ <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/tests-amour">

87+ Test Amour

88+ </a>

89+ </div>

90+ </div>

91+ </li>

92+ <li class="g">

93+ <h3 class="r">

94+ <a href="http://www.google.com/images?q=toto">

95+ <b>This</b>

96+ </a>

97+ </h3>

98+ </li>

99+ <li class="g">

100+ <h3 class="r">

101+ <a href="http://www.google.com/search?q=toto">

102+ <b>This</b> is

103+ </a>

104+ </h3>

105+ </li>

106+ <li class="g">

107+ <h3 class="r">

108+ <a href="€">

109+ <b>This</b> is <b>the</b>

110+ </a>

111+ </h3>

112+ </li>

113+ <li class="g">

114+ <h3 class="r">

115+ <a href="/url?q=url">

116+ <b>This</b> is <b>the</b>

117+ </a>

118+ </h3>

119+ </li>

120+ <p class="_Bmc" style="margin:3px 8px">

121+ <a href="/search?num=20&safe=off&q=t&revid=1754833769&sa=X&ei=-&ved=">

122+ suggestion <b>title</b>

123+ </a>

124+ </p>

125+ """

126+ response = mock.Mock(text=html)

127+ results = google.response(response)

128+ self.assertEqual(type(results), list)

129+ self.assertEqual(len(results), 2)

130+ self.assertEqual(results[0]['title'], 'This is the title')

131+ self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')

132+ self.assertEqual(results[0]['content'], 'This should be the content.')

133+ self.assertEqual(results[1]['suggestion'], 'suggestion title')

134+

135+ html = """

136+ <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">

137+ </li>

138+ """

139+ response = mock.Mock(text=html)

140+ results = google.response(response)

141+ self.assertEqual(type(results), list)

142+ self.assertEqual(len(results), 0)

143+

144+ def test_parse_images(self):

145+ html = """

146+ <li>

147+ <div>

148+ <a href="http://www.google.com/url?q=http://this.is.the.url/">

149+ <img style="margin:3px 0;margin-right:6px;padding:0" height="90"

150+ src="https://this.is.the.image/image.jpg" width="60" align="middle" alt="" border="0">

151+ </a>

152+ </div>

153+ </li>

154+ """

155+ dom = lxml.html.fromstring(html)

156+ results = google.parse_images(dom)

157+ self.assertEqual(type(results), list)

158+ self.assertEqual(len(results), 1)

159+ self.assertEqual(results[0]['url'], 'http://this.is.the.url/')

160+ self.assertEqual(results[0]['title'], '')

161+ self.assertEqual(results[0]['content'], '')

162+ self.assertEqual(results[0]['img_src'], 'https://this.is.the.image/image.jpg')

searx/tests/engines/test_mediawiki.pynew file+130

@@ -0,0 +1,130 @@

1+# -*- coding: utf-8 -*-

2+from collections import defaultdict

3+import mock

4+from searx.engines import mediawiki

5+from searx.testing import SearxTestCase

8+class TestMediawikiEngine(SearxTestCase):

10+ def test_request(self):

11+ query = 'test_query'

12+ dicto = defaultdict(dict)

13+ dicto['pageno'] = 1

14+ dicto['language'] = 'fr_FR'

15+ params = mediawiki.request(query, dicto)

16+ self.assertIn('url', params)

17+ self.assertIn(query, params['url'])

18+ self.assertIn('wikipedia.org', params['url'])

19+ self.assertIn('fr', params['url'])

20+

21+ dicto['language'] = 'all'

22+ params = mediawiki.request(query, dicto)

23+ self.assertIn('en', params['url'])

24+

25+ mediawiki.base_url = "http://test.url/"

26+ mediawiki.search_url = mediawiki.base_url +\

27+ 'w/api.php?action=query'\

28+ '&list=search'\

29+ '&{query}'\

30+ '&srprop=timestamp'\

31+ '&format=json'\

32+ '&sroffset={offset}'\

33+ '&srlimit={limit}' # noqa

34+ params = mediawiki.request(query, dicto)

35+ self.assertIn('test.url', params['url'])

36+

37+ def test_response(self):

38+ dicto = defaultdict(dict)

39+ dicto['language'] = 'fr'

40+ mediawiki.base_url = "https://{language}.wikipedia.org/"

41+

42+ self.assertRaises(AttributeError, mediawiki.response, None)

43+ self.assertRaises(AttributeError, mediawiki.response, [])

44+ self.assertRaises(AttributeError, mediawiki.response, '')

45+ self.assertRaises(AttributeError, mediawiki.response, '[]')

46+

47+ response = mock.Mock(text='{}', search_params=dicto)

48+ self.assertEqual(mediawiki.response(response), [])

49+

50+ response = mock.Mock(text='{"data": []}', search_params=dicto)

51+ self.assertEqual(mediawiki.response(response), [])

52+

53+ json = """

54+ {

55+ "query-continue": {

56+ "search": {

57+ "sroffset": 1

58+ }

59+ },

60+ "query": {

61+ "searchinfo": {

62+ "totalhits": 29721

63+ },

64+ "search": [

65+ {

66+ "ns": 0,

67+ "title": "This is the title étude",

68+ "timestamp": "2014-12-19T17:42:52Z"

69+ }

70+ ]

71+ }

72+ }

73+ """

74+ response = mock.Mock(text=json, search_params=dicto)

75+ results = mediawiki.response(response)

76+ self.assertEqual(type(results), list)

77+ self.assertEqual(len(results), 1)

78+ self.assertEqual(results[0]['title'], u'This is the title étude')

79+ self.assertIn('fr.wikipedia.org', results[0]['url'])

80+ self.assertIn('This_is_the_title', results[0]['url'])

81+ self.assertIn('%C3%A9tude', results[0]['url'])

82+ self.assertEqual(results[0]['content'], '')

83+

84+ json = """

85+ {

86+ "query-continue": {

87+ "search": {

88+ "sroffset": 1

89+ }

90+ },

91+ "query": {

92+ "searchinfo": {

93+ "totalhits": 29721

94+ },

95+ "search": [

96+ ]

97+ }

98+ }

99+ """

100+ response = mock.Mock(text=json, search_params=dicto)

101+ results = mediawiki.response(response)

102+ self.assertEqual(type(results), list)

103+ self.assertEqual(len(results), 0)

104+

105+ json = """

106+ {

107+ "query-continue": {

108+ "search": {

109+ "sroffset": 1

110+ }

111+ },

112+ "query": {

113+ }

114+ }

115+ """

116+ response = mock.Mock(text=json, search_params=dicto)

117+ results = mediawiki.response(response)

118+ self.assertEqual(type(results), list)

119+ self.assertEqual(len(results), 0)

120+

121+ json = """

122+ {"toto":[

123+ {"id":200,"name":"Artist Name",

124+ "link":"http:\/\/www.mediawiki.com\/artist\/1217","type":"artist"}

125+ ]}

126+ """

127+ response = mock.Mock(text=json, search_params=dicto)

128+ results = mediawiki.response(response)

129+ self.assertEqual(type(results), list)

130+ self.assertEqual(len(results), 0)

searx/tests/engines/test_openstreetmap.pynew file+199

@@ -0,0 +1,199 @@

1+# -*- coding: utf-8 -*-

2+from collections import defaultdict

3+import mock

4+from searx.engines import openstreetmap

5+from searx.testing import SearxTestCase

8+class TestOpenstreetmapEngine(SearxTestCase):

10+ def test_request(self):

11+ query = 'test_query'

12+ dicto = defaultdict(dict)

13+ dicto['pageno'] = 1

14+ params = openstreetmap.request(query, dicto)

15+ self.assertIn('url', params)

16+ self.assertIn(query, params['url'])

17+ self.assertIn('openstreetmap.org', params['url'])

18+

19+ def test_response(self):

20+ self.assertRaises(AttributeError, openstreetmap.response, None)

21+ self.assertRaises(AttributeError, openstreetmap.response, [])

22+ self.assertRaises(AttributeError, openstreetmap.response, '')

23+ self.assertRaises(AttributeError, openstreetmap.response, '[]')

24+

25+ response = mock.Mock(text='{}')

26+ self.assertEqual(openstreetmap.response(response), [])

27+

28+ response = mock.Mock(text='{"data": []}')

29+ self.assertEqual(openstreetmap.response(response), [])

30+

31+ json = """

32+ [

33+ {

34+ "place_id": "127732055",

35+ "licence": "Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright",

36+ "osm_type": "relation",

37+ "osm_id": "7444",

38+ "boundingbox": [

39+ "48.8155755",

40+ "48.902156",

41+ "2.224122",

42+ "2.4697602"

43+ ],

44+ "lat": "48.8565056",

45+ "lon": "2.3521334",

46+ "display_name": "This is the title",

47+ "class": "place",

48+ "type": "city",

49+ "importance": 0.96893459932191,

50+ "icon": "https://nominatim.openstreetmap.org/images/mapicons/poi_place_city.p.20.png",

51+ "address": {

52+ "city": "Paris",

53+ "county": "Paris",

54+ "state": "Île-de-France",

55+ "country": "France",

56+ "country_code": "fr"

57+ },

58+ "geojson": {

59+ "type": "Polygon",

60+ "coordinates": [

61+ [

62+ [

63+ 2.224122,

64+ 48.854199

65+ ]

66+ ]

67+ ]

68+ }

69+ }

70+ ]

71+ """

72+ response = mock.Mock(text=json)

73+ results = openstreetmap.response(response)

74+ self.assertEqual(type(results), list)

75+ self.assertEqual(len(results), 1)

76+ self.assertEqual(results[0]['title'], 'This is the title')

77+ self.assertEqual(results[0]['url'], 'https://openstreetmap.org/relation/7444')

78+ self.assertIn('coordinates', results[0]['geojson'])

79+ self.assertEqual(results[0]['geojson']['coordinates'][0][0][0], 2.224122)

80+ self.assertEqual(results[0]['geojson']['coordinates'][0][0][1], 48.854199)

81+ self.assertEqual(results[0]['address'], None)

82+ self.assertIn('48.8155755', results[0]['boundingbox'])

83+ self.assertIn('48.902156', results[0]['boundingbox'])

84+ self.assertIn('2.224122', results[0]['boundingbox'])

85+ self.assertIn('2.4697602', results[0]['boundingbox'])

86+

87+ json = """

88+ [

89+ {

90+ "place_id": "127732055",

91+ "licence": "Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright",

92+ "osm_type": "relation",

93+ "osm_id": "7444",

94+ "boundingbox": [

95+ "48.8155755",

96+ "48.902156",

97+ "2.224122",

98+ "2.4697602"

99+ ],

100+ "lat": "48.8565056",

101+ "lon": "2.3521334",

102+ "display_name": "This is the title",

103+ "class": "tourism",

104+ "type": "city",

105+ "importance": 0.96893459932191,

106+ "icon": "https://nominatim.openstreetmap.org/images/mapicons/poi_place_city.p.20.png",

107+ "address": {

108+ "city": "Paris",

109+ "county": "Paris",

110+ "state": "Île-de-France",

111+ "country": "France",

112+ "country_code": "fr",

113+ "address29": "Address"

114+ },

115+ "geojson": {

116+ "type": "Polygon",

117+ "coordinates": [

118+ [

119+ [

120+ 2.224122,

121+ 48.854199

122+ ]

123+ ]

124+ ]

125+ }

126+ },

127+ {

128+ "place_id": "127732055",

129+ "licence": "Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright",

130+ "osm_type": "relation",

131+ "osm_id": "7444",

132+ "boundingbox": [

133+ "48.8155755",

134+ "48.902156",

135+ "2.224122",

136+ "2.4697602"

137+ ],

138+ "lat": "48.8565056",

139+ "lon": "2.3521334",

140+ "display_name": "This is the title",

141+ "class": "tourism",

142+ "type": "city",

143+ "importance": 0.96893459932191,

144+ "icon": "https://nominatim.openstreetmap.org/images/mapicons/poi_place_city.p.20.png",

145+ "address": {

146+ "city": "Paris",

147+ "county": "Paris",

148+ "state": "Île-de-France",

149+ "country": "France",

150+ "postcode": 75000,

151+ "country_code": "fr"

152+ },

153+ "geojson": {

154+ "type": "Polygon",

155+ "coordinates": [

156+ [

157+ [

158+ 2.224122,

159+ 48.854199

160+ ]

161+ ]

162+ ]

163+ }

164+ },

165+ {

166+ "place_id": "127732055",

167+ "licence": "Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright",

168+ "osm_type": "node",

169+ "osm_id": "7444",

170+ "boundingbox": [

171+ "48.8155755",

172+ "48.902156",

173+ "2.224122",

174+ "2.4697602"

175+ ],

176+ "lat": "48.8565056",

177+ "lon": "2.3521334",

178+ "display_name": "This is the title",

179+ "class": "tourism",

180+ "type": "city",

181+ "importance": 0.96893459932191,

182+ "icon": "https://nominatim.openstreetmap.org/images/mapicons/poi_place_city.p.20.png",

183+ "address": {

184+ "city": "Paris",

185+ "county": "Paris",

186+ "state": "Île-de-France",

187+ "country": "France",

188+ "country_code": "fr",

189+ "address29": "Address"

190+ }

191+ }

192+ ]

193+ """

194+ response = mock.Mock(text=json)

195+ results = openstreetmap.response(response)

196+ self.assertEqual(type(results), list)

197+ self.assertEqual(len(results), 3)

198+ self.assertIn('48.8565056', results[2]['geojson']['coordinates'])

199+ self.assertIn('2.3521334', results[2]['geojson']['coordinates'])

searx/tests/engines/test_photon.pynew file+166

@@ -0,0 +1,166 @@

1+# -*- coding: utf-8 -*-

2+from collections import defaultdict

3+import mock

4+from searx.engines import photon

5+from searx.testing import SearxTestCase

8+class TestPhotonEngine(SearxTestCase):

10+ def test_request(self):

11+ query = 'test_query'

12+ dicto = defaultdict(dict)

13+ dicto['pageno'] = 1

14+ dicto['language'] = 'all'

15+ params = photon.request(query, dicto)

16+ self.assertIn('url', params)

17+ self.assertIn(query, params['url'])

18+ self.assertIn('photon.komoot.de', params['url'])

19+

20+ dicto['language'] = 'all'

21+ params = photon.request(query, dicto)

22+ self.assertNotIn('lang', params['url'])

23+

24+ dicto['language'] = 'al'

25+ params = photon.request(query, dicto)

26+ self.assertNotIn('lang', params['url'])

27+

28+ dicto['language'] = 'fr'

29+ params = photon.request(query, dicto)

30+ self.assertIn('fr', params['url'])

31+

32+ def test_response(self):

33+ self.assertRaises(AttributeError, photon.response, None)

34+ self.assertRaises(AttributeError, photon.response, [])

35+ self.assertRaises(AttributeError, photon.response, '')

36+ self.assertRaises(AttributeError, photon.response, '[]')

37+

38+ response = mock.Mock(text='{}')

39+ self.assertEqual(photon.response(response), [])

40+

41+ response = mock.Mock(text='{"data": []}')

42+ self.assertEqual(photon.response(response), [])

43+

44+ json = """

45+ {

46+ "features": [

47+ {

48+ "properties": {

49+ "osm_key": "waterway",

50+ "extent": [

51+ -1.4508446,

52+ 51.1614997,

53+ -1.4408036,

54+ 51.1525635

55+ ],

56+ "name": "This is the title",

57+ "state": "England",

58+ "osm_id": 114823817,

59+ "osm_type": "W",

60+ "osm_value": "river",

61+ "city": "Test Valley",

62+ "country": "United Kingdom"

63+ },

64+ "type": "Feature",

65+ "geometry": {

66+ "type": "Point",

67+ "coordinates": [

68+ -1.4458571,

69+ 51.1576661

70+ ]

71+ }

72+ },

73+ {

74+ "properties": {

75+ "osm_key": "place",

76+ "street": "Rue",

77+ "state": "Ile-de-France",

78+ "osm_id": 129211377,

79+ "osm_type": "R",

80+ "housenumber": "10",

81+ "postcode": "75011",

82+ "osm_value": "house",

83+ "city": "Paris",

84+ "country": "France"

85+ },

86+ "type": "Feature",

87+ "geometry": {

88+ "type": "Point",

89+ "coordinates": [

90+ 2.3725025,

91+ 48.8654481

92+ ]

93+ }

94+ },

95+ {

96+ "properties": {

97+ "osm_key": "amenity",

98+ "street": "Allée",

99+ "name": "Bibliothèque",

100+ "state": "Ile-de-France",

101+ "osm_id": 1028573132,

102+ "osm_type": "N",

103+ "postcode": "75001",

104+ "osm_value": "library",

105+ "city": "Paris",

106+ "country": "France"

107+ },

108+ "type": "Feature",

109+ "geometry": {

110+ "type": "Point",

111+ "coordinates": [

112+ 2.3445634,

113+ 48.862494

114+ ]

115+ }

116+ },

117+ {

118+ "properties": {

119+ "osm_key": "amenity",

120+ "osm_id": 1028573132,

121+ "osm_type": "Y",

122+ "postcode": "75001",

123+ "osm_value": "library",

124+ "city": "Paris",

125+ "country": "France"

126+ },

127+ "type": "Feature",

128+ "geometry": {

129+ "type": "Point",

130+ "coordinates": [

131+ 2.3445634,

132+ 48.862494

133+ ]

134+ }

135+ },

136+ {

137+ }

138+ ],

139+ "type": "FeatureCollection"

140+ }

141+ """

142+ response = mock.Mock(text=json)

143+ results = photon.response(response)

144+ self.assertEqual(type(results), list)

145+ self.assertEqual(len(results), 3)

146+ self.assertEqual(results[0]['title'], 'This is the title')

147+ self.assertEqual(results[0]['content'], '')

148+ self.assertEqual(results[0]['longitude'], -1.4458571)

149+ self.assertEqual(results[0]['latitude'], 51.1576661)

150+ self.assertIn(-1.4508446, results[0]['boundingbox'])

151+ self.assertIn(51.1614997, results[0]['boundingbox'])

152+ self.assertIn(-1.4408036, results[0]['boundingbox'])

153+ self.assertIn(51.1525635, results[0]['boundingbox'])

154+ self.assertIn('type', results[0]['geojson'])

155+ self.assertEqual(results[0]['geojson']['type'], 'Point')

156+ self.assertEqual(results[0]['address'], None)

157+ self.assertEqual(results[0]['osm']['type'], 'way')

158+ self.assertEqual(results[0]['osm']['id'], 114823817)

159+ self.assertEqual(results[0]['url'], 'https://openstreetmap.org/way/114823817')

160+ self.assertEqual(results[1]['osm']['type'], 'relation')

161+ self.assertEqual(results[2]['address']['name'], u'Bibliothèque')

162+ self.assertEqual(results[2]['address']['house_number'], None)

163+ self.assertEqual(results[2]['address']['locality'], 'Paris')

164+ self.assertEqual(results[2]['address']['postcode'], '75001')

165+ self.assertEqual(results[2]['address']['country'], 'France')

166+ self.assertEqual(results[2]['osm']['type'], 'node')

searx/tests/engines/test_startpage.pynew file+140

@@ -0,0 +1,140 @@

1+# -*- coding: utf-8 -*-

2+from collections import defaultdict

3+import mock

4+from searx.engines import startpage

5+from searx.testing import SearxTestCase

8+class TestStartpageEngine(SearxTestCase):

10+ def test_request(self):

11+ query = 'test_query'

12+ dicto = defaultdict(dict)

13+ dicto['pageno'] = 1

14+ dicto['language'] = 'fr_FR'

15+ params = startpage.request(query, dicto)

16+ self.assertIn('url', params)

17+ self.assertIn('startpage.com', params['url'])

18+ self.assertIn('data', params)

19+ self.assertIn('query', params['data'])

20+ self.assertIn(query, params['data']['query'])

21+ self.assertIn('with_language', params['data'])

22+ self.assertIn('lang_fr', params['data']['with_language'])

23+

24+ dicto['language'] = 'all'

25+ params = startpage.request(query, dicto)

26+ self.assertNotIn('with_language', params['data'])

27+

28+ def test_response(self):

29+ self.assertRaises(AttributeError, startpage.response, None)

30+ self.assertRaises(AttributeError, startpage.response, [])

31+ self.assertRaises(AttributeError, startpage.response, '')

32+ self.assertRaises(AttributeError, startpage.response, '[]')

33+

34+ response = mock.Mock(content='<html></html>')

35+ self.assertEqual(startpage.response(response), [])

36+

37+ html = """

38+ <div class='result' style=' *width : auto; *margin-right : 10%;'>

39+ <h3>

40+ <a href='http://this.should.be.the.link/' id='title_2' name='title_2' >

41+ This should be the title

42+ </a>

43+ <span id='title_stars_2' name='title_stars_2'> </span>

44+ </h3>

45+ <p class='desc'>

46+ This should be the content.

47+ </p>

48+ <p>

49+ <span class='url'>www.speed<b>test</b>.net/fr/

50+ </span>

51+ -

52+ <A class="proxy" id="proxy_link" HREF="https://ixquick-proxy.com/do/spg/proxy?ep=&edata=&ek=&ekdata="

53+ class='proxy'>

54+ Navigation avec Ixquick Proxy

55+ </A>

56+ -

57+ <A HREF="https://ixquick-proxy.com/do/spg/highlight.pl?l=francais&c=hf&cat=web&q=test&rl=NONE&rid=

58+ &hlq=https://startpage.com/do/search&mtabp=-1&mtcmd=process_search&mtlanguage=francais&mtengine0=

59+ &mtcat=web&u=http:%2F%2Fwww.speedtest.net%2Ffr%2F" class='proxy'>

60+ Mis en surbrillance

61+ </A>

62+ </p>

63+ </div>

64+ """

65+ response = mock.Mock(content=html)

66+ results = startpage.response(response)

67+ self.assertEqual(type(results), list)

68+ self.assertEqual(len(results), 1)

69+ self.assertEqual(results[0]['title'], 'This should be the title')

70+ self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')

71+ self.assertEqual(results[0]['content'], 'This should be the content.')

72+

73+ html = """

74+ <div class='result' style=' *width : auto; *margin-right : 10%;'>

75+ <h3>

76+ <a href='http://www.google.com/aclk?sa=l&ai=C' id='title_2' name='title_2' >

77+ This should be the title

78+ </a>

79+ <span id='title_stars_2' name='title_stars_2'> </span>

80+ </h3>

81+ <p class='desc'>

82+ This should be the content.

83+ </p>

84+ <p>

85+ <span class='url'>www.speed<b>test</b>.net/fr/

86+ </span>

87+ -

88+ <A class="proxy" id="proxy_link" HREF="https://ixquick-proxy.com/do/spg/proxy?ep=&edata=&ek=&ekdata="

89+ class='proxy'>

90+ Navigation avec Ixquick Proxy

91+ </A>

92+ -

93+ <A HREF="https://ixquick-proxy.com/do/spg/highlight.pl?l=francais&c=hf&cat=web&q=test&rl=NONE&rid=

94+ &hlq=https://startpage.com/do/search&mtabp=-1&mtcmd=process_search&mtlanguage=francais&mtengine0=

95+ &mtcat=web&u=http:%2F%2Fwww.speedtest.net%2Ffr%2F" class='proxy'>

96+ Mis en surbrillance

97+ </A>

98+ </p>

99+ </div>

100+ <div class='result' style=' *width : auto; *margin-right : 10%;'>

101+ <h3>

102+ <span id='title_stars_2' name='title_stars_2'> </span>

103+ </h3>

104+ <p class='desc'>

105+ This should be the content.

106+ </p>

107+ <p>

108+ <span class='url'>www.speed<b>test</b>.net/fr/

109+ </span>

110+ </p>

111+ </div>

112+ <div class='result' style=' *width : auto; *margin-right : 10%;'>

113+ <h3>

114+ <a href='http://this.should.be.the.link/' id='title_2' name='title_2' >

115+ This should be the title

116+ </a>

117+ <span id='title_stars_2' name='title_stars_2'> </span>

118+ </h3>

119+ <p>

120+ <span class='url'>www.speed<b>test</b>.net/fr/

121+ </span>

122+ -

123+ <A class="proxy" id="proxy_link" HREF="https://ixquick-proxy.com/do/spg/proxy?ep=&edata=&ek=&ekdata="

124+ class='proxy'>

125+ Navigation avec Ixquick Proxy

126+ </A>

127+ -

128+ <A HREF="https://ixquick-proxy.com/do/spg/highlight.pl?l=francais&c=hf&cat=web&q=test&rl=NONE&rid=

129+ &hlq=https://startpage.com/do/search&mtabp=-1&mtcmd=process_search&mtlanguage=francais&mtengine0=

130+ &mtcat=web&u=http:%2F%2Fwww.speedtest.net%2Ffr%2F" class='proxy'>

131+ Mis en surbrillance

132+ </A>

133+ </p>

134+ </div>

135+ """

136+ response = mock.Mock(content=html)

137+ results = startpage.response(response)

138+ self.assertEqual(type(results), list)

139+ self.assertEqual(len(results), 1)

140+ self.assertEqual(results[0]['content'], '')

searx/tests/engines/test_subtitleseeker.pynew file+169

@@ -0,0 +1,169 @@

1+from collections import defaultdict

2+import mock

3+from searx.engines import subtitleseeker

4+from searx.testing import SearxTestCase

7+class TestSubtitleseekerEngine(SearxTestCase):

9+ def test_request(self):

10+ query = 'test_query'

11+ dicto = defaultdict(dict)

12+ dicto['pageno'] = 1

13+ params = subtitleseeker.request(query, dicto)

14+ self.assertTrue('url' in params)

15+ self.assertTrue(query in params['url'])

16+ self.assertTrue('subtitleseeker.com' in params['url'])

17+

18+ def test_response(self):

19+ dicto = defaultdict(dict)

20+ dicto['language'] = 'fr_FR'

21+ response = mock.Mock(search_params=dicto)

22+

23+ self.assertRaises(AttributeError, subtitleseeker.response, None)

24+ self.assertRaises(AttributeError, subtitleseeker.response, [])

25+ self.assertRaises(AttributeError, subtitleseeker.response, '')

26+ self.assertRaises(AttributeError, subtitleseeker.response, '[]')

27+

28+ response = mock.Mock(text='<html></html>', search_params=dicto)

29+ self.assertEqual(subtitleseeker.response(response), [])

30+

31+ html = """

32+ <div class="boxRows">

33+ <div class="boxRowsInner" style="width:600px;">

34+ <img src="http://static.subtitleseeker.com/images/movie.gif"

35+ style="width:16px; height:16px;" class="icon">

36+ <a href="http://this.is.the.url/"

37+ class="blue" title="Title subtitle" >

38+ This is the Title

39+ </a>

40+ <br><br>

41+ <span class="f10b grey-dark arial" style="padding:0px 0px 5px 20px">

42+ "Alternative Title"

43+ </span>

44+ </div>

45+ <div class="boxRowsInner f12b red" style="width:70px;">

46+ 1998

47+ </div>

48+ <div class="boxRowsInner grey-web f12" style="width:120px;">

49+ <img src="http://static.subtitleseeker.com/images/basket_put.png"

50+ style="width:16px; height:16px;" class="icon">

51+ 1039 Subs

52+ </div>

53+ <div class="boxRowsInner grey-web f10" style="width:130px;">

54+ <img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"

55+ style="width:16px; height:16px;" class="icon">

56+ 1 hours ago

57+ </div>

58+ <div class="clear"></div>

59+ </div>

60+ """

61+ response = mock.Mock(text=html, search_params=dicto)

62+ results = subtitleseeker.response(response)

63+ self.assertEqual(type(results), list)

64+ self.assertEqual(len(results), 1)

65+ self.assertEqual(results[0]['title'], 'This is the Title')

66+ self.assertEqual(results[0]['url'], 'http://this.is.the.url/French/')

67+ self.assertIn('1998', results[0]['content'])

68+ self.assertIn('1039 Subs', results[0]['content'])

69+ self.assertIn('Alternative Title', results[0]['content'])

70+

71+ html = """

72+ <div class="boxRows">

73+ <div class="boxRowsInner" style="width:600px;">

74+ <img src="http://static.subtitleseeker.com/images/movie.gif"

75+ style="width:16px; height:16px;" class="icon">

76+ <a href="http://this.is.the.url/"

77+ class="blue" title="Title subtitle" >

78+ This is the Title

79+ </a>

80+ </div>

81+ <div class="boxRowsInner f12b red" style="width:70px;">

82+ 1998

83+ </div>

84+ <div class="boxRowsInner grey-web f12" style="width:120px;">

85+ <img src="http://static.subtitleseeker.com/images/basket_put.png"

86+ style="width:16px; height:16px;" class="icon">

87+ 1039 Subs

88+ </div>

89+ <div class="boxRowsInner grey-web f10" style="width:130px;">

90+ <img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"

91+ style="width:16px; height:16px;" class="icon">

92+ 1 hours ago

93+ </div>

94+ <div class="clear"></div>

95+ </div>

96+ """

97+ dicto['language'] = 'all'

98+ response = mock.Mock(text=html, search_params=dicto)

99+ results = subtitleseeker.response(response)

100+ self.assertEqual(type(results), list)

101+ self.assertEqual(len(results), 1)

102+ self.assertEqual(results[0]['title'], 'This is the Title')

103+ self.assertEqual(results[0]['url'], 'http://this.is.the.url/')

104+ self.assertIn('1998', results[0]['content'])

105+ self.assertIn('1039 Subs', results[0]['content'])

106+

107+ html = """

108+ <div class="boxRows">

109+ <div class="boxRowsInner" style="width:600px;">

110+ <img src="http://static.subtitleseeker.com/images/movie.gif"

111+ style="width:16px; height:16px;" class="icon">

112+ <a href="http://this.is.the.url/"

113+ class="blue" title="Title subtitle" >

114+ This is the Title

115+ </a>

116+ </div>

117+ <div class="boxRowsInner f12b red" style="width:70px;">

118+ 1998

119+ </div>

120+ <div class="boxRowsInner grey-web f12" style="width:120px;">

121+ <img src="http://static.subtitleseeker.com/images/basket_put.png"

122+ style="width:16px; height:16px;" class="icon">

123+ 1039 Subs

124+ </div>

125+ <div class="boxRowsInner grey-web f10" style="width:130px;">

126+ <img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"

127+ style="width:16px; height:16px;" class="icon">

128+ 1 hours ago

129+ </div>

130+ <div class="clear"></div>

131+ </div>

132+ """

133+ subtitleseeker.language = 'English'

134+ response = mock.Mock(text=html, search_params=dicto)

135+ results = subtitleseeker.response(response)

136+ self.assertEqual(type(results), list)

137+ self.assertEqual(len(results), 1)

138+ self.assertEqual(results[0]['title'], 'This is the Title')

139+ self.assertEqual(results[0]['url'], 'http://this.is.the.url/English/')

140+ self.assertIn('1998', results[0]['content'])

141+ self.assertIn('1039 Subs', results[0]['content'])

142+

143+ html = """

144+ <div class="boxRowsInner" style="width:600px;">

145+ <img src="http://static.subtitleseeker.com/images/movie.gif"

146+ style="width:16px; height:16px;" class="icon">

147+ <a href="http://this.is.the.url/"

148+ class="blue" title="Title subtitle" >

149+ This is the Title

150+ </a>

151+ </div>

152+ <div class="boxRowsInner f12b red" style="width:70px;">

153+ 1998

154+ </div>

155+ <div class="boxRowsInner grey-web f12" style="width:120px;">

156+ <img src="http://static.subtitleseeker.com/images/basket_put.png"

157+ style="width:16px; height:16px;" class="icon">

158+ 1039 Subs

159+ </div>

160+ <div class="boxRowsInner grey-web f10" style="width:130px;">

161+ <img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png"

162+ style="width:16px; height:16px;" class="icon">

163+ 1 hours ago

164+ </div>

165+ """

166+ response = mock.Mock(text=html, search_params=dicto)

167+ results = subtitleseeker.response(response)

168+ self.assertEqual(type(results), list)

169+ self.assertEqual(len(results), 0)

searx/tests/engines/test_twitter.pynew file+502

@@ -0,0 +1,502 @@

1+# -*- coding: utf-8 -*-

2+from collections import defaultdict

3+import mock

4+from searx.engines import twitter

5+from searx.testing import SearxTestCase

8+class TestTwitterEngine(SearxTestCase):

10+ def test_request(self):

11+ query = 'test_query'

12+ dicto = defaultdict(dict)

13+ dicto['pageno'] = 0

14+ dicto['language'] = 'fr_FR'

15+ params = twitter.request(query, dicto)

16+ self.assertIn('url', params)

17+ self.assertIn(query, params['url'])

18+ self.assertIn('twitter.com', params['url'])

19+ self.assertIn('cookies', params)

20+ self.assertIn('lang', params['cookies'])

21+ self.assertIn('fr', params['cookies']['lang'])

22+

23+ dicto['language'] = 'all'

24+ params = twitter.request(query, dicto)

25+ self.assertIn('cookies', params)

26+ self.assertIn('lang', params['cookies'])

27+ self.assertIn('en', params['cookies']['lang'])

28+

29+ def test_response(self):

30+ self.assertRaises(AttributeError, twitter.response, None)

31+ self.assertRaises(AttributeError, twitter.response, [])

32+ self.assertRaises(AttributeError, twitter.response, '')

33+ self.assertRaises(AttributeError, twitter.response, '[]')

34+

35+ response = mock.Mock(text='<html></html>')

36+ self.assertEqual(twitter.response(response), [])

37+

38+ html = """

39+ <li class="js-stream-item stream-item stream-item expanding-stream-item" data-item-id="563005573290287105"

40+ id="stream-item-tweet-563005573290287105" data-item-type="tweet">

41+ <div class="tweet original-tweet js-stream-tweet js-actionable-tweet js-profile-popup-actionable

42+ js-original-tweet has-cards has-native-media" data-tweet-id="563005573290287105" data-disclosure-type=""

43+ data-item-id="563005573290287105" data-screen-name="Jalopnik" data-name="Jalopnik"

44+ data-user-id="3060631" data-has-native-media="true" data-has-cards="true" data-card-type="photo"

45+ data-expanded-footer="<div class="js-tweet-details-fixer

46+ tweet-details-fixer">




47+ <div class="cards-media-container js-media-container"><div

48+ data-card-url="//twitter.com/Jalopnik/status/563005573290287105/photo/1" data-card-type="

49+ photo" class="cards-base cards-multimedia" data-element-context="platform_photo_card

50+ ">


 <a class="media media-thumbnail twitter-timeline-link is-preview

51+ " data-url="https://pbs.twimg.com/media/B9Aylf5IMAAuziP.jpg:large"

52+ data-resolved-url-large="https://pbs.twimg.com/media/B9Aylf5IMAAuziP.jpg:large"

53+ href="//twitter.com/Jalopnik/status/563005573290287105/photo/1">


54+ <div class="">
 <img src="

55+ https://pbs.twimg.com/media/B9Aylf5IMAAuziP.jpg"

56+ alt="Embedded image permalink" width="636" height="309">


57+ </div>

 </a>

 <div class="cards-content">


58+ <div class="byline">
 
 </div>
 
 </div>


59+ 
</div>




</div>



 <div

60+ class="js-machine-translated-tweet-container"></div>
 <div

61+ class="js-tweet-stats-container tweet-stats-container ">
 </div>



62+ <div class="client-and-actions">
 <span class="metadata">


63+ <span>5:06 PM - 4 Feb 2015</span>

 &middot; <a

64+ class="permalink-link js-permalink js-nav" href="/Jalopnik/status/563005573290287105

65+ "tabindex="-1">Details</a>
 

 
 


66+ 

 </span>
</div>


</div>
" data-you-follow="false"

67+ data-you-block="false">

68+ <div class="context">

69+ </div>

70+ <div class="content">

71+ <div class="stream-item-header">

72+ <a class="account-group js-account-group js-action-profile js-user-profile-link js-nav"

73+ href="/Jalopnik" data-user-id="3060631">

74+ <img class="avatar js-action-profile-avatar"

75+ src="https://pbs.twimg.com/profile_images/2976430168/5cd4a59_bigger.jpeg" alt="">

76+ <strong class="fullname js-action-profile-name show-popup-with-id" data-aria-label-part>

77+ Jalopnik

78+ </strong>

79+ <span>&rlm;</span>

80+ <span class="username js-action-profile-name" data-aria-label-part>

81+ <s>@</s><b>TitleName</b>

82+ </span>

83+ </a>

84+ <small class="time">

85+ <a href="/this.is.the.url"

86+ class="tweet-timestamp js-permalink js-nav js-tooltip" title="5:06 PM - 4 Feb 2015" >

87+ <span class="u-hiddenVisually" data-aria-label-part="last">17 minutes ago</span>

88+ </a>

89+ </small>

90+ </div>

91+ <p class="js-tweet-text tweet-text" lang="en" data-aria-label-part="0">

92+ This is the content étude à€

93+ <a href="http://t.co/nRWsqQAwBL" rel="nofollow" dir="ltr"

94+ data-expanded-url="http://jalo.ps/ReMENu4" class="twitter-timeline-link"

95+ target="_blank" title="http://jalo.ps/ReMENu4" >

96+ <span class="tco-ellipsis">

97+ </span>

98+ <span class="invisible">http://</span><span class="js-display-url">link.in.tweet</span>

99+ <span class="invisible"></span>

100+ <span class="tco-ellipsis">

101+ <span class="invisible"> </span>

102+ </span>

103+ </a>

104+ <a href="http://t.co/rbFsfeE0l3" class="twitter-timeline-link u-hidden"

105+ data-pre-embedded="true" dir="ltr">

106+ pic.twitter.com/rbFsfeE0l3

107+ </a>

108+ </p>

109+ <div class="expanded-content js-tweet-details-dropdown">

110+ </div>

111+ <div class="stream-item-footer">

112+ <a class="details with-icn js-details" href="/Jalopnik/status/563005573290287105">

113+ <span class="Icon Icon--photo">

114+ </span>

115+ <b>

116+ <span class="expand-stream-item js-view-details">

117+ View photo

118+ </span>

119+ <span class="collapse-stream-item js-hide-details">

120+ Hide photo

121+ </span>

122+ </b>

123+ </a>

124+ <span class="ProfileTweet-action--reply u-hiddenVisually">

125+ <span class="ProfileTweet-actionCount" aria-hidden="true" data-tweet-stat-count="0">

126+ <span class="ProfileTweet-actionCountForAria" >0 replies</span>

127+ </span>

128+ </span>

129+ <span class="ProfileTweet-action--retweet u-hiddenVisually">

130+ <span class="ProfileTweet-actionCount" data-tweet-stat-count="8">

131+ <span class="ProfileTweet-actionCountForAria" data-aria-label-part>8 retweets</span>

132+ </span>

133+ </span>

134+ <span class="ProfileTweet-action--favorite u-hiddenVisually">

135+ <span class="ProfileTweet-actionCount" data-tweet-stat-count="14">

136+ <span class="ProfileTweet-actionCountForAria" data-aria-label-part>14 favorites</span>

137+ </span>

138+ </span>

139+ <div role="group" aria-label="Tweet actions" class="ProfileTweet-actionList u-cf js-actions">

140+ <div class="ProfileTweet-action ProfileTweet-action--reply">

141+ <button class="ProfileTweet-actionButton u-textUserColorHover js-actionButton

142+ js-actionReply" data-modal="ProfileTweet-reply" type="button" title="Reply">

143+ <span class="Icon Icon--reply">

144+ </span>

145+ <span class="u-hiddenVisually">Reply</span>

146+ <span class="ProfileTweet-actionCount u-textUserColorHover

147+ ProfileTweet-actionCount--isZero">

148+ <span class="ProfileTweet-actionCountForPresentation" aria-hidden="true">

149+ </span>

150+ </span>

151+ </button>

152+ </div>

153+ <div class="ProfileTweet-action ProfileTweet-action--retweet js-toggleState js-toggleRt">

154+ <button class="ProfileTweet-actionButton js-actionButton js-actionRetweet js-tooltip"

155+ title="Retweet" data-modal="ProfileTweet-retweet" type="button">

156+ <span class="Icon Icon--retweet">

157+ </span>

158+ <span class="u-hiddenVisually">Retweet</span>

159+ <span class="ProfileTweet-actionCount">

160+ <span class="ProfileTweet-actionCountForPresentation">8</span>

161+ </span>

162+ </button>

163+ <button class="ProfileTweet-actionButtonUndo js-actionButton js-actionRetweet"

164+ data-modal="ProfileTweet-retweet" title="Undo retweet" type="button">

165+ <span class="Icon Icon--retweet">

166+ </span>

167+ <span class="u-hiddenVisually">Retweeted</span>

168+ <span class="ProfileTweet-actionCount">

169+ <span class="ProfileTweet-actionCountForPresentation">8</span>

170+ </span>

171+ </button>

172+ </div>

173+ <div class="ProfileTweet-action ProfileTweet-action--favorite js-toggleState">

174+ <button class="ProfileTweet-actionButton js-actionButton js-actionFavorite js-tooltip"

175+ title="Favorite" type="button">

176+ <span class="Icon Icon--favorite">

177+ </span>

178+ <span class="u-hiddenVisually">Favorite</span>

179+ <span class="ProfileTweet-actionCount">

180+ <span class="ProfileTweet-actionCountForPresentation">14</span>

181+ </span>

182+ </button>

183+ <button class="ProfileTweet-actionButtonUndo u-linkClean js-actionButton

184+ js-actionFavorite" title="Undo favorite" type="button">

185+ <span class="Icon Icon--favorite">

186+ </span>

187+ <span class="u-hiddenVisually">Favorited</span>

188+ <span class="ProfileTweet-actionCount">

189+ <span class="ProfileTweet-actionCountForPresentation">

190+ 14

191+ </span>

192+ </span>

193+ </button>

194+ </div>

195+ <div class="ProfileTweet-action ProfileTweet-action--more js-more-ProfileTweet-actions">

196+ <div class="dropdown">

197+ <button class="ProfileTweet-actionButton u-textUserColorHover dropdown-toggle

198+ js-tooltip js-dropdown-toggle" type="button" title="More">

199+ <span class="Icon Icon--dots">

200+ </span>

201+ <span class="u-hiddenVisually">More</span>

202+ </button>

203+ <div class="dropdown-menu">

204+ <div class="dropdown-caret">

205+ <div class="caret-outer">

206+ </div>

207+ <div class="caret-inner">

208+ </div>

209+ </div>

210+ <ul>

211+ <li class="share-via-dm js-actionShareViaDM" data-nav="share_tweet_dm">

212+ <button type="button" class="dropdown-link">

213+ Share via Direct Message

214+ </button>

215+ </li>

216+ <li class="embed-link js-actionEmbedTweet" data-nav="embed_tweet">

217+ <button type="button" class="dropdown-link">

218+ Embed Tweet

219+ </button>

220+ </li>

221+ <li class="mute-user-item pretty-link">

222+ <button type="button" class="dropdown-link">

223+ Mute

224+ </button>

225+ </li>

226+ <li class="unmute-user-item pretty-link">

227+ <button type="button" class="dropdown-link">

228+ Unmute

229+ </button>

230+ </li>

231+ <li class="block-or-report-link js-actionBlockOrReport"

232+ data-nav="block_or_report">

233+ <button type="button" class="dropdown-link">

234+ Block or report

235+ </button>

236+ </li>

237+ </ul>

238+ </div>

239+ </div>

240+ </div>

241+ </div>

242+ </div>

243+ </div>

244+ </div>

245+ </li>

246+ """

247+ response = mock.Mock(text=html)

248+ results = twitter.response(response)

249+ self.assertEqual(type(results), list)

250+ self.assertEqual(len(results), 1)

251+ self.assertEqual(results[0]['title'], '@TitleName')

252+ self.assertEqual(results[0]['url'], 'https://twitter.com/this.is.the.url')

253+ self.assertIn(u'This is the content', results[0]['content'])

254+ # self.assertIn(u'This is the content étude à€', results[0]['content'])

255+

256+ html = """

257+ <li class="js-stream-item stream-item stream-item expanding-stream-item" data-item-id="563005573290287105"

258+ id="stream-item-tweet-563005573290287105" data-item-type="tweet">

259+ <div class="tweet original-tweet js-stream-tweet js-actionable-tweet js-profile-popup-actionable

260+ js-original-tweet has-cards has-native-media" data-tweet-id="563005573290287105" data-disclosure-type=""

261+ data-item-id="563005573290287105" data-screen-name="Jalopnik" data-name="Jalopnik"

262+ data-user-id="3060631" data-has-native-media="true" data-has-cards="true" data-card-type="photo"

263+ data-expanded-footer="<div class="js-tweet-details-fixer

264+ tweet-details-fixer">




265+ <div class="cards-media-container js-media-container"><div

266+ data-card-url="//twitter.com/Jalopnik/status/563005573290287105/photo/1" data-card-type="

267+ photo" class="cards-base cards-multimedia" data-element-context="platform_photo_card

268+ ">


 <a class="media media-thumbnail twitter-timeline-link is-preview

269+ " data-url="https://pbs.twimg.com/media/B9Aylf5IMAAuziP.jpg:large"

270+ data-resolved-url-large="https://pbs.twimg.com/media/B9Aylf5IMAAuziP.jpg:large"

271+ href="//twitter.com/Jalopnik/status/563005573290287105/photo/1">


272+ <div class="">
 <img src="

273+ https://pbs.twimg.com/media/B9Aylf5IMAAuziP.jpg"

274+ alt="Embedded image permalink" width="636" height="309">


275+ </div>

 </a>

 <div class="cards-content">


276+ <div class="byline">
 
 </div>
 
 </div>


277+ 
</div>




</div>



 <div

278+ class="js-machine-translated-tweet-container"></div>
 <div

279+ class="js-tweet-stats-container tweet-stats-container ">
 </div>



280+ <div class="client-and-actions">
 <span class="metadata">


281+ <span>5:06 PM - 4 Feb 2015</span>

 &middot; <a

282+ class="permalink-link js-permalink js-nav" href="/Jalopnik/status/563005573290287105

283+ "tabindex="-1">Details</a>
 

 
 


284+ 

 </span>
</div>


</div>
" data-you-follow="false"

285+ data-you-block="false">

286+ <div class="context">

287+ </div>

288+ <div class="content">

289+ <div class="stream-item-header">

290+ <a class="account-group js-account-group js-action-profile js-user-profile-link js-nav"

291+ href="/Jalopnik" data-user-id="3060631">

292+ <img class="avatar js-action-profile-avatar"

293+ src="https://pbs.twimg.com/profile_images/2976430168/5cd4a59_bigger.jpeg" alt="">

294+ <strong class="fullname js-action-profile-name show-popup-with-id" data-aria-label-part>

295+ Jalopnik

296+ </strong>

297+ <span>&rlm;</span>

298+ <span class="username js-action-profile-name" data-aria-label-part>

299+ <s>@</s><b>TitleName</b>

300+ </span>

301+ </a>

302+ <small class="time">

303+ <a href="/this.is.the.url"

304+ class="tweet-timestamp js-permalink js-nav js-tooltip" title="5:06 PM - 4 Feb 2015" >

305+ <span class="_timestamp js-short-timestamp js-relative-timestamp" data-time="1423065963"

306+ data-time-ms="1423065963000" data-long-form="true" aria-hidden="true">

307+ 17m

308+ </span>

309+ <span class="u-hiddenVisually" data-aria-label-part="last">17 minutes ago</span>

310+ </a>

311+ </small>

312+ </div>

313+ <p class="js-tweet-text tweet-text" lang="en" data-aria-label-part="0">

314+ This is the content étude à€

315+ <a href="http://t.co/nRWsqQAwBL" rel="nofollow" dir="ltr"

316+ data-expanded-url="http://jalo.ps/ReMENu4" class="twitter-timeline-link"

317+ target="_blank" title="http://jalo.ps/ReMENu4" >

318+ <span class="tco-ellipsis">

319+ </span>

320+ <span class="invisible">http://</span><span class="js-display-url">link.in.tweet</span>

321+ <span class="invisible"></span>

322+ <span class="tco-ellipsis">

323+ <span class="invisible"> </span>

324+ </span>

325+ </a>

326+ <a href="http://t.co/rbFsfeE0l3" class="twitter-timeline-link u-hidden"

327+ data-pre-embedded="true" dir="ltr">

328+ pic.twitter.com/rbFsfeE0l3

329+ </a>

330+ </p>

331+ <div class="expanded-content js-tweet-details-dropdown">

332+ </div>

333+ <div class="stream-item-footer">

334+ <a class="details with-icn js-details" href="/Jalopnik/status/563005573290287105">

335+ <span class="Icon Icon--photo">

336+ </span>

337+ <b>

338+ <span class="expand-stream-item js-view-details">

339+ View photo

340+ </span>

341+ <span class="collapse-stream-item js-hide-details">

342+ Hide photo

343+ </span>

344+ </b>

345+ </a>

346+ <span class="ProfileTweet-action--reply u-hiddenVisually">

347+ <span class="ProfileTweet-actionCount" aria-hidden="true" data-tweet-stat-count="0">

348+ <span class="ProfileTweet-actionCountForAria" >0 replies</span>

349+ </span>

350+ </span>

351+ <span class="ProfileTweet-action--retweet u-hiddenVisually">

352+ <span class="ProfileTweet-actionCount" data-tweet-stat-count="8">

353+ <span class="ProfileTweet-actionCountForAria" data-aria-label-part>8 retweets</span>

354+ </span>

355+ </span>

356+ <span class="ProfileTweet-action--favorite u-hiddenVisually">

357+ <span class="ProfileTweet-actionCount" data-tweet-stat-count="14">

358+ <span class="ProfileTweet-actionCountForAria" data-aria-label-part>14 favorites</span>

359+ </span>

360+ </span>

361+ <div role="group" aria-label="Tweet actions" class="ProfileTweet-actionList u-cf js-actions">

362+ <div class="ProfileTweet-action ProfileTweet-action--reply">

363+ <button class="ProfileTweet-actionButton u-textUserColorHover js-actionButton

364+ js-actionReply" data-modal="ProfileTweet-reply" type="button" title="Reply">

365+ <span class="Icon Icon--reply">

366+ </span>

367+ <span class="u-hiddenVisually">Reply</span>

368+ <span class="ProfileTweet-actionCount u-textUserColorHover

369+ ProfileTweet-actionCount--isZero">

370+ <span class="ProfileTweet-actionCountForPresentation" aria-hidden="true">

371+ </span>

372+ </span>

373+ </button>

374+ </div>

375+ <div class="ProfileTweet-action ProfileTweet-action--retweet js-toggleState js-toggleRt">

376+ <button class="ProfileTweet-actionButton js-actionButton js-actionRetweet js-tooltip"

377+ title="Retweet" data-modal="ProfileTweet-retweet" type="button">

378+ <span class="Icon Icon--retweet">

379+ </span>

380+ <span class="u-hiddenVisually">Retweet</span>

381+ <span class="ProfileTweet-actionCount">

382+ <span class="ProfileTweet-actionCountForPresentation">8</span>

383+ </span>

384+ </button>

385+ <button class="ProfileTweet-actionButtonUndo js-actionButton js-actionRetweet"

386+ data-modal="ProfileTweet-retweet" title="Undo retweet" type="button">

387+ <span class="Icon Icon--retweet">

388+ </span>

389+ <span class="u-hiddenVisually">Retweeted</span>

390+ <span class="ProfileTweet-actionCount">

391+ <span class="ProfileTweet-actionCountForPresentation">8</span>

392+ </span>

393+ </button>

394+ </div>

395+ <div class="ProfileTweet-action ProfileTweet-action--favorite js-toggleState">

396+ <button class="ProfileTweet-actionButton js-actionButton js-actionFavorite js-tooltip"

397+ title="Favorite" type="button">

398+ <span class="Icon Icon--favorite">

399+ </span>

400+ <span class="u-hiddenVisually">Favorite</span>

401+ <span class="ProfileTweet-actionCount">

402+ <span class="ProfileTweet-actionCountForPresentation">14</span>

403+ </span>

404+ </button>

405+ <button class="ProfileTweet-actionButtonUndo u-linkClean js-actionButton

406+ js-actionFavorite" title="Undo favorite" type="button">

407+ <span class="Icon Icon--favorite">

408+ </span>

409+ <span class="u-hiddenVisually">Favorited</span>

410+ <span class="ProfileTweet-actionCount">

411+ <span class="ProfileTweet-actionCountForPresentation">

412+ 14

413+ </span>

414+ </span>

415+ </button>

416+ </div>

417+ <div class="ProfileTweet-action ProfileTweet-action--more js-more-ProfileTweet-actions">

418+ <div class="dropdown">

419+ <button class="ProfileTweet-actionButton u-textUserColorHover dropdown-toggle

420+ js-tooltip js-dropdown-toggle" type="button" title="More">

421+ <span class="Icon Icon--dots">

422+ </span>

423+ <span class="u-hiddenVisually">More</span>

424+ </button>

425+ <div class="dropdown-menu">

426+ <div class="dropdown-caret">

427+ <div class="caret-outer">

428+ </div>

429+ <div class="caret-inner">

430+ </div>

431+ </div>

432+ <ul>

433+ <li class="share-via-dm js-actionShareViaDM" data-nav="share_tweet_dm">

434+ <button type="button" class="dropdown-link">

435+ Share via Direct Message

436+ </button>

437+ </li>

438+ <li class="embed-link js-actionEmbedTweet" data-nav="embed_tweet">

439+ <button type="button" class="dropdown-link">

440+ Embed Tweet

441+ </button>

442+ </li>

443+ <li class="mute-user-item pretty-link">

444+ <button type="button" class="dropdown-link">

445+ Mute

446+ </button>

447+ </li>

448+ <li class="unmute-user-item pretty-link">

449+ <button type="button" class="dropdown-link">

450+ Unmute

451+ </button>

452+ </li>

453+ <li class="block-or-report-link js-actionBlockOrReport"

454+ data-nav="block_or_report">

455+ <button type="button" class="dropdown-link">

456+ Block or report

457+ </button>

458+ </li>

459+ </ul>

460+ </div>

461+ </div>

462+ </div>

463+ </div>

464+ </div>

465+ </div>

466+ </div>

467+ </li>

468+ """

469+ response = mock.Mock(text=html)

470+ results = twitter.response(response)

471+ self.assertEqual(type(results), list)

472+ self.assertEqual(len(results), 1)

473+ self.assertEqual(results[0]['title'], '@TitleName')

474+ self.assertEqual(results[0]['url'], 'https://twitter.com/this.is.the.url')

475+ self.assertIn(u'This is the content', results[0]['content'])

476+

477+ html = """

478+ <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">

479+ <div Class="sa_mc">

480+ <div class="sb_tlst">

481+ <h2>

482+ <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">

483+ <strong>This</strong> should be the title</a>

484+ </h2>

485+ </div>

486+ <div class="sb_meta">

487+ <cite>

488+ <strong>this</strong>.meta.com</cite>

489+ <span class="c_tlbxTrg">

490+ <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">

491+ </span>

492+ </span>

493+ </div>

494+ <p>

495+ <strong>This</strong> should be the content.</p>

496+ </div>

497+ </li>

498+ """

499+ response = mock.Mock(text=html)

500+ results = twitter.response(response)

501+ self.assertEqual(type(results), list)

502+ self.assertEqual(len(results), 0)

searx/tests/engines/test_yacy.pynew file+96

@@ -0,0 +1,96 @@

1+from collections import defaultdict

2+import mock

3+from searx.engines import yacy

4+from searx.testing import SearxTestCase

7+class TestYacyEngine(SearxTestCase):

9+ def test_request(self):

10+ query = 'test_query'

11+ dicto = defaultdict(dict)

12+ dicto['pageno'] = 1

13+ dicto['language'] = 'fr_FR'

14+ params = yacy.request(query, dicto)

15+ self.assertIn('url', params)

16+ self.assertIn(query, params['url'])

17+ self.assertIn('localhost', params['url'])

18+ self.assertIn('fr', params['url'])

19+

20+ dicto['language'] = 'all'

21+ params = yacy.request(query, dicto)

22+ self.assertIn('url', params)

23+ self.assertNotIn('lr=lang_', params['url'])

24+

25+ def test_response(self):

26+ self.assertRaises(AttributeError, yacy.response, None)

27+ self.assertRaises(AttributeError, yacy.response, [])

28+ self.assertRaises(AttributeError, yacy.response, '')

29+ self.assertRaises(AttributeError, yacy.response, '[]')

30+

31+ response = mock.Mock(text='{}')

32+ self.assertEqual(yacy.response(response), [])

33+

34+ response = mock.Mock(text='{"data": []}')

35+ self.assertEqual(yacy.response(response), [])

36+

37+ json = """

38+ {

39+ "channels": [

40+ {

41+ "title": "YaCy P2P-Search for test",

42+ "description": "Search for test",

43+ "link": "http://search.yacy.de:7001/yacysearch.html?query=test&resource=global&contentdom=0",

44+ "image": {

45+ "url": "http://search.yacy.de:7001/env/grafics/yacy.png",

46+ "title": "Search for test",

47+ "link": "http://search.yacy.de:7001/yacysearch.html?query=test&resource=global&contentdom=0"

48+ },

49+ "totalResults": "249",

50+ "startIndex": "0",

51+ "itemsPerPage": "5",

52+ "searchTerms": "test",

53+ "items": [

54+ {

55+ "title": "This is the title",

56+ "link": "http://this.is.the.url",

57+ "code": "",

58+ "description": "This should be the content",

59+ "pubDate": "Sat, 08 Jun 2013 02:00:00 +0200",

60+ "size": "44213",

61+ "sizename": "43 kbyte",

62+ "guid": "lzh_1T_5FP-A",

63+ "faviconCode": "XTS4uQ_5FP-A",

64+ "host": "www.gamestar.de",

65+ "path": "/spiele/city-of-heroes-freedom/47019.html",

66+ "file": "47019.html",

67+ "urlhash": "lzh_1T_5FP-A",

68+ "ranking": "0.20106804"

69+ },

70+ {

71+ "title": "This is the title2",

72+ "icon": "/ViewImage.png?maxwidth=96&maxheight=96&code=7EbAbW6BpPOA",

73+ "image": "http://image.url/image.png",

74+ "cache": "/ViewImage.png?quadratic=&url=http://golem.ivwbox.de/cgi-bin/ivw/CP/G_INET?d=14071378",

75+ "url": "http://this.is.the.url",

76+ "urlhash": "7EbAbW6BpPOA",

77+ "host": "www.golem.de",

78+ "width": "-1",

79+ "height": "-1"

80+ }

81+ ]

82+ }

83+ ]

84+ }

85+ """

86+ response = mock.Mock(text=json)

87+ results = yacy.response(response)

88+ self.assertEqual(type(results), list)

89+ self.assertEqual(len(results), 2)

90+ self.assertEqual(results[0]['title'], 'This is the title')

91+ self.assertEqual(results[0]['url'], 'http://this.is.the.url')

92+ self.assertEqual(results[0]['content'], 'This should be the content')

93+ self.assertEqual(results[1]['img_src'], 'http://image.url/image.png')

94+ self.assertEqual(results[1]['content'], '')

95+ self.assertEqual(results[1]['url'], 'http://this.is.the.url')

96+ self.assertEqual(results[1]['title'], 'This is the title2')

searx/tests/engines/test_yahoo.pynew file+154

@@ -0,0 +1,154 @@

1+# -*- coding: utf-8 -*-

2+from collections import defaultdict

3+import mock

4+from searx.engines import yahoo

5+from searx.testing import SearxTestCase

8+class TestYahooEngine(SearxTestCase):

10+ def test_parse_url(self):

11+ test_url = 'http://r.search.yahoo.com/_ylt=A0LEb9JUSKcAEGRXNyoA;_ylu=X3oDMTEzZm1qazYwBHNlYwNzcgRwb3MDMQRjb' +\

12+ '2xvA2Jm2dGlkA1NNRTcwM18x/RV=2/RE=1423106085/RO=10/RU=https%3a%2f%2fthis.is.the.url%2f/RK=0/RS=' +\

13+ 'dtcJsfP4mEeBOjnVfUQ-'

14+ url = yahoo.parse_url(test_url)

15+ self.assertEqual('https://this.is.the.url/', url)

16+

17+ test_url = 'http://r.search.yahoo.com/_ylt=A0LElb9JUSKcAEGRXNyoA;_ylu=X3oDMTEzZm1qazYwBHNlYwNzcgRwb3MDMQRjb' +\

18+ '2xvA2Jm2dGlkA1NNRTcwM18x/RV=2/RE=1423106085/RO=10/RU=https%3a%2f%2fthis.is.the.url%2f/RS=' +\

19+ 'dtcJsfP4mEeBOjnVfUQ-'

20+ url = yahoo.parse_url(test_url)

21+ self.assertEqual('https://this.is.the.url/', url)

22+

23+ test_url = 'https://this.is.the.url/'

24+ url = yahoo.parse_url(test_url)

25+ self.assertEqual('https://this.is.the.url/', url)

26+

27+ def test_request(self):

28+ query = 'test_query'

29+ dicto = defaultdict(dict)

30+ dicto['pageno'] = 1

31+ dicto['language'] = 'fr_FR'

32+ params = yahoo.request(query, dicto)

33+ self.assertIn('url', params)

34+ self.assertIn(query, params['url'])

35+ self.assertIn('search.yahoo.com', params['url'])

36+ self.assertIn('fr', params['url'])

37+ self.assertIn('cookies', params)

38+ self.assertIn('sB', params['cookies'])

39+ self.assertIn('fr', params['cookies']['sB'])

40+

41+ dicto['language'] = 'all'

42+ params = yahoo.request(query, dicto)

43+ self.assertIn('cookies', params)

44+ self.assertIn('sB', params['cookies'])

45+ self.assertIn('en', params['cookies']['sB'])

46+ self.assertIn('en', params['url'])

47+

48+ def test_response(self):

49+ self.assertRaises(AttributeError, yahoo.response, None)

50+ self.assertRaises(AttributeError, yahoo.response, [])

51+ self.assertRaises(AttributeError, yahoo.response, '')

52+ self.assertRaises(AttributeError, yahoo.response, '[]')

53+

54+ response = mock.Mock(text='<html></html>')

55+ self.assertEqual(yahoo.response(response), [])

56+

57+ html = """

58+ <div class="res">

59+ <div>

60+ <h3>

61+ <a id="link-1" class="yschttl spt" href="http://r.search.yahoo.com/_ylt=A0LEVzClb9JUSKcAEGRXNyoA;

62+ _ylu=X3oDMTEzZm1qazYwBHNlYwNzcgRwb3MDMQRjb2xvA2JmMQR2dGlkA1NNRTcwM18x/RV=2/RE=1423106085/RO=10

63+ /RU=https%3a%2f%2fthis.is.the.url%2f/RK=0/RS=dtcJsfP4mEeBOjnVfUQ-"target="_blank" data-bk="5063.1">

64+ <b>This</b> is the title

65+ </a>

66+ </h3>

67+ </div>

68+ <span class="url" dir="ltr">www.<b>test</b>.com</span>

69+ <div class="abstr">

70+ <b>This</b> is the content

71+ </div>

72+ </div>

73+ <div id="satat" data-bns="Yahoo" data-bk="124.1">

74+ <h2>Also Try</h2>

75+ <table>

76+ <tbody>

77+ <tr>

78+ <td>

79+ <a id="srpnat0" class="" href="https://search.yahoo.com/search=rs-bottom" >

80+ <span>

81+ <b></b>This is <b>the suggestion</b>

82+ </span>

83+ </a>

84+ </td>

85+ </tr>

86+ </tbody>

87+ </table>

88+ </div>

89+ """

90+ response = mock.Mock(text=html)

91+ results = yahoo.response(response)

92+ self.assertEqual(type(results), list)

93+ self.assertEqual(len(results), 2)

94+ self.assertEqual(results[0]['title'], 'This is the title')

95+ self.assertEqual(results[0]['url'], 'https://this.is.the.url/')

96+ self.assertEqual(results[0]['content'], 'This is the content')

97+ self.assertEqual(results[1]['suggestion'], 'This is the suggestion')

98+

99+ html = """

100+ <div class="res">

101+ <div>

102+ <h3>

103+ <a id="link-1" class="yschttl spt" href="http://r.search.yahoo.com/_ylt=A0LEVzClb9JUSKcAEGRXNyoA;

104+ _ylu=X3oDMTEzZm1qazYwBHNlYwNzcgRwb3MDMQRjb2xvA2JmMQR2dGlkA1NNRTcwM18x/RV=2/RE=1423106085/RO=10

105+ /RU=https%3a%2f%2fthis.is.the.url%2f/RK=0/RS=dtcJsfP4mEeBOjnVfUQ-"target="_blank" data-bk="5063.1">

106+ <b>This</b> is the title

107+ </a>

108+ </h3>

109+ </div>

110+ <span class="url" dir="ltr">www.<b>test</b>.com</span>

111+ <div class="abstr">

112+ <b>This</b> is the content

113+ </div>

114+ </div>

115+ <div class="res">

116+ <div>

117+ <h3>

118+ <a id="link-1" class="yschttl spt">

119+ <b>This</b> is the title

120+ </a>

121+ </h3>

122+ </div>

123+ <span class="url" dir="ltr">www.<b>test</b>.com</span>

124+ <div class="abstr">

125+ <b>This</b> is the content

126+ </div>

127+ </div>

128+ <div class="res">

129+ <div>

130+ <h3>

131+ </h3>

132+ </div>

133+ <span class="url" dir="ltr">www.<b>test</b>.com</span>

134+ <div class="abstr">

135+ <b>This</b> is the content

136+ </div>

137+ </div>

138+ """

139+ response = mock.Mock(text=html)

140+ results = yahoo.response(response)

141+ self.assertEqual(type(results), list)

142+ self.assertEqual(len(results), 1)

143+ self.assertEqual(results[0]['title'], 'This is the title')

144+ self.assertEqual(results[0]['url'], 'https://this.is.the.url/')

145+ self.assertEqual(results[0]['content'], 'This is the content')

146+

147+ html = """

148+ <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">

149+ </li>

150+ """

151+ response = mock.Mock(text=html)

152+ results = yahoo.response(response)

153+ self.assertEqual(type(results), list)

154+ self.assertEqual(len(results), 0)

searx/tests/engines/test_yahoo_news.pynew file+143

@@ -0,0 +1,143 @@

1+# -*- coding: utf-8 -*-

2+from collections import defaultdict

3+from datetime import datetime

4+import mock

5+from searx.engines import yahoo_news

6+from searx.testing import SearxTestCase

9+class TestYahooNewsEngine(SearxTestCase):

10+

11+ def test_request(self):

12+ query = 'test_query'

13+ dicto = defaultdict(dict)

14+ dicto['pageno'] = 1

15+ dicto['language'] = 'fr_FR'

16+ params = yahoo_news.request(query, dicto)

17+ self.assertIn('url', params)

18+ self.assertIn(query, params['url'])

19+ self.assertIn('news.search.yahoo.com', params['url'])

20+ self.assertIn('fr', params['url'])

21+ self.assertIn('cookies', params)

22+ self.assertIn('sB', params['cookies'])

23+ self.assertIn('fr', params['cookies']['sB'])

24+

25+ dicto['language'] = 'all'

26+ params = yahoo_news.request(query, dicto)

27+ self.assertIn('cookies', params)

28+ self.assertIn('sB', params['cookies'])

29+ self.assertIn('en', params['cookies']['sB'])

30+ self.assertIn('en', params['url'])

31+

32+ def test_response(self):

33+ self.assertRaises(AttributeError, yahoo_news.response, None)

34+ self.assertRaises(AttributeError, yahoo_news.response, [])

35+ self.assertRaises(AttributeError, yahoo_news.response, '')

36+ self.assertRaises(AttributeError, yahoo_news.response, '[]')

37+

38+ response = mock.Mock(text='<html></html>')

39+ self.assertEqual(yahoo_news.response(response), [])

40+

41+ html = """

42+ <div class="res">

43+ <div>

44+ <h3>

45+ <a class="yschttl spt" href="http://this.is.the.url" target="_blank">

46+ This is

47+ the <b>title</b>...

48+ </a>

49+ </h3>

50+ </div>

51+ <span class="url">Business via Yahoo! Finance</span>   <span class="timestamp">Feb 03 09:45am</span>

52+ <div class="abstr">

53+ This is the content

54+ </div>

55+ </div>

56+ """

57+ response = mock.Mock(text=html)

58+ results = yahoo_news.response(response)

59+ self.assertEqual(type(results), list)

60+ self.assertEqual(len(results), 1)

61+ self.assertEqual(results[0]['title'], 'This is the title...')

62+ self.assertEqual(results[0]['url'], 'http://this.is.the.url/')

63+ self.assertEqual(results[0]['content'], 'This is the content')

64+

65+ html = """

66+ <div class="res">

67+ <div>

68+ <h3>

69+ <a class="yschttl spt" href="http://this.is.the.url" target="_blank">

70+ This is

71+ the <b>title</b>...

72+ </a>

73+ </h3>

74+ </div>

75+ <span class="url">Business via Yahoo!</span>   <span class="timestamp">2 hours, 22 minutes ago</span>

76+ <div class="abstr">

77+ This is the content

78+ </div>

79+ </div>

80+ <div class="res">

81+ <div>

82+ <h3>

83+ <a class="yschttl spt" href="http://this.is.the.url" target="_blank">

84+ This is

85+ the <b>title</b>...

86+ </a>

87+ </h3>

88+ </div>

89+ <span class="url">Business via Yahoo!</span>   <span class="timestamp">22 minutes ago</span>

90+ <div class="abstr">

91+ This is the content

92+ </div>

93+ </div>

94+ <div class="res">

95+ <div>

96+ <h3>

97+ <a class="yschttl spt" href="http://this.is.the.url" target="_blank">

98+ This is

99+ the <b>title</b>...

100+ </a>

101+ </h3>

102+ </div>

103+ <span class="url">Business via Yahoo!</span>   <span class="timestamp">Feb 03 09:45am 1900</span>

104+ <div class="abstr">

105+ This is the content

106+ </div>

107+ </div>

108+ """

109+ response = mock.Mock(text=html)

110+ results = yahoo_news.response(response)

111+ self.assertEqual(type(results), list)

112+ self.assertEqual(len(results), 3)

113+ self.assertEqual(results[0]['title'], 'This is the title...')

114+ self.assertEqual(results[0]['url'], 'http://this.is.the.url/')

115+ self.assertEqual(results[0]['content'], 'This is the content')

116+ self.assertEqual(results[2]['publishedDate'].year, datetime.now().year)

117+

118+ html = """

119+ <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">

120+ <div Class="sa_mc">

121+ <div class="sb_tlst">

122+ <h2>

123+ <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">

124+ <strong>This</strong> should be the title</a>

125+ </h2>

126+ </div>

127+ <div class="sb_meta">

128+ <cite>

129+ <strong>this</strong>.meta.com</cite>

130+ <span class="c_tlbxTrg">

131+ <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">

132+ </span>

133+ </span>

134+ </div>

135+ <p>

136+ <strong>This</strong> should be the content.</p>

137+ </div>

138+ </li>

139+ """

140+ response = mock.Mock(text=html)

141+ results = yahoo_news.response(response)

142+ self.assertEqual(type(results), list)

143+ self.assertEqual(len(results), 0)

searx/tests/test_engines.py+15−1

@@ -3,25 +3,39 @@ from searx.tests.engines.test_bing_images import * # noqa

3from searx.tests.engines.test_bing_news import * # noqa

4from searx.tests.engines.test_blekko_images import * # noqa

5from searx.tests.engines.test_btdigg import * # noqa

6+from searx.tests.engines.test_currency_convert import * # noqa

6from searx.tests.engines.test_dailymotion import * # noqa

7from searx.tests.engines.test_dailymotion import * # noqa

7from searx.tests.engines.test_deezer import * # noqa

8from searx.tests.engines.test_deezer import * # noqa

8from searx.tests.engines.test_deviantart import * # noqa

9from searx.tests.engines.test_deviantart import * # noqa

9from searx.tests.engines.test_digg import * # noqa

10from searx.tests.engines.test_digg import * # noqa

11+from searx.tests.engines.test_duckduckgo import * # noqa

12+from searx.tests.engines.test_duckduckgo_definitions import * # noqa

10from searx.tests.engines.test_dummy import * # noqa

13from searx.tests.engines.test_dummy import * # noqa

14+from searx.tests.engines.test_faroo import * # noqa

11from searx.tests.engines.test_flickr import * # noqa

15from searx.tests.engines.test_flickr import * # noqa

12from searx.tests.engines.test_flickr_noapi import * # noqa

16from searx.tests.engines.test_flickr_noapi import * # noqa

13from searx.tests.engines.test_gigablast import * # noqa

17from searx.tests.engines.test_gigablast import * # noqa

14from searx.tests.engines.test_github import * # noqa

18from searx.tests.engines.test_github import * # noqa

15−from searx.tests.engines.test_www1x import * # noqa

19+from searx.tests.engines.test_google import * # noqa

16from searx.tests.engines.test_google_images import * # noqa

20from searx.tests.engines.test_google_images import * # noqa

17from searx.tests.engines.test_google_news import * # noqa

21from searx.tests.engines.test_google_news import * # noqa

18from searx.tests.engines.test_kickass import * # noqa

22from searx.tests.engines.test_kickass import * # noqa

23+from searx.tests.engines.test_mediawiki import * # noqa

19from searx.tests.engines.test_mixcloud import * # noqa

24from searx.tests.engines.test_mixcloud import * # noqa

25+from searx.tests.engines.test_openstreetmap import * # noqa

26+from searx.tests.engines.test_photon import * # noqa

20from searx.tests.engines.test_piratebay import * # noqa

27from searx.tests.engines.test_piratebay import * # noqa

21from searx.tests.engines.test_searchcode_code import * # noqa

28from searx.tests.engines.test_searchcode_code import * # noqa

22from searx.tests.engines.test_searchcode_doc import * # noqa

29from searx.tests.engines.test_searchcode_doc import * # noqa

23from searx.tests.engines.test_soundcloud import * # noqa

30from searx.tests.engines.test_soundcloud import * # noqa

24from searx.tests.engines.test_stackoverflow import * # noqa

31from searx.tests.engines.test_stackoverflow import * # noqa

32+from searx.tests.engines.test_startpage import * # noqa

33+from searx.tests.engines.test_subtitleseeker import * # noqa

34+from searx.tests.engines.test_twitter import * # noqa

25from searx.tests.engines.test_vimeo import * # noqa

35from searx.tests.engines.test_vimeo import * # noqa

36+from searx.tests.engines.test_www1x import * # noqa

26from searx.tests.engines.test_www500px import * # noqa

37from searx.tests.engines.test_www500px import * # noqa

38+from searx.tests.engines.test_yacy import * # noqa

39+from searx.tests.engines.test_yahoo import * # noqa

27from searx.tests.engines.test_youtube import * # noqa

40from searx.tests.engines.test_youtube import * # noqa

41+from searx.tests.engines.test_yahoo_news import * # noqa