bd22e9a · searx · git.iko.soy

manage.sh+4−1

compare with master

unifiedsplit

@@ -16,7 +16,10 @@ update_dev_packages() {

16

17pep8_check() {

18 echo '[!] Running pep8 check'

19+ # ignored rules:

20+ # E402 module level import not at top of file

21+ # W503 line break before binary operator

19− pep8 --max-line-length=120 "$SEARX_DIR" "$BASE_DIR/tests"

22+ pep8 --max-line-length=120 --ignore "E402,W503" "$SEARX_DIR" "$BASE_DIR/tests"

20}

23}

21

24

22unit_tests() {

25unit_tests() {

requirements-dev.txt+1−1

@@ -1,7 +1,7 @@

1babel==2.2.0

2−flake8==2.5.1

3mock==1.0.1

2mock==1.0.1

4nose2[coverage-plugin]

3nose2[coverage-plugin]

4+pep8==1.7.0

5plone.testing==4.0.15

6robotframework-selenium2library==1.7.4

7robotsuite==1.7.0

searx/autocomplete.py+2−4

@@ -114,8 +114,7 @@ def dbpedia(query):

114 # dbpedia autocompleter, no HTTPS

115 autocomplete_url = 'http://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'

116

117− response = get(autocomplete_url

118− + urlencode(dict(QueryString=query)))

117+ response = get(autocomplete_url + urlencode(dict(QueryString=query)))

119

118

120 results = []

119 results = []

121

120

@@ -141,8 +140,7 @@ def google(query):

141 # google autocompleter

140 # google autocompleter

142 autocomplete_url = 'https://suggestqueries.google.com/complete/search?client=toolbar&'

141 autocomplete_url = 'https://suggestqueries.google.com/complete/search?client=toolbar&'

143

142

144− response = get(autocomplete_url

143+ response = get(autocomplete_url + urlencode(dict(q=query)))

145− + urlencode(dict(q=query)))

146

144

147 results = []

145 results = []

148

146

searx/engines/blekko_images.py+1−1

@@ -37,7 +37,7 @@ def request(query, params):

37 c=c)

38

39 if params['pageno'] != 1:

40− params['url'] += '&page={pageno}'.format(pageno=(params['pageno']-1))

40+ params['url'] += '&page={pageno}'.format(pageno=(params['pageno'] - 1))

41

42 # let Blekko know we wan't have profiling

43 params['cookies']['tag_lesslogging'] = '1'

searx/engines/btdigg.py+1−1

@@ -29,7 +29,7 @@ search_url = url + '/search?q={search_term}&p={pageno}'

29# do search-request

30def request(query, params):

31 params['url'] = search_url.format(search_term=quote(query),

32− pageno=params['pageno']-1)

32+ pageno=params['pageno'] - 1)

33

34 return params

35

searx/engines/deviantart.py+1−1

@@ -24,7 +24,7 @@ paging = True

24

25# search-url

26base_url = 'https://www.deviantart.com/'

27−search_url = base_url+'browse/all/?offset={offset}&{query}'

27+search_url = base_url + 'browse/all/?offset={offset}&{query}'

28

29

30# do search-request

searx/engines/digg.py+1−1

@@ -22,7 +22,7 @@ paging = True

22

23# search-url

24base_url = 'https://digg.com/'

25−search_url = base_url+'api/search/{query}.json?position={position}&format=html'

25+search_url = base_url + 'api/search/{query}.json?position={position}&format=html'

26

27# specific xpath variables

28results_xpath = '//article'

searx/engines/faroo.py+1−1

@@ -88,7 +88,7 @@ def response(resp):

88 for result in search_res['results']:

89 if result['news']:

90 # timestamp (milliseconds since 1970)

91− publishedDate = datetime.datetime.fromtimestamp(result['date']/1000.0) # noqa

91+ publishedDate = datetime.datetime.fromtimestamp(result['date'] / 1000.0) # noqa

92

93 # append news result

94 results.append({'url': result['url'],

searx/engines/searchcode_code.py+2−2

@@ -20,7 +20,7 @@ paging = True

20

21# search-url

22url = 'https://searchcode.com/'

23−search_url = url+'api/codesearch_I/?{query}&p={pageno}'

23+search_url = url + 'api/codesearch_I/?{query}&p={pageno}'

24

25# special code-endings which are not recognised by the file ending

26code_endings = {'cs': 'c#',

@@ -32,7 +32,7 @@ code_endings = {'cs': 'c#',

32# do search-request

33def request(query, params):

34 params['url'] = search_url.format(query=urlencode({'q': query}),

35− pageno=params['pageno']-1)

35+ pageno=params['pageno'] - 1)

36

37 # Disable SSL verification

38 # error: (60) SSL certificate problem: unable to get local issuer

searx/engines/searchcode_doc.py+2−2

@@ -19,13 +19,13 @@ paging = True

19

20# search-url

21url = 'https://searchcode.com/'

22−search_url = url+'api/search_IV/?{query}&p={pageno}'

22+search_url = url + 'api/search_IV/?{query}&p={pageno}'

23

24

25# do search-request

26def request(query, params):

27 params['url'] = search_url.format(query=urlencode({'q': query}),

28− pageno=params['pageno']-1)

28+ pageno=params['pageno'] - 1)

29

30 # Disable SSL verification

31 # error: (60) SSL certificate problem: unable to get local issuer

searx/engines/stackoverflow.py+1−1

@@ -22,7 +22,7 @@ paging = True

22

23# search-url

24url = 'https://stackoverflow.com/'

25−search_url = url+'search?{query}&page={pageno}'

25+search_url = url + 'search?{query}&page={pageno}'

26

27# specific xpath variables

28results_xpath = '//div[contains(@class,"question-summary")]'

searx/engines/startpage.py+4−4

@@ -90,8 +90,8 @@ def response(resp):

90

91 # check if search result starts with something like: "2 Sep 2014 ... "

92 if re.match("^([1-9]|[1-2][0-9]|3[0-1]) [A-Z][a-z]{2} [0-9]{4} \.\.\. ", content):

93− date_pos = content.find('...')+4

93+ date_pos = content.find('...') + 4

94− date_string = content[0:date_pos-5]

94+ date_string = content[0:date_pos - 5]

95 published_date = parser.parse(date_string, dayfirst=True)

96

97 # fix content string

@@ -99,8 +99,8 @@ def response(resp):

99

100 # check if search result starts with something like: "5 days ago ... "

101 elif re.match("^[0-9]+ days? ago \.\.\. ", content):

102− date_pos = content.find('...')+4

102+ date_pos = content.find('...') + 4

103− date_string = content[0:date_pos-5]

103+ date_string = content[0:date_pos - 5]

104

105 # calculate datetime

106 published_date = datetime.now() - timedelta(days=int(re.match(r'\d+', date_string).group()))

searx/engines/wikidata.py+2−2

@@ -295,7 +295,7 @@ def get_geolink(claims, propertyName, defaultValue=''):

295 if precision < 0.0003:

296 zoom = 19

297 else:

298− zoom = int(15 - precision*8.8322 + precision*precision*0.625447)

298+ zoom = int(15 - precision * 8.8322 + precision * precision * 0.625447)

299

300 url = url_map\

301 .replace('{latitude}', str(value.get('latitude', 0)))\

@@ -318,6 +318,6 @@ def get_wikilink(result, wikiid):

318

319def get_wiki_firstlanguage(result, wikipatternid):

320 for k in result.get('sitelinks', {}).keys():

321− if k.endswith(wikipatternid) and len(k) == (2+len(wikipatternid)):

321+ if k.endswith(wikipatternid) and len(k) == (2 + len(wikipatternid)):

322 return k[0:2]

323 return None

searx/engines/www1x.py+1−1

@@ -22,7 +22,7 @@ paging = False

22

23# search-url

24base_url = 'https://1x.com'

25−search_url = base_url+'/backend/search.php?{query}'

25+search_url = base_url + '/backend/search.php?{query}'

26

27

28# do search-request

searx/engines/xpath.py+2−2

@@ -43,7 +43,7 @@ def extract_url(xpath_results, search_url):

43 if url.startswith('//'):

44 # add http or https to this kind of url //example.com/

45 parsed_search_url = urlparse(search_url)

46− url = parsed_search_url.scheme+url

46+ url = parsed_search_url.scheme + url

47 elif url.startswith('/'):

48 # fix relative url to the search engine

49 url = urljoin(search_url, url)

@@ -69,7 +69,7 @@ def normalize_url(url):

69 p = parsed_url.path

70 mark = p.find('/**')

71 if mark != -1:

72− return unquote(p[mark+3:]).decode('utf-8')

72+ return unquote(p[mark + 3:]).decode('utf-8')

73

74 return url

75

searx/engines/yandex.py+1−1

@@ -38,7 +38,7 @@ content_xpath = './/div[@class="serp-item__text"]//text()'

38def request(query, params):

39 lang = params['language'].split('_')[0]

40 host = base_url.format(tld=language_map.get(lang) or default_tld)

41− params['url'] = host + search_url.format(page=params['pageno']-1,

41+ params['url'] = host + search_url.format(page=params['pageno'] - 1,

42 query=urlencode({'text': query}))

43 return params

44

searx/plugins/https_rewrite.py+2−2

@@ -103,10 +103,10 @@ def load_single_https_ruleset(rules_path):

103 # into a valid python regex group

104 rule_from = ruleset.attrib['from'].replace('$', '\\')

105 if rule_from.endswith('\\'):

106− rule_from = rule_from[:-1]+'$'

106+ rule_from = rule_from[:-1] + '$'

107 rule_to = ruleset.attrib['to'].replace('$', '\\')

108 if rule_to.endswith('\\'):

109− rule_to = rule_to[:-1]+'$'

109+ rule_to = rule_to[:-1] + '$'

110

111 # TODO, not working yet because of the hack above,

112 # currently doing that in webapp.py

searx/poolrequests.py+1−1

@@ -92,7 +92,7 @@ def head(url, **kwargs):

92 return request('head', url, **kwargs)

93

94

95−def post(url, data=None, **kwargs):

95+def post(url, data=None, **kwargs):

96 return request('post', url, data=data, **kwargs)

97

98