aaae9a2 · searx · git.iko.soy

.github/workflows/data-update.yml+8−8

compare with master

unifiedsplit

@@ -11,12 +11,12 @@ jobs:

11 strategy:

12 matrix:

13 fetch:

14− - ahmia_blacklist

14+ - update_ahmia_blacklist.py

15− - currencies

15+ - update_currencies.py

16− - external_bangs

16+ - update_external_bangs.py

17− - firefox_version

17+ - update_firefox_version.py

18− - languages

18+ - update_languages.py

19− - wikidata_units

19+ - update_wikidata_units.py

20 steps:

21 - name: Checkout

22 uses: actions/checkout@v2

@@ -45,10 +45,10 @@ jobs:

45

46 - name: Fetch data

47 env:

48− FETCH_SCRIPT: utils/fetch_${{ matrix.fetch }}.py

48+ FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}

49 run: |

50 source local/py3/bin/activate

51− python $FETCH_SCRIPT

51+ $FETCH_SCRIPT

52

53 - name: Create Pull Request

54 id: cpr

Makefile+2−2

@@ -195,8 +195,8 @@ PYLINT_FILES=\

195 searx/engines/google_videos.py \

196 searx/engines/google_images.py \

197 searx/engines/mediathekviewweb.py \

198− utils/fetch_external_bangs.py \

199− searx/engines/google_scholar.py

198+ searx/engines/google_scholar.py \

199+ searx_extra/update/update_external_bangs.py

200

201test.pylint: pyenvinstall

202 $(call cmd,pylint,$(PYLINT_FILES))

docs/index.rst+1

@@ -30,6 +30,7 @@ anyone, you can set up your own, see :ref:`installation`.

30 user/index

31 admin/index

32 dev/index

33+ searx_extra/index

33 utils/index

34 utils/index

34 blog/index

35 blog/index

35

36

docs/searx_extra/index.rstnew file+14

@@ -0,0 +1,14 @@

1+.. _searx_extra:

2+

3+======================================================

4+Tooling box ``searx_extra`` for developers and users

5+======================================================

6+

7+In the folder :origin:`searx_extra/` we maintain some tools useful for

8+developers and users.

9+

10+.. toctree::

11+ :maxdepth: 2

12+ :caption: Contents

13+

14+ standalone_searx.py

docs/searx_extra/standalone_searx.py.rstnew file+9

@@ -0,0 +1,9 @@

1+

2+.. _standalone_searx.py:

3+

4+===================================

5+``searx_extra/standalone_searx.py``

6+===================================

7+

8+.. automodule:: searx_extra.standalone_searx

9+ :members:

docs/utils/index.rst+4−6

@@ -1,12 +1,11 @@

1.. _searx_utils:

2.. _toolboxing:

3

4−=======================

4+========================================

5−Tooling box ``utils/*``

5+Tooling box ``utils`` for administrators

6−=======================

6+========================================

7

8−In the folder :origin:`utils/` we maintain some tools useful for admins and

8+In the folder :origin:`utils/` we maintain some tools useful for administrators.

9−developers.

10

9

11.. toctree::

10.. toctree::

12 :maxdepth: 2

11 :maxdepth: 2

@@ -16,7 +15,6 @@ developers.

16 filtron.sh

15 filtron.sh

17 morty.sh

16 morty.sh

18 lxc.sh

17 lxc.sh

19− standalone_searx.py

20

18

21.. _toolboxing common:

19.. _toolboxing common:

22

20

docs/utils/standalone_searx.py.rstdeleted−11

@@ -1,11 +0,0 @@

1−

2−.. _standalone_searx.py:

3−

4−=============================

5−``utils/standalone_searx.py``

6−=============================

7−

8−.. automodule:: standalone_searx

9− :members:

10−

11−

searx_extra/__init__.pynew file

No changes to the content.

utils/google_search.py → searx_extra/google_search.pyrenamed

The content is the same.

utils/standalone_searx.py → searx_extra/standalone_searx.pyrenamed+1−1

@@ -15,7 +15,7 @@ Example to use this script:

15

16.. code:: bash

17

18− $ python3 utils/standalone_searx.py rain

18+ $ python3 searx_extra/standalone_searx.py rain

19

20Example to run it from python:

21

searx_extra/update/__init__.pynew file

No changes to the content.

utils/fetch_ahmia_blacklist.py → searx_extra/update/update_ahmia_blacklist.pyrenamed+1−4

@@ -4,11 +4,8 @@

4# More info in https://ahmia.fi/blacklist/

5

6# set path

7−from sys import path

8−from os.path import realpath, dirname, join

7+from os.path import join

9−path.append(realpath(dirname(realpath(__file__)) + '/../'))

10

8

11−#

12import requests

9import requests

13from searx import searx_dir

10from searx import searx_dir

14

11

utils/fetch_currencies.py → searx_extra/update/update_currencies.pyrenamed−1

@@ -7,7 +7,6 @@ import json

7# set path

8from sys import path

9from os.path import realpath, dirname, join

10−path.append(realpath(dirname(realpath(__file__)) + '/../'))

11

10

12from searx import searx_dir, settings

11from searx import searx_dir, settings

13from searx.engines.wikidata import send_wikidata_query

12from searx.engines.wikidata import send_wikidata_query

utils/fetch_engine_descriptions.py → searx_extra/update/update_engine_descriptions.pyrenamed+1−5

@@ -3,13 +3,9 @@

3import sys

4import json

5from urllib.parse import quote, urlparse

6−from os.path import realpath, dirname

6+import detect_language

7−import cld3

8from lxml.html import fromstring

7from lxml.html import fromstring

9

8

10−# set path

11−sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))

12−

13from searx.engines.wikidata import send_wikidata_query

9from searx.engines.wikidata import send_wikidata_query

14from searx.utils import extract_text

10from searx.utils import extract_text

15import searx

11import searx

utils/fetch_external_bangs.py → searx_extra/update/update_external_bangs.pyrenamed+1−5

@@ -13,16 +13,12 @@ but most probably it will requires to update RE_BANG_VERSION

13"""

14# pylint: disable=C0116

15

16−import sys

17import json

16import json

18import re

17import re

19−from os.path import realpath, dirname, join

18+from os.path import join

20

19

21import requests

20import requests

22

21

23−# set path

24−sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))

25−

26from searx import searx_dir # pylint: disable=E0401 C0413

22from searx import searx_dir # pylint: disable=E0401 C0413

27

23

28

24

utils/fetch_firefox_version.py → searx_extra/update/update_firefox_version.pyrenamed+1−6

@@ -1,14 +1,9 @@

1#!/usr/bin/env python

2

3−# set path

4−from sys import path

5−from os.path import realpath, dirname, join

6−path.append(realpath(dirname(realpath(__file__)) + '/../'))

7−

8−#

9import json

3import json

10import requests

4import requests

11import re

5import re

6+from os.path import dirname, join

12from urllib.parse import urlparse, urljoin

7from urllib.parse import urlparse, urljoin

13from distutils.version import LooseVersion, StrictVersion

8from distutils.version import LooseVersion, StrictVersion

14from lxml import html

9from lxml import html

utils/fetch_languages.py → searx_extra/update/update_languages.pyrenamed+1−3

@@ -1,4 +1,4 @@

1−# -*- coding: utf-8 -*-

1+#!/usr/bin/env python

2

3# This script generates languages.py from intersecting each engine's supported languages.

4#

@@ -7,11 +7,9 @@

7import json

8from pathlib import Path

9from pprint import pformat

10−from sys import path

11from babel import Locale, UnknownLocaleError

10from babel import Locale, UnknownLocaleError

12from babel.languages import get_global

11from babel.languages import get_global

13

12

14−path.append('../searx') # noqa

15from searx import settings, searx_dir

13from searx import settings, searx_dir

16from searx.engines import initialize_engines, engines

14from searx.engines import initialize_engines, engines

17

15

utils/update-translations.sh → searx_extra/update/update_translations.shrenamed

The content is the same.

utils/fetch_wikidata_units.py → searx_extra/update/update_wikidata_units.pyrenamed+1−3

@@ -4,9 +4,7 @@ import json

4import collections

5

6# set path

7−from sys import path

8−from os.path import realpath, dirname, join

7+from os.path import join

9−path.append(realpath(dirname(realpath(__file__)) + '/../'))

10

8

11from searx import searx_dir

9from searx import searx_dir

12from searx.engines.wikidata import send_wikidata_query

10from searx.engines.wikidata import send_wikidata_query

setup.py+1−1

@@ -41,7 +41,7 @@ setup(

41 author='Adam Tauber',

42 author_email='asciimoo@gmail.com',

43 license='GNU Affero General Public License',

44− packages=find_packages(exclude=["tests*"]),

44+ packages=find_packages(exclude=["tests*", "searx_extra"]),

45 zip_safe=False,

46 install_requires=requirements,

47 extras_require={

tests/unit/test_standalone_searx.py+1−19

@@ -1,7 +1,6 @@

1# -*- coding: utf-8 -*-

2"""Test utils/standalone_searx.py"""

3import datetime

4−import importlib.util

5import io

4import io

6import sys

5import sys

7

6

@@ -10,16 +9,7 @@ from nose2.tools import params

10

9

11from searx.search import SearchQuery, EngineRef, initialize

10from searx.search import SearchQuery, EngineRef, initialize

12from searx.testing import SearxTestCase

11from searx.testing import SearxTestCase

13−

14−

15−def get_standalone_searx_module():

16− """Get standalone_searx module."""

17− module_name = 'utils.standalone_searx'

12+from searx_extra import standalone_searx as sas

18− filename = 'utils/standalone_searx.py'

19− spec = importlib.util.spec_from_file_location(module_name, filename)

20− sas = importlib.util.module_from_spec(spec)

21− spec.loader.exec_module(sas)

22− return sas

23

13

24

14

25class StandaloneSearx(SearxTestCase):

15class StandaloneSearx(SearxTestCase):

@@ -33,7 +23,6 @@ class StandaloneSearx(SearxTestCase):

33

23

34 def test_parse_argument_no_args(self):

24 def test_parse_argument_no_args(self):

35 """Test parse argument without args."""

25 """Test parse argument without args."""

36− sas = get_standalone_searx_module()

37 with patch.object(sys, 'argv', ['standalone_searx']), \

26 with patch.object(sys, 'argv', ['standalone_searx']), \

38 self.assertRaises(SystemExit):

27 self.assertRaises(SystemExit):

39 sys.stderr = io.StringIO()

28 sys.stderr = io.StringIO()

@@ -42,7 +31,6 @@ class StandaloneSearx(SearxTestCase):

42

31

43 def test_parse_argument_basic_args(self):

32 def test_parse_argument_basic_args(self):

44 """Test parse argument with basic args."""

33 """Test parse argument with basic args."""

45− sas = get_standalone_searx_module()

46 query = 'red box'

34 query = 'red box'

47 exp_dict = {

35 exp_dict = {

48 'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1,

36 'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1,

@@ -56,7 +44,6 @@ class StandaloneSearx(SearxTestCase):

56

44

57 def test_to_dict(self):

45 def test_to_dict(self):

58 """test to_dict."""

46 """test to_dict."""

59− sas = get_standalone_searx_module()

60 self.assertEqual(

47 self.assertEqual(

61 sas.to_dict(

48 sas.to_dict(

62 sas.get_search_query(sas.parse_argument(['red box']))),

49 sas.get_search_query(sas.parse_argument(['red box']))),

@@ -72,7 +59,6 @@ class StandaloneSearx(SearxTestCase):

72

59

73 def test_to_dict_with_mock(self):

60 def test_to_dict_with_mock(self):

74 """test to dict."""

61 """test to dict."""

75− sas = get_standalone_searx_module()

76 with patch.object(sas.searx.search, 'Search') as mock_s:

62 with patch.object(sas.searx.search, 'Search') as mock_s:

77 m_search = mock_s().search()

63 m_search = mock_s().search()

78 m_sq = Mock()

64 m_sq = Mock()

@@ -97,7 +83,6 @@ class StandaloneSearx(SearxTestCase):

97

83

98 def test_get_search_query(self):

84 def test_get_search_query(self):

99 """test get_search_query."""

85 """test get_search_query."""

100− sas = get_standalone_searx_module()

101 args = sas.parse_argument(['rain', ])

86 args = sas.parse_argument(['rain', ])

102 search_q = sas.get_search_query(args)

87 search_q = sas.get_search_query(args)

103 self.assertTrue(search_q)

88 self.assertTrue(search_q)

@@ -106,7 +91,6 @@ class StandaloneSearx(SearxTestCase):

106

91

107 def test_no_parsed_url(self):

92 def test_no_parsed_url(self):

108 """test no_parsed_url func"""

93 """test no_parsed_url func"""

109− sas = get_standalone_searx_module()

110 self.assertEqual(

94 self.assertEqual(

111 sas.no_parsed_url([{'parsed_url': 'http://example.com'}]),

95 sas.no_parsed_url([{'parsed_url': 'http://example.com'}]),

112 [{}]

96 [{}]

@@ -119,11 +103,9 @@ class StandaloneSearx(SearxTestCase):

119 )

103 )

120 def test_json_serial(self, arg, exp_res):

104 def test_json_serial(self, arg, exp_res):

121 """test json_serial func"""

105 """test json_serial func"""

122− sas = get_standalone_searx_module()

123 self.assertEqual(sas.json_serial(arg), exp_res)

106 self.assertEqual(sas.json_serial(arg), exp_res)

124

107

125 def test_json_serial_error(self):

108 def test_json_serial_error(self):

126 """test error on json_serial."""

109 """test error on json_serial."""

127− sas = get_standalone_searx_module()

128 with self.assertRaises(TypeError):

110 with self.assertRaises(TypeError):

129 sas.json_serial('a')

111 sas.json_serial('a')