b8cd326 · searx · git.iko.soy

git.iko.soy/searx

Add searx_extra package

Split the utils directory into: * searx_extra contains update scripts, standalone_searx.py * utils contains the files to build and setup searx.

commitb8cd3264644208d7afa1a239f829222d45226334 authorAlexandre Flament <alex@al-f.net> dateThu 25 Feb 2021, 17:42 (6 years ago) parent111d38c

21 files changed+47−75

compare with master

unifiedsplit

.github/workflows/data-update.yml+8−8

@@ -11,12 +11,12 @@ jobs:

1111 strategy:

1212 matrix:

1313 fetch:

14− - ahmia_blacklist

15− - currencies

16− - external_bangs

17− - firefox_version

18− - languages

19− - wikidata_units

14+ - update_ahmia_blacklist.py

15+ - update_currencies.py

16+ - update_external_bangs.py

17+ - update_firefox_version.py

18+ - update_languages.py

19+ - update_wikidata_units.py

2020 steps:

2121 - name: Checkout

2222 uses: actions/checkout@v2

@@ -45,10 +45,10 @@ jobs:

4545

4646 - name: Fetch data

4747 env:

48− FETCH_SCRIPT: utils/fetch_${{ matrix.fetch }}.py

48+ FETCH_SCRIPT: ./searx_extra/update/${{ matrix.fetch }}

4949 run: |

5050 source local/py3/bin/activate

51− python $FETCH_SCRIPT

51+ $FETCH_SCRIPT

5252

5353 - name: Create Pull Request

5454 id: cpr

Makefile+2−2

@@ -195,8 +195,8 @@ PYLINT_FILES=\

195195 searx/engines/google_videos.py \

196196 searx/engines/google_images.py \

197197 searx/engines/mediathekviewweb.py \

198− utils/fetch_external_bangs.py \

199− searx/engines/google_scholar.py

198+ searx/engines/google_scholar.py \

199+ searx_extra/update/update_external_bangs.py

200200

201201test.pylint: pyenvinstall

202202 $(call cmd,pylint,$(PYLINT_FILES))

docs/index.rst+1

@@ -30,6 +30,7 @@ anyone, you can set up your own, see :ref:`installation`.

3030 user/index

3131 admin/index

3232 dev/index

33+ searx_extra/index

3334 utils/index

3435 blog/index

3536

docs/searx_extra/index.rstnew file+14

@@ -0,0 +1,14 @@

1+.. _searx_extra:

3+======================================================

4+Tooling box ``searx_extra`` for developers and users

5+======================================================

7+In the folder :origin:`searx_extra/` we maintain some tools useful for

8+developers and users.

10+.. toctree::

11+ :maxdepth: 2

12+ :caption: Contents

13+

14+ standalone_searx.py

docs/searx_extra/standalone_searx.py.rstnew file+9

@@ -0,0 +1,9 @@

2+.. _standalone_searx.py:

4+===================================

5+``searx_extra/standalone_searx.py``

6+===================================

8+.. automodule:: searx_extra.standalone_searx

9+ :members:

docs/utils/index.rst+4−6

@@ -1,12 +1,11 @@

11.. _searx_utils:

22.. _toolboxing:

4−=======================

5−Tooling box ``utils/*``

6−=======================

4+========================================

5+Tooling box ``utils`` for administrators

6+========================================

8−In the folder :origin:`utils/` we maintain some tools useful for admins and

9−developers.

8+In the folder :origin:`utils/` we maintain some tools useful for administrators.

109

1110.. toctree::

1211 :maxdepth: 2

@@ -16,7 +15,6 @@ developers.

1615 filtron.sh

1716 morty.sh

1817 lxc.sh

19− standalone_searx.py

2018

2119.. _toolboxing common:

2220

docs/utils/standalone_searx.py.rstdeleted−11

@@ -1,11 +0,0 @@

1−

2−.. _standalone_searx.py:

3−

4−=============================

5−``utils/standalone_searx.py``

6−=============================

7−

8−.. automodule:: standalone_searx

9− :members:

10−

11−

searx_extra/__init__.pynew file

No changes to the content.

utils/google_search.py → searx_extra/google_search.pyrenamed

The content is the same.

utils/standalone_searx.py → searx_extra/standalone_searx.pyrenamed+1−1

@@ -15,7 +15,7 @@ Example to use this script:

1515

1616.. code:: bash

1717

18− $ python3 utils/standalone_searx.py rain

18+ $ python3 searx_extra/standalone_searx.py rain

1919

2020Example to run it from python:

2121

searx_extra/update/__init__.pynew file

No changes to the content.

utils/fetch_ahmia_blacklist.py → searx_extra/update/update_ahmia_blacklist.pyrenamed+1−4

@@ -4,11 +4,8 @@

44# More info in https://ahmia.fi/blacklist/

66# set path

7−from sys import path

8−from os.path import realpath, dirname, join

9−path.append(realpath(dirname(realpath(__file__)) + '/../'))

7+from os.path import join

108

11−#

129import requests

1310from searx import searx_dir

1411

utils/fetch_currencies.py → searx_extra/update/update_currencies.pyrenamed−1

@@ -7,7 +7,6 @@ import json

77# set path

88from sys import path

99from os.path import realpath, dirname, join

10−path.append(realpath(dirname(realpath(__file__)) + '/../'))

1110

1211from searx import searx_dir, settings

1312from searx.engines.wikidata import send_wikidata_query

utils/fetch_engine_descriptions.py → searx_extra/update/update_engine_descriptions.pyrenamed+1−5

@@ -3,13 +3,9 @@

33import sys

44import json

55from urllib.parse import quote, urlparse

6−from os.path import realpath, dirname

7−import cld3

6+import detect_language

87from lxml.html import fromstring

10−# set path

11−sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))

12−

139from searx.engines.wikidata import send_wikidata_query

1410from searx.utils import extract_text

1511import searx

utils/fetch_external_bangs.py → searx_extra/update/update_external_bangs.pyrenamed+1−5

@@ -13,16 +13,12 @@ but most probably it will requires to update RE_BANG_VERSION

1313"""

1414# pylint: disable=C0116

1515

16−import sys

1716import json

1817import re

19−from os.path import realpath, dirname, join

18+from os.path import join

2019

2120import requests

2221

23−# set path

24−sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))

25−

2622from searx import searx_dir # pylint: disable=E0401 C0413

2723

2824

utils/fetch_firefox_version.py → searx_extra/update/update_firefox_version.pyrenamed+1−6

@@ -1,14 +1,9 @@

11#!/usr/bin/env python

3−# set path

4−from sys import path

5−from os.path import realpath, dirname, join

6−path.append(realpath(dirname(realpath(__file__)) + '/../'))

7−

8−#

93import json

104import requests

115import re

6+from os.path import dirname, join

127from urllib.parse import urlparse, urljoin

138from distutils.version import LooseVersion, StrictVersion

149from lxml import html

utils/fetch_languages.py → searx_extra/update/update_languages.pyrenamed+1−3

@@ -1,4 +1,4 @@

1−# -*- coding: utf-8 -*-

1+#!/usr/bin/env python

33# This script generates languages.py from intersecting each engine's supported languages.

44#

@@ -7,11 +7,9 @@

77import json

88from pathlib import Path

99from pprint import pformat

10−from sys import path

1110from babel import Locale, UnknownLocaleError

1211from babel.languages import get_global

1312

14−path.append('../searx') # noqa

1513from searx import settings, searx_dir

1614from searx.engines import initialize_engines, engines

1715

utils/update-translations.sh → searx_extra/update/update_translations.shrenamed

The content is the same.

utils/fetch_wikidata_units.py → searx_extra/update/update_wikidata_units.pyrenamed+1−3

@@ -4,9 +4,7 @@ import json

44import collections

66# set path

7−from sys import path

8−from os.path import realpath, dirname, join

9−path.append(realpath(dirname(realpath(__file__)) + '/../'))

7+from os.path import join

108

119from searx import searx_dir

1210from searx.engines.wikidata import send_wikidata_query

setup.py+1−1

@@ -41,7 +41,7 @@ setup(

4141 author='Adam Tauber',

4242 author_email='asciimoo@gmail.com',

4343 license='GNU Affero General Public License',

44− packages=find_packages(exclude=["tests*"]),

44+ packages=find_packages(exclude=["tests*", "searx_extra"]),

4545 zip_safe=False,

4646 install_requires=requirements,

4747 extras_require={

tests/unit/test_standalone_searx.py+1−19

@@ -1,7 +1,6 @@

11# -*- coding: utf-8 -*-

22"""Test utils/standalone_searx.py"""

33import datetime

4−import importlib.util

54import io

65import sys

@@ -10,16 +9,7 @@ from nose2.tools import params

109

1110from searx.search import SearchQuery, EngineRef, initialize

1211from searx.testing import SearxTestCase

13−

14−

15−def get_standalone_searx_module():

16− """Get standalone_searx module."""

17− module_name = 'utils.standalone_searx'

18− filename = 'utils/standalone_searx.py'

19− spec = importlib.util.spec_from_file_location(module_name, filename)

20− sas = importlib.util.module_from_spec(spec)

21− spec.loader.exec_module(sas)

22− return sas

12+from searx_extra import standalone_searx as sas

2313

2414

2515class StandaloneSearx(SearxTestCase):

@@ -33,7 +23,6 @@ class StandaloneSearx(SearxTestCase):

3323

3424 def test_parse_argument_no_args(self):

3525 """Test parse argument without args."""

36− sas = get_standalone_searx_module()

3726 with patch.object(sys, 'argv', ['standalone_searx']), \

3827 self.assertRaises(SystemExit):

3928 sys.stderr = io.StringIO()

@@ -42,7 +31,6 @@ class StandaloneSearx(SearxTestCase):

4231

4332 def test_parse_argument_basic_args(self):

4433 """Test parse argument with basic args."""

45− sas = get_standalone_searx_module()

4634 query = 'red box'

4735 exp_dict = {

4836 'query': query, 'category': 'general', 'lang': 'all', 'pageno': 1,

@@ -56,7 +44,6 @@ class StandaloneSearx(SearxTestCase):

5644

5745 def test_to_dict(self):

5846 """test to_dict."""

59− sas = get_standalone_searx_module()

6047 self.assertEqual(

6148 sas.to_dict(

6249 sas.get_search_query(sas.parse_argument(['red box']))),

@@ -72,7 +59,6 @@ class StandaloneSearx(SearxTestCase):

7259

7360 def test_to_dict_with_mock(self):

7461 """test to dict."""

75− sas = get_standalone_searx_module()

7662 with patch.object(sas.searx.search, 'Search') as mock_s:

7763 m_search = mock_s().search()

7864 m_sq = Mock()

@@ -97,7 +83,6 @@ class StandaloneSearx(SearxTestCase):

9783

9884 def test_get_search_query(self):

9985 """test get_search_query."""

100− sas = get_standalone_searx_module()

10186 args = sas.parse_argument(['rain', ])

10287 search_q = sas.get_search_query(args)

10388 self.assertTrue(search_q)

@@ -106,7 +91,6 @@ class StandaloneSearx(SearxTestCase):

10691

10792 def test_no_parsed_url(self):

10893 """test no_parsed_url func"""

109− sas = get_standalone_searx_module()

11094 self.assertEqual(

11195 sas.no_parsed_url([{'parsed_url': 'http://example.com'}]),

11296 [{}]

@@ -119,11 +103,9 @@ class StandaloneSearx(SearxTestCase):

119103 )

120104 def test_json_serial(self, arg, exp_res):

121105 """test json_serial func"""

122− sas = get_standalone_searx_module()

123106 self.assertEqual(sas.json_serial(arg), exp_res)

124107

125108 def test_json_serial_error(self):

126109 """test error on json_serial."""

127− sas = get_standalone_searx_module()

128110 with self.assertRaises(TypeError):

129111 sas.json_serial('a')