cdc2f33 · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

Merge pull request #2074 from asciimoo/external-plugins

cdc2f33Noémi Ványi6 years ago

docs/dev/plugins.rst+7

compare with master

unifiedsplit

@@ -30,6 +30,13 @@ Example plugin

30 ctx['search'].suggestions.add('example')

31 return True

33+External plugins

34+================

35+

36+External plugins are standard python modules implementing all the requirements of the standard plugins.

37+Plugins can be enabled by adding them to :ref:`settings.yml`'s ``plugins`` section.

38+Example external plugin can be found `here <https://github.com/asciimoo/searx_external_plugin_example>`_.

39+

33Register your plugin

40Register your plugin

34====================

41====================

searx/__init__.py+4

@@ -30,6 +30,7 @@ except:

31searx_dir = abspath(dirname(__file__))

32engine_dir = dirname(realpath(__file__))

33+static_path = abspath(join(dirname(__file__), 'static'))

35def check_settings_yml(file_name):

36def check_settings_yml(file_name):

@@ -55,6 +56,9 @@ if not settings_path:

55with open(settings_path, 'r', encoding='utf-8') as settings_yaml:

56with open(settings_path, 'r', encoding='utf-8') as settings_yaml:

56 settings = safe_load(settings_yaml)

57 settings = safe_load(settings_yaml)

59+if settings['ui']['static_path']:

60+ static_path = settings['ui']['static_path']

61+

58'''

62'''

59enable debug if

63enable debug if

60the environnement variable SEARX_DEBUG is 1 or true

64the environnement variable SEARX_DEBUG is 1 or true

searx/plugins/__init__.py+94−3

@@ -14,8 +14,16 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

16'''

17+

18+from hashlib import sha256

19+from importlib import import_module

20+from os import listdir, makedirs, remove, stat, utime

21+from os.path import abspath, basename, dirname, exists, join

22+from shutil import copyfile

17−from sys import exit, version_info

23+from sys import version_info

24+from traceback import print_exc

25+

18−from searx import logger

26+from searx import logger, settings, static_path

20if version_info[0] == 3:

28if version_info[0] == 3:

21 unicode = str

29 unicode = str

@@ -54,7 +62,9 @@ class PluginStore():

54 for plugin in self.plugins:

62 for plugin in self.plugins:

55 yield plugin

63 yield plugin

57− def register(self, *plugins):

65+ def register(self, *plugins, external=False):

66+ if external:

67+ plugins = load_external_plugins(plugins)

58 for plugin in plugins:

68 for plugin in plugins:

59 for plugin_attr, plugin_attr_type in required_attrs:

69 for plugin_attr, plugin_attr_type in required_attrs:

60 if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):

70 if not hasattr(plugin, plugin_attr) or not isinstance(getattr(plugin, plugin_attr), plugin_attr_type):

@@ -77,6 +87,84 @@ class PluginStore():

77 return ret

87 return ret

90+def load_external_plugins(plugin_names):

91+ plugins = []

92+ for name in plugin_names:

93+ logger.debug('loading plugin: {0}'.format(name))

94+ try:

95+ pkg = import_module(name)

96+ except Exception as e:

97+ logger.critical('failed to load plugin module {0}: {1}'.format(name, e))

98+ exit(3)

99+

100+ pkg.__base_path = dirname(abspath(pkg.__file__))

101+

102+ prepare_package_resources(pkg, name)

103+

104+ plugins.append(pkg)

105+ logger.debug('plugin "{0}" loaded'.format(name))

106+ return plugins

107+

108+

109+def sync_resource(base_path, resource_path, name, target_dir, plugin_dir):

110+ dep_path = join(base_path, resource_path)

111+ file_name = basename(dep_path)

112+ resource_path = join(target_dir, file_name)

113+ if not exists(resource_path) or sha_sum(dep_path) != sha_sum(resource_path):

114+ try:

115+ copyfile(dep_path, resource_path)

116+ # copy atime_ns and mtime_ns, so the weak ETags (generated by

117+ # the HTTP server) do not change

118+ dep_stat = stat(dep_path)

119+ utime(resource_path, ns=(dep_stat.st_atime_ns, dep_stat.st_mtime_ns))

120+ except:

121+ logger.critical('failed to copy plugin resource {0} for plugin {1}'.format(file_name, name))

122+ exit(3)

123+

124+ # returning with the web path of the resource

125+ return join('plugins/external_plugins', plugin_dir, file_name)

126+

127+

128+def prepare_package_resources(pkg, name):

129+ plugin_dir = 'plugin_' + name

130+ target_dir = join(static_path, 'plugins/external_plugins', plugin_dir)

131+ try:

132+ makedirs(target_dir, exist_ok=True)

133+ except:

134+ logger.critical('failed to create resource directory {0} for plugin {1}'.format(target_dir, name))

135+ exit(3)

136+

137+ resources = []

138+

139+ if hasattr(pkg, 'js_dependencies'):

140+ resources.extend(map(basename, pkg.js_dependencies))

141+ pkg.js_dependencies = tuple([

142+ sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)

143+ for x in pkg.js_dependencies

144+ ])

145+ if hasattr(pkg, 'css_dependencies'):

146+ resources.extend(map(basename, pkg.css_dependencies))

147+ pkg.css_dependencies = tuple([

148+ sync_resource(pkg.__base_path, x, name, target_dir, plugin_dir)

149+ for x in pkg.css_dependencies

150+ ])

151+

152+ for f in listdir(target_dir):

153+ if basename(f) not in resources:

154+ resource_path = join(target_dir, basename(f))

155+ try:

156+ remove(resource_path)

157+ except:

158+ logger.critical('failed to remove unused resource file {0} for plugin {1}'.format(resource_path, name))

159+ exit(3)

160+

161+

162+def sha_sum(filename):

163+ with open(filename, "rb") as f:

164+ bytes = f.read()

165+ return sha256(bytes).hexdigest()

166+

167+

80plugins = PluginStore()

168plugins = PluginStore()

81plugins.register(oa_doi_rewrite)

169plugins.register(oa_doi_rewrite)

82plugins.register(https_rewrite)

170plugins.register(https_rewrite)

@@ -86,3 +174,6 @@ plugins.register(self_info)

86plugins.register(search_on_category_select)

174plugins.register(search_on_category_select)

87plugins.register(tracker_url_remover)

175plugins.register(tracker_url_remover)

88plugins.register(vim_hotkeys)

176plugins.register(vim_hotkeys)

177+# load external plugins

178+if 'plugins' in settings:

179+ plugins.register(*settings['plugins'], external=True)

searx/settings.yml+8

@@ -57,6 +57,14 @@ outgoing: # communication with search engines

57# - 1.1.1.1

58# - 1.1.1.2

60+# External plugin configuration

61+# See http://asciimoo.github.io/searx/dev/plugins.html for more details

62+#

63+# plugins:

64+# - plugin1

65+# - plugin2

66+# - ...

67+

60engines:

68engines:

61 - name: apk mirror

69 - name: apk mirror

62 engine: apkmirror

70 engine: apkmirror

searx/static/plugins/external_plugins/.gitignorenew file+3

@@ -0,0 +1,3 @@

1+*

2+*/

3+!.gitignore

searx/webapp.py+1−1

@@ -58,7 +58,7 @@ import flask_babel

58from flask_babel import Babel, gettext, format_date, format_decimal

59from flask.ctx import has_request_context

60from flask.json import jsonify

61−from searx import brand

61+from searx import brand, static_path

62from searx import settings, searx_dir, searx_debug

63from searx.exceptions import SearxParameterException

64from searx.engines import (