New plugin to scan remote Web sites (URL) (issue #981)

NEWS

@@ -7,6 +7,7 @@ Version 2.9.2
 
 Enhancements and new features:
 
+    * New plugin to scan remote Web sites (URL) (issue #981) 
     * Use -> and <- arrows keys to switch between processing sort (issue #1075)
     * Add trends in the Curses interface (issue #1077)
 

conf/glances.conf

@@ -217,12 +217,14 @@ refresh=30
 timeout=3
 # If port_default_gateway is True, add the default gateway on top of the scan list
 port_default_gateway=True
+#
 # Define the scan list (1 < x < 255)
 # port_x_host (name or IP) is mandatory
 # port_x_port (TCP port number) is optional (if not set, use ICMP)
 # port_x_description is optional (if not set, define to host:port)
 # port_x_timeout is optional and overwrite the default timeout value
 # port_x_rtt_warning is optional and defines the warning threshold in ms
+#
 #port_1_host=192.168.0.1
 #port_1_port=80
 #port_1_description=Home Box
@@ -232,10 +234,25 @@ port_default_gateway=True
 #port_3_host=www.google.com
 #port_3_description=Internet ICMP
 #port_3_rtt_warning=1000
-#port_4_host=www.google.com
 #port_4_description=Internet Web
+#port_4_host=www.google.com
 #port_4_port=80
 #port_4_rtt_warning=1000
+#
+# Define Web (URL) monitoring list (1 < x < 255)
+# web_x_url is the URL to monitor (example: http://my.site.com/folder)
+# web_x_description is optional (if not set, define to URL)
+# web_x_timeout is optional and overwrite the default timeout value
+# web_x_rtt_warning is optional and defines the warning respond time in ms (approximatively)
+#
+#web_1_url=https://blog.nicolargo.com
+#web_1_description=My Blog
+#web_1_rtt_warning=3000
+#web_2_url=https://github.com
+#web_3_url=http://www.google.fr
+#web_3_description=Google Fr
+#web_4_url=https://blog.nicolargo.com/nonexist
+#web_4_description=Intranet
 
 [docker]
 # Thresholds for CPU and MEM (in %)

docs/aoa/ports.rst

@@ -7,9 +7,9 @@ Ports
 
 .. image:: ../_static/ports.png
 
-This plugin aims at providing a list of hosts/port to scan.
+This plugin aims at providing a list of hosts/port and URL to scan.
 
-You can define ``ICMP`` or ``TCP`` ports scan.
+You can define ``ICMP`` or ``TCP`` ports scans and URL (head only) check.
 
 The list should be defined in the ``[ports]`` section of the Glances
 configuration file.
@@ -24,12 +24,14 @@ configuration file.
     timeout=3
     # If port_default_gateway is True, add the default gateway on top of the scan list
     port_default_gateway=True
+    #
     # Define the scan list (1 < x < 255)
     # port_x_host (name or IP) is mandatory
     # port_x_port (TCP port number) is optional (if not set, use ICMP)
     # port_x_description is optional (if not set, define to host:port)
     # port_x_timeout is optional and overwrite the default timeout value
     # port_x_rtt_warning is optional and defines the warning threshold in ms
+    #
     port_1_host=192.168.0.1
     port_1_port=80
     port_1_description=Home Box
@@ -43,3 +45,16 @@ configuration file.
     port_4_description=Internet Web
     port_4_port=80
     port_4_rtt_warning=1000
+    #
+    # Define Web (URL) monitoring list (1 < x < 255)
+    # web_x_url is the URL to monitor (example: http://my.site.com/folder)
+    # web_x_description is optional (if not set, define to URL)
+    # web_x_timeout is optional and overwrite the default timeout value
+    # web_x_rtt_warning is optional and defines the warning respond time in ms (approximatively)
+    #
+    web_1_url=https://blog.nicolargo.com
+    web_1_description=My Blog
+    web_1_rtt_warning=3000
+    web_2_url=https://github.com
+    web_3_url=http://www.google.fr
+    web_3_description=Google Fr

glances/compat.py

@@ -36,6 +36,7 @@ if PY3:
     from urllib.request import urlopen
     from urllib.error import HTTPError, URLError
     from statistics import mean
+    from urllib.parse import urlparse
 
     input = input
     range = range
@@ -91,6 +92,7 @@ else:
     from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler, SimpleXMLRPCServer
     from xmlrpclib import Fault, ProtocolError, ServerProxy, Transport, Server
     from urllib2 import urlopen, HTTPError, URLError
+    from urlparse import urlparse
 
     input = raw_input
     range = xrange

glances/plugins/glances_ports.py

@@ -24,9 +24,17 @@ import subprocess
 import threading
 import socket
 import time
+import numbers
+
+try:
+    import requests
+    requests_tag = True
+except ImportError:
+    requests_tag = False
 
 from glances.globals import WINDOWS
 from glances.ports_list import GlancesPortsList
+from glances.web_list import GlancesWebList
 from glances.timer import Timer, Counter
 from glances.compat import bool_type
 from glances.logger import logger
@@ -47,7 +55,7 @@ class Plugin(GlancesPlugin):
         self.display_curse = True
 
         # Init stats
-        self.stats = GlancesPortsList(config=config, args=args).get_ports_list()
+        self.stats = GlancesPortsList(config=config, args=args).get_ports_list() + GlancesWebList(config=config, args=args).get_web_list()
 
         # Init global Timer
         self.timer_ports = Timer(0)
@@ -93,7 +101,7 @@ class Plugin(GlancesPlugin):
 
         return self.stats
 
-    def get_alert(self, port, header="", log=False):
+    def get_ports_alert(self, port, header="", log=False):
         """Return the alert status relative to the port scan return value."""
 
         if port['status'] is None:
@@ -107,6 +115,18 @@ class Plugin(GlancesPlugin):
 
         return 'OK'
 
+    def get_web_alert(self, web, header="", log=False):
+        """Return the alert status relative to the web/url scan return value."""
+
+        if web['status'] is None:
+            return 'CAREFUL'
+        elif web['status'] not in [200, 301, 302]:
+            return 'CRITICAL'
+        elif web['rtt_warning'] is not None and web['elapsed'] > web['rtt_warning']:
+            return 'WARNING'
+
+        return 'OK'
+
     def msg_curse(self, args=None, max_width=None):
         """Return the dict to display in the curse interface."""
         # Init the return message
@@ -118,23 +138,36 @@ class Plugin(GlancesPlugin):
 
         # Build the string message
         for p in self.stats:
-            if p['host'] is None:
-                status = 'None'
-            elif p['status'] is None:
-                status = 'Scanning'
-            elif isinstance(p['status'], bool_type) and p['status'] is True:
-                status = 'Open'
-            elif p['status'] == 0:
-                status = 'Timeout'
-            else:
-                # Convert second to ms
-                status = '{0:.0f}ms'.format(p['status'] * 1000.0)
+            if 'host' in p:
+                if p['host'] is None:
+                    status = 'None'
+                elif p['status'] is None:
+                    status = 'Scanning'
+                elif isinstance(p['status'], bool_type) and p['status'] is True:
+                    status = 'Open'
+                elif p['status'] == 0:
+                    status = 'Timeout'
+                else:
+                    # Convert second to ms
+                    status = '{0:.0f}ms'.format(p['status'] * 1000.0)
 
-            msg = '{:14.14} '.format(p['description'])
-            ret.append(self.curse_add_line(msg))
-            msg = '{:>8}'.format(status)
-            ret.append(self.curse_add_line(msg, self.get_alert(p)))
-            ret.append(self.curse_new_line())
+                msg = '{:14.14} '.format(p['description'])
+                ret.append(self.curse_add_line(msg))
+                msg = '{:>8}'.format(status)
+                ret.append(self.curse_add_line(msg, self.get_ports_alert(p)))
+                ret.append(self.curse_new_line())
+            elif 'url' in p:
+                msg = '{:14.14} '.format(p['description'])
+                ret.append(self.curse_add_line(msg))
+                if isinstance(p['status'], numbers.Number):
+                    status = 'Code {}'.format(p['status'])
+                elif p['status'] is None:
+                    status = 'Scanning'
+                else:
+                    status = p['status']
+                msg = '{:>8}'.format(status)
+                ret.append(self.curse_add_line(msg, self.get_web_alert(p)))
+                ret.append(self.curse_new_line())
 
         # Delete the last empty line
         try:
@@ -144,18 +177,10 @@ class Plugin(GlancesPlugin):
 
         return ret
 
-    def _port_scan_all(self, stats):
-        """Scan all host/port of the given stats"""
-        for p in stats:
-            self._port_scan(p)
-            # Had to wait between two scans
-            # If not, result are not ok
-            time.sleep(1)
-
 
 class ThreadScanner(threading.Thread):
     """
-    Specific thread for the port scanner.
+    Specific thread for the port/web scanner.
 
     stats is a list of dict
     """
@@ -176,12 +201,18 @@ class ThreadScanner(threading.Thread):
         Infinite loop, should be stopped by calling the stop() method"""
 
         for p in self._stats:
-            self._port_scan(p)
+            # End of the thread has been asked
             if self.stopped():
                 break
-            # Had to wait between two scans
-            # If not, result are not ok
-            time.sleep(1)
+            # Scan a port (ICMP or TCP)
+            if 'port' in p:
+                self._port_scan(p)
+                # Had to wait between two scans
+                # If not, result are not ok
+                time.sleep(1)
+            # Scan an URL
+            elif 'url' in p and requests_tag:
+                self._web_scan(p)
 
     @property
     def stats(self):
@@ -202,6 +233,21 @@ class ThreadScanner(threading.Thread):
         """Return True is the thread is stopped"""
         return self._stopper.isSet()
 
+    def _web_scan(self, web):
+        """Scan the  Web/URL (dict) and update the status key"""
+        try:
+            req = requests.head(web['url'],
+                                allow_redirects=True,
+                                timeout=web['timeout'])
+        except Exception as e:
+            logger.debug(e)
+            web['status'] = 'Error'
+            web['elapsed'] = 0
+        else:
+            web['status'] = req.status_code
+            web['elapsed'] = req.elapsed.total_seconds()
+        return web
+
     def _port_scan(self, port):
         """Scan the port structure (dict) and update the status key"""
         if int(port['port']) == 0:

glances/web_list.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Glances.
+#
+# Copyright (C) 2017 Nicolargo <nicolas@nicolargo.com>
+#
+# Glances is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Glances is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Manage the Glances web/url list (Ports plugin)."""
+
+from glances.compat import range, urlparse
+from glances.logger import logger
+
+
+class GlancesWebList(object):
+
+    """Manage the Web/Url list for the ports plugin."""
+
+    _section = "ports"
+    _default_refresh = 60
+    _default_timeout = 3
+
+    def __init__(self, config=None, args=None):
+        # web_list is a list of dict (JSON compliant)
+        # [ {'url': 'http://blog.nicolargo.com',
+        #    'refresh': 30,
+        #    'description': 'My blog',
+        #    'status': 404} ... ]
+        # Load the configuration file
+        self._web_list = self.load(config)
+
+    def load(self, config):
+        """Load the web list from the configuration file."""
+        web_list = []
+
+        if config is None:
+            logger.debug("No configuration file available. Cannot load ports list.")
+        elif not config.has_section(self._section):
+            logger.debug("No [%s] section in the configuration file. Cannot load ports list." % self._section)
+        else:
+            logger.debug("Start reading the [%s] section in the configuration file" % self._section)
+
+            refresh = int(config.get_value(self._section, 'refresh', default=self._default_refresh))
+            timeout = int(config.get_value(self._section, 'timeout', default=self._default_timeout))
+
+            # Read the web/url list
+            for i in range(1, 256):
+                new_web = {}
+                postfix = 'web_%s_' % str(i)
+
+                # Read mandatories configuration key: host
+                new_web['url'] = config.get_value(self._section, '%s%s' % (postfix, 'url'))
+                if new_web['url'] is None:
+                    continue
+                url_parse = urlparse(new_web['url'])
+                if not bool(url_parse.scheme) or not bool(url_parse.netloc):
+                    logger.error('Bad URL (%s) in the [%s] section of configuration file.' % (new_web['url'],
+                                                                                              self._section))
+                    continue
+
+                # Read optionals configuration keys
+                # Default description is the URL without the http://
+                new_web['description'] = config.get_value(self._section,
+                                                          '%sdescription' % postfix,
+                                                          default="%s" % url_parse.netloc)
+
+                # Default status
+                new_web['status'] = None
+                new_web['elapsed'] = 0
+
+                # Refresh rate in second
+                new_web['refresh'] = refresh
+
+                # Timeout in second
+                new_web['timeout'] = int(config.get_value(self._section,
+                                                          '%stimeout' % postfix,
+                                                          default=timeout))
+
+                # RTT warning
+                new_web['rtt_warning'] = config.get_value(self._section,
+                                                          '%srtt_warning' % postfix,
+                                                          default=None)
+                if new_web['rtt_warning'] is not None:
+                    # Convert to second
+                    new_web['rtt_warning'] = int(new_web['rtt_warning']) / 1000.0
+
+                # Add the server to the list
+                logger.debug("Add Web URL %s to the static list" % new_web['url'])
+                web_list.append(new_web)
+
+            # Ports list loaded
+            logger.debug("Web list loaded: %s" % web_list)
+
+        return web_list
+
+    def get_web_list(self):
+        """Return the current server list (dict of dict)."""
+        return self._web_list
+
+    def set_server(self, pos, key, value):
+        """Set the key to the value for the pos (position in the list)."""
+        self._web_list[pos][key] = value