From 80d5479466ed6d9de3ffe4dfae1bbb9055793e82 Mon Sep 17 00:00:00 2001 From: nicolargo Date: Wed, 23 Dec 2020 10:56:02 +0100 Subject: [PATCH 1/2] First version of the new influxdb2 exporter --- conf/glances.conf | 25 +++++ docs/gw/influxdb.rst | 47 ++++++++++ glances/exports/glances_influxdb.py | 2 +- glances/exports/glances_influxdb2.py | 135 +++++++++++++++++++++++++++ 4 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 glances/exports/glances_influxdb2.py diff --git a/conf/glances.conf b/conf/glances.conf index 587a2b26..2df9783d 100644 --- a/conf/glances.conf +++ b/conf/glances.conf @@ -398,6 +398,10 @@ height=600 style=DarkStyle [influxdb] +# !!! +# Will be DEPRECATED in future release. +# Please have a look on the new influxdb2 export module (compatible with InfluxDB 1.8.x and 2.x) +# !!! # Configuration for the --export influxdb option # https://influxdb.com/ host=localhost @@ -418,6 +422,27 @@ prefix=localhost # You can also use dynamic values #tags=system:`uname -s` +[influxdb2] +# Configuration for the --export influxdb2 option +# https://influxdb.com/ +host=localhost +port=8086 +protocol=http +org=nicolargo +bucket=glances +token=EjFUTWe8U-MIseEAkaVIgVnej_TrnbdvEcRkaB1imstW7gapSqy6_6-8XD-yd51V0zUUpDy-kAdVD1purDLuxA== +# Prefix will be added for all measurement name +# Ex: prefix=foo +# => foo.cpu +# => foo.mem +# You can also use dynamic values +#prefix=`hostname` +prefix=localhost +# Tags will be added for all measurements +#tags=foo:bar,spam:eggs +# You can also use dynamic values +#tags=system:`uname -s` + [cassandra] # Configuration for the --export cassandra option # Also works for the ScyllaDB diff --git a/docs/gw/influxdb.rst b/docs/gw/influxdb.rst index 43db9df9..e931c70c 100644 --- a/docs/gw/influxdb.rst +++ b/docs/gw/influxdb.rst @@ -4,6 +4,10 @@ InfluxDB ======== You can export statistics to an ``InfluxDB`` server (time series server). + +InfluxDB (up to version 1.7.x) +------------------------------ + The connection should be defined in the Glances configuration file as following: @@ -16,6 +20,13 @@ following: user=root password=root db=glances + # Prefix will be added for all measurement name + # Ex: prefix=foo + # => foo.cpu + # => foo.mem + # You can also use dynamic values + #prefix=`hostname` + prefix=localhost # Tags will be added for all measurements #tags=foo:bar,spam:eggs # You can also use dynamic values @@ -33,6 +44,42 @@ configuration file (no limit on columns number). Note: if you want to use SSL, please set 'protocol=https'. + +InfluxDB v2 (from InfluxDB v1.8.x/Flux and InfluxDB v2.x) +--------------------------------------------------------- + +The connection should be defined in the Glances configuration file as +following: + +.. code-block:: ini + + [influxdb] + host=localhost + port=8086 + protocol=http + org=nicolargo + bucket=glances + token=EjFUTWe8U-MIseEAkaVIgVnej_TrnbdvEcRkaB1imstW7gapSqy6_6-8XD-yd51V0zUUpDy-kAdVD1purDLuxA== + # Prefix will be added for all measurement name + # Ex: prefix=foo + # => foo.cpu + # => foo.mem + # You can also use dynamic values + #prefix=`hostname` + prefix=localhost + # Tags will be added for all measurements + #tags=foo:bar,spam:eggs + # You can also use dynamic values + #tags=system:`uname -s` + +and run Glances with: + +.. code-block:: console + + $ glances --export influxdb2 + +Note: if you want to use SSL, please set 'protocol=https'. + Grafana ------- diff --git a/glances/exports/glances_influxdb.py b/glances/exports/glances_influxdb.py index 5df70832..2362e7fd 100644 --- a/glances/exports/glances_influxdb.py +++ b/glances/exports/glances_influxdb.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see . -"""InfluxDB interface class.""" +"""InfluxDB (up to InfluxDB 1.7.x) interface class.""" import sys diff --git a/glances/exports/glances_influxdb2.py b/glances/exports/glances_influxdb2.py new file mode 100644 index 00000000..267c5bc8 --- /dev/null +++ b/glances/exports/glances_influxdb2.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Glances. +# +# Copyright (C) 2020 Nicolargo +# +# Glances is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Glances is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +"""InfluxDB (from to InfluxDB 1.8+) interface class.""" + +import sys + +from glances.logger import logger +from glances.exports.glances_export import GlancesExport + +from influxdb_client import InfluxDBClient, WriteOptions + + +class Export(GlancesExport): + """This class manages the InfluxDB export module.""" + + def __init__(self, config=None, args=None): + """Init the InfluxDB export IF.""" + super(Export, self).__init__(config=config, args=args) + + # Mandatories configuration keys (additional to host and port) + self.org = None + self.bucket = None + self.token = None + + # Optionals configuration keys + self.protocol = 'http' + self.prefix = None + self.tags = None + + # Load the InfluxDB configuration file + self.export_enable = self.load_conf('influxdb2', + mandatories=['host', 'port', + 'user', 'password', + 'org', 'bucket', 'token'], + options=['protocol', + 'prefix', + 'tags']) + if not self.export_enable: + sys.exit(2) + + # Init the InfluxDB client + self.client = self.init() + + def init(self): + """Init the connection to the InfluxDB server.""" + if not self.export_enable: + return None + + url = '{}://{}:{}'.format(self.protocol, self.host, self.port) + try: + client = InfluxDBClient(url=url, + enable_gzip=False, + org=self.org, + token=self.token) + except Exception as e: + logger.critical("Cannot connect to InfluxDB server '%s' (%s)" % (url, e)) + sys.exit(2) + else: + logger.info("Connected to InfluxDB server ({})".format(client.health())) + + + # Create the write client + write_client = client.write_api(write_options=WriteOptions(batch_size=500, + flush_interval=10_000, + jitter_interval=2_000, + retry_interval=5_000, + max_retries=5, + max_retry_delay=30_000, + exponential_base=2)) + return write_client + + def _normalize(self, name, columns, points): + """Normalize data for the InfluxDB's data model.""" + + for i, _ in enumerate(points): + # Supported type: + # https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/ + if points[i] is None: + # Ignore points with None value + del(points[i]) + del(columns[i]) + continue + try: + points[i] = float(points[i]) + except (TypeError, ValueError): + pass + else: + continue + try: + points[i] = str(points[i]) + except (TypeError, ValueError): + pass + else: + continue + + return [{'measurement': name, + 'tags': self.parse_tags(self.tags), + 'fields': dict(zip(columns, points))}] + + def export(self, name, columns, points): + """Write the points to the InfluxDB server.""" + # Manage prefix + if self.prefix is not None: + name = self.prefix + '.' + name + # Write input to the InfluxDB database + if len(points) == 0: + logger.debug("Cannot export empty {} stats to InfluxDB".format(name)) + else: + try: + self.client.write(self.bucket, + self.org, + self._normalize(name, columns, points), + time_precision="s") + except Exception as e: + # Log level set to debug instead of error (see: issue #1561) + logger.debug("Cannot export {} stats to InfluxDB ({})".format(name, e)) + else: + logger.debug("Export {} stats to InfluxDB".format(name)) From 6ce88cffd3bb0cf8cd6d1757977ea6f9a8b63f7e Mon Sep 17 00:00:00 2001 From: nicolargo Date: Thu, 24 Dec 2020 09:49:45 +0100 Subject: [PATCH 2/2] Improve log message --- docs/gw/influxdb.rst | 3 +++ glances/exports/glances_influxdb2.py | 11 ++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/gw/influxdb.rst b/docs/gw/influxdb.rst index e931c70c..b8717c48 100644 --- a/docs/gw/influxdb.rst +++ b/docs/gw/influxdb.rst @@ -48,6 +48,9 @@ Note: if you want to use SSL, please set 'protocol=https'. InfluxDB v2 (from InfluxDB v1.8.x/Flux and InfluxDB v2.x) --------------------------------------------------------- +Note: The InfluxDB v2 client (https://pypi.org/project/influxdb-client/) +is only available for Python 3.6 or higher. + The connection should be defined in the Glances configuration file as following: diff --git a/glances/exports/glances_influxdb2.py b/glances/exports/glances_influxdb2.py index 267c5bc8..1e619642 100644 --- a/glances/exports/glances_influxdb2.py +++ b/glances/exports/glances_influxdb2.py @@ -73,16 +73,17 @@ class Export(GlancesExport): logger.critical("Cannot connect to InfluxDB server '%s' (%s)" % (url, e)) sys.exit(2) else: - logger.info("Connected to InfluxDB server ({})".format(client.health())) + logger.info("Connected to InfluxDB server version {} ({})".format(client.health().version, + client.health().message)) # Create the write client write_client = client.write_api(write_options=WriteOptions(batch_size=500, - flush_interval=10_000, - jitter_interval=2_000, - retry_interval=5_000, + flush_interval=10000, + jitter_interval=2000, + retry_interval=5000, max_retries=5, - max_retry_delay=30_000, + max_retry_delay=30000, exponential_base=2)) return write_client