diff --git a/glances/core/glances_logging.py b/glances/core/glances_logging.py index b5e604bb..633bbd16 100644 --- a/glances/core/glances_logging.py +++ b/glances/core/glances_logging.py @@ -43,6 +43,9 @@ LOGGING_CFG = { }, 'short': { 'format': '%(levelname)s: %(message)s' + }, + 'free': { + 'format': '%(message)s' } }, 'handlers': { @@ -56,7 +59,7 @@ LOGGING_CFG = { 'console': { 'level': 'CRITICAL', 'class': 'logging.StreamHandler', - 'formatter': 'short' + 'formatter': 'free' } }, 'loggers': { diff --git a/glances/core/glances_stats.py b/glances/core/glances_stats.py index bd07d290..9372723a 100644 --- a/glances/core/glances_stats.py +++ b/glances/core/glances_stats.py @@ -225,9 +225,12 @@ class GlancesStats(object): def end(self): """End of the Glances stats.""" - # Close the export module + # Close export modules for e in self._exports: self._exports[e].exit() + # Close plugins + for p in self._plugins: + self._plugins[p].exit() class GlancesStatsServer(GlancesStats): diff --git a/glances/plugins/glances_docker.py b/glances/plugins/glances_docker.py index fb6c9fda..995a6ce7 100644 --- a/glances/plugins/glances_docker.py +++ b/glances/plugins/glances_docker.py @@ -19,14 +19,16 @@ """Docker plugin.""" -import numbers import os import re +import threading +import time # Import Glances libs from glances.core.glances_logging import logger from glances.core.glances_timer import getTimeSinceLastUpdate from glances.plugins.glances_plugin import GlancesPlugin +from glances.core.glances_timer import Timer # Docker-py library (optional and Linux-only) # https://github.com/docker/docker-py @@ -60,6 +62,17 @@ class Plugin(GlancesPlugin): # Init the Docker API self.docker_client = False + # Dict of thread (to grab stats asynchroniously, one thread is created by container) + # key: Container Id + # value: instance of ThreadDockerGrabber + self.thread_list = {} + + def exit(self): + """Overwrite the exit method to close threads""" + logger.debug("Stop the Docker plugin") + for t in self.thread_list.values(): + t.stop() + def get_key(self): """Return the key of the list.""" return 'name' @@ -148,6 +161,8 @@ class Plugin(GlancesPlugin): if self.input_method == 'local': # Update stats + + # Docker version # Exemple: { # "KernelVersion": "3.16.4-tinycore64", # "Arch": "amd64", @@ -161,7 +176,10 @@ class Plugin(GlancesPlugin): self.stats['version'] = self.docker_client.version() except Exception as e: # Correct issue#649 - logger.error("Plugin can not get Docker version ({})".format(e)) + logger.error("{} plugin - Can not get Docker version ({})".format(self.plugin_name, e)) + return self.stats + + # Container globals information # Example: [{u'Status': u'Up 36 seconds', # u'Created': 1420378904, # u'Image': u'nginx:1', @@ -170,42 +188,44 @@ class Plugin(GlancesPlugin): # u'Command': u"nginx -g 'daemon off;'", # u'Names': [u'/webstack_nginx_1'], # u'Id': u'b0da859e84eb4019cf1d965b15e9323006e510352c402d2f442ea632d61faaa5'}] - self.stats['containers'] = self.docker_client.containers() + + # Update current containers list + try: + self.stats['containers'] = self.docker_client.containers() + except Exception as e: + logger.error("{} plugin - Can not get containers list ({})".format(self.plugin_name, e)) + return self.stats + + # Start new thread for new container + for container in self.stats['containers']: + if container['Id'] not in self.thread_list: + # Thread did not exist in the internal dict + # Create it and add it to the internal dict + logger.debug("{} plugin - Create thread for container {}".format(self.plugin_name, container['Id'][:12])) + t = ThreadDockerGrabber(self.docker_client, container['Id'], self.args.time) + self.thread_list[container['Id']] = t + t.start() + + # Stop threads for non-existing containers + nonexisting_containers = list(set(self.thread_list.keys()) - set([c['Id'] for c in self.stats['containers']])) + for container_id in nonexisting_containers: + # Stop the thread + logger.debug("{} plugin - Stop thread for old container {}".format(self.plugin_name, container_id[:12])) + self.thread_list[container_id].stop() + # Delete the item from the dict + del(self.thread_list[container_id]) + # Get stats for all containers - for c in self.stats['containers']: - if not hasattr(self, 'docker_stats'): - # Create a dict with all the containers' stats instance - self.docker_stats = {} - - # TODO: Find a way to correct this - # The following optimization is not compatible with the network stats - # The self.docker_client.stats method should be call every time in order to have network stats refreshed - # Nevertheless, if we call it every time, Glances is slow... - if c['Id'] not in self.docker_stats: - # Create the stats instance for the current container - try: - self.docker_stats[c['Id']] = self.docker_client.stats(c['Id'], decode=True) - logger.debug("Create Docker stats object for container {0}".format(c['Id'])) - except Exception as e: - # Correct Issue #602 - logger.error("Cannot call Docker stats method {0}".format(e)) - - # Get the docker stats - try: - # self.docker_stats[c['Id']] = self.docker_client.stats(c['Id'], decode=True) - all_stats = self.docker_stats[c['Id']].next() - except Exception: - all_stats = {} - - c['cpu'] = self.get_docker_cpu(c['Id'], all_stats) - c['memory'] = self.get_docker_memory(c['Id'], all_stats) - # c['network'] = self.get_docker_network(c['Id'], all_stats) + for container in self.stats['containers']: + # The key is the container name and not the Id + container['key'] = self.get_key() # Export name (first name in the list, without the /) - c['name'] = c['Names'][0][1:] + container['name'] = container['Names'][0][1:] - # The key is the container name and not the Id - c['key'] = self.get_key() + container['cpu'] = self.get_docker_cpu(container['Id'], self.thread_list[container['Id']].stats) + container['memory'] = self.get_docker_memory(container['Id'], self.thread_list[container['Id']].stats) + container['network'] = self.get_docker_network(container['Id'], self.thread_list[container['Id']].stats) elif self.input_method == 'snmp': # Update stats using SNMP @@ -214,28 +234,6 @@ class Plugin(GlancesPlugin): return self.stats - def get_docker_cpu_old(self, container_id): - """Return the container CPU usage by reading /sys/fs/cgroup/. - - Input: id is the full container id - Output: a dict {'total': 1.49, 'user': 0.65, 'system': 0.84} - """ - ret = {} - # Read the stats - try: - with open('/sys/fs/cgroup/cpuacct/docker/' + container_id + '/cpuacct.stat', 'r') as f: - for line in f: - m = re.search(r"(system|user)\s+(\d+)", line) - if m: - ret[m.group(1)] = int(m.group(2)) - except IOError as e: - logger.error("Can not grab container CPU stat ({0})".format(e)) - return ret - if isinstance(ret["system"], numbers.Number) and isinstance(ret["user"], numbers.Number): - ret["total"] = ret["system"] + ret["user"] - # Return the stats - return ret - def get_docker_cpu(self, container_id, all_stats): """Return the container CPU usage. @@ -257,13 +255,7 @@ class Plugin(GlancesPlugin): cpu_new['nb_core'] = len(all_stats['cpu_stats']['cpu_usage']['percpu_usage']) except KeyError as e: # all_stats do not have CPU information - logger.debug("Can not grab CPU usage for container {0} ({1}). Trying fallback method.".format(container_id, e)) - # Trying fallback to old grab method - ret = self.get_docker_cpu_old(container_id) - # Get the user ticks - ticks = self.get_user_ticks() - for k in ret.keys(): - ret[k] = float(ret[k]) / ticks + logger.debug("Can not grab CPU usage for container {0} ({1})".format(container_id, e)) else: # Previous CPU stats stored in the cpu_old variable if not hasattr(self, 'cpu_old'): @@ -292,26 +284,6 @@ class Plugin(GlancesPlugin): # Return the stats return ret - def get_docker_memory_old(self, container_id): - """Return the container MEMORY usage by reading /sys/fs/cgroup/. - - Input: id is the full container id - Output: a dict {'rss': 1015808, 'cache': 356352} - """ - ret = {} - # Read the stats - try: - with open('/sys/fs/cgroup/memory/docker/' + container_id + '/memory.stat', 'r') as f: - for line in f: - m = re.search(r"(rss|cache)\s+(\d+)", line) - if m: - ret[m.group(1)] = int(m.group(2)) - except IOError as e: - logger.error("Can not grab container MEM stat ({0})".format(e)) - return ret - # Return the stats - return ret - def get_docker_memory(self, container_id, all_stats): """Return the container MEMORY. @@ -328,9 +300,7 @@ class Plugin(GlancesPlugin): ret['max_usage'] = all_stats['memory_stats']['max_usage'] except KeyError as e: # all_stats do not have MEM information - logger.debug("Can not grab MEM usage for container {0} ({1}). Trying fallback method.".format(container_id, e)) - # Trying fallback to old grab method - ret = self.get_docker_memory_old(container_id) + logger.debug("Can not grab MEM usage for container {0} ({1})".format(container_id, e)) # Return the stats return ret @@ -417,10 +387,10 @@ class Plugin(GlancesPlugin): ret.append(self.curse_add_line(msg)) msg = '{0:>7}'.format('MEM') ret.append(self.curse_add_line(msg)) - # msg = '{0:>6}'.format('Rx/s') - # ret.append(self.curse_add_line(msg)) - # msg = '{0:>6}'.format('Tx/s') - # ret.append(self.curse_add_line(msg)) + msg = '{0:>6}'.format('Rx/s') + ret.append(self.curse_add_line(msg)) + msg = '{0:>6}'.format('Tx/s') + ret.append(self.curse_add_line(msg)) msg = ' {0:8}'.format('Command') ret.append(self.curse_add_line(msg)) # Data @@ -455,13 +425,13 @@ class Plugin(GlancesPlugin): msg = '{0:>7}'.format('?') ret.append(self.curse_add_line(msg)) # NET RX/TX - # for r in ['rx', 'tx']: - # try: - # value = self.auto_unit(int(container['network'][r] // container['network']['time_since_update'] * 8)) + "b" - # msg = '{0:>6}'.format(value) - # except KeyError: - # msg = '{0:>6}'.format('?') - # ret.append(self.curse_add_line(msg)) + for r in ['rx', 'tx']: + try: + value = self.auto_unit(int(container['network'][r] // container['network']['time_since_update'] * 8)) + "b" + msg = '{0:>6}'.format(value) + except KeyError: + msg = '{0:>6}'.format('?') + ret.append(self.curse_add_line(msg)) # Command msg = ' {0}'.format(container['Command']) ret.append(self.curse_add_line(msg)) @@ -474,3 +444,58 @@ class Plugin(GlancesPlugin): return 'CAREFUL' else: return 'OK' + +class ThreadDockerGrabber(threading.Thread): + """ + Specific thread to grab docker stats. + + stats is a dict + """ + + def __init__(self, docker_client, container_id, refresh_time=3): + """Init the class: + docker_client: instance of Docker-py client + container_id: Id of the container""" + logger.debug("docker plugin - Create thread for container {}".format(container_id[:12])) + super(ThreadDockerGrabber, self).__init__() + # Refresh time for sub thread + self._refresh_time = refresh_time + # Event needed to stop properly the thread + self._stop = threading.Event() + # The docker-py return stats as a stream + self._container_id = container_id + self._stats_stream = docker_client.stats(container_id, decode=True) + # The class return the stats as a dict + self._stats = {} + + def run(self): + """Function called to grab stats. + Infinite loop, should be stopped by calling the stop() method""" + + for i in self._stats_stream: + self._stats = i + # countdown = Timer(self._refresh_time) + # while not countdown.finished() and not is_stopped: + # is_stopped = self.stopped() + # time.sleep(0.1) + # if is_stopped: + # break + time.sleep(0.1) + if self.stopped(): + break + + @property + def stats(self): + return self._stats + + @stats.setter + def stats(self, value): + self._stats = value + + def stop(self, timeout=None): + logger.debug("docker plugin - Close thread for container {}".format(self._container_id[:12])) + self._stop.set() + super(ThreadDockerGrabber, self).join(timeout) + + def stopped(self): + return self._stop.isSet() diff --git a/glances/plugins/glances_plugin.py b/glances/plugins/glances_plugin.py index e23595ad..5380e1b8 100644 --- a/glances/plugins/glances_plugin.py +++ b/glances/plugins/glances_plugin.py @@ -71,6 +71,10 @@ class GlancesPlugin(object): # Init the views self.views = dict() + def exit(self): + """Method to be called when Glances exit""" + logger.debug("Stop the {} plugin".format(self.plugin_name)) + def __repr__(self): """Return the raw stats.""" return self.stats