From fb93c969356df0af0348779f88d8b7420aa46fe6 Mon Sep 17 00:00:00 2001 From: nicolargo Date: Sun, 21 Jan 2018 22:53:22 +0100 Subject: [PATCH] Ok but miss some extended stats (see line 348) --- glances/amps_list.py | 3 +- glances/globals.py | 1 + glances/logs.py | 2 +- glances/processes.py | 388 ++++++++++--------------------------------- 4 files changed, 95 insertions(+), 299 deletions(-) diff --git a/glances/amps_list.py b/glances/amps_list.py index e9eff323..3669da66 100644 --- a/glances/amps_list.py +++ b/glances/amps_list.py @@ -105,8 +105,7 @@ class AmpsList(object): def update(self): """Update the command result attributed.""" # Search application monitored processes by a regular expression - processlist = glances_processes.getalllist() - logger.info(processlist) + processlist = glances_processes.getlist() # Iter upon the AMPs dict for k, v in iteritems(self.get()): if not v.enable(): diff --git a/glances/globals.py b/glances/globals.py index f335d737..4edbdf6f 100644 --- a/glances/globals.py +++ b/glances/globals.py @@ -25,6 +25,7 @@ import sys # Operating system flag # Note: Somes libs depends of OS +# Note2: Included in PsUtil 4.0 or higher BSD = sys.platform.find('bsd') != -1 LINUX = sys.platform.startswith('linux') MACOS = sys.platform.startswith('darwin') diff --git a/glances/logs.py b/glances/logs.py index 780d9883..9f3d7e54 100644 --- a/glances/logs.py +++ b/glances/logs.py @@ -109,7 +109,7 @@ class GlancesLogs(object): If 'item' is not a 'new one', update the existing item. If event < peak_time the the alert is not setoff. """ - proc_list = proc_list or glances_processes.getalllist() + proc_list = proc_list or glances_processes.getlist() # Add or update the log item_index = self.__itemexist__(item_type) diff --git a/glances/processes.py b/glances/processes.py index 1fcde6fb..91cbc615 100644 --- a/glances/processes.py +++ b/glances/processes.py @@ -20,7 +20,7 @@ import operator import os -from glances.compat import iteritems, itervalues, listitems +from glances.compat import iteritems, itervalues, listitems, iterkeys from glances.globals import BSD, LINUX, MACOS, SUNOS, WINDOWS from glances.timer import Timer, getTimeSinceLastUpdate from glances.filter import GlancesFilter @@ -29,16 +29,6 @@ from glances.logger import logger import psutil -def is_kernel_thread(proc): - """Return True if proc is a kernel thread, False instead.""" - try: - return os.getpgid(proc.pid) == 0 - # Python >= 3.3 raises ProcessLookupError, which inherits OSError - except OSError: - # return False is process is dead - return False - - class GlancesProcesses(object): """Get processed stats using the psutil library.""" @@ -66,7 +56,6 @@ class GlancesProcesses(object): # Init stats self.auto_sort = True self._sort_key = 'cpu_percent' - self.allprocesslist = [] self.processlist = [] self.reset_processcount() @@ -94,12 +83,27 @@ class GlancesProcesses(object): self.reset_max_values() def reset_processcount(self): + """Reset the global process count""" self.processcount = {'total': 0, 'running': 0, 'sleeping': 0, 'thread': 0, 'pid_max': None} + def update_processcount(self, plist): + """Update the global process count from the current processes list""" + # Update the maximum process ID (pid) number + self.processcount['pid_max'] = self.pid_max + # For each key in the processcount dict + # count the number of processes with the same status + for k in iterkeys(self.processcount): + self.processcount[k] = len(list(filter(lambda v: v['status'] is k, + plist))) + # Compute thread + self.processcount['thread'] = sum(i['num_threads'] for i in plist) + # Compute total + self.processcount['total'] = len(plist) + def enable(self): """Enable process stats.""" self.disable_tag = False @@ -224,110 +228,6 @@ class GlancesProcesses(object): for k in self._max_values_list: self._max_values[k] = 0.0 - def __get_mandatory_stats(self, proc, procstat): - """ - Get mandatory_stats: for all processes. - Needed for the sorting/filter step. - - Stats grabbed inside this method: - * 'name', 'cpu_times', 'status', 'ppid' - * 'username', 'cpu_percent', 'memory_percent' - """ - procstat['mandatory_stats'] = True - - # Name, cpu_times, status and ppid stats are in the same /proc file - # Optimisation fir issue #958 - try: - procstat.update(proc.as_dict( - attrs=['name', 'cpu_times', 'status', 'ppid'], - ad_value='')) - except (psutil.NoSuchProcess, psutil.AccessDenied): - # Try/catch for issue #432 (process no longer exist) - # Try/catch for issue #1120 (only see on Macos) - return None - else: - procstat['status'] = str(procstat['status'])[:1].upper() - - try: - procstat.update(proc.as_dict( - attrs=['username', 'cpu_percent', 'memory_percent'], - ad_value='')) - except (psutil.NoSuchProcess, psutil.AccessDenied): - # Try/catch for issue #432 (process no longer exist) - return None - - if procstat['cpu_percent'] == '' or procstat['memory_percent'] == '': - # Do not display process if we cannot get the basic - # cpu_percent or memory_percent stats - return None - - # Compute the maximum value for cpu_percent and memory_percent - for k in self._max_values_list: - if procstat[k] > self.get_max_values(k): - self.set_max_values(k, procstat[k]) - - # Process command line (cached with internal cache) - if procstat['pid'] not in self.cmdline_cache: - # Patch for issue #391 - try: - self.cmdline_cache[procstat['pid']] = proc.cmdline() - except (AttributeError, EnvironmentError, UnicodeDecodeError, - psutil.AccessDenied, psutil.NoSuchProcess): - self.cmdline_cache[procstat['pid']] = "" - procstat['cmdline'] = self.cmdline_cache[procstat['pid']] - - # Process IO - # procstat['io_counters'] is a list: - # [read_bytes, write_bytes, read_bytes_old, write_bytes_old, io_tag] - # If io_tag = 0 > Access denied (display "?") - # If io_tag = 1 > No access denied (display the IO rate) - # Availability: all platforms except macOS and Illumos/Solaris - try: - # Get the process IO counters - proc_io = proc.io_counters() - io_new = [proc_io.read_bytes, proc_io.write_bytes] - except (psutil.AccessDenied, psutil.NoSuchProcess, NotImplementedError): - # Access denied to process IO (no root account) - # NoSuchProcess (process die between first and second grab) - # Put 0 in all values (for sort) and io_tag = 0 (for display) - procstat['io_counters'] = [0, 0] + [0, 0] - io_tag = 0 - except AttributeError: - return procstat - else: - # For IO rate computation - # Append saved IO r/w bytes - try: - procstat['io_counters'] = io_new + self.io_old[procstat['pid']] - except KeyError: - procstat['io_counters'] = io_new + [0, 0] - # then save the IO r/w bytes - self.io_old[procstat['pid']] = io_new - io_tag = 1 - - # Append the IO tag (for display) - procstat['io_counters'] += [io_tag] - - return procstat - - def __get_standard_stats(self, proc, procstat): - """ - Get standard_stats: only for displayed processes. - - Stats grabbed inside this method: - * nice and memory_info - """ - procstat['standard_stats'] = True - - # Process nice and memory_info (issue #926) - try: - procstat.update( - proc.as_dict(attrs=['nice', 'memory_info'])) - except psutil.NoSuchProcess: - pass - - return procstat - def __get_extended_stats(self, proc, procstat): """ Get extended stats, only for top processes (see issue #403). @@ -377,175 +277,12 @@ class GlancesProcesses(object): return procstat - def __get_process_stats(self, proc, - mandatory_stats=True, - standard_stats=True, - extended_stats=False): - """Get stats of a running processes.""" - # Process ID (always) - procstat = proc.as_dict(attrs=['pid']) - - if mandatory_stats: - procstat = self.__get_mandatory_stats(proc, procstat) - - if procstat is not None and standard_stats: - procstat = self.__get_standard_stats(proc, procstat) - - if procstat is not None and extended_stats and not self.disable_extended_tag: - procstat = self.__get_extended_stats(proc, procstat) - - return procstat - def update(self): """Update the processes stats.""" # Reset the stats self.processlist = [] self.reset_processcount() - # Do not process if disable tag is set - if self.disable_tag: - return - - # Get the time since last update - time_since_update = getTimeSinceLastUpdate('process_disk') - - # Reset the max dict - self.reset_max_values() - - # Update the maximum process ID (pid) number - self.processcount['pid_max'] = self.pid_max - - # Build an internal dict with only mandatories stats (sort keys) - processdict = {} - excluded_processes = set() - for proc in psutil.process_iter(): - # Ignore kernel threads if needed - if self.no_kernel_threads and not WINDOWS and is_kernel_thread(proc): - continue - - # If self.max_processes is None: Only retrieve mandatory stats - # Else: retrieve mandatory and standard stats - s = self.__get_process_stats(proc, - mandatory_stats=True, - standard_stats=self.max_processes is None) - # Check if s is note None (issue #879) - # ignore the 'idle' process on Windows and *BSD - # ignore the 'kernel_task' process on macOS - # waiting for upstream patch from psutil - if (s is None or - BSD and s['name'] == 'idle' or - WINDOWS and s['name'] == 'System Idle Process' or - MACOS and s['name'] == 'kernel_task'): - continue - # Continue to the next process if it has to be filtered - if self._filter.is_filtered(s): - excluded_processes.add(proc) - continue - - # Ok add the process to the list - processdict[proc] = s - # Update processcount (global statistics) - try: - self.processcount[str(proc.status())] += 1 - except KeyError: - # Key did not exist, create it - try: - self.processcount[str(proc.status())] = 1 - except psutil.NoSuchProcess: - pass - except psutil.NoSuchProcess: - pass - else: - self.processcount['total'] += 1 - # Update thread number (global statistics) - try: - self.processcount['thread'] += proc.num_threads() - except Exception: - pass - - if self._enable_tree: - self.process_tree = ProcessTreeNode.build_tree(processdict, - self.sort_key, - self.sort_reverse, - self.no_kernel_threads, - excluded_processes) - - for i, node in enumerate(self.process_tree): - # Only retreive stats for visible processes (max_processes) - if self.max_processes is not None and i >= self.max_processes: - break - - # add standard stats - new_stats = self.__get_process_stats(node.process, - mandatory_stats=False, - standard_stats=True, - extended_stats=False) - if new_stats is not None: - node.stats.update(new_stats) - - # Add a specific time_since_update stats for bitrate - node.stats['time_since_update'] = time_since_update - - else: - # Process optimization - # Only retreive stats for visible processes (max_processes) - if self.max_processes is not None: - # Sort the internal dict and cut the top N (Return a list of tuple) - # tuple=key (proc), dict (returned by __get_process_stats) - try: - processiter = sorted(iteritems(processdict), - key=lambda x: x[1][self.sort_key], - reverse=self.sort_reverse) - except (KeyError, TypeError) as e: - logger.error("Cannot sort process list by {}: {}".format(self.sort_key, e)) - logger.error('{}'.format(listitems(processdict)[0])) - # Fallback to all process (issue #423) - processloop = iteritems(processdict) - first = False - else: - processloop = processiter[0:self.max_processes] - first = True - else: - # Get all processes stats - processloop = iteritems(processdict) - first = False - - for i in processloop: - # Already existing mandatory stats - procstat = i[1] - if self.max_processes is not None: - # Update with standard stats - # and extended stats but only for TOP (first) process - s = self.__get_process_stats(i[0], - mandatory_stats=False, - standard_stats=True, - extended_stats=first) - if s is None: - continue - procstat.update(s) - # Add a specific time_since_update stats for bitrate - procstat['time_since_update'] = time_since_update - # Update process list - self.processlist.append(procstat) - # Next... - first = False - - # Build the all processes list used by the AMPs - self.allprocesslist = [p for p in itervalues(processdict)] - - # Clean internals caches if timeout is reached - if self.cache_timer.finished(): - self.username_cache = {} - self.cmdline_cache = {} - # Restart the timer - self.cache_timer.reset() - - def update_NEW(self): - """Update the processes stats.""" - # Reset the stats - self.processlist = [] - self.reset_processcount() - # Do not process if disable tag is set if self.disable_tag: return @@ -557,20 +294,84 @@ class GlancesProcesses(object): mandatories_attr = ['cmdline', 'cpu_percent', 'cpu_times', 'memory_info', 'memory_percent', 'name', 'nice', 'pid', - 'ppid', 'status', 'username'] - # io_counters is not available on macOS and Illumos/Solaris - if not MACOS and not SUNOS: + 'ppid', 'status', 'username', + 'status', 'num_threads', 'gids'] + # io_counters availability: Linux, BSD, Windows, AIX + if LINUX or BSD or WINDOWS: mandatories_attr += ['io_counters'] - # and build the processes stats list - self.processlist = [p.info for p in sorted(psutil.process_iter(attrs=mandatories_attr, - ad_value=None), - key=lambda p: p.info['cpu_percent'])] - # Update the maximum process ID (pid) number - self.processcount['pid_max'] = self.pid_max + # and build the processes stats list + self.processlist = [p.info for p in psutil.process_iter(attrs=mandatories_attr, + ad_value=None) + # OS specifics processes filter + if not (BSD and p.info['name'] == 'idle') and + not (WINDOWS and p.info['name'] == 'System Idle Process') and + not (MACOS and p.info['name'] == 'kernel_task') and + # Kernel threads filter + not (self.no_kernel_threads and LINUX and p.info['gids'].real == 0) and + # User filter + not (self._filter.is_filtered(p.info)) + ] + + # Sort the processes list by the current sort_key + self.processlist = sorted(self.processlist, + key=lambda p: p[self.sort_key], + reverse=True) + + # Update the processcount + self.update_processcount(self.processlist) # Loop over processes and add metadata + first = True for proc in self.processlist: + if first and not self.disable_extended_tag: + # Get extended stats, only for top processes (see issue #403). + # - cpu_affinity (Linux, Windows, FreeBSD) + # - ionice (Linux and Windows > Vista) + # - memory_full_info (Linux) + # - num_ctx_switches (not available on Illumos/Solaris) + # - num_fds (Unix-like) + # - num_handles (Windows) + # - num_threads (not available on *BSD) + # - memory_maps (only swap, Linux) + # https://www.cyberciti.biz/faq/linux-which-process-is-using-swap/ + # - connections (TCP and UDP) + extended = {} + try: + top_process = psutil.Process(proc['pid']) + extended_stats = ['cpu_affinity', 'ionice', + 'memory_full_info', 'num_ctx_switches', + 'num_fds', 'num_threads'] + if WINDOWS: + extended_stats += ['num_handles'] + extended = top_process.as_dict(attrs=extended_stats) + # !!! TODO + # if LINUX: + # try: + # procstat['memory_swap'] = sum([v.swap for v in proc.memory_maps()]) + # except psutil.NoSuchProcess: + # pass + # except (psutil.AccessDenied, TypeError, NotImplementedError): + # # NotImplementedError: /proc/${PID}/smaps file doesn't exist + # # on kernel < 2.6.14 or CONFIG_MMU kernel configuration option + # # is not enabled (see psutil #533/glances #413). + # # XXX: Remove TypeError once we'll drop psutil < 3.0.0. + # procstat['memory_swap'] = None + # try: + # procstat['tcp'] = len(proc.connections(kind="tcp")) + # procstat['udp'] = len(proc.connections(kind="udp")) + # except psutil.AccessDenied: + # procstat['tcp'] = None + # procstat['udp'] = None + except (psutil.NoSuchProcess, ValueError, AttributeError) as e: + logger.error('Can not grab extended stats ({})'.format(e)) + extended['extended_stats'] = False + else: + logger.debug('Grab extended stats for process {}'.format(proc['pid'])) + extended['extended_stats'] = True + proc.update(extended) + first = False + # Time since last update (for disk_io rate computation) proc['time_since_update'] = time_since_update @@ -598,23 +399,18 @@ class GlancesProcesses(object): else: proc['io_counters'] = [0, 0] + [0, 0] io_tag = 0 - # Append the IO tag (for display) proc['io_counters'] += [io_tag] - # Compute the maximum value for keys in self._max_values_list - # Compute max + # Compute the maximum value for keys in self._max_values_list (CPU, MEM) for k in self._max_values_list: - self.set_max_values(k, max(i[k] for i in self.processlist)) + if self.processlist != []: + self.set_max_values(k, max(i[k] for i in self.processlist)) def getcount(self): """Get the number of processes.""" return self.processcount - def getalllist(self): - """Get the allprocesslist.""" - return self.allprocesslist - def getlist(self, sortedby=None): """Get the processlist.""" return self.processlist