Merge pull request #2745 from nicolargo/2637-docker-memory-usage-is-incorrect

Make docker memory usage match `docker stats`

NEWS.rst

@@ -23,6 +23,7 @@ Under development: https://github.com/nicolargo/glances/issues?q=is%3Aopen+is%3A
         alias=sda1:InternalDisk,sdb1:ExternalDisk
 
     * Alert data model change from a list of list to a list of dict #2633
+    * Docker memory usage uses the same algorithm than docker stats #2637
 
 ===============
 Version 3.4.0.5

docs/aoa/containers.rst

@@ -14,6 +14,8 @@ You can install this dependency using:
 
 .. image:: ../_static/containers.png
 
+Note: Memory usage is compute as following "display memory usage = memory usage - inactive_file"
+
 It is possible to define limits and actions from the configuration file
 under the ``[containers]`` section:
 

glances/outputs/static/js/components/plugin-containers.vue

@@ -112,14 +112,21 @@ export default {
             const { sorter } = this;
             const containers = (this.stats || []).map(
                 (containerData) => {
-                    // prettier-ignore
+                    let memory_usage_no_cache = '?'
+                    if (containerData.memory.usage != undefined) {
+                        memory_usage_no_cache = containerData.memory.usage;
+                        if (containerData.memory.inactive_file != undefined) {
+                            memory_usage_no_cache = memory_usage_no_cache - containerData.memory.inactive_file;
+                        }
+                    }
+                
                     return {
                         'id': containerData.id,
                         'name': containerData.name,
                         'status': containerData.status,
                         'uptime': containerData.uptime,
                         'cpu_percent': containerData.cpu.total,
-                        'memory_usage': containerData.memory.usage != undefined ? containerData.memory.usage : '?',
+                        'memory_usage': memory_usage_no_cache,
                         'limit': containerData.memory.limit != undefined ? containerData.memory.limit : '?',
                         'io_rx': containerData.io_rx != undefined ? containerData.io_rx : '?',
                         'io_wx': containerData.io_wx != undefined ? containerData.io_wx : '?',

glances/plugins/containers/__init__.py

@@ -256,6 +256,11 @@ class PluginModel(GlancesPluginModel):
         """Return the user ticks by reading the environment variable."""
         return os.sysconf(os.sysconf_names['SC_CLK_TCK'])
 
+    def memory_usage_no_cache(self, mem):
+        """Return the 'real' memory usage by removing inactive_file to usage"""
+        # Ref: https://github.com/docker/docker-py/issues/3210
+        return mem['usage'] - (mem['inactive_file'] if 'inactive_file' in mem else 0)
+
     def update_views(self):
         """Update stats views."""
         # Call the father's method
@@ -281,11 +286,17 @@ class PluginModel(GlancesPluginModel):
             if 'memory' in i and 'usage' in i['memory']:
                 # Looking for specific MEM container threshold in the conf file
                 alert = self.get_alert(
-                    i['memory']['usage'], maximum=i['memory']['limit'], header=i['name'] + '_mem', action_key=i['name']
+                    self.memory_usage_no_cache(i['memory']),
+                    maximum=i['memory']['limit'],
+                    header=i['name'] + '_mem', action_key=i['name']
                 )
                 if alert == 'DEFAULT':
                     # Not found ? Get back to default MEM threshold value
-                    alert = self.get_alert(i['memory']['usage'], maximum=i['memory']['limit'], header='mem')
+                    alert = self.get_alert(
+                        self.memory_usage_no_cache(i['memory']),
+                        maximum=i['memory']['limit'],
+                        header='mem'
+                    )
                 self.views[i[self.get_key()]]['mem']['decoration'] = alert
 
         # Display Engine and Pod name ?
@@ -383,8 +394,8 @@ class PluginModel(GlancesPluginModel):
             ret.append(self.curse_add_line(msg, self.get_views(item=container['name'], key='cpu', option='decoration')))
             # MEM
             try:
-                msg = '{:>7}'.format(self.auto_unit(container['memory']['usage']))
-            except (KeyError, TypeError):
+                msg = '{:>7}'.format(self.auto_unit(self.memory_usage_no_cache(container['memory'])))
+            except KeyError:
                 msg = '{:>7}'.format('_')
             ret.append(self.curse_add_line(msg, self.get_views(item=container['name'], key='mem', option='decoration')))
             try:

glances/plugins/containers/engines/docker.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Glances.
 #
-# SPDX-FileCopyrightText: 2022 Nicolas Hennion <nicolas@nicolargo.com>
+# SPDX-FileCopyrightText: 2023 Nicolas Hennion <nicolas@nicolargo.com>
 #
 # SPDX-License-Identifier: LGPL-3.0-only
 #
@@ -29,7 +29,7 @@ else:
 
 
 class DockerStatsFetcher:
-    MANDATORY_MEMORY_FIELDS = ["usage", 'limit']
+    MANDATORY_MEMORY_FIELDS = ['usage', 'limit']
 
     def __init__(self, container):
         self._container = container
@@ -120,7 +120,9 @@ class DockerStatsFetcher:
     def _get_memory_stats(self):
         """Return the container MEMORY.
 
-        Output: a dict {'rss': 1015808, 'cache': 356352,  'usage': ..., 'max_usage': ...}
+        Output: a dict {'usage': ..., 'limit': ..., 'inactive_file': ...}
+
+        Note:the displayed memory usage is 'usage - inactive_file'
         """
         memory_stats = self._streamer.stats.get('memory_stats')
 
@@ -130,15 +132,10 @@ class DockerStatsFetcher:
             return None
 
         stats = {field: memory_stats[field] for field in self.MANDATORY_MEMORY_FIELDS}
-        try:
-            # Issue #1857 - Some stats are not always available in ['memory_stats']['stats']
-            detailed_stats = memory_stats['stats']
-            stats['rss'] = detailed_stats.get('rss') or detailed_stats.get('total_rss')
-            stats['max_usage'] = detailed_stats.get('max_usage')
-            stats['cache'] = detailed_stats.get('cache')
-        except (KeyError, TypeError) as e:
-            self._log_debug("Can't grab MEM usage", e)  # stats do not have MEM information
-            return None
+
+        # Optional field stats: inactive_file
+        if memory_stats.get('stats', {}).get('inactive_file') is not None:
+            stats['inactive_file'] = memory_stats['stats']['inactive_file']
 
         # Return the stats
         return stats

glances/plugins/containers/engines/podman.py

@@ -139,7 +139,7 @@ class PodmanPodStatsFetcher:
     def _get_memory_stats(self, stats):
         """Return the container MEMORY.
 
-        Output: a dict {'rss': 1015808, 'cache': 356352,  'usage': ..., 'max_usage': ...}
+        Output: a dict {'usage': ..., 'limit': ...}
         """
         if "MemUsage" not in stats or "/" not in stats["MemUsage"]:
             self._log_debug("Missing MEM usage fields")
@@ -155,7 +155,7 @@ class PodmanPodStatsFetcher:
             self._log_debug("Compute MEM usage failed", e)
             return None
 
-        return {"usage": usage, "limit": limit}
+        return {'usage': usage, 'limit': limit, 'inactive_file': 0}
 
     def _get_network_stats(self, stats):
         """Return the container network usage using the Docker API (v1.0 or higher).