check, and limit, memory usage of tests (#6964)

.github/workflows/build-test-ubuntu-core-full_node.yml

@@ -97,9 +97,14 @@ jobs:
     - name: Install developer requirements
       run: |
         . ./activate
-        venv/bin/python -m pip install pytest pytest-asyncio pytest-xdist
+        venv/bin/python -m pip install pytest pytest-asyncio pytest-xdist pytest-monitor
 
     - name: Test core-full_node code with pytest
       run: |
         . ./activate
         ./venv/bin/py.test tests/core/full_node/test_address_manager.py tests/core/full_node/test_block_store.py tests/core/full_node/test_coin_store.py tests/core/full_node/test_full_node.py tests/core/full_node/test_full_node_store.py tests/core/full_node/test_initial_freeze.py tests/core/full_node/test_mempool.py tests/core/full_node/test_mempool_performance.py tests/core/full_node/test_node_load.py tests/core/full_node/test_sync_store.py tests/core/full_node/test_transactions.py -s -v --durations 0 
+    - name: Check resource usage
+      run: |
+        sqlite3 -readonly -separator " " .pymon "select item,cpu_usage,total_time,mem_usage from TEST_METRICS order by mem_usage desc;" >metrics.out
+        cat metrics.out
+        ./tests/check_pytest_monitor_output.py <metrics.out

tests/check_pytest_monitor_output.py

@@ -0,0 +1,19 @@
+#!/bin/env python3
+import sys
+
+ret = 0
+
+# example input line
+# test_non_tx_aggregate_limits 0.997759588095738 1.45325589179993 554.45703125
+for ln in sys.stdin:
+    line = ln.strip().split()
+
+    print(f"{float(line[1]) * 100.0: 7.2f}% CPU {float(line[2]):6.2f}s {line[3]:6.5} MB RAM {line[0]}")
+    if float(line[3]) > 1500:
+        print("   ERROR: ^^ exceeded RAM limit ^^ \n")
+        ret += 1
+
+if ret > 0:
+    print("some tests used too much RAM")
+
+sys.exit(ret)