Add CLN regtest to integration tests

This commit is contained in:
Reckless_Satoshi 2023-11-13 14:40:47 +00:00 committed by Reckless_Satoshi
parent ebd0a287c3
commit 605a37bb87
16 changed files with 272 additions and 167 deletions

View File

@ -15,68 +15,64 @@ concurrency:
cancel-in-progress: true
jobs:
build:
test:
runs-on: ubuntu-latest
strategy:
max-parallel: 4
max-parallel: 2
matrix:
python-version: ["3.11.6", "3.12"]
lnd-version: ["v0.17.0-beta","v0.17.1-beta.rc1"]
python-tag: ['3.11.6-slim-bookworm', '3.12-slim-bookworm']
lnd-version: ["v0.17.0-beta"] # , "v0.17.0-beta.rc1"]
cln-version: ["v23.08.1"]
ln-vendor: ["LND", "CLN"]
steps:
- name: 'Checkout'
uses: actions/checkout@v4
- name: 'Compose Eegtest Orchestration'
- name: Update Python version in Dockerfile
run: |
sed -i "1s/FROM python:.*/FROM python:${{ matrix.python-tag }}/" Dockerfile
sed -i '/RUN pip install --no-cache-dir -r requirements.txt/a COPY requirements_dev.txt .\nRUN pip install --no-cache-dir -r requirements_dev.txt' Dockerfile
- uses: satackey/action-docker-layer-caching@v0.0.11
continue-on-error: true
with:
key: coordinator-docker-cache-${{ hashFiles('./Dockerfile') }}
restore-keys: |
coordinator-docker-cache-
- name: 'Compose Regtest Orchestration'
uses: isbang/compose-action@v1.5.1
with:
compose-file: "docker-test.yml"
env: "tests/compose.env"
compose-file: "./docker-tests.yml"
down-flags: "--volumes"
# Ideally we run only coordinator-${{ matrix.ln-vendor }} , at the moment some tests fail if LND is not around.
services: |
bitcoind
postgres
redis
coordinator-CLN
coordinator-LND
robot-LND
coordinator
env:
LND_VERSION: ${{ matrix.lnd-version }}
CLN_VERSION: ${{ matrix.cln-version }}
BITCOIND_VERSION: ${{ matrix.bitcoind-version }}
ROBOSATS_ENVS_FILE: ".env-sample"
# - name: 'Set up Python ${{ matrix.python-version }}'
# uses: actions/setup-python@v4
# with:
# python-version: ${{ matrix.python-version }}
# - name: 'Cache pip dependencies'
# uses: actions/cache@v3
# with:
# path: ~/.cache/pip
# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
# restore-keys: |
# ${{ runner.os }}-pip-
# - name: 'Install Python Dependencies'
# run: |
# python -m pip install --upgrade pip
# pip install -r requirements.txt
# pip install -r requirements_dev.txt
# - name: 'Install LND/CLN gRPC Dependencies'
# run: bash ./scripts/generate_grpc.sh
# - name: 'Create .env File'
# run: |
# mv .env-sample .env
# sed -i "s/USE_TOR=True/USE_TOR=False/" .env
# - name: 'Wait for PostgreSQL to become ready'
# run: |
# sudo apt-get install -y postgresql-client
# until pg_isready -h localhost -p 5432 -U postgres; do sleep 2; done
- name: Wait for coordinator (django server)
run: |
while [ "$(docker inspect --format "{{.State.Health.Status}}" coordinator)" != "healthy" ]; do
echo "Waiting for coordinator to be healthy..."
sleep 5
done
- name: 'Run tests with coverage'
run: |
docker exec coordinator coverage run manage.py test
docker exec coordinator coverage report
# jobs:
# test:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v2
# - name: Run Docker Compose
# run: |
# docker-compose up -d
# docker-compose run web python manage.py test
env:
LNVENDOR: ${{ matrix.ln-vendor }}
DEVELOPMENT: True
USE_TOR: False

View File

@ -26,7 +26,7 @@ jobs:
with:
python-version: '3.11.6'
cache: pip
- run: pip install black==22.8.0 flake8==5.0.4 isort==5.10.1
- run: pip install requirements_dev.txt
- name: Run linters
uses: wearerequired/lint-action@v2
with:

View File

@ -38,8 +38,8 @@ jobs:
fi
django-test:
uses: RoboSats/robosats/.github/workflows/django-test.yml@main
integration-tests:
uses: RoboSats/robosats/.github/workflows/integration-tests.yml@main
needs: check-versions
frontend-build:

View File

@ -69,6 +69,16 @@ class CLNNode:
print(f"Cannot get CLN version: {e}")
return None
@classmethod
def get_info(cls):
try:
nodestub = node_pb2_grpc.NodeStub(cls.node_channel)
request = node_pb2.GetinfoRequest()
response = nodestub.Getinfo(request)
return response
except Exception as e:
print(f"Cannot get CLN node id: {e}")
@classmethod
def decode_payreq(cls, invoice):
"""Decodes a lightning payment request (invoice)"""

View File

@ -28,8 +28,8 @@ class InfoSerializer(serializers.Serializer):
lifetime_volume = serializers.FloatField(
help_text="Total volume in BTC since exchange's inception"
)
lnd_version = serializers.CharField(required=False)
cln_version = serializers.CharField(required=False)
lnd_version = serializers.CharField()
cln_version = serializers.CharField()
robosats_running_commit_hash = serializers.CharField()
alternative_site = serializers.CharField()
alternative_name = serializers.CharField()

View File

@ -21,8 +21,6 @@ from api.utils import (
verify_signed_message,
weighted_median,
)
from tests.mocks.cln import MockNodeStub
from tests.mocks.lnd import MockVersionerStub
class TestUtils(TestCase):
@ -96,15 +94,13 @@ class TestUtils(TestCase):
mock_response_blockchain.json.assert_called_once()
mock_response_yadio.json.assert_called_once()
@patch("api.lightning.lnd.verrpc_pb2_grpc.VersionerStub", MockVersionerStub)
def test_get_lnd_version(self):
version = get_lnd_version()
self.assertEqual(version, "v0.17.0-beta")
self.assertTrue(isinstance(version, str))
@patch("api.lightning.cln.node_pb2_grpc.NodeStub", MockNodeStub)
def test_get_cln_version(self):
version = get_cln_version()
self.assertEqual(version, "v23.08")
self.assertTrue(isinstance(version, str))
@patch(
"builtins.open", new_callable=mock_open, read_data="00000000000000000000 dev"

View File

@ -181,7 +181,7 @@ def get_lnd_version():
return LNDNode.get_version()
except Exception:
return None
return "No LND"
cln_version_cache = {}
@ -194,7 +194,7 @@ def get_cln_version():
return CLNNode.get_version()
except Exception:
return None
return "No CLN"
robosats_commit_cache = {}

View File

@ -2,20 +2,29 @@
# docker-compose -f docker-tests.yml --env-file tests/compose.env up -d
# Some useful handy commands that hopefully are never needed
# docker exec -it btc bitcoin-cli -chain=regtest -rpcpassword=test -rpcuser=test createwallet default
# docker exec -it btc bitcoin-cli -chain=regtest -rpcpassword=test -rpcuser=test -generate 101
# docker exec -it coordinator-lnd lncli --network=regtest getinfo
# docker exec -it robot-lnd lncli --network=regtest --rpcserver localhost:10010 getinfo
# docker exec -it coordinator-LND lncli --network=regtest getinfo
# docker exec -it robot-LND lncli --network=regtest --rpcserver localhost:10010 getinfo
version: '3.9'
services:
bitcoind:
image: ruimarinho/bitcoin-core:${BITCOIND_TAG}
image: ruimarinho/bitcoin-core:${BITCOIND_VERSION:-24.0.1}-alpine
container_name: btc
restart: always
ports:
- "8000:8000"
- "8080:8080"
- "8081:8081"
- "10009:10009"
- "10010:10010"
- "9999:9999"
- "9998:9998"
- "5432:5432"
- "6379:6379"
volumes:
- bitcoin:/bitcoin/.bitcoin/
command:
@ -35,9 +44,9 @@ services:
--zmqpubrawtx=tcp://0.0.0.0:28333
--listenonion=0
coordinator-lnd:
image: lightninglabs/lnd:${LND_TAG}
container_name: coordinator-lnd
coordinator-LND:
image: lightninglabs/lnd:${LND_VERSION:-v0.17.0-beta}
container_name: coordinator-LND
restart: always
volumes:
- bitcoin:/root/.bitcoin/
@ -47,11 +56,11 @@ services:
--noseedbackup
--nobootstrap
--restlisten=localhost:8081
--no-rest-tls
--debuglevel=debug
--maxpendingchannels=10
--rpclisten=0.0.0.0:10009
--listen=0.0.0.0:9735
--no-rest-tls
--color=#4126a7
--alias=RoboSats
--bitcoin.active
@ -67,9 +76,24 @@ services:
- bitcoind
network_mode: service:bitcoind
robot-lnd:
image: lightninglabs/lnd:${LND_TAG}
container_name: robot-lnd
coordinator-CLN:
image: elementsproject/lightningd:${CLN_VERSION:-v23.08.1}
restart: always
container_name: coordinator-CLN
environment:
LIGHTNINGD_NETWORK: 'regtest'
volumes:
- cln:/root/.lightning
- ./docker/cln/plugins/cln-grpc-hold:/root/.lightning/plugins/cln-grpc-hold
- bitcoin:/root/.bitcoin
command: --regtest --wumbo --bitcoin-rpcuser=test --bitcoin-rpcpassword=test --rest-host=0.0.0.0 --bind-addr=127.0.0.1:9737 --grpc-port=9999 --grpc-hold-port=9998 --important-plugin=/root/.lightning/plugins/cln-grpc-hold --database-upgrade=true
depends_on:
- bitcoind
network_mode: service:bitcoind
robot-LND:
image: lightninglabs/lnd:${LND_VERSION:-v0.17.0-beta}
container_name: robot-LND
restart: always
volumes:
- bitcoin:/root/.bitcoin/
@ -78,6 +102,7 @@ services:
command:
--noseedbackup
--nobootstrap
--restlisten=localhost:8080
--no-rest-tls
--debuglevel=debug
--maxpendingchannels=10
@ -99,7 +124,7 @@ services:
network_mode: service:bitcoind
redis:
image: redis:${REDIS_TAG}
image: redis:${REDIS_VERSION:-7.2.1}-alpine
container_name: redis
restart: always
volumes:
@ -116,11 +141,11 @@ services:
TESTING: True
USE_TOR: False
MACAROON_PATH: 'data/chain/bitcoin/regtest/admin.macaroon'
CLN_DIR: '/cln/regtest/'
env_file:
- ${ROBOSATS_ENVS_FILE}
depends_on:
- redis
- coordinator-lnd
- postgres
network_mode: service:bitcoind
volumes:
@ -128,76 +153,22 @@ services:
- lnd:/lnd
- lndrobot:/lndrobot
- cln:/cln
healthcheck:
test: ["CMD", "curl", "localhost:8000"]
interval: 5s
timeout: 5s
retries: 3
postgres:
image: postgres:${POSTGRES_TAG:-14.2-alpine}
image: postgres:${POSTGRES_VERSION:-14.2}-alpine
container_name: sql
restart: always
environment:
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_DB: ${POSTGRES_DB}
POSTGRES_PASSWORD: 'example'
POSTGRES_USER: 'postgres'
POSTGRES_DB: 'postgres'
network_mode: service:bitcoind
# clean-orders:
# image: robosats-image
# restart: always
# container_name: clord
# command: python3 manage.py clean_orders
# environment:
# SKIP_COLLECT_STATIC: "true"
# POSTGRES_HOST: 'postgres'
# env_file:
# - ${ROBOSATS_ENVS_FILE}
# follow-invoices:
# image: robosats-image
# container_name: invo
# restart: always
# env_file:
# - ${ROBOSATS_ENVS_FILE}
# environment:
# SKIP_COLLECT_STATIC: "true"
# POSTGRES_HOST: 'postgres'
# command: python3 manage.py follow_invoices
# telegram-watcher:
# image: robosats-image
# container_name: tg
# restart: always
# environment:
# SKIP_COLLECT_STATIC: "true"
# POSTGRES_HOST: 'postgres'
# env_file:
# - ${ROBOSATS_ENVS_FILE}
# command: python3 manage.py telegram_watcher
# celery:
# image: robosats-image
# container_name: cele
# restart: always
# env_file:
# - ${ROBOSATS_ENVS_FILE}
# environment:
# SKIP_COLLECT_STATIC: "true"
# POSTGRES_HOST: 'postgres'
# command: celery -A robosats worker --loglevel=WARNING
# depends_on:
# - redis
# celery-beat:
# image: robosats-image
# container_name: beat
# restart: always
# env_file:
# - ${ROBOSATS_ENVS_FILE}
# environment:
# SKIP_COLLECT_STATIC: "true"
# POSTGRES_HOST: 'postgres'
# command: celery -A robosats beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler
# depends_on:
# - redis
volumes:
redisdata:
bitcoin:

BIN
docker/cln/plugins/cln-grpc-hold Executable file

Binary file not shown.

View File

@ -8,4 +8,5 @@ omit = [
# omit test and mocks from coverage reports
"tests/*",
"*mocks*",
"manage.py",
]

View File

@ -1,7 +1,7 @@
coverage==7.3.2
black==23.3.0
isort==5.12.0
flake8==6.1.0
pyflakes==3.1.0
coverage==7.3.2
drf-openapi-tester==2.3.3
pre-commit==3.5.0

View File

@ -1161,10 +1161,12 @@ components:
- alternative_site
- bond_size
- book_liquidity
- cln_version
- current_swap_fee_rate
- last_day_nonkyc_btc_premium
- last_day_volume
- lifetime_volume
- lnd_version
- maker_fee
- network
- node_alias

View File

@ -1,10 +1,7 @@
ROBOSATS_ENVS_FILE=".env-sample"
BITCOIND_TAG='24.0.1-alpine'
LND_TAG='v0.17.0-beta'
REDIS_TAG='7.2.1-alpine@sha256:7f5a0dfbf379db69dc78434091dce3220e251022e71dcdf36207928cbf9010de'
POSTGRES_TAG='14.2-alpine'
POSTGRES_DB='postgres'
POSTGRES_USER='postgres'
POSTGRES_PASSWORD='example'
BITCOIND_VERSION='24.0.1'
LND_VERSION='v0.17.0-beta'
CLN_VERSION='v23.08.1'
REDIS_VERSION='7.2.1'
POSTGRES_VERSION='14.2'

View File

@ -1,24 +1,35 @@
import codecs
import sys
import time
import requests
from requests.auth import HTTPBasicAuth
from requests.exceptions import ReadTimeout
wait_step = 0.2
def get_node(name="robot"):
"""
We have two regtest LND nodes: "coordinator" (the robosats backend) and "robot" (the robosats user)
"""
if name == "robot":
with open("/lndrobot/data/chain/bitcoin/regtest/admin.macaroon", "rb") as f:
macaroon = f.read()
return {"port": 8080, "headers": {"Grpc-Metadata-macaroon": macaroon.hex()}}
macaroon = codecs.encode(
open("/lndrobot/data/chain/bitcoin/regtest/admin.macaroon", "rb").read(),
"hex",
)
port = 8080
elif name == "coordinator":
with open("/lnd/data/chain/bitcoin/regtest/admin.macaroon", "rb") as f:
macaroon = f.read()
return {"port": 8081, "headers": {"Grpc-Metadata-macaroon": macaroon.hex()}}
macaroon = codecs.encode(
open("/lnd/data/chain/bitcoin/regtest/admin.macaroon", "rb").read(), "hex"
)
port = 8081
return {"port": port, "headers": {"Grpc-Metadata-macaroon": macaroon}}
def get_node_id(node_name):
def get_lnd_node_id(node_name):
node = get_node(node_name)
response = requests.get(
f'http://localhost:{node["port"]}/v1/getinfo', headers=node["headers"]
@ -27,13 +38,99 @@ def get_node_id(node_name):
return data["identity_pubkey"]
def get_cln_node_id():
from api.lightning.cln import CLNNode
response = CLNNode.get_info()
return response.id.hex()
def wait_for_lnd_node_sync(node_name):
node = get_node(node_name)
waited = 0
while True:
response = requests.get(
f'http://localhost:{node["port"]}/v1/getinfo', headers=node["headers"]
)
if response.json()["synced_to_chain"]:
return
else:
sys.stdout.write(
f"\rWaiting for {node_name} node chain sync {round(waited,1)}s"
)
sys.stdout.flush()
waited += wait_step
time.sleep(wait_step)
def wait_for_lnd_active_channels(node_name):
node = get_node(node_name)
waited = 0
while True:
response = requests.get(
f'http://localhost:{node["port"]}/v1/getinfo', headers=node["headers"]
)
if response.json()["num_active_channels"] > 0:
return
else:
sys.stdout.write(
f"\rWaiting for {node_name} node channels to be active {round(waited,1)}s"
)
sys.stdout.flush()
waited += wait_step
time.sleep(wait_step)
def wait_for_cln_node_sync():
from api.lightning.cln import CLNNode
waited = 0
while True:
response = CLNNode.get_info()
if response.warning_bitcoind_sync or response.warning_lightningd_sync:
sys.stdout.write(
f"\rWaiting for coordinator CLN node sync {round(waited,1)}s"
)
sys.stdout.flush()
waited += wait_step
time.sleep(wait_step)
else:
return
def wait_for_cln_active_channels():
from api.lightning.cln import CLNNode
waited = 0
while True:
response = CLNNode.get_info()
if response.num_active_channels > 0:
return
else:
sys.stdout.write(
f"\rWaiting for coordinator CLN node channels to be active {round(waited,1)}s"
)
sys.stdout.flush()
waited += wait_step
time.sleep(wait_step)
def connect_to_node(node_name, node_id, ip_port):
node = get_node(node_name)
data = {"addr": {"pubkey": node_id, "host": ip_port}}
response = requests.post(
f'http://localhost:{node["port"]}/v1/peers', json=data, headers=node["headers"]
)
return response.json()
while True:
response = requests.post(
f'http://localhost:{node["port"]}/v1/peers',
json=data,
headers=node["headers"],
)
if response.json() == {}:
return response.json()
else:
if "already connected to peer" in response.json()["message"]:
return response.json()
print(f"Could not connect to coordinator node: {response.json()}")
time.sleep(wait_step)
def open_channel(node_name, node_id, local_funding_amount, push_sat):
@ -60,6 +157,7 @@ def create_address(node_name):
def generate_blocks(address, num_blocks):
print(f"Mining {num_blocks} blocks")
data = {
"jsonrpc": "1.0",
"id": "curltest",

View File

@ -44,8 +44,8 @@ class CoordinatorInfoTest(BaseAPITestCase):
self.assertEqual(data["last_day_nonkyc_btc_premium"], 0)
self.assertEqual(data["last_day_volume"], 0)
self.assertEqual(data["lifetime_volume"], 0)
self.assertEqual(data["lnd_version"], "v0.17.0-beta")
self.assertEqual(data["cln_version"], "v23.08")
self.assertTrue(isinstance(data["lnd_version"], str))
self.assertTrue(isinstance(data["cln_version"], str))
self.assertEqual(
data["robosats_running_commit_hash"], "00000000000000000000 dev"
)

View File

@ -14,12 +14,19 @@ from tests.node_utils import (
connect_to_node,
create_address,
generate_blocks,
get_node_id,
get_cln_node_id,
get_lnd_node_id,
open_channel,
pay_invoice,
wait_for_cln_active_channels,
wait_for_cln_node_sync,
wait_for_lnd_active_channels,
wait_for_lnd_node_sync,
)
from tests.test_api import BaseAPITestCase
LNVENDOR = config("LNVENDOR", cast=str, default="LND")
def read_file(file_path):
"""
@ -49,6 +56,20 @@ class TradeTest(BaseAPITestCase):
"longitude": 135.503,
}
def wait_nodes_sync():
wait_for_lnd_node_sync("robot")
if LNVENDOR == "LND":
wait_for_lnd_node_sync("coordinator")
elif LNVENDOR == "CLN":
wait_for_cln_node_sync()
def wait_active_channels():
wait_for_lnd_active_channels("robot")
if LNVENDOR == "LND":
wait_for_lnd_active_channels("coordinator")
elif LNVENDOR == "CLN":
wait_for_cln_active_channels()
@classmethod
def setUpTestData(cls):
"""
@ -61,19 +82,32 @@ class TradeTest(BaseAPITestCase):
cache_market()
# Fund two LN nodes in regtest and open channels
coordinator_node_id = get_node_id("coordinator")
connect_to_node("robot", coordinator_node_id, "localhost:9735")
# Coordinator is either LND or CLN. Robot user is always LND.
if LNVENDOR == "LND":
coordinator_node_id = get_lnd_node_id("coordinator")
coordinator_port = 9735
elif LNVENDOR == "CLN":
coordinator_node_id = get_cln_node_id()
coordinator_port = 9737
print("Coordinator Node ID: ", coordinator_node_id)
funding_address = create_address("robot")
generate_blocks(funding_address, 101)
cls.wait_nodes_sync()
time.sleep(
2
) # channels cannot be created until the node is fully sync. We just created 101 blocks.
# Open channel between Robot user and coordinator
print(f"\nOpening channel from Robot user node to coordinator {LNVENDOR} node")
connect_to_node("robot", coordinator_node_id, f"localhost:{coordinator_port}")
open_channel("robot", coordinator_node_id, 100_000_000, 50_000_000)
# Generate 6 blocks so the channel becomes active
generate_blocks(funding_address, 6)
# Generate 10 blocks so the channel becomes active and wait for sync
generate_blocks(funding_address, 10)
# Wait a tiny bit so payments can be done in the new channel
cls.wait_nodes_sync()
cls.wait_active_channels()
time.sleep(1)
def test_login_superuser(self):
"""