From ebd0a287c329f84a76029c499076ccf83ae3317b Mon Sep 17 00:00:00 2001 From: Reckless_Satoshi Date: Sun, 12 Nov 2023 12:39:39 +0000 Subject: [PATCH] Add dev dependencies and regtest test environments --- .github/workflows/django-test.yml | 85 ++++++------ api/lightning/lnd.py | 4 +- api/models/order.py | 2 +- api/serializers.py | 18 ++- docker-tests.yml | 206 ++++++++++++++++++++++++++++++ requirements.txt | 4 - requirements_dev.txt | 7 + robosats/middleware.py | 2 + scripts/entrypoint.sh | 6 + tests/api_specs.yaml | 21 ++- tests/compose.env | 10 ++ tests/mocks/cln.py | 26 +--- tests/mocks/lnd.py | 9 +- tests/node_utils.py | 98 ++++++++++++++ tests/test_coordinator_info.py | 9 +- tests/test_trade_pipeline.py | 122 +++++++++++------- 16 files changed, 485 insertions(+), 144 deletions(-) create mode 100644 docker-tests.yml create mode 100644 requirements_dev.txt create mode 100644 tests/compose.env create mode 100644 tests/node_utils.py diff --git a/.github/workflows/django-test.yml b/.github/workflows/django-test.yml index c35202c4..1507a26a 100644 --- a/.github/workflows/django-test.yml +++ b/.github/workflows/django-test.yml @@ -17,61 +17,66 @@ concurrency: jobs: build: runs-on: ubuntu-latest - env: - DEVELOPMENT: 1 strategy: max-parallel: 4 matrix: python-version: ["3.11.6", "3.12"] - - services: - db: - image: postgres:14.2 - env: - POSTGRES_DB: postgres - POSTGRES_USER: postgres - POSTGRES_PASSWORD: example - ports: - - 5432:5432 - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + lnd-version: ["v0.17.0-beta","v0.17.1-beta.rc1"] steps: - name: 'Checkout' uses: actions/checkout@v4 - - name: 'Set up Python ${{ matrix.python-version }}' - uses: actions/setup-python@v4 + - name: 'Compose Eegtest Orchestration' + uses: isbang/compose-action@v1.5.1 with: - python-version: ${{ matrix.python-version }} + compose-file: "docker-test.yml" + env: "tests/compose.env" - - name: 'Cache pip dependencies' - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- + # - name: 'Set up Python ${{ matrix.python-version }}' + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ matrix.python-version }} - - name: 'Install Python Dependencies' - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + # - name: 'Cache pip dependencies' + # uses: actions/cache@v3 + # with: + # path: ~/.cache/pip + # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + # restore-keys: | + # ${{ runner.os }}-pip- - - name: 'Install LND/CLN gRPC Dependencies' - run: bash ./scripts/generate_grpc.sh + # - name: 'Install Python Dependencies' + # run: | + # python -m pip install --upgrade pip + # pip install -r requirements.txt + # pip install -r requirements_dev.txt - - name: 'Create .env File' - run: | - mv .env-sample .env - sed -i "s/USE_TOR='True'/USE_TOR='False'/" .env + # - name: 'Install LND/CLN gRPC Dependencies' + # run: bash ./scripts/generate_grpc.sh - - name: 'Wait for PostgreSQL to become ready' - run: | - sudo apt-get install -y postgresql-client - until pg_isready -h localhost -p 5432 -U postgres; do sleep 2; done + # - name: 'Create .env File' + # run: | + # mv .env-sample .env + # sed -i "s/USE_TOR=True/USE_TOR=False/" .env + + # - name: 'Wait for PostgreSQL to become ready' + # run: | + # sudo apt-get install -y postgresql-client + # until pg_isready -h localhost -p 5432 -U postgres; do sleep 2; done - name: 'Run tests with coverage' run: | - pip install coverage - coverage run manage.py test - coverage report \ No newline at end of file + docker exec coordinator coverage run manage.py test + docker exec coordinator coverage report + + +# jobs: +# test: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v2 +# - name: Run Docker Compose +# run: | +# docker-compose up -d +# docker-compose run web python manage.py test \ No newline at end of file diff --git a/api/lightning/lnd.py b/api/lightning/lnd.py index 8a6ac59f..92336d9f 100644 --- a/api/lightning/lnd.py +++ b/api/lightning/lnd.py @@ -37,7 +37,7 @@ except Exception: # Read macaroon from file or .env variable string encoded as base64 try: - with open(os.path.join(config("LND_DIR"), config("MACAROON_path")), "rb") as f: + with open(os.path.join(config("LND_DIR"), config("MACAROON_PATH")), "rb") as f: MACAROON = f.read() except Exception: MACAROON = b64decode(config("LND_MACAROON_BASE64")) @@ -49,7 +49,7 @@ MAX_SWAP_AMOUNT = config("MAX_SWAP_AMOUNT", cast=int, default=500_000) # Logger function used to build tests/mocks/lnd.py def log(name, request, response): - if not config("LOG_LND", cast=bool, default=True): + if not config("LOG_LND", cast=bool, default=False): return current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") log_message = f"######################################\nEvent: {name}\nTime: {current_time}\nRequest:\n{request}\nResponse:\n{response}\nType: {type(response)}\n" diff --git a/api/models/order.py b/api/models/order.py index a8964997..50161281 100644 --- a/api/models/order.py +++ b/api/models/order.py @@ -10,7 +10,7 @@ from django.db.models.signals import pre_delete from django.dispatch import receiver from django.utils import timezone -if config("COORDINATOR_TESTING", cast=bool, default=False): +if config("TESTING", cast=bool, default=False): import random import string diff --git a/api/serializers.py b/api/serializers.py index 7da7c2bc..078374c9 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -28,8 +28,8 @@ class InfoSerializer(serializers.Serializer): lifetime_volume = serializers.FloatField( help_text="Total volume in BTC since exchange's inception" ) - lnd_version = serializers.CharField() - cln_version = serializers.CharField() + lnd_version = serializers.CharField(required=False) + cln_version = serializers.CharField(required=False) robosats_running_commit_hash = serializers.CharField() alternative_site = serializers.CharField() alternative_name = serializers.CharField() @@ -170,11 +170,15 @@ class OrderDetailSerializer(serializers.ModelSerializer): "- **'Inactive'** (seen more than 10 min ago)\n\n" "Note: When you make a request to this route, your own status get's updated and can be seen by your counterparty", ) - taker_status = serializers.BooleanField( + taker_status = serializers.CharField( required=False, - help_text="True if you are either a taker or maker, False otherwise", + help_text="Status of the maker:\n" + "- **'Active'** (seen within last 2 min)\n" + "- **'Seen Recently'** (seen within last 10 min)\n" + "- **'Inactive'** (seen more than 10 min ago)\n\n" + "Note: When you make a request to this route, your own status get's updated and can be seen by your counterparty", ) - price_now = serializers.IntegerField( + price_now = serializers.FloatField( required=False, help_text="Price of the order in the order's currency at the time of request (upto 5 significant digits)", ) @@ -274,11 +278,11 @@ class OrderDetailSerializer(serializers.ModelSerializer): required=False, help_text="in percentage, the swap fee rate the platform charges", ) - latitude = serializers.CharField( + latitude = serializers.FloatField( required=False, help_text="Latitude of the order for F2F payments", ) - longitude = serializers.CharField( + longitude = serializers.FloatField( required=False, help_text="Longitude of the order for F2F payments", ) diff --git a/docker-tests.yml b/docker-tests.yml new file mode 100644 index 00000000..73c7caed --- /dev/null +++ b/docker-tests.yml @@ -0,0 +1,206 @@ +# Spin up a regtest lightning network to run integration tests +# docker-compose -f docker-tests.yml --env-file tests/compose.env up -d + +# Some useful handy commands that hopefully are never needed +# docker exec -it btc bitcoin-cli -chain=regtest -rpcpassword=test -rpcuser=test createwallet default +# docker exec -it btc bitcoin-cli -chain=regtest -rpcpassword=test -rpcuser=test -generate 101 + +# docker exec -it coordinator-lnd lncli --network=regtest getinfo +# docker exec -it robot-lnd lncli --network=regtest --rpcserver localhost:10010 getinfo + +version: '3.9' +services: + bitcoind: + image: ruimarinho/bitcoin-core:${BITCOIND_TAG} + container_name: btc + restart: always + ports: + - "8000:8000" + volumes: + - bitcoin:/bitcoin/.bitcoin/ + command: + --txindex=1 + --printtoconsole + --regtest=1 + --server=1 + --rest=1 + --rpcuser=test + --rpcpassword=test + --logips=1 + --debug=1 + --rpcport=18443 + --rpcallowip=172.0.0.0/8 + --rpcallowip=192.168.0.0/16 + --zmqpubrawblock=tcp://0.0.0.0:28332 + --zmqpubrawtx=tcp://0.0.0.0:28333 + --listenonion=0 + + coordinator-lnd: + image: lightninglabs/lnd:${LND_TAG} + container_name: coordinator-lnd + restart: always + volumes: + - bitcoin:/root/.bitcoin/ + - lnd:/home/lnd/.lnd + - lnd:/root/.lnd + command: + --noseedbackup + --nobootstrap + --restlisten=localhost:8081 + --no-rest-tls + --debuglevel=debug + --maxpendingchannels=10 + --rpclisten=0.0.0.0:10009 + --listen=0.0.0.0:9735 + --color=#4126a7 + --alias=RoboSats + --bitcoin.active + --bitcoin.regtest + --bitcoin.node=bitcoind + --bitcoind.rpchost=127.0.0.1 + --bitcoind.rpcuser=test + --bitcoind.rpcpass=test + --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 + --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 + --protocol.wumbo-channels + depends_on: + - bitcoind + network_mode: service:bitcoind + + robot-lnd: + image: lightninglabs/lnd:${LND_TAG} + container_name: robot-lnd + restart: always + volumes: + - bitcoin:/root/.bitcoin/ + - lndrobot:/home/lnd/.lnd + - lndrobot:/root/.lnd + command: + --noseedbackup + --nobootstrap + --no-rest-tls + --debuglevel=debug + --maxpendingchannels=10 + --rpclisten=0.0.0.0:10010 + --listen=0.0.0.0:9736 + --color=#4126a7 + --alias=Robot + --bitcoin.active + --bitcoin.regtest + --bitcoin.node=bitcoind + --bitcoind.rpchost=127.0.0.1 + --bitcoind.rpcuser=test + --bitcoind.rpcpass=test + --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 + --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 + --protocol.wumbo-channels + depends_on: + - bitcoind + network_mode: service:bitcoind + + redis: + image: redis:${REDIS_TAG} + container_name: redis + restart: always + volumes: + - redisdata:/data + network_mode: service:bitcoind + + coordinator: + build: . + image: robosats-image + container_name: coordinator + restart: always + environment: + DEVELOPMENT: True + TESTING: True + USE_TOR: False + MACAROON_PATH: 'data/chain/bitcoin/regtest/admin.macaroon' + env_file: + - ${ROBOSATS_ENVS_FILE} + depends_on: + - redis + - coordinator-lnd + - postgres + network_mode: service:bitcoind + volumes: + - .:/usr/src/robosats + - lnd:/lnd + - lndrobot:/lndrobot + - cln:/cln + + postgres: + image: postgres:${POSTGRES_TAG:-14.2-alpine} + container_name: sql + restart: always + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_DB: ${POSTGRES_DB} + network_mode: service:bitcoind + + # clean-orders: + # image: robosats-image + # restart: always + # container_name: clord + # command: python3 manage.py clean_orders + # environment: + # SKIP_COLLECT_STATIC: "true" + # POSTGRES_HOST: 'postgres' + # env_file: + # - ${ROBOSATS_ENVS_FILE} + + # follow-invoices: + # image: robosats-image + # container_name: invo + # restart: always + # env_file: + # - ${ROBOSATS_ENVS_FILE} + # environment: + # SKIP_COLLECT_STATIC: "true" + # POSTGRES_HOST: 'postgres' + # command: python3 manage.py follow_invoices + + # telegram-watcher: + # image: robosats-image + # container_name: tg + # restart: always + # environment: + # SKIP_COLLECT_STATIC: "true" + # POSTGRES_HOST: 'postgres' + # env_file: + # - ${ROBOSATS_ENVS_FILE} + # command: python3 manage.py telegram_watcher + + # celery: + # image: robosats-image + # container_name: cele + # restart: always + # env_file: + # - ${ROBOSATS_ENVS_FILE} + # environment: + # SKIP_COLLECT_STATIC: "true" + # POSTGRES_HOST: 'postgres' + # command: celery -A robosats worker --loglevel=WARNING + # depends_on: + # - redis + + # celery-beat: + # image: robosats-image + # container_name: beat + # restart: always + # env_file: + # - ${ROBOSATS_ENVS_FILE} + # environment: + # SKIP_COLLECT_STATIC: "true" + # POSTGRES_HOST: 'postgres' + # command: celery -A robosats beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler + # depends_on: + # - redis + +volumes: + redisdata: + bitcoin: + lnd: + cln: + lndrobot: \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index aebdba92..dcd4c8f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,9 +26,5 @@ python-gnupg==0.5.1 daphne==4.0.0 drf-spectacular==0.26.2 drf-spectacular-sidecar==2023.5.1 -black==23.3.0 -isort==5.12.0 -flake8==6.1.0 -pyflakes==3.1.0 django-cors-headers==4.3.0 base91==1.0.1 diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 00000000..10c8b42d --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,7 @@ +black==23.3.0 +isort==5.12.0 +flake8==6.1.0 +pyflakes==3.1.0 +coverage==7.3.2 +drf-openapi-tester==2.3.3 +pre-commit==3.5.0 \ No newline at end of file diff --git a/robosats/middleware.py b/robosats/middleware.py index 16f3109f..7e2a5df1 100644 --- a/robosats/middleware.py +++ b/robosats/middleware.py @@ -161,6 +161,8 @@ class RobotTokenSHA256AuthenticationMiddleWare: resized_img.save(f, format="WEBP", quality=80) user.robot.avatar = "static/assets/avatars/" + nickname + ".webp" + + update_last_login(None, user) user.save() response = self.get_response(request) diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 750a8226..ebb419f6 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -10,6 +10,12 @@ else python manage.py collectstatic --noinput fi +# Collect static files +if [ $DEVELOPMENT ]; then + echo "Installing python development dependencies" + pip install -r requirements_dev.txt +fi + # Print first start up message when pb2/grpc files if they do exist if [ ! -f "/usr/src/robosats/api/lightning/lightning_pb2.py" ]; then echo "Looks like the first run of this container. pb2 and gRPC files were not detected on the attached volume, copying them into the attached volume /robosats/api/lightning ." diff --git a/tests/api_specs.yaml b/tests/api_specs.yaml index b2768cd1..012d5051 100644 --- a/tests/api_specs.yaml +++ b/tests/api_specs.yaml @@ -1161,12 +1161,10 @@ components: - alternative_site - bond_size - book_liquidity - - cln_version - current_swap_fee_rate - last_day_nonkyc_btc_premium - last_day_volume - lifetime_volume - - lnd_version - maker_fee - network - node_alias @@ -1486,10 +1484,17 @@ components: Note: When you make a request to this route, your own status get's updated and can be seen by your counterparty taker_status: - type: boolean - description: True if you are either a taker or maker, False otherwise + type: string + description: |- + Status of the maker: + - **'Active'** (seen within last 2 min) + - **'Seen Recently'** (seen within last 10 min) + - **'Inactive'** (seen more than 10 min ago) + + Note: When you make a request to this route, your own status get's updated and can be seen by your counterparty price_now: - type: integer + type: number + format: double description: Price of the order in the order's currency at the time of request (upto 5 significant digits) premium_percentile: @@ -1657,10 +1662,12 @@ components: description: The network eg. 'testnet', 'mainnet'. Only if status = `14` (Successful Trade) and is_buyer = `true` latitude: - type: string + type: number + format: double description: Latitude of the order for F2F payments longitude: - type: string + type: number + format: double description: Longitude of the order for F2F payments required: - expires_at diff --git a/tests/compose.env b/tests/compose.env new file mode 100644 index 00000000..ad69231d --- /dev/null +++ b/tests/compose.env @@ -0,0 +1,10 @@ +ROBOSATS_ENVS_FILE=".env-sample" + +BITCOIND_TAG='24.0.1-alpine' +LND_TAG='v0.17.0-beta' +REDIS_TAG='7.2.1-alpine@sha256:7f5a0dfbf379db69dc78434091dce3220e251022e71dcdf36207928cbf9010de' +POSTGRES_TAG='14.2-alpine' + +POSTGRES_DB='postgres' +POSTGRES_USER='postgres' +POSTGRES_PASSWORD='example' \ No newline at end of file diff --git a/tests/mocks/cln.py b/tests/mocks/cln.py index cc2a8d10..331e68aa 100644 --- a/tests/mocks/cln.py +++ b/tests/mocks/cln.py @@ -1,6 +1,7 @@ from unittest.mock import MagicMock # Mock up of CLN gRPC responses +# Unfinished, during integration tests we SHOULD spin up a regtest CLN instance class MockNodeStub: @@ -31,28 +32,3 @@ class MockNodeStub: response.binding.address = "127.0.0.1" response.binding.port = 9736 return response - - -class MockHoldStub: - def __init__(self, channel): - pass - - def HoldInvoiceLookup(self, request): - response = MagicMock() - return response - - def HoldInvoice(self, request): - response = MagicMock() - return response - - def HoldInvoiceSettle(self, request): - response = MagicMock() - return response - - def HoldInvoiceCancel(self, request): - response = MagicMock() - return response - - def DecodeBolt11(self, request): - response = MagicMock() - return response diff --git a/tests/mocks/lnd.py b/tests/mocks/lnd.py index ea52b1c5..4698c13f 100644 --- a/tests/mocks/lnd.py +++ b/tests/mocks/lnd.py @@ -1,6 +1,7 @@ from unittest.mock import MagicMock # Mock up of LND gRPC responses +# Unfinished, during integration tests we spin up a regtest LND instance class MockLightningStub: @@ -22,7 +23,7 @@ class MockLightningStub: response = MagicMock() if ( request.pay_req - == "lntb17310n1pj552mdpp50p2utzh7mpsf3uq7u7cws4a96tj3kyq54hchdkpw8zecamx9klrqd2j2pshjmt9de6zqun9vejhyetwvdjn5gphxs6nsvfe893z6wphvfsj6dryvymj6wp5xvuz6wp5xcukvdec8yukgcf49cs9g6rfwvs8qcted4jkuapq2ay5cnpqgefy2326g5syjn3qt984253q2aq5cnz92skzqcmgv43kkgr0dcs9ymmzdafkzarnyp5kvgr5dpjjqmr0vd4jqampwvs8xatrvdjhxumxw4kzugzfwss8w6tvdssxyefqw4hxcmmrddjkggpgveskjmpfyp6kumr9wdejq7t0w5sxx6r9v96zqmmjyp3kzmnrv4kzqatwd9kxzar9wfskcmre9ccqz52xqzwzsp5hkzegrhn6kegr33z8qfxtcudaklugygdrakgyy7va0wt2qs7drfq9qyyssqc6rztchzl4m7mlulrhlcajszcl9fan8908k9n5x7gmz8g8d6ht5pj4l8r0dushq6j5s8x7yv9a5klz0kfxwy8v6ze6adyrrp4wu0q0sq3t604x" + == "lntb17310n1pj552mdpp50p2utgzfwss8w6tvdssxyefqw4hxcmmrddjkggpgveskjmpfyp6kumr9wdejq7t0w5sxx6r9v96zqmmjyp3kzmnrv4kzqatwd9kxzar9wfskcmre9ccqz52xqzwzsp5hkzegrhn6kegr33z8qfxtcudaklugygdrakgyy7va0wt2qs7drfq9qyyssqc6rztchzl4m7mlulrhlcajszcl9fan8908k9n5x7gmz8g8d6ht5pj4l8r0dushq6j5s8x7yv9a5klz0kfxwy8v6ze6adyrrp4wu0q0sq3t604x" ): response.destination = ( "033b58d7681fe5dd2fb21fd741996cda5449616f77317dd1156b80128d6a71b807" @@ -35,7 +36,9 @@ class MockLightningStub: response.expiry = 450 response.description = "Payment reference: 7458199b-87ba-4da7-8438-8469f7899da5. This payment WILL FREEZE IN YOUR WALLET, check on RoboSats if the lock was successful. It will be unlocked (fail) unless you cheat or cancel unilaterally." response.cltv_expiry = 650 - response.payment_addr = '\275\205\224\016\363\325\262\201\306"8\022e\343\215\355\277\304\021\r\037l\202\023\314\353\334\265\002\036h\322' + response.payment_addr = ( + "\275\205\224\016\363\325\262\201\353\334\265\002\036h\322" + ) response.num_msat = 1731000 return response @@ -86,7 +89,7 @@ class MockInvoicesStub: def AddHoldInvoice(self, request): response = MagicMock() - # if request.value == 1731: + print(request) response.payment_request = "lntb17310n1pj552mdpp50p2utzh7mpsf3uq7u7cws4a96tj3kyq54hchdkpw8zecamx9klrqd2j2pshjmt9de6zqun9vejhyetwvdjn5gphxs6nsvfe893z6wphvfsj6dryvymj6wp5xvuz6wp5xcukvdec8yukgcf49cs9g6rfwvs8qcted4jkuapq2ay5cnpqgefy2326g5syjn3qt984253q2aq5cnz92skzqcmgv43kkgr0dcs9ymmzdafkzarnyp5kvgr5dpjjqmr0vd4jqampwvs8xatrvdjhxumxw4kzugzfwss8w6tvdssxyefqw4hxcmmrddjkggpgveskjmpfyp6kumr9wdejq7t0w5sxx6r9v96zqmmjyp3kzmnrv4kzqatwd9kxzar9wfskcmre9ccqz52xqzwzsp5hkzegrhn6kegr33z8qfxtcudaklugygdrakgyy7va0wt2qs7drfq9qyyssqc6rztchzl4m7mlulrhlcajszcl9fan8908k9n5x7gmz8g8d6ht5pj4l8r0dushq6j5s8x7yv9a5klz0kfxwy8v6ze6adyrrp4wu0q0sq3t604x" response.add_index = 1 response.payment_addr = b'\275\205\224\016\363\325\262\201\306"8\022e\343\215\355\277\304\021\r\037l\202\023\314\353\334\265\002\036h\322' diff --git a/tests/node_utils.py b/tests/node_utils.py new file mode 100644 index 00000000..71d43fb2 --- /dev/null +++ b/tests/node_utils.py @@ -0,0 +1,98 @@ +import requests +from requests.auth import HTTPBasicAuth +from requests.exceptions import ReadTimeout + + +def get_node(name="robot"): + """ + We have two regtest LND nodes: "coordinator" (the robosats backend) and "robot" (the robosats user) + """ + if name == "robot": + with open("/lndrobot/data/chain/bitcoin/regtest/admin.macaroon", "rb") as f: + macaroon = f.read() + return {"port": 8080, "headers": {"Grpc-Metadata-macaroon": macaroon.hex()}} + + elif name == "coordinator": + with open("/lnd/data/chain/bitcoin/regtest/admin.macaroon", "rb") as f: + macaroon = f.read() + return {"port": 8081, "headers": {"Grpc-Metadata-macaroon": macaroon.hex()}} + + +def get_node_id(node_name): + node = get_node(node_name) + response = requests.get( + f'http://localhost:{node["port"]}/v1/getinfo', headers=node["headers"] + ) + data = response.json() + return data["identity_pubkey"] + + +def connect_to_node(node_name, node_id, ip_port): + node = get_node(node_name) + data = {"addr": {"pubkey": node_id, "host": ip_port}} + response = requests.post( + f'http://localhost:{node["port"]}/v1/peers', json=data, headers=node["headers"] + ) + return response.json() + + +def open_channel(node_name, node_id, local_funding_amount, push_sat): + node = get_node(node_name) + data = { + "node_pubkey_string": node_id, + "local_funding_amount": local_funding_amount, + "push_sat": push_sat, + } + response = requests.post( + f'http://localhost:{node["port"]}/v1/channels', + json=data, + headers=node["headers"], + ) + return response.json() + + +def create_address(node_name): + node = get_node(node_name) + response = requests.get( + f'http://localhost:{node["port"]}/v1/newaddress', headers=node["headers"] + ) + return response.json()["address"] + + +def generate_blocks(address, num_blocks): + data = { + "jsonrpc": "1.0", + "id": "curltest", + "method": "generatetoaddress", + "params": [num_blocks, address], + } + response = requests.post( + "http://localhost:18443", json=data, auth=HTTPBasicAuth("test", "test") + ) + return response.json() + + +def pay_invoice(node_name, invoice): + node = get_node(node_name) + data = {"payment_request": invoice} + try: + requests.post( + f'http://localhost:{node["port"]}/v1/channels/transactions', + json=data, + headers=node["headers"], + timeout=1, + ) + except ReadTimeout: + # Request to pay hodl invoice has timed out: that's good! + return + + +def add_invoice(node_name, amount): + node = get_node(node_name) + data = {"value": amount} + response = requests.post( + f'http://localhost:{node["port"]}/v1/invoices', + json=data, + headers=node["headers"], + ) + return response.json()["payment_request"] diff --git a/tests/test_coordinator_info.py b/tests/test_coordinator_info.py index c4708f84..564d6932 100644 --- a/tests/test_coordinator_info.py +++ b/tests/test_coordinator_info.py @@ -1,5 +1,4 @@ import json -from unittest.mock import patch from decouple import config from django.conf import settings @@ -7,8 +6,6 @@ from django.contrib.auth.models import User from django.test import Client from django.urls import reverse -from tests.mocks.cln import MockNodeStub -from tests.mocks.lnd import MockVersionerStub from tests.test_api import BaseAPITestCase FEE = config("FEE", cast=float, default=0.2) @@ -31,8 +28,6 @@ class CoordinatorInfoTest(BaseAPITestCase): self.client = Client() User.objects.create_superuser(self.su_name, "super@user.com", self.su_pass) - @patch("api.lightning.cln.node_pb2_grpc.NodeStub", MockNodeStub) - @patch("api.lightning.lnd.verrpc_pb2_grpc.VersionerStub", MockVersionerStub) def test_info(self): path = reverse("info") @@ -56,7 +51,9 @@ class CoordinatorInfoTest(BaseAPITestCase): ) self.assertEqual(data["version"], settings.VERSION) self.assertEqual(data["node_id"], NODE_ID) - self.assertEqual(data["network"], "testnet") + self.assertEqual( + data["network"], "testnet" + ) # tests take place in regtest, but this attribute is read from .env self.assertAlmostEqual(data["maker_fee"], MAKER_FEE) self.assertAlmostEqual(data["taker_fee"], TAKER_FEE) self.assertAlmostEqual(data["bond_size"], BOND_SIZE) diff --git a/tests/test_trade_pipeline.py b/tests/test_trade_pipeline.py index 6109d9d1..d3e44cbe 100644 --- a/tests/test_trade_pipeline.py +++ b/tests/test_trade_pipeline.py @@ -1,7 +1,7 @@ import json +import time from datetime import datetime from decimal import Decimal -from unittest.mock import patch from decouple import config from django.contrib.auth.models import User @@ -10,10 +10,13 @@ from django.urls import reverse from api.management.commands.follow_invoices import Command as FollowInvoices from api.models import Currency, Order from api.tasks import cache_market -from tests.mocks.cln import MockHoldStub # , MockNodeStub -from tests.mocks.lnd import ( # MockRouterStub,; MockSignerStub,; MockVersionerStub, - MockInvoicesStub, - MockLightningStub, +from tests.node_utils import ( + connect_to_node, + create_address, + generate_blocks, + get_node_id, + open_channel, + pay_invoice, ) from tests.test_api import BaseAPITestCase @@ -57,6 +60,21 @@ class TradeTest(BaseAPITestCase): # Fetch currency prices from external APIs cache_market() + # Fund two LN nodes in regtest and open channels + coordinator_node_id = get_node_id("coordinator") + connect_to_node("robot", coordinator_node_id, "localhost:9735") + + funding_address = create_address("robot") + generate_blocks(funding_address, 101) + + time.sleep( + 2 + ) # channels cannot be created until the node is fully sync. We just created 101 blocks. + open_channel("robot", coordinator_node_id, 100_000_000, 50_000_000) + + # Generate 6 blocks so the channel becomes active + generate_blocks(funding_address, 6) + def test_login_superuser(self): """ Test the login functionality for the superuser. @@ -65,7 +83,9 @@ class TradeTest(BaseAPITestCase): data = {"username": self.su_name, "password": self.su_pass} response = self.client.post(path, data) self.assertEqual(response.status_code, 302) - self.assertResponse(response) + self.assertResponse( + response + ) # should skip given that /coordinator/login is not documented def test_cache_market(self): """ @@ -102,13 +122,15 @@ class TradeTest(BaseAPITestCase): else: headers = {"HTTP_AUTHORIZATION": f"Token {b91_token}"} - return headers, pub_key, enc_priv_key + return headers - def assert_robot(self, response, pub_key, enc_priv_key, robot_index): + def assert_robot(self, response, robot_index): """ Assert that the robot is created correctly. """ nickname = read_file(f"tests/robots/{robot_index}/nickname") + pub_key = read_file(f"tests/robots/{robot_index}/pub_key") + enc_priv_key = read_file(f"tests/robots/{robot_index}/enc_priv_key") data = json.loads(response.content.decode()) @@ -148,18 +170,17 @@ class TradeTest(BaseAPITestCase): Creates the robots in /tests/robots/{robot_index} """ path = reverse("robot") - headers, pub_key, enc_priv_key = self.get_robot_auth(robot_index, True) + headers = self.get_robot_auth(robot_index, True) - response = self.client.get(path, **headers) - - self.assert_robot(response, pub_key, enc_priv_key, robot_index) + return self.client.get(path, **headers) def test_create_robots(self): """ Test the creation of two robots to be used in the trade tests """ - self.create_robot(robot_index=1) - self.create_robot(robot_index=2) + for robot_index in [1, 2]: + response = self.create_robot(robot_index) + self.assert_robot(response, robot_index) def make_order(self, maker_form, robot_index=1): """ @@ -167,7 +188,7 @@ class TradeTest(BaseAPITestCase): """ path = reverse("make") # Get valid robot auth headers - headers, _, _ = self.get_robot_auth(robot_index, True) + headers = self.get_robot_auth(robot_index, True) response = self.client.post(path, maker_form, **headers) return response @@ -181,6 +202,8 @@ class TradeTest(BaseAPITestCase): data = json.loads(response.content.decode()) # Checks + self.assertResponse(response) + self.assertIsInstance(data["id"], int, "Order ID is not an integer") self.assertEqual( data["status"], @@ -255,13 +278,10 @@ class TradeTest(BaseAPITestCase): return data - @patch("api.lightning.cln.hold_pb2_grpc.HoldStub", MockHoldStub) - @patch("api.lightning.lnd.lightning_pb2_grpc.LightningStub", MockLightningStub) - @patch("api.lightning.lnd.invoices_pb2_grpc.InvoicesStub", MockInvoicesStub) def get_order(self, order_id, robot_index=1, first_encounter=False): path = reverse("order") params = f"?order_id={order_id}" - headers, _, _ = self.get_robot_auth(robot_index, first_encounter) + headers = self.get_robot_auth(robot_index, first_encounter) response = self.client.get(path + params, **headers) return response @@ -279,6 +299,8 @@ class TradeTest(BaseAPITestCase): data = json.loads(response.content.decode()) self.assertEqual(response.status_code, 200) + self.assertResponse(response) + self.assertEqual(data["id"], order_made_data["id"]) self.assertTrue( isinstance(datetime.fromisoformat(data["created_at"]), datetime) @@ -301,13 +323,8 @@ class TradeTest(BaseAPITestCase): self.assertFalse(data["maker_locked"]) self.assertFalse(data["taker_locked"]) self.assertFalse(data["escrow_locked"]) - self.assertEqual( - data["bond_invoice"], - "lntb17310n1pj552mdpp50p2utzh7mpsf3uq7u7cws4a96tj3kyq54hchdkpw8zecamx9klrqd2j2pshjmt9de6zqun9vejhyetwvdjn5gphxs6nsvfe893z6wphvfsj6dryvymj6wp5xvuz6wp5xcukvdec8yukgcf49cs9g6rfwvs8qcted4jkuapq2ay5cnpqgefy2326g5syjn3qt984253q2aq5cnz92skzqcmgv43kkgr0dcs9ymmzdafkzarnyp5kvgr5dpjjqmr0vd4jqampwvs8xatrvdjhxumxw4kzugzfwss8w6tvdssxyefqw4hxcmmrddjkggpgveskjmpfyp6kumr9wdejq7t0w5sxx6r9v96zqmmjyp3kzmnrv4kzqatwd9kxzar9wfskcmre9ccqz52xqzwzsp5hkzegrhn6kegr33z8qfxtcudaklugygdrakgyy7va0wt2qs7drfq9qyyssqc6rztchzl4m7mlulrhlcajszcl9fan8908k9n5x7gmz8g8d6ht5pj4l8r0dushq6j5s8x7yv9a5klz0kfxwy8v6ze6adyrrp4wu0q0sq3t604x", - ) self.assertTrue(isinstance(data["bond_satoshis"], int)) - @patch("api.lightning.lnd.invoices_pb2_grpc.InvoicesStub", MockInvoicesStub) def check_for_locked_bonds(self): # A background thread checks every 5 second the status of invoices. We invoke directly during test. # It will ask LND via gRPC. In our test, the request/response from LND is mocked, and it will return fake invoice status "ACCEPTED" @@ -320,7 +337,11 @@ class TradeTest(BaseAPITestCase): order_made_data = json.loads(order_made_response.content.decode()) # Maker's first order fetch. Should trigger maker bond hold invoice generation. - self.get_order(order_made_data["id"]) + response = self.get_order(order_made_data["id"]) + invoice = response.json()["bond_invoice"] + + # Lock the invoice from the robot's node + pay_invoice("robot", invoice) # Check for invoice locked (the mocked LND will return ACCEPTED) self.check_for_locked_bonds() @@ -336,6 +357,8 @@ class TradeTest(BaseAPITestCase): data = json.loads(response.content.decode()) self.assertEqual(response.status_code, 200) + self.assertResponse(response) + self.assertEqual(data["id"], data["id"]) self.assertEqual(data["status_message"], Order.Status(Order.Status.PUB).label) self.assertTrue(data["maker_locked"]) @@ -352,13 +375,13 @@ class TradeTest(BaseAPITestCase): self.assertTrue(isinstance(public_data["price_now"], float)) self.assertTrue(isinstance(data["satoshis_now"], int)) - @patch("api.lightning.cln.hold_pb2_grpc.HoldStub", MockHoldStub) - @patch("api.lightning.lnd.lightning_pb2_grpc.LightningStub", MockLightningStub) - @patch("api.lightning.lnd.invoices_pb2_grpc.InvoicesStub", MockInvoicesStub) + # @patch("api.lightning.cln.hold_pb2_grpc.HoldStub", MockHoldStub) + # @patch("api.lightning.lnd.lightning_pb2_grpc.LightningStub", MockLightningStub) + # @patch("api.lightning.lnd.invoices_pb2_grpc.InvoicesStub", MockInvoicesStub) def take_order(self, order_id, amount, robot_index=2): path = reverse("order") params = f"?order_id={order_id}" - headers, _, _ = self.get_robot_auth(robot_index, first_encounter=True) + headers = self.get_robot_auth(robot_index, first_encounter=True) body = {"action": "take", "amount": amount} response = self.client.post(path + params, body, **headers) @@ -372,28 +395,29 @@ class TradeTest(BaseAPITestCase): response = self.take_order(data_publised["id"], take_amount, taker_index) return response - # def test_make_and_take_order(self): - # maker_index = 1 - # taker_index = 2 - # maker_form = self.maker_form_with_range - # self.create_robot(taker_index) #### WEEEE SHOULD NOT BE NEEDED >??? WHY ROBOT HAS NO LOGIN TIME?? - # response = self.make_and_take_order(maker_form, 80, maker_index, taker_index) - # data = json.loads(response.content.decode()) + def test_make_and_take_order(self): + maker_index = 1 + taker_index = 2 + maker_form = self.maker_form_with_range - # print(data) + response = self.make_and_take_order(maker_form, 80, maker_index, taker_index) + data = json.loads(response.content.decode()) - # self.assertEqual( - # data["ur_nick"], read_file(f"tests/robots/{taker_index}/nickname") - # ) - # self.assertEqual( - # data["taker_nick"], read_file(f"tests/robots/{taker_index}/nickname") - # ) - # self.assertEqual( - # data["maker_nick"], read_file(f"tests/robots/{maker_index}/nickname") - # ) - # self.assertFalse(data["is_maker"]) - # self.assertTrue(data["is_taker"]) - # self.assertTrue(data["is_participant"]) + self.assertEqual(response.status_code, 200) + self.assertResponse(response) + + self.assertEqual( + data["ur_nick"], read_file(f"tests/robots/{taker_index}/nickname") + ) + self.assertEqual( + data["taker_nick"], read_file(f"tests/robots/{taker_index}/nickname") + ) + self.assertEqual( + data["maker_nick"], read_file(f"tests/robots/{maker_index}/nickname") + ) + self.assertFalse(data["is_maker"]) + self.assertTrue(data["is_taker"]) + self.assertTrue(data["is_participant"]) # a = { # "maker_status": "Active",