Add dev dependencies and regtest test environments

.github/workflows/django-test.yml

@@ -17,61 +17,66 @@ concurrency:
 jobs:
   build:
     runs-on: ubuntu-latest
-    env:
-      DEVELOPMENT: 1
     strategy:
       max-parallel: 4
       matrix:
         python-version: ["3.11.6", "3.12"]
-
+        lnd-version: ["v0.17.0-beta","v0.17.1-beta.rc1"]
-    services:
-      db:
-        image: postgres:14.2
-        env:
-          POSTGRES_DB: postgres
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: example
-        ports:
-          - 5432:5432
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
     steps:
     - name: 'Checkout'
       uses: actions/checkout@v4
 
-    - name: 'Set up Python ${{ matrix.python-version }}'
+    - name: 'Compose Eegtest Orchestration'
-      uses: actions/setup-python@v4
+      uses: isbang/compose-action@v1.5.1
       with:
-        python-version: ${{ matrix.python-version }}
+        compose-file: "docker-test.yml"
+        env: "tests/compose.env"
 
-    - name: 'Cache pip dependencies'
+    # - name: 'Set up Python ${{ matrix.python-version }}'
-      uses: actions/cache@v3
+    #   uses: actions/setup-python@v4
-      with:
+    #   with:
-        path: ~/.cache/pip
+    #     python-version: ${{ matrix.python-version }}
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
 
-    - name: 'Install Python Dependencies'
+    # - name: 'Cache pip dependencies'
-      run: |
+    #   uses: actions/cache@v3
-        python -m pip install --upgrade pip
+    #   with:
-        pip install -r requirements.txt
+    #     path: ~/.cache/pip
+    #     key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+    #     restore-keys: |
+    #       ${{ runner.os }}-pip-
 
-    - name: 'Install LND/CLN gRPC Dependencies'
+    # - name: 'Install Python Dependencies'
-      run: bash ./scripts/generate_grpc.sh
+    #   run: |
+    #     python -m pip install --upgrade pip
+    #     pip install -r requirements.txt
+    #     pip install -r requirements_dev.txt
 
-    - name: 'Create .env File'
+    # - name: 'Install LND/CLN gRPC Dependencies'
-      run: |
+    #   run: bash ./scripts/generate_grpc.sh
-        mv .env-sample .env
-        sed -i "s/USE_TOR='True'/USE_TOR='False'/" .env
 
-    - name: 'Wait for PostgreSQL to become ready'
+    # - name: 'Create .env File'
-      run: |
+    #   run: |
-        sudo apt-get install -y postgresql-client
+    #     mv .env-sample .env
-        until pg_isready -h localhost -p 5432 -U postgres; do sleep 2; done
+    #     sed -i "s/USE_TOR=True/USE_TOR=False/" .env
+
+    # - name: 'Wait for PostgreSQL to become ready'
+    #   run: |
+    #     sudo apt-get install -y postgresql-client
+    #     until pg_isready -h localhost -p 5432 -U postgres; do sleep 2; done
 
     - name: 'Run tests with coverage'
       run: |
-        pip install coverage
+        docker exec coordinator coverage run manage.py test
-        coverage run manage.py test
+        docker exec coordinator coverage report
-        coverage report
+
+
+# jobs:
+#   test:
+#     runs-on: ubuntu-latest
+#     steps:
+#       - uses: actions/checkout@v2
+#       - name: Run Docker Compose
+#         run: |
+#           docker-compose up -d
+#           docker-compose run web python manage.py test

api/lightning/lnd.py

@@ -37,7 +37,7 @@ except Exception:
 
 # Read macaroon from file or .env variable string encoded as base64
 try:
-    with open(os.path.join(config("LND_DIR"), config("MACAROON_path")), "rb") as f:
+    with open(os.path.join(config("LND_DIR"), config("MACAROON_PATH")), "rb") as f:
         MACAROON = f.read()
 except Exception:
     MACAROON = b64decode(config("LND_MACAROON_BASE64"))
@@ -49,7 +49,7 @@ MAX_SWAP_AMOUNT = config("MAX_SWAP_AMOUNT", cast=int, default=500_000)
 
 # Logger function used to build tests/mocks/lnd.py
 def log(name, request, response):
-    if not config("LOG_LND", cast=bool, default=True):
+    if not config("LOG_LND", cast=bool, default=False):
         return
     current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     log_message = f"######################################\nEvent: {name}\nTime: {current_time}\nRequest:\n{request}\nResponse:\n{response}\nType: {type(response)}\n"

api/models/order.py

@@ -10,7 +10,7 @@ from django.db.models.signals import pre_delete
 from django.dispatch import receiver
 from django.utils import timezone
 
-if config("COORDINATOR_TESTING", cast=bool, default=False):
+if config("TESTING", cast=bool, default=False):
     import random
     import string
 

api/serializers.py

@@ -28,8 +28,8 @@ class InfoSerializer(serializers.Serializer):
     lifetime_volume = serializers.FloatField(
         help_text="Total volume in BTC since exchange's inception"
     )
-    lnd_version = serializers.CharField()
+    lnd_version = serializers.CharField(required=False)
-    cln_version = serializers.CharField()
+    cln_version = serializers.CharField(required=False)
     robosats_running_commit_hash = serializers.CharField()
     alternative_site = serializers.CharField()
     alternative_name = serializers.CharField()
@@ -170,11 +170,15 @@ class OrderDetailSerializer(serializers.ModelSerializer):
         "- **'Inactive'** (seen more than 10 min ago)\n\n"
         "Note: When you make a request to this route, your own status get's updated and can be seen by your counterparty",
     )
-    taker_status = serializers.BooleanField(
+    taker_status = serializers.CharField(
         required=False,
-        help_text="True if you are either a taker or maker, False otherwise",
+        help_text="Status of the maker:\n"
+        "- **'Active'** (seen within last 2 min)\n"
+        "- **'Seen Recently'** (seen within last 10 min)\n"
+        "- **'Inactive'** (seen more than 10 min ago)\n\n"
+        "Note: When you make a request to this route, your own status get's updated and can be seen by your counterparty",
     )
-    price_now = serializers.IntegerField(
+    price_now = serializers.FloatField(
         required=False,
         help_text="Price of the order in the order's currency at the time of request (upto 5 significant digits)",
     )
@@ -274,11 +278,11 @@ class OrderDetailSerializer(serializers.ModelSerializer):
         required=False,
         help_text="in percentage, the swap fee rate the platform charges",
     )
-    latitude = serializers.CharField(
+    latitude = serializers.FloatField(
         required=False,
         help_text="Latitude of the order for F2F payments",
     )
-    longitude = serializers.CharField(
+    longitude = serializers.FloatField(
         required=False,
         help_text="Longitude of the order for F2F payments",
     )

docker-tests.yml

@@ -0,0 +1,206 @@
+# Spin up a regtest lightning network to run integration tests
+# docker-compose -f docker-tests.yml --env-file tests/compose.env up -d
+
+# Some useful handy commands that hopefully are never needed
+# docker exec -it btc bitcoin-cli -chain=regtest -rpcpassword=test -rpcuser=test createwallet default
+# docker exec -it btc bitcoin-cli -chain=regtest -rpcpassword=test -rpcuser=test -generate 101
+
+# docker exec -it coordinator-lnd lncli --network=regtest getinfo
+# docker exec -it robot-lnd lncli --network=regtest --rpcserver localhost:10010 getinfo
+
+version: '3.9'
+services:
+  bitcoind:
+    image: ruimarinho/bitcoin-core:${BITCOIND_TAG}
+    container_name: btc
+    restart: always
+    ports:
+      - "8000:8000"
+    volumes:
+      - bitcoin:/bitcoin/.bitcoin/
+    command:
+      --txindex=1
+      --printtoconsole
+      --regtest=1
+      --server=1
+      --rest=1
+      --rpcuser=test
+      --rpcpassword=test
+      --logips=1
+      --debug=1
+      --rpcport=18443
+      --rpcallowip=172.0.0.0/8
+      --rpcallowip=192.168.0.0/16
+      --zmqpubrawblock=tcp://0.0.0.0:28332
+      --zmqpubrawtx=tcp://0.0.0.0:28333
+      --listenonion=0
+
+  coordinator-lnd:
+    image: lightninglabs/lnd:${LND_TAG}
+    container_name: coordinator-lnd
+    restart: always
+    volumes:
+      - bitcoin:/root/.bitcoin/
+      - lnd:/home/lnd/.lnd
+      - lnd:/root/.lnd
+    command:
+      --noseedbackup
+      --nobootstrap
+      --restlisten=localhost:8081
+      --no-rest-tls
+      --debuglevel=debug
+      --maxpendingchannels=10
+      --rpclisten=0.0.0.0:10009
+      --listen=0.0.0.0:9735
+      --color=#4126a7
+      --alias=RoboSats
+      --bitcoin.active
+      --bitcoin.regtest
+      --bitcoin.node=bitcoind
+      --bitcoind.rpchost=127.0.0.1
+      --bitcoind.rpcuser=test
+      --bitcoind.rpcpass=test
+      --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332
+      --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333
+      --protocol.wumbo-channels
+    depends_on:
+      - bitcoind
+    network_mode: service:bitcoind
+
+  robot-lnd:
+    image: lightninglabs/lnd:${LND_TAG}
+    container_name: robot-lnd
+    restart: always
+    volumes:
+      - bitcoin:/root/.bitcoin/
+      - lndrobot:/home/lnd/.lnd
+      - lndrobot:/root/.lnd
+    command:
+      --noseedbackup
+      --nobootstrap
+      --no-rest-tls
+      --debuglevel=debug
+      --maxpendingchannels=10
+      --rpclisten=0.0.0.0:10010
+      --listen=0.0.0.0:9736
+      --color=#4126a7
+      --alias=Robot
+      --bitcoin.active
+      --bitcoin.regtest
+      --bitcoin.node=bitcoind
+      --bitcoind.rpchost=127.0.0.1
+      --bitcoind.rpcuser=test
+      --bitcoind.rpcpass=test
+      --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332
+      --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333
+      --protocol.wumbo-channels
+    depends_on:
+      - bitcoind
+    network_mode: service:bitcoind
+
+  redis:
+    image: redis:${REDIS_TAG}
+    container_name: redis
+    restart: always
+    volumes:
+      - redisdata:/data
+    network_mode: service:bitcoind
+
+  coordinator:
+    build: .
+    image: robosats-image
+    container_name: coordinator
+    restart: always
+    environment:
+      DEVELOPMENT: True
+      TESTING: True
+      USE_TOR: False
+      MACAROON_PATH: 'data/chain/bitcoin/regtest/admin.macaroon'
+    env_file:
+      - ${ROBOSATS_ENVS_FILE}
+    depends_on:
+      - redis
+      - coordinator-lnd
+      - postgres
+    network_mode: service:bitcoind
+    volumes:
+      - .:/usr/src/robosats
+      - lnd:/lnd
+      - lndrobot:/lndrobot
+      - cln:/cln
+
+  postgres:
+    image: postgres:${POSTGRES_TAG:-14.2-alpine}
+    container_name: sql
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_DB: ${POSTGRES_DB}
+    network_mode: service:bitcoind
+
+  # clean-orders:
+  #   image: robosats-image
+  #   restart: always
+  #   container_name: clord
+  #   command: python3 manage.py clean_orders
+  #   environment:
+  #     SKIP_COLLECT_STATIC: "true"
+  #     POSTGRES_HOST: 'postgres'
+  #   env_file:
+  #     - ${ROBOSATS_ENVS_FILE}
+
+  # follow-invoices:
+  #   image: robosats-image
+  #   container_name: invo
+  #   restart: always
+  #   env_file:
+  #     - ${ROBOSATS_ENVS_FILE}
+  #   environment:
+  #     SKIP_COLLECT_STATIC: "true"
+  #     POSTGRES_HOST: 'postgres'
+  #   command: python3 manage.py follow_invoices
+
+  # telegram-watcher:
+  #   image: robosats-image
+  #   container_name: tg
+  #   restart: always
+  #   environment:
+  #     SKIP_COLLECT_STATIC: "true"
+  #     POSTGRES_HOST: 'postgres'
+  #   env_file:
+  #     - ${ROBOSATS_ENVS_FILE}
+  #   command: python3 manage.py telegram_watcher
+
+  # celery:
+  #   image: robosats-image
+  #   container_name: cele
+  #   restart: always
+  #   env_file:
+  #     - ${ROBOSATS_ENVS_FILE}
+  #   environment:
+  #     SKIP_COLLECT_STATIC: "true"
+  #     POSTGRES_HOST: 'postgres'
+  #   command: celery -A robosats worker --loglevel=WARNING
+  #   depends_on:
+  #     - redis
+
+  # celery-beat:
+  #   image: robosats-image
+  #   container_name: beat
+  #   restart: always
+  #   env_file:
+  #     - ${ROBOSATS_ENVS_FILE}
+  #   environment:
+  #     SKIP_COLLECT_STATIC: "true"
+  #     POSTGRES_HOST: 'postgres'
+  #   command: celery -A robosats beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler
+  #   depends_on:
+  #     - redis
+
+volumes:
+  redisdata:
+  bitcoin:
+  lnd:
+  cln:
+  lndrobot:

requirements.txt

@@ -26,9 +26,5 @@ python-gnupg==0.5.1
 daphne==4.0.0
 drf-spectacular==0.26.2
 drf-spectacular-sidecar==2023.5.1
-black==23.3.0
-isort==5.12.0
-flake8==6.1.0
-pyflakes==3.1.0
 django-cors-headers==4.3.0
 base91==1.0.1

requirements_dev.txt

@@ -0,0 +1,7 @@
+black==23.3.0
+isort==5.12.0
+flake8==6.1.0
+pyflakes==3.1.0
+coverage==7.3.2
+drf-openapi-tester==2.3.3
+pre-commit==3.5.0

robosats/middleware.py

@@ -161,6 +161,8 @@ class RobotTokenSHA256AuthenticationMiddleWare:
                     resized_img.save(f, format="WEBP", quality=80)
 
             user.robot.avatar = "static/assets/avatars/" + nickname + ".webp"
+
+            update_last_login(None, user)
             user.save()
 
         response = self.get_response(request)

scripts/entrypoint.sh

@@ -10,6 +10,12 @@ else
     python manage.py collectstatic --noinput
 fi
 
+# Collect static files
+if [ $DEVELOPMENT ]; then
+    echo "Installing python development dependencies"
+    pip install -r requirements_dev.txt
+fi
+
 # Print first start up message when pb2/grpc files if they do exist
 if [ ! -f "/usr/src/robosats/api/lightning/lightning_pb2.py" ]; then
     echo "Looks like the first run of this container. pb2 and gRPC files were not detected on the attached volume, copying them into the attached volume /robosats/api/lightning ."

tests/api_specs.yaml

@@ -1161,12 +1161,10 @@ components:
       - alternative_site
       - bond_size
       - book_liquidity
-      - cln_version
       - current_swap_fee_rate
       - last_day_nonkyc_btc_premium
       - last_day_volume
       - lifetime_volume
-      - lnd_version
       - maker_fee
       - network
       - node_alias
@@ -1486,10 +1484,17 @@ components:
 
             Note: When you make a request to this route, your own status get's updated and can be seen by your counterparty
         taker_status:
-          type: boolean
+          type: string
-          description: True if you are either a taker or maker, False otherwise
+          description: |-
+            Status of the maker:
+            - **'Active'** (seen within last 2 min)
+            - **'Seen Recently'** (seen within last 10 min)
+            - **'Inactive'** (seen more than 10 min ago)
+
+            Note: When you make a request to this route, your own status get's updated and can be seen by your counterparty
         price_now:
-          type: integer
+          type: number
+          format: double
           description: Price of the order in the order's currency at the time of request
             (upto 5 significant digits)
         premium_percentile:
@@ -1657,10 +1662,12 @@ components:
           description: The network eg. 'testnet', 'mainnet'. Only if status = `14`
             (Successful Trade) and is_buyer = `true`
         latitude:
-          type: string
+          type: number
+          format: double
           description: Latitude of the order for F2F payments
         longitude:
-          type: string
+          type: number
+          format: double
           description: Longitude of the order for F2F payments
       required:
       - expires_at

tests/compose.env

@@ -0,0 +1,10 @@
+ROBOSATS_ENVS_FILE=".env-sample"
+
+BITCOIND_TAG='24.0.1-alpine'
+LND_TAG='v0.17.0-beta'
+REDIS_TAG='7.2.1-alpine@sha256:7f5a0dfbf379db69dc78434091dce3220e251022e71dcdf36207928cbf9010de'
+POSTGRES_TAG='14.2-alpine'
+
+POSTGRES_DB='postgres'
+POSTGRES_USER='postgres'
+POSTGRES_PASSWORD='example'

tests/mocks/cln.py

@@ -1,6 +1,7 @@
 from unittest.mock import MagicMock
 
 # Mock up of CLN gRPC responses
+# Unfinished, during integration tests we SHOULD spin up a regtest CLN instance
 
 
 class MockNodeStub:
@@ -31,28 +32,3 @@ class MockNodeStub:
         response.binding.address = "127.0.0.1"
         response.binding.port = 9736
         return response
-
-
-class MockHoldStub:
-    def __init__(self, channel):
-        pass
-
-    def HoldInvoiceLookup(self, request):
-        response = MagicMock()
-        return response
-
-    def HoldInvoice(self, request):
-        response = MagicMock()
-        return response
-
-    def HoldInvoiceSettle(self, request):
-        response = MagicMock()
-        return response
-
-    def HoldInvoiceCancel(self, request):
-        response = MagicMock()
-        return response
-
-    def DecodeBolt11(self, request):
-        response = MagicMock()
-        return response

tests/mocks/lnd.py

@@ -1,6 +1,7 @@
 from unittest.mock import MagicMock
 
 # Mock up of LND gRPC responses
+# Unfinished, during integration tests we spin up a regtest LND instance
 
 
 class MockLightningStub:
@@ -22,7 +23,7 @@ class MockLightningStub:
         response = MagicMock()
         if (
             request.pay_req
-            == "lntb17310n1pj552mdpp50p2utzh7mpsf3uq7u7cws4a96tj3kyq54hchdkpw8zecamx9klrqd2j2pshjmt9de6zqun9vejhyetwvdjn5gphxs6nsvfe893z6wphvfsj6dryvymj6wp5xvuz6wp5xcukvdec8yukgcf49cs9g6rfwvs8qcted4jkuapq2ay5cnpqgefy2326g5syjn3qt984253q2aq5cnz92skzqcmgv43kkgr0dcs9ymmzdafkzarnyp5kvgr5dpjjqmr0vd4jqampwvs8xatrvdjhxumxw4kzugzfwss8w6tvdssxyefqw4hxcmmrddjkggpgveskjmpfyp6kumr9wdejq7t0w5sxx6r9v96zqmmjyp3kzmnrv4kzqatwd9kxzar9wfskcmre9ccqz52xqzwzsp5hkzegrhn6kegr33z8qfxtcudaklugygdrakgyy7va0wt2qs7drfq9qyyssqc6rztchzl4m7mlulrhlcajszcl9fan8908k9n5x7gmz8g8d6ht5pj4l8r0dushq6j5s8x7yv9a5klz0kfxwy8v6ze6adyrrp4wu0q0sq3t604x"
+            == "lntb17310n1pj552mdpp50p2utgzfwss8w6tvdssxyefqw4hxcmmrddjkggpgveskjmpfyp6kumr9wdejq7t0w5sxx6r9v96zqmmjyp3kzmnrv4kzqatwd9kxzar9wfskcmre9ccqz52xqzwzsp5hkzegrhn6kegr33z8qfxtcudaklugygdrakgyy7va0wt2qs7drfq9qyyssqc6rztchzl4m7mlulrhlcajszcl9fan8908k9n5x7gmz8g8d6ht5pj4l8r0dushq6j5s8x7yv9a5klz0kfxwy8v6ze6adyrrp4wu0q0sq3t604x"
         ):
             response.destination = (
                 "033b58d7681fe5dd2fb21fd741996cda5449616f77317dd1156b80128d6a71b807"
@@ -35,7 +36,9 @@ class MockLightningStub:
             response.expiry = 450
             response.description = "Payment reference: 7458199b-87ba-4da7-8438-8469f7899da5. This payment WILL FREEZE IN YOUR WALLET, check on RoboSats if the lock was successful. It will be unlocked (fail) unless you cheat or cancel unilaterally."
             response.cltv_expiry = 650
-            response.payment_addr = '\275\205\224\016\363\325\262\201\306"8\022e\343\215\355\277\304\021\r\037l\202\023\314\353\334\265\002\036h\322'
+            response.payment_addr = (
+                "\275\205\224\016\363\325\262\201\353\334\265\002\036h\322"
+            )
             response.num_msat = 1731000
 
             return response
@@ -86,7 +89,7 @@ class MockInvoicesStub:
 
     def AddHoldInvoice(self, request):
         response = MagicMock()
-        # if request.value == 1731:
+        print(request)
         response.payment_request = "lntb17310n1pj552mdpp50p2utzh7mpsf3uq7u7cws4a96tj3kyq54hchdkpw8zecamx9klrqd2j2pshjmt9de6zqun9vejhyetwvdjn5gphxs6nsvfe893z6wphvfsj6dryvymj6wp5xvuz6wp5xcukvdec8yukgcf49cs9g6rfwvs8qcted4jkuapq2ay5cnpqgefy2326g5syjn3qt984253q2aq5cnz92skzqcmgv43kkgr0dcs9ymmzdafkzarnyp5kvgr5dpjjqmr0vd4jqampwvs8xatrvdjhxumxw4kzugzfwss8w6tvdssxyefqw4hxcmmrddjkggpgveskjmpfyp6kumr9wdejq7t0w5sxx6r9v96zqmmjyp3kzmnrv4kzqatwd9kxzar9wfskcmre9ccqz52xqzwzsp5hkzegrhn6kegr33z8qfxtcudaklugygdrakgyy7va0wt2qs7drfq9qyyssqc6rztchzl4m7mlulrhlcajszcl9fan8908k9n5x7gmz8g8d6ht5pj4l8r0dushq6j5s8x7yv9a5klz0kfxwy8v6ze6adyrrp4wu0q0sq3t604x"
         response.add_index = 1
         response.payment_addr = b'\275\205\224\016\363\325\262\201\306"8\022e\343\215\355\277\304\021\r\037l\202\023\314\353\334\265\002\036h\322'

tests/node_utils.py

@@ -0,0 +1,98 @@
+import requests
+from requests.auth import HTTPBasicAuth
+from requests.exceptions import ReadTimeout
+
+
+def get_node(name="robot"):
+    """
+    We have two regtest LND nodes: "coordinator" (the robosats backend) and "robot" (the robosats user)
+    """
+    if name == "robot":
+        with open("/lndrobot/data/chain/bitcoin/regtest/admin.macaroon", "rb") as f:
+            macaroon = f.read()
+            return {"port": 8080, "headers": {"Grpc-Metadata-macaroon": macaroon.hex()}}
+
+    elif name == "coordinator":
+        with open("/lnd/data/chain/bitcoin/regtest/admin.macaroon", "rb") as f:
+            macaroon = f.read()
+            return {"port": 8081, "headers": {"Grpc-Metadata-macaroon": macaroon.hex()}}
+
+
+def get_node_id(node_name):
+    node = get_node(node_name)
+    response = requests.get(
+        f'http://localhost:{node["port"]}/v1/getinfo', headers=node["headers"]
+    )
+    data = response.json()
+    return data["identity_pubkey"]
+
+
+def connect_to_node(node_name, node_id, ip_port):
+    node = get_node(node_name)
+    data = {"addr": {"pubkey": node_id, "host": ip_port}}
+    response = requests.post(
+        f'http://localhost:{node["port"]}/v1/peers', json=data, headers=node["headers"]
+    )
+    return response.json()
+
+
+def open_channel(node_name, node_id, local_funding_amount, push_sat):
+    node = get_node(node_name)
+    data = {
+        "node_pubkey_string": node_id,
+        "local_funding_amount": local_funding_amount,
+        "push_sat": push_sat,
+    }
+    response = requests.post(
+        f'http://localhost:{node["port"]}/v1/channels',
+        json=data,
+        headers=node["headers"],
+    )
+    return response.json()
+
+
+def create_address(node_name):
+    node = get_node(node_name)
+    response = requests.get(
+        f'http://localhost:{node["port"]}/v1/newaddress', headers=node["headers"]
+    )
+    return response.json()["address"]
+
+
+def generate_blocks(address, num_blocks):
+    data = {
+        "jsonrpc": "1.0",
+        "id": "curltest",
+        "method": "generatetoaddress",
+        "params": [num_blocks, address],
+    }
+    response = requests.post(
+        "http://localhost:18443", json=data, auth=HTTPBasicAuth("test", "test")
+    )
+    return response.json()
+
+
+def pay_invoice(node_name, invoice):
+    node = get_node(node_name)
+    data = {"payment_request": invoice}
+    try:
+        requests.post(
+            f'http://localhost:{node["port"]}/v1/channels/transactions',
+            json=data,
+            headers=node["headers"],
+            timeout=1,
+        )
+    except ReadTimeout:
+        # Request to pay hodl invoice has timed out: that's good!
+        return
+
+
+def add_invoice(node_name, amount):
+    node = get_node(node_name)
+    data = {"value": amount}
+    response = requests.post(
+        f'http://localhost:{node["port"]}/v1/invoices',
+        json=data,
+        headers=node["headers"],
+    )
+    return response.json()["payment_request"]

tests/test_coordinator_info.py

@@ -1,5 +1,4 @@
 import json
-from unittest.mock import patch
 
 from decouple import config
 from django.conf import settings
@@ -7,8 +6,6 @@ from django.contrib.auth.models import User
 from django.test import Client
 from django.urls import reverse
 
-from tests.mocks.cln import MockNodeStub
-from tests.mocks.lnd import MockVersionerStub
 from tests.test_api import BaseAPITestCase
 
 FEE = config("FEE", cast=float, default=0.2)
@@ -31,8 +28,6 @@ class CoordinatorInfoTest(BaseAPITestCase):
         self.client = Client()
         User.objects.create_superuser(self.su_name, "super@user.com", self.su_pass)
 
-    @patch("api.lightning.cln.node_pb2_grpc.NodeStub", MockNodeStub)
-    @patch("api.lightning.lnd.verrpc_pb2_grpc.VersionerStub", MockVersionerStub)
     def test_info(self):
         path = reverse("info")
 
@@ -56,7 +51,9 @@ class CoordinatorInfoTest(BaseAPITestCase):
         )
         self.assertEqual(data["version"], settings.VERSION)
         self.assertEqual(data["node_id"], NODE_ID)
-        self.assertEqual(data["network"], "testnet")
+        self.assertEqual(
+            data["network"], "testnet"
+        )  # tests take place in regtest, but this attribute is read from .env
         self.assertAlmostEqual(data["maker_fee"], MAKER_FEE)
         self.assertAlmostEqual(data["taker_fee"], TAKER_FEE)
         self.assertAlmostEqual(data["bond_size"], BOND_SIZE)

tests/test_trade_pipeline.py

@@ -1,7 +1,7 @@
 import json
+import time
 from datetime import datetime
 from decimal import Decimal
-from unittest.mock import patch
 
 from decouple import config
 from django.contrib.auth.models import User
@@ -10,10 +10,13 @@ from django.urls import reverse
 from api.management.commands.follow_invoices import Command as FollowInvoices
 from api.models import Currency, Order
 from api.tasks import cache_market
-from tests.mocks.cln import MockHoldStub  # , MockNodeStub
+from tests.node_utils import (
-from tests.mocks.lnd import (  # MockRouterStub,; MockSignerStub,; MockVersionerStub,
+    connect_to_node,
-    MockInvoicesStub,
+    create_address,
-    MockLightningStub,
+    generate_blocks,
+    get_node_id,
+    open_channel,
+    pay_invoice,
 )
 from tests.test_api import BaseAPITestCase
 
@@ -57,6 +60,21 @@ class TradeTest(BaseAPITestCase):
         # Fetch currency prices from external APIs
         cache_market()
 
+        # Fund two LN nodes in regtest and open channels
+        coordinator_node_id = get_node_id("coordinator")
+        connect_to_node("robot", coordinator_node_id, "localhost:9735")
+
+        funding_address = create_address("robot")
+        generate_blocks(funding_address, 101)
+
+        time.sleep(
+            2
+        )  # channels cannot be created until the node is fully sync. We just created 101 blocks.
+        open_channel("robot", coordinator_node_id, 100_000_000, 50_000_000)
+
+        # Generate 6 blocks so the channel becomes active
+        generate_blocks(funding_address, 6)
+
     def test_login_superuser(self):
         """
         Test the login functionality for the superuser.
@@ -65,7 +83,9 @@ class TradeTest(BaseAPITestCase):
         data = {"username": self.su_name, "password": self.su_pass}
         response = self.client.post(path, data)
         self.assertEqual(response.status_code, 302)
-        self.assertResponse(response)
+        self.assertResponse(
+            response
+        )  # should skip given that /coordinator/login is not documented
 
     def test_cache_market(self):
         """
@@ -102,13 +122,15 @@ class TradeTest(BaseAPITestCase):
         else:
             headers = {"HTTP_AUTHORIZATION": f"Token {b91_token}"}
 
-        return headers, pub_key, enc_priv_key
+        return headers
 
-    def assert_robot(self, response, pub_key, enc_priv_key, robot_index):
+    def assert_robot(self, response, robot_index):
         """
         Assert that the robot is created correctly.
         """
         nickname = read_file(f"tests/robots/{robot_index}/nickname")
+        pub_key = read_file(f"tests/robots/{robot_index}/pub_key")
+        enc_priv_key = read_file(f"tests/robots/{robot_index}/enc_priv_key")
 
         data = json.loads(response.content.decode())
 
@@ -148,18 +170,17 @@ class TradeTest(BaseAPITestCase):
         Creates the robots in /tests/robots/{robot_index}
         """
         path = reverse("robot")
-        headers, pub_key, enc_priv_key = self.get_robot_auth(robot_index, True)
+        headers = self.get_robot_auth(robot_index, True)
 
-        response = self.client.get(path, **headers)
+        return self.client.get(path, **headers)
-
-        self.assert_robot(response, pub_key, enc_priv_key, robot_index)
 
     def test_create_robots(self):
         """
         Test the creation of two robots to be used in the trade tests
         """
-        self.create_robot(robot_index=1)
+        for robot_index in [1, 2]:
-        self.create_robot(robot_index=2)
+            response = self.create_robot(robot_index)
+            self.assert_robot(response, robot_index)
 
     def make_order(self, maker_form, robot_index=1):
         """
@@ -167,7 +188,7 @@ class TradeTest(BaseAPITestCase):
         """
         path = reverse("make")
         # Get valid robot auth headers
-        headers, _, _ = self.get_robot_auth(robot_index, True)
+        headers = self.get_robot_auth(robot_index, True)
 
         response = self.client.post(path, maker_form, **headers)
         return response
@@ -181,6 +202,8 @@ class TradeTest(BaseAPITestCase):
         data = json.loads(response.content.decode())
 
         # Checks
+        self.assertResponse(response)
+
         self.assertIsInstance(data["id"], int, "Order ID is not an integer")
         self.assertEqual(
             data["status"],
@@ -255,13 +278,10 @@ class TradeTest(BaseAPITestCase):
 
         return data
 
-    @patch("api.lightning.cln.hold_pb2_grpc.HoldStub", MockHoldStub)
-    @patch("api.lightning.lnd.lightning_pb2_grpc.LightningStub", MockLightningStub)
-    @patch("api.lightning.lnd.invoices_pb2_grpc.InvoicesStub", MockInvoicesStub)
     def get_order(self, order_id, robot_index=1, first_encounter=False):
         path = reverse("order")
         params = f"?order_id={order_id}"
-        headers, _, _ = self.get_robot_auth(robot_index, first_encounter)
+        headers = self.get_robot_auth(robot_index, first_encounter)
         response = self.client.get(path + params, **headers)
 
         return response
@@ -279,6 +299,8 @@ class TradeTest(BaseAPITestCase):
         data = json.loads(response.content.decode())
 
         self.assertEqual(response.status_code, 200)
+        self.assertResponse(response)
+
         self.assertEqual(data["id"], order_made_data["id"])
         self.assertTrue(
             isinstance(datetime.fromisoformat(data["created_at"]), datetime)
@@ -301,13 +323,8 @@ class TradeTest(BaseAPITestCase):
         self.assertFalse(data["maker_locked"])
         self.assertFalse(data["taker_locked"])
         self.assertFalse(data["escrow_locked"])
-        self.assertEqual(
-            data["bond_invoice"],
-            "lntb17310n1pj552mdpp50p2utzh7mpsf3uq7u7cws4a96tj3kyq54hchdkpw8zecamx9klrqd2j2pshjmt9de6zqun9vejhyetwvdjn5gphxs6nsvfe893z6wphvfsj6dryvymj6wp5xvuz6wp5xcukvdec8yukgcf49cs9g6rfwvs8qcted4jkuapq2ay5cnpqgefy2326g5syjn3qt984253q2aq5cnz92skzqcmgv43kkgr0dcs9ymmzdafkzarnyp5kvgr5dpjjqmr0vd4jqampwvs8xatrvdjhxumxw4kzugzfwss8w6tvdssxyefqw4hxcmmrddjkggpgveskjmpfyp6kumr9wdejq7t0w5sxx6r9v96zqmmjyp3kzmnrv4kzqatwd9kxzar9wfskcmre9ccqz52xqzwzsp5hkzegrhn6kegr33z8qfxtcudaklugygdrakgyy7va0wt2qs7drfq9qyyssqc6rztchzl4m7mlulrhlcajszcl9fan8908k9n5x7gmz8g8d6ht5pj4l8r0dushq6j5s8x7yv9a5klz0kfxwy8v6ze6adyrrp4wu0q0sq3t604x",
-        )
         self.assertTrue(isinstance(data["bond_satoshis"], int))
 
-    @patch("api.lightning.lnd.invoices_pb2_grpc.InvoicesStub", MockInvoicesStub)
     def check_for_locked_bonds(self):
         # A background thread checks every 5 second the status of invoices. We invoke directly during test.
         # It will ask LND via gRPC. In our test, the request/response from LND is mocked, and it will return fake invoice status "ACCEPTED"
@@ -320,7 +337,11 @@ class TradeTest(BaseAPITestCase):
         order_made_data = json.loads(order_made_response.content.decode())
 
         # Maker's first order fetch. Should trigger maker bond hold invoice generation.
-        self.get_order(order_made_data["id"])
+        response = self.get_order(order_made_data["id"])
+        invoice = response.json()["bond_invoice"]
+
+        # Lock the invoice from the robot's node
+        pay_invoice("robot", invoice)
 
         # Check for invoice locked (the mocked LND will return ACCEPTED)
         self.check_for_locked_bonds()
@@ -336,6 +357,8 @@ class TradeTest(BaseAPITestCase):
         data = json.loads(response.content.decode())
 
         self.assertEqual(response.status_code, 200)
+        self.assertResponse(response)
+
         self.assertEqual(data["id"], data["id"])
         self.assertEqual(data["status_message"], Order.Status(Order.Status.PUB).label)
         self.assertTrue(data["maker_locked"])
@@ -352,13 +375,13 @@ class TradeTest(BaseAPITestCase):
         self.assertTrue(isinstance(public_data["price_now"], float))
         self.assertTrue(isinstance(data["satoshis_now"], int))
 
-    @patch("api.lightning.cln.hold_pb2_grpc.HoldStub", MockHoldStub)
+    # @patch("api.lightning.cln.hold_pb2_grpc.HoldStub", MockHoldStub)
-    @patch("api.lightning.lnd.lightning_pb2_grpc.LightningStub", MockLightningStub)
+    # @patch("api.lightning.lnd.lightning_pb2_grpc.LightningStub", MockLightningStub)
-    @patch("api.lightning.lnd.invoices_pb2_grpc.InvoicesStub", MockInvoicesStub)
+    # @patch("api.lightning.lnd.invoices_pb2_grpc.InvoicesStub", MockInvoicesStub)
     def take_order(self, order_id, amount, robot_index=2):
         path = reverse("order")
         params = f"?order_id={order_id}"
-        headers, _, _ = self.get_robot_auth(robot_index, first_encounter=True)
+        headers = self.get_robot_auth(robot_index, first_encounter=True)
         body = {"action": "take", "amount": amount}
         response = self.client.post(path + params, body, **headers)
 
@@ -372,28 +395,29 @@ class TradeTest(BaseAPITestCase):
         response = self.take_order(data_publised["id"], take_amount, taker_index)
         return response
 
-    # def test_make_and_take_order(self):
+    def test_make_and_take_order(self):
-    #     maker_index = 1
+        maker_index = 1
-    #     taker_index = 2
+        taker_index = 2
-    #     maker_form = self.maker_form_with_range
+        maker_form = self.maker_form_with_range
-    #     self.create_robot(taker_index) #### WEEEE SHOULD NOT BE NEEDED >??? WHY ROBOT HAS NO LOGIN TIME??
-    #     response = self.make_and_take_order(maker_form, 80, maker_index, taker_index)
-    #     data = json.loads(response.content.decode())
 
-    #     print(data)
+        response = self.make_and_take_order(maker_form, 80, maker_index, taker_index)
+        data = json.loads(response.content.decode())
 
-    #     self.assertEqual(
+        self.assertEqual(response.status_code, 200)
-    #         data["ur_nick"], read_file(f"tests/robots/{taker_index}/nickname")
+        self.assertResponse(response)
-    #     )
+
-    #     self.assertEqual(
+        self.assertEqual(
-    #         data["taker_nick"], read_file(f"tests/robots/{taker_index}/nickname")
+            data["ur_nick"], read_file(f"tests/robots/{taker_index}/nickname")
-    #     )
+        )
-    #     self.assertEqual(
+        self.assertEqual(
-    #         data["maker_nick"], read_file(f"tests/robots/{maker_index}/nickname")
+            data["taker_nick"], read_file(f"tests/robots/{taker_index}/nickname")
-    #     )
+        )
-    #     self.assertFalse(data["is_maker"])
+        self.assertEqual(
-    #     self.assertTrue(data["is_taker"])
+            data["maker_nick"], read_file(f"tests/robots/{maker_index}/nickname")
-    #     self.assertTrue(data["is_participant"])
+        )
+        self.assertFalse(data["is_maker"])
+        self.assertTrue(data["is_taker"])
+        self.assertTrue(data["is_participant"])
 
     #     a = {
     #         "maker_status": "Active",