From aea467a95253ac40d252a32461078a7772a40076 Mon Sep 17 00:00:00 2001 From: Mariano Sorgente Date: Wed, 22 Jan 2020 17:53:12 +0900 Subject: [PATCH 01/38] Fix node starting script (#85) --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8947f3bbdf96..c3e4f4ca381d 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ sh install.sh ```bash sudo yum update -sudo yum install gcc-c++ cmake3 wget git openssl openssl-devel +sudo yum install gcc-c++ cmake3 wget git openssl openssl-devel sudo yum install python3 python3-devel libffi-devel gmp-devel # CMake - add a symlink for cmake3 - required by blspy @@ -191,7 +191,7 @@ To run a full node on port 8444, and connect to the testnet, run the following c This will also start an ssh server in port 8222 for the UI, which you can connect to to see the state of the node. ```bash -python -m src.server.start_full_node "127.0.0.1" 8444 -id 1 -r 8555 & +./scripts/run_full_node.sh ssh -p 8222 localhost ``` @@ -200,7 +200,7 @@ Instead of running only a full node (as in 4a), you can also run a farmer. Farmers are entities in the network who use their hard drive space to try to create blocks (like Bitcoin's miners), and earn block rewards. First, you must generate some hard drive plots, which can take a long time depending on the [size of the plots](https://github.com/Chia-Network/chia-blockchain/wiki/k-sizes) -(the k variable). Then, run the farmer + full node with the following script. A full node is also started, +(the k variable). Then, run the farmer + full node with the following script. A full node is also started, which you can ssh into to view the node UI (previous ssh command). ```bash python -m scripts.create_plots -k 20 -n 10 @@ -225,7 +225,7 @@ UPnP is enabled by default, to open port 8444 for incoming connections. If this you can disable it in the configuration. Some routers may require port forwarding, or enabling UPnP in the router configuration. -Due to the nature of proof of space lookups by the harvester in the current alpha you should limit +Due to the nature of proof of space lookups by the harvester in the current alpha you should limit the number of plots on a physical drive to 50 or less. This limit should significantly increase before beta. You can also run the simulation, which runs all servers and multiple full nodes, locally, at once. From b85144423594bab65e9fe2520206a7ae87e3fd13 Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Mon, 27 Jan 2020 20:36:50 -0800 Subject: [PATCH 02/38] RHEL8 farmer/fullnode install instructions --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 578f1deecadd..f977d54765ee 100644 --- a/README.md +++ b/README.md @@ -63,9 +63,28 @@ git clone https://github.com/Chia-Network/chia-blockchain.git cd chia-blockchain sh install.sh + . .venv/bin/activate ``` +### RHEL 8.0 +```bash +sudo yum update +sudo yum install gcc-c++ cmake3 git openssl openssl-devel +sudo yum install wget make libffi-devel gmp-devel + +# Install Python 3.7.5 (current rpm's are 3.6.x) +wget https://www.python.org/ftp/python/3.7.5/Python-3.7.5.tgz +tar -zxvf Python-3.7.5.tgz; cd Python-3.7.5 +./configure --enable-optimizations; sudo make install; cd .. + +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain + +sh install.sh + +. .venv/bin/activate +``` ### Windows (WSL + Ubuntu) #### Install WSL + Ubuntu 18.04 LTS, upgrade to Ubuntu 19.x @@ -89,6 +108,7 @@ git clone https://github.com/Chia-Network/chia-blockchain.git cd chia-blockchain sudo sh install.sh + . .venv/bin/activate ``` From 3e06d9bbbe422a4515366c194886224c2378efed Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Mon, 27 Jan 2020 20:45:48 -0800 Subject: [PATCH 03/38] Add RHEL 8 VDF install instructions --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index f977d54765ee..6c905e93cd53 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,14 @@ cd chia-blockchain sh install_timelord.sh ``` +### RHEL 8.0 +``` +sudo yum install mpfr-devel boost boost-devel +cd chia-blockchain + +sh install_timelord.sh +``` ### Windows (WSL + Ubuntu) #### Install WSL + Ubuntu upgraded to 19.x ```bash From a679aedcf521001a8379933573a0f8678477ec70 Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Mon, 27 Jan 2020 20:46:56 -0800 Subject: [PATCH 04/38] Typo in RHEL VDF instructions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6c905e93cd53..7215181e4069 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ cd chia-blockchain sh install_timelord.sh ``` ### RHEL 8.0 -``` +```bash sudo yum install mpfr-devel boost boost-devel cd chia-blockchain From 1e2e5575f33528ce070ba4d225e1a273707b53b8 Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Mon, 27 Jan 2020 20:49:38 -0800 Subject: [PATCH 05/38] Missing sqlite req for RHEL 8 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7215181e4069..8a791f101c42 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ sh install.sh ```bash sudo yum update sudo yum install gcc-c++ cmake3 git openssl openssl-devel -sudo yum install wget make libffi-devel gmp-devel +sudo yum install wget make libffi-devel gmp-devel sqlite-devel # Install Python 3.7.5 (current rpm's are 3.6.x) wget https://www.python.org/ftp/python/3.7.5/Python-3.7.5.tgz From fb2ccbfb756df115808adb44d90554d0007140e3 Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Mon, 27 Jan 2020 21:12:35 -0800 Subject: [PATCH 06/38] Lower boost requirements to 1.66 per RHEL Boost 1.66 is good enough --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8a791f101c42..ee473187e412 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ sh install.sh ## Step 2: Install timelord (optional) Note: this step is needed only if you intend to run a timelord or a local simulation. -These assume you've already successfully installed harvester, farmer, plotting, and full node above. boost 1.67 or newer is required on all platforms. +These assume you've already successfully installed harvester, farmer, plotting, and full node above. boost 1.66 or newer is required on all platforms. ### Ubuntu/Debian ```bash cd chia-blockchain From ea94c5c2bf3254d9f92e2f56c635e56fccea1927 Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Mon, 27 Jan 2020 22:19:47 -0800 Subject: [PATCH 07/38] Install actually RHEL 8.1 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ee473187e412..851c622e0d90 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ sh install.sh . .venv/bin/activate ``` -### RHEL 8.0 +### RHEL 8.1 ```bash sudo yum update @@ -174,7 +174,7 @@ cd chia-blockchain sh install_timelord.sh ``` -### RHEL 8.0 +### RHEL 8.1 ```bash sudo yum install mpfr-devel boost boost-devel From 3a7cab800d6fb8b06fd3845eaf7f71ae41ea9949 Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Mon, 27 Jan 2020 23:19:56 -0800 Subject: [PATCH 08/38] Specify CentOS 7.7 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 851c622e0d90..4e63c4f711b9 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ sh install.sh . .venv/bin/activate ``` -### CentOS 7 +### CentOS 7.7 ```bash sudo yum update @@ -156,7 +156,7 @@ cd chia-blockchain sh install_timelord.sh ``` -### Amazon Linux 2 and CentOS 7 +### Amazon Linux 2 and CentOS 7.7 ```bash #Only for Amazon Linux 2 sudo amazon-linux-extras install epel From 22991687d8025265bfe276c100dd0b669d818281 Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Sat, 1 Feb 2020 23:14:12 -0800 Subject: [PATCH 09/38] UI should say "Synced" instead of "Not Syncing" UI spends most of it's life Synced so change from the negative tense --- src/ui/prompt_ui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ui/prompt_ui.py b/src/ui/prompt_ui.py index f45fab311b44..699768d185e4 100644 --- a/src/ui/prompt_ui.py +++ b/src/ui/prompt_ui.py @@ -340,7 +340,7 @@ class FullNodeUI: else: self.syncing.text = f"Syncing" else: - self.syncing.text = "Not syncing" + self.syncing.text = "Synced" total_iters = self.lca_block.challenge.total_iters From 5e924d2100f8a38a5d9cbe505f76af785cd547b3 Mon Sep 17 00:00:00 2001 From: wjblanke Date: Thu, 6 Feb 2020 09:27:56 -0800 Subject: [PATCH 10/38] Softaes14wjb (#95) Software AES fallback for runtime (eg Celeron) and compiletime (eg Raspberry Pi via cmake try_run) --- lib/chiapos/CMakeLists.txt | 14 + lib/chiapos/src/aes.hpp | 661 ++++++++++++++++++--------- lib/chiapos/src/aesni.hpp | 270 +++++++++++ lib/chiapos/src/cmake_aesni_test.cpp | 16 + 4 files changed, 742 insertions(+), 219 deletions(-) create mode 100644 lib/chiapos/src/aesni.hpp create mode 100644 lib/chiapos/src/cmake_aesni_test.cpp diff --git a/lib/chiapos/CMakeLists.txt b/lib/chiapos/CMakeLists.txt index 4a1d1c1d7cb5..c45cf137ae68 100644 --- a/lib/chiapos/CMakeLists.txt +++ b/lib/chiapos/CMakeLists.txt @@ -32,6 +32,20 @@ add_subdirectory(lib/pybind11) pybind11_add_module(chiapos ${CMAKE_CURRENT_SOURCE_DIR}/python-bindings/chiapos.cpp) set (CMAKE_CXX_FLAGS "-g -O3 -Wall -msse2 -msse -march=native -std=c++1z -maes") +try_run(CMAKE_AESNI_TEST_RUN_RESULT + CMAKE_AESNI_TEST_COMPILE_RESULT + ${CMAKE_CURRENT_BINARY_DIR}/cmake_aesni_test + ${CMAKE_CURRENT_SOURCE_DIR}/src/cmake_aesni_test.cpp) + +# Did compilation succeed and process return 0 (success)? +IF("${CMAKE_AESNI_TEST_COMPILE_RESULT}" AND ("${CMAKE_AESNI_TEST_RUN_RESULT}" EQUAL 0)) + message(STATUS "AESNI Enabled") + set (CMAKE_CXX_FLAGS "-g -O3 -Wall -msse2 -msse -march=native -std=c++17 -maes") +ELSE() + message(STATUS "AESNI Disabled") + add_compile_definitions (DISABLE_AESNI) + set (CMAKE_CXX_FLAGS "-g -O3 -Wall -march=native -std=c++17") +ENDIF() add_executable(ProofOfSpace src/cli.cpp diff --git a/lib/chiapos/src/aes.hpp b/lib/chiapos/src/aes.hpp index def86c3b9121..129782be3c43 100644 --- a/lib/chiapos/src/aes.hpp +++ b/lib/chiapos/src/aes.hpp @@ -1,270 +1,493 @@ -// Copyright 2018 Chia Network Inc - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at - -// http://www.apache.org/licenses/LICENSE-2.0 - -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Some public domain code is taken from pycrypto: -// https://github.com/dlitz/pycrypto/blob/master/src/AESNI.c -// -// AESNI.c: AES using AES-NI instructions -// -// Written in 2013 by Sebastian Ramacher - -#ifndef SRC_CPP_AES_HPP_ -#define SRC_CPP_AES_HPP_ - -#include // for memcmp -#include // for intrinsics for AES-NI - -/** - * Encrypts a message of 128 bits with a 128 bit key, using - * 10 rounds of AES128 (9 full rounds and one final round). Uses AES-NI - * assembly instructions. - */ -#define DO_ENC_BLOCK_128(m, k) \ - do \ - { \ - m = _mm_xor_si128(m, k[0]); \ - m = _mm_aesenc_si128(m, k[1]); \ - m = _mm_aesenc_si128(m, k[2]); \ - m = _mm_aesenc_si128(m, k[3]); \ - m = _mm_aesenc_si128(m, k[4]); \ - m = _mm_aesenc_si128(m, k[5]); \ - m = _mm_aesenc_si128(m, k[6]); \ - m = _mm_aesenc_si128(m, k[7]); \ - m = _mm_aesenc_si128(m, k[8]); \ - m = _mm_aesenc_si128(m, k[9]); \ - m = _mm_aesenclast_si128(m, k[10]); \ - } while (0) - -/** - * Encrypts a message of 128 bits with a 256 bit key, using - * 13 rounds of AES256 (13 full rounds and one final round). Uses - * AES-NI assembly instructions. - */ -#define DO_ENC_BLOCK_256(m, k) \ - do {\ - m = _mm_xor_si128(m, k[ 0]); \ - m = _mm_aesenc_si128(m, k[ 1]); \ - m = _mm_aesenc_si128(m, k[ 2]); \ - m = _mm_aesenc_si128(m, k[ 3]); \ - m = _mm_aesenc_si128(m, k[ 4]); \ - m = _mm_aesenc_si128(m, k[ 5]); \ - m = _mm_aesenc_si128(m, k[ 6]); \ - m = _mm_aesenc_si128(m, k[ 7]); \ - m = _mm_aesenc_si128(m, k[ 8]); \ - m = _mm_aesenc_si128(m, k[ 9]); \ - m = _mm_aesenc_si128(m, k[ 10]);\ - m = _mm_aesenc_si128(m, k[ 11]);\ - m = _mm_aesenc_si128(m, k[ 12]);\ - m = _mm_aesenc_si128(m, k[ 13]);\ - m = _mm_aesenclast_si128(m, k[ 14]);\ - }while(0) - -/** - * Encrypts a message of 128 bits with a 128 bit key, using - * 2 full rounds of AES128. Uses AES-NI assembly instructions. - */ -#define DO_ENC_BLOCK_2ROUND(m, k) \ - do \ - { \ - m = _mm_xor_si128(m, k[0]); \ - m = _mm_aesenc_si128(m, k[1]); \ - m = _mm_aesenc_si128(m, k[2]); \ - } while (0) -/** - * Decrypts a ciphertext of 128 bits with a 128 bit key, using - * 10 rounds of AES128 (9 full rounds and one final round). - * Uses AES-NI assembly instructions. - */ -#define DO_DEC_BLOCK(m, k) \ - do \ - { \ - m = _mm_xor_si128(m, k[10 + 0]); \ - m = _mm_aesdec_si128(m, k[10 + 1]); \ - m = _mm_aesdec_si128(m, k[10 + 2]); \ - m = _mm_aesdec_si128(m, k[10 + 3]); \ - m = _mm_aesdec_si128(m, k[10 + 4]); \ - m = _mm_aesdec_si128(m, k[10 + 5]); \ - m = _mm_aesdec_si128(m, k[10 + 6]); \ - m = _mm_aesdec_si128(m, k[10 + 7]); \ - m = _mm_aesdec_si128(m, k[10 + 8]); \ - m = _mm_aesdec_si128(m, k[10 + 9]); \ - m = _mm_aesdeclast_si128(m, k[0]); \ - } while (0) - -/** - * Decrypts a ciphertext of 128 bits with a 128 bit key, using - * 2 full rounds of AES128. Uses AES-NI assembly instructions. - * Will not work unless key schedule is modified. - */ /* -#define DO_DEC_BLOCK_2ROUND(m, k) \ - do \ - { \ - m = _mm_xor_si128(m, k[2 + 0]); \ - m = _mm_aesdec_si128(m, k[2 + 1]); \ - m = _mm_aesdec_si128(m, k[2 + 2]); \ - } while (0) + +The code in this file is originally from the Tiny AES project, which is in the +public domain. + +https://github.com/kokke/tiny-AES-c + +It has been heavily modified by Chia. + +*** + +This is an implementation of the AES algorithm, specifically ECB, CTR and CBC mode. +Block size can be chosen in aes.h - available choices are AES128, AES192, AES256. + +The implementation is verified against the test vectors in: + National Institute of Standards and Technology Special Publication 800-38A 2001 ED + +ECB-AES128 +---------- + + plain-text: + 6bc1bee22e409f96e93d7e117393172a + ae2d8a571e03ac9c9eb76fac45af8e51 + 30c81c46a35ce411e5fbc1191a0a52ef + f69f2445df4f9b17ad2b417be66c3710 + + key: + 2b7e151628aed2a6abf7158809cf4f3c + + resulting cipher + 3ad77bb40d7a3660a89ecaf32466ef97 + f5d3d58503b9699de785895a96fdbaaf + 43b1cd7f598ece23881b00e3ed030688 + 7b0c785e27e8ad3f8223207104725dd4 + + +NOTE: String length must be evenly divisible by 16byte (str_len % 16 == 0) + You should pad the end of the string with zeros if this is not the case. + For AES192/256 the key size is proportionally larger. + */ -static __m128i key_schedule[20]; // The expanded key -static __m128i aes128_keyexpand(__m128i key) { - key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); - key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); - return _mm_xor_si128(key, _mm_slli_si128(key, 4)); +/*****************************************************************************/ +/* Includes: */ +/*****************************************************************************/ +#include +#include // CBC mode, for memset + +//#define DISABLE_AESNI + +#ifndef DISABLE_AESNI +#include +#include "aesni.hpp" + +bool bHasAES=false; +bool bCheckedAES=false; +#endif // DISABLE_AESNI + +/*****************************************************************************/ +/* Defines: */ +/*****************************************************************************/ +// The number of columns comprising a state in AES. This is a constant in AES. Value=4 +#define Nb 4 + +/*****************************************************************************/ +/* Private variables: */ +/*****************************************************************************/ +// state - array holding the intermediate results during decryption. +typedef uint8_t state_t[4][4]; + +// The lookup-tables are marked const so they can be placed in read-only storage instead of RAM +// The numbers below can be computed dynamically trading ROM for RAM - +// This can be useful in (embedded) bootloader applications, where ROM is often limited. +static const uint8_t sbox[256] = { + //0 1 2 3 4 5 6 7 8 9 A B C D E F + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 }; + +static const uint8_t rsbox[256] = { + 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, + 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, + 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, + 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, + 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, + 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, + 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, + 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, + 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, + 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d }; + +// The round constant word array, Rcon[i], contains the values given by +// x to the power (i-1) being powers of x (x is denoted as {02}) in the field GF(2^8) +static const uint8_t Rcon[11] = { + 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36 }; + +/* + * Jordan Goulder points out in PR #12 (https://github.com/kokke/tiny-AES-C/pull/12), + * that you can remove most of the elements in the Rcon array, because they are unused. + * + * From Wikipedia's article on the Rijndael key schedule @ https://en.wikipedia.org/wiki/Rijndael_key_schedule#Rcon + * + * "Only the first some of these constants are actually used – up to rcon[10] for AES-128 (as 11 round keys are needed), + * up to rcon[8] for AES-192, up to rcon[7] for AES-256. rcon[0] is not used in AES algorithm." + */ + +/*****************************************************************************/ +/* Private functions: */ +/*****************************************************************************/ + +#define getSBoxValue(num) (sbox[(num)]) + + +// This function adds the round key to state. +// The round key is added to the state by an XOR function. +static void AddRoundKey(uint8_t round, state_t* state, const uint8_t* RoundKey) +{ + uint8_t i,j; + for (i = 0; i < 4; ++i) + { + for (j = 0; j < 4; ++j) + { + (*state)[i][j] ^= RoundKey[(round * Nb * 4) + (i * Nb) + j]; + } + } } -#define KEYEXP128_H(K1, K2, I, S) _mm_xor_si128(aes128_keyexpand(K1), \ - _mm_shuffle_epi32(_mm_aeskeygenassist_si128(K2, I), S)) +// The SubBytes Function Substitutes the values in the +// state matrix with values in an S-box. +static void SubBytes(state_t* state) +{ + uint8_t i, j; + for (i = 0; i < 4; ++i) + { + for (j = 0; j < 4; ++j) + { + (*state)[j][i] = getSBoxValue((*state)[j][i]); + } + } +} -#define KEYEXP128(K, I) KEYEXP128_H(K, K, I, 0xff) -#define KEYEXP256(K1, K2, I) KEYEXP128_H(K1, K2, I, 0xff) -#define KEYEXP256_2(K1, K2) KEYEXP128_H(K1, K2, 0x00, 0xaa) +// The ShiftRows() function shifts the rows in the state to the left. +// Each row is shifted with different offset. +// Offset = Row number. So the first row is not shifted. +static void ShiftRows(state_t* state) +{ + uint8_t temp; -// public API + // Rotate first row 1 columns to left + temp = (*state)[0][1]; + (*state)[0][1] = (*state)[1][1]; + (*state)[1][1] = (*state)[2][1]; + (*state)[2][1] = (*state)[3][1]; + (*state)[3][1] = temp; + + // Rotate second row 2 columns to left + temp = (*state)[0][2]; + (*state)[0][2] = (*state)[2][2]; + (*state)[2][2] = temp; + + temp = (*state)[1][2]; + (*state)[1][2] = (*state)[3][2]; + (*state)[3][2] = temp; + + // Rotate third row 3 columns to left + temp = (*state)[0][3]; + (*state)[0][3] = (*state)[3][3]; + (*state)[3][3] = (*state)[2][3]; + (*state)[2][3] = (*state)[1][3]; + (*state)[1][3] = temp; +} + +static uint8_t xtime(uint8_t x) +{ + return ((x<<1) ^ (((x>>7) & 1) * 0x1b)); +} + +// MixColumns function mixes the columns of the state matrix +static void MixColumns(state_t* state) +{ + uint8_t i; + uint8_t Tmp, Tm, t; + for (i = 0; i < 4; ++i) + { + t = (*state)[i][0]; + Tmp = (*state)[i][0] ^ (*state)[i][1] ^ (*state)[i][2] ^ (*state)[i][3] ; + Tm = (*state)[i][0] ^ (*state)[i][1] ; Tm = xtime(Tm); (*state)[i][0] ^= Tm ^ Tmp ; + Tm = (*state)[i][1] ^ (*state)[i][2] ; Tm = xtime(Tm); (*state)[i][1] ^= Tm ^ Tmp ; + Tm = (*state)[i][2] ^ (*state)[i][3] ; Tm = xtime(Tm); (*state)[i][2] ^= Tm ^ Tmp ; + Tm = (*state)[i][3] ^ t ; Tm = xtime(Tm); (*state)[i][3] ^= Tm ^ Tmp ; + } +} + +uint8_t RoundKey128[176]; // AES_KEYLEN 16 Key length in bytes +uint8_t RoundKey256[240]; // AES_KEYLEN 32 Key length in bytes + +#define KEYNR256 14 +#define KEYNR128 10 + +#define KEYNK256 8 +#define KEYNK128 4 + +#define ENCRYPTNR256 14 +#define ENCRYPTNR128 3 + +// This function produces Nb(Nr+1) round keys. The round keys are used in each round to decrypt the states. +static void KeyExpansion(uint8_t* RoundKey, const uint8_t* Key, int keyNr, int keyNk) +{ + unsigned i, j, k; + uint8_t tempa[4]; // Used for the column/row operations + + // The first round key is the key itself. + for (i = 0; i < keyNk; ++i) + { + RoundKey[(i * 4) + 0] = Key[(i * 4) + 0]; + RoundKey[(i * 4) + 1] = Key[(i * 4) + 1]; + RoundKey[(i * 4) + 2] = Key[(i * 4) + 2]; + RoundKey[(i * 4) + 3] = Key[(i * 4) + 3]; + } + + // All other round keys are found from the previous round keys. + for (i = keyNk; i < Nb * (keyNr + 1); ++i) + { + { + k = (i - 1) * 4; + tempa[0]=RoundKey[k + 0]; + tempa[1]=RoundKey[k + 1]; + tempa[2]=RoundKey[k + 2]; + tempa[3]=RoundKey[k + 3]; + + } + + if (i % keyNk == 0) + { + // This function shifts the 4 bytes in a word to the left once. + // [a0,a1,a2,a3] becomes [a1,a2,a3,a0] + + // Function RotWord() + { + const uint8_t u8tmp = tempa[0]; + tempa[0] = tempa[1]; + tempa[1] = tempa[2]; + tempa[2] = tempa[3]; + tempa[3] = u8tmp; + } + + // SubWord() is a function that takes a four-byte input word and + // applies the S-box to each of the four bytes to produce an output word. + + // Function Subword() + { + tempa[0] = getSBoxValue(tempa[0]); + tempa[1] = getSBoxValue(tempa[1]); + tempa[2] = getSBoxValue(tempa[2]); + tempa[3] = getSBoxValue(tempa[3]); + } + + tempa[0] = tempa[0] ^ Rcon[i/keyNk]; + } + + // AES256 only + if ((keyNk==8)&&(i % keyNk == 4)) + { + // Function Subword() + { + tempa[0] = getSBoxValue(tempa[0]); + tempa[1] = getSBoxValue(tempa[1]); + tempa[2] = getSBoxValue(tempa[2]); + tempa[3] = getSBoxValue(tempa[3]); + } + } + + j = i * 4; k=(i - keyNk) * 4; + RoundKey[j + 0] = RoundKey[k + 0] ^ tempa[0]; + RoundKey[j + 1] = RoundKey[k + 1] ^ tempa[1]; + RoundKey[j + 2] = RoundKey[k + 2] ^ tempa[2]; + RoundKey[j + 3] = RoundKey[k + 3] ^ tempa[3]; + } +} /* * Loads an AES key. Can either be a 16 byte or 32 byte bytearray. */ void aes_load_key(uint8_t *enc_key, int keylen) { - switch (keylen) { - case 16: { - /* 128 bit key setup */ - key_schedule[0] = _mm_loadu_si128((const __m128i*) enc_key); - key_schedule[1] = KEYEXP128(key_schedule[0], 0x01); - key_schedule[2] = KEYEXP128(key_schedule[1], 0x02); - key_schedule[3] = KEYEXP128(key_schedule[2], 0x04); - key_schedule[4] = KEYEXP128(key_schedule[3], 0x08); - key_schedule[5] = KEYEXP128(key_schedule[4], 0x10); - key_schedule[6] = KEYEXP128(key_schedule[5], 0x20); - key_schedule[7] = KEYEXP128(key_schedule[6], 0x40); - key_schedule[8] = KEYEXP128(key_schedule[7], 0x80); - key_schedule[9] = KEYEXP128(key_schedule[8], 0x1B); - key_schedule[10] = KEYEXP128(key_schedule[9], 0x36); + +#ifndef DISABLE_AESNI + if(!bCheckedAES) + { + uint32_t eax, ebx, ecx, edx; + + eax = ebx = ecx = edx = 0; + __get_cpuid(1, &eax, &ebx, &ecx, &edx); + bHasAES=(ecx & bit_AES) > 0; + bCheckedAES=true; + } + + if(bHasAES) + return ni_aes_load_key(enc_key, keylen); +#endif // DISABLE_AESNI + + switch(keylen){ + case 32: + KeyExpansion(RoundKey256, enc_key, KEYNR256, KEYNK256); break; - } - case 32: { - /* 256 bit key setup */ - key_schedule[0] = _mm_loadu_si128((const __m128i*) enc_key); - key_schedule[1] = _mm_loadu_si128((const __m128i*) (enc_key+16)); - key_schedule[2] = KEYEXP256(key_schedule[0], key_schedule[1], 0x01); - key_schedule[3] = KEYEXP256_2(key_schedule[1], key_schedule[2]); - key_schedule[4] = KEYEXP256(key_schedule[2], key_schedule[3], 0x02); - key_schedule[5] = KEYEXP256_2(key_schedule[3], key_schedule[4]); - key_schedule[6] = KEYEXP256(key_schedule[4], key_schedule[5], 0x04); - key_schedule[7] = KEYEXP256_2(key_schedule[5], key_schedule[6]); - key_schedule[8] = KEYEXP256(key_schedule[6], key_schedule[7], 0x08); - key_schedule[9] = KEYEXP256_2(key_schedule[7], key_schedule[8]); - key_schedule[10] = KEYEXP256(key_schedule[8], key_schedule[9], 0x10); - key_schedule[11] = KEYEXP256_2(key_schedule[9], key_schedule[10]); - key_schedule[12] = KEYEXP256(key_schedule[10], key_schedule[11], 0x20); - key_schedule[13] = KEYEXP256_2(key_schedule[11], key_schedule[12]); - key_schedule[14] = KEYEXP256(key_schedule[12], key_schedule[13], 0x40); + case 16: + KeyExpansion(RoundKey128, enc_key, KEYNR128, KEYNK128); break; - } } } -// Declares a global variable for efficiency. -__m128i m_global; +/* +* XOR 128 bits +*/ +static inline void xor128(const uint8_t *in1, const uint8_t *in2, uint8_t *out) { + for(int i=0;i<16;i++) { + out[i]=in1[i]^in2[i]; + } +} /* * Encrypts a plaintext using AES256. */ -static inline void aes256_enc(const uint8_t *plainText, uint8_t *cipherText) { - m_global = _mm_loadu_si128(reinterpret_cast(plainText)); +static inline void aes256_enc(const uint8_t *in, uint8_t *out) { + +#ifndef DISABLE_AESNI + if(bHasAES) + return ni_aes256_enc(in, out); +#endif // DISABLE_AESNI + + memcpy(out,in,16); + + state_t *state=(state_t*)out; + + uint8_t round = 0; - DO_ENC_BLOCK_256(m_global, key_schedule); + // Add the First round key to the state before starting the rounds. + AddRoundKey(0, state, RoundKey256); - _mm_storeu_si128(reinterpret_cast<__m128i *>(cipherText), m_global); + // There will be Nr rounds. + // The first Nr-1 rounds are identical. + // These Nr-1 rounds are executed in the loop below. + for (round = 1; round < ENCRYPTNR256; ++round) + { + SubBytes(state); + ShiftRows(state); + MixColumns(state); + AddRoundKey(round, state, RoundKey256); + } + + // The last round is given below. + // The MixColumns function is not here in the last round. + SubBytes(state); + ShiftRows(state); + AddRoundKey(ENCRYPTNR256, state, RoundKey256); } - -/* - * Encrypts a plaintext using AES128 with 2 rounds. - */ -static inline void aes128_enc(const uint8_t *plainText, uint8_t *cipherText) { - m_global = _mm_loadu_si128(reinterpret_cast(plainText)); - - // Uses the 2 round encryption innstead of the full 10 round encryption - DO_ENC_BLOCK_2ROUND(m_global, key_schedule); - - _mm_storeu_si128(reinterpret_cast<__m128i *>(cipherText), m_global); -} - + /* * Encrypts an integer using AES128 with 2 rounds. */ -static inline __m128i aes128_enc_int(__m128i plainText) { - // Uses the 2 round encryption innstead of the full 10 round encryption - DO_ENC_BLOCK_2ROUND(plainText, key_schedule); - return plainText; +static inline void aes128_enc(uint8_t *in, uint8_t *out) { + +#ifndef DISABLE_AESNI + if(bHasAES) + return ni_aes128_enc(in, out); +#endif // DISABLE_AESNI + + memcpy(out,in,16); + + state_t *state=(state_t*)out; + + uint8_t round = 0; + + // Add the First round key to the state before starting the rounds. + AddRoundKey(0, state, RoundKey128); + + // There will be Nr rounds. + // The first Nr-1 rounds are identical. + // These Nr-1 rounds are executed in the loop below. + for (round = 1; round < ENCRYPTNR128; ++round) + { + SubBytes(state); + ShiftRows(state); + MixColumns(state); + AddRoundKey(round, state, RoundKey128); + } } - -__m128i m1; -__m128i m2; -__m128i m3; -__m128i m4; - + /* * Uses AES cache mode to map a 2 block ciphertext into 128 bit result. */ static inline void aes128_2b(uint8_t *block1, uint8_t *block2, uint8_t *res) { - m1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block1)); - m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block2)); - m3 = aes128_enc_int(m1); // E(L) - m3 = aes128_enc_int(_mm_xor_si128(m3, m2)); - _mm_storeu_si128(reinterpret_cast<__m128i *>(res), m3); -} + +#ifndef DISABLE_AESNI + if(bHasAES) + return ni_aes128_2b(block1, block2, res); +#endif // DISABLE_AESNI + + uint8_t m1[16]; + uint8_t m2[16]; + uint8_t m3[16]; + uint8_t intermediate[16]; + memcpy(m1,block1,16); + memcpy(m2,block2,16); + + aes128_enc(m1,m3); + xor128(m3, m2, intermediate); + aes128_enc(intermediate,m3); + + memcpy(res,m3,16); +} + /* * Uses AES cache mode to map a 3 block ciphertext into 128 bit result. */ static inline void aes128_3b(uint8_t *block1, uint8_t* block2, uint8_t *block3, uint8_t* res) { - m1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block1)); - m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block2)); + +#ifndef DISABLE_AESNI + if(bHasAES) + return ni_aes128_3b(block1, block2, block3, res); +#endif // DISABLE_AESNI + + uint8_t m1[16]; + uint8_t m2[16]; + uint8_t m3[16]; - m1 = aes128_enc_int(m1); // E(La) - m2 = aes128_enc_int(m2); // E(Ra) + memcpy(m1,block1,16); + memcpy(m2,block2,16); - m1 = _mm_xor_si128(m1, m2); - m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block3)); - - m2 = aes128_enc_int(m2); - m1 = _mm_xor_si128(m1, m2); - m3 = aes128_enc_int(m1); - _mm_storeu_si128(reinterpret_cast<__m128i *>(res), m3); + aes128_enc(m1,m1); // E(La) + aes128_enc(m2,m2); // E(Ra) + + xor128(m1, m2, m1); + memcpy(m2,block3,16); + + aes128_enc(m2,m2); + xor128(m1, m2, m1); + aes128_enc(m1,m3); + memcpy(res,m3,16); } /* * Uses AES cache mode to map a 4 block ciphertext into 128 bit result. */ static inline void aes128_4b(uint8_t *block1, uint8_t* block2, uint8_t *block3, uint8_t* block4, uint8_t* res) { - m1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block1)); - m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block3)); - m3 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block2)); - m4 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block4)); + +#ifndef DISABLE_AESNI + if(bHasAES) + return ni_aes128_4b(block1, block2, block3, block4, res); +#endif // DISABLE_AESNI - m1 = aes128_enc_int(m1); // E(La) - m1 = _mm_xor_si128(m1, m3); - m1 = aes128_enc_int(m1); // E(E(La) ^ Lb) - m2 = aes128_enc_int(m2); // E(Ra) + uint8_t m1[16]; + uint8_t m2[16]; + uint8_t m3[16]; + uint8_t m4[16]; - m1 = _mm_xor_si128(m1, m2); // xor e(Ra) - m1 = _mm_xor_si128(m1, m4); // xor Rb + memcpy(m1,block1,16); + memcpy(m2,block2,16); + memcpy(m3,block3,16); + memcpy(m4,block4,16); - m3 = aes128_enc_int(m1); - _mm_storeu_si128(reinterpret_cast<__m128i *>(res), m3); + aes128_enc(m1,m1); // E(La) + xor128(m1, m3, m1); + aes128_enc(m1,m1); // E(E(La) ^ Lb) + aes128_enc(m2,m2); // E(Ra) + + xor128(m1, m2, m1); // xor e(Ra) + xor128(m1, m4, m1); // xor Rb + + aes128_enc(m1,m3); // E(La) + memcpy(res,m3,16); } -#endif // SRC_CPP_AES_HPP_ diff --git a/lib/chiapos/src/aesni.hpp b/lib/chiapos/src/aesni.hpp new file mode 100644 index 000000000000..d3d469d5353c --- /dev/null +++ b/lib/chiapos/src/aesni.hpp @@ -0,0 +1,270 @@ +// Copyright 2018 Chia Network Inc + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Some public domain code is taken from pycrypto: +// https://github.com/dlitz/pycrypto/blob/master/src/AESNI.c +// +// AESNI.c: AES using AES-NI instructions +// +// Written in 2013 by Sebastian Ramacher + +#ifndef SRC_CPP_AES_HPP_ +#define SRC_CPP_AES_HPP_ + +#include // for memcmp +#include // for intrinsics for AES-NI + +/** + * Encrypts a message of 128 bits with a 128 bit key, using + * 10 rounds of AES128 (9 full rounds and one final round). Uses AES-NI + * assembly instructions. + */ +#define DO_ENC_BLOCK_128(m, k) \ + do \ + { \ + m = _mm_xor_si128(m, k[0]); \ + m = _mm_aesenc_si128(m, k[1]); \ + m = _mm_aesenc_si128(m, k[2]); \ + m = _mm_aesenc_si128(m, k[3]); \ + m = _mm_aesenc_si128(m, k[4]); \ + m = _mm_aesenc_si128(m, k[5]); \ + m = _mm_aesenc_si128(m, k[6]); \ + m = _mm_aesenc_si128(m, k[7]); \ + m = _mm_aesenc_si128(m, k[8]); \ + m = _mm_aesenc_si128(m, k[9]); \ + m = _mm_aesenclast_si128(m, k[10]); \ + } while (0) + +/** + * Encrypts a message of 128 bits with a 256 bit key, using + * 13 rounds of AES256 (13 full rounds and one final round). Uses + * AES-NI assembly instructions. + */ +#define DO_ENC_BLOCK_256(m, k) \ + do {\ + m = _mm_xor_si128(m, k[ 0]); \ + m = _mm_aesenc_si128(m, k[ 1]); \ + m = _mm_aesenc_si128(m, k[ 2]); \ + m = _mm_aesenc_si128(m, k[ 3]); \ + m = _mm_aesenc_si128(m, k[ 4]); \ + m = _mm_aesenc_si128(m, k[ 5]); \ + m = _mm_aesenc_si128(m, k[ 6]); \ + m = _mm_aesenc_si128(m, k[ 7]); \ + m = _mm_aesenc_si128(m, k[ 8]); \ + m = _mm_aesenc_si128(m, k[ 9]); \ + m = _mm_aesenc_si128(m, k[ 10]);\ + m = _mm_aesenc_si128(m, k[ 11]);\ + m = _mm_aesenc_si128(m, k[ 12]);\ + m = _mm_aesenc_si128(m, k[ 13]);\ + m = _mm_aesenclast_si128(m, k[ 14]);\ + }while(0) + +/** + * Encrypts a message of 128 bits with a 128 bit key, using + * 2 full rounds of AES128. Uses AES-NI assembly instructions. + */ +#define DO_ENC_BLOCK_2ROUND(m, k) \ + do \ + { \ + m = _mm_xor_si128(m, k[0]); \ + m = _mm_aesenc_si128(m, k[1]); \ + m = _mm_aesenc_si128(m, k[2]); \ + } while (0) +/** + * Decrypts a ciphertext of 128 bits with a 128 bit key, using + * 10 rounds of AES128 (9 full rounds and one final round). + * Uses AES-NI assembly instructions. + */ +#define DO_DEC_BLOCK(m, k) \ + do \ + { \ + m = _mm_xor_si128(m, k[10 + 0]); \ + m = _mm_aesdec_si128(m, k[10 + 1]); \ + m = _mm_aesdec_si128(m, k[10 + 2]); \ + m = _mm_aesdec_si128(m, k[10 + 3]); \ + m = _mm_aesdec_si128(m, k[10 + 4]); \ + m = _mm_aesdec_si128(m, k[10 + 5]); \ + m = _mm_aesdec_si128(m, k[10 + 6]); \ + m = _mm_aesdec_si128(m, k[10 + 7]); \ + m = _mm_aesdec_si128(m, k[10 + 8]); \ + m = _mm_aesdec_si128(m, k[10 + 9]); \ + m = _mm_aesdeclast_si128(m, k[0]); \ + } while (0) + +/** + * Decrypts a ciphertext of 128 bits with a 128 bit key, using + * 2 full rounds of AES128. Uses AES-NI assembly instructions. + * Will not work unless key schedule is modified. + */ +/* +#define DO_DEC_BLOCK_2ROUND(m, k) \ + do \ + { \ + m = _mm_xor_si128(m, k[2 + 0]); \ + m = _mm_aesdec_si128(m, k[2 + 1]); \ + m = _mm_aesdec_si128(m, k[2 + 2]); \ + } while (0) +*/ + +static __m128i key_schedule[20]; // The expanded key + +static __m128i aes128_keyexpand(__m128i key) { + key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); + key = _mm_xor_si128(key, _mm_slli_si128(key, 4)); + return _mm_xor_si128(key, _mm_slli_si128(key, 4)); +} + +#define KEYEXP128_H(K1, K2, I, S) _mm_xor_si128(aes128_keyexpand(K1), \ + _mm_shuffle_epi32(_mm_aeskeygenassist_si128(K2, I), S)) + +#define KEYEXP128(K, I) KEYEXP128_H(K, K, I, 0xff) +#define KEYEXP256(K1, K2, I) KEYEXP128_H(K1, K2, I, 0xff) +#define KEYEXP256_2(K1, K2) KEYEXP128_H(K1, K2, 0x00, 0xaa) + +// public API + +/* + * Loads an AES key. Can either be a 16 byte or 32 byte bytearray. + */ +void ni_aes_load_key(uint8_t *enc_key, int keylen) { + switch (keylen) { + case 16: { + /* 128 bit key setup */ + key_schedule[0] = _mm_loadu_si128((const __m128i*) enc_key); + key_schedule[1] = KEYEXP128(key_schedule[0], 0x01); + key_schedule[2] = KEYEXP128(key_schedule[1], 0x02); + key_schedule[3] = KEYEXP128(key_schedule[2], 0x04); + key_schedule[4] = KEYEXP128(key_schedule[3], 0x08); + key_schedule[5] = KEYEXP128(key_schedule[4], 0x10); + key_schedule[6] = KEYEXP128(key_schedule[5], 0x20); + key_schedule[7] = KEYEXP128(key_schedule[6], 0x40); + key_schedule[8] = KEYEXP128(key_schedule[7], 0x80); + key_schedule[9] = KEYEXP128(key_schedule[8], 0x1B); + key_schedule[10] = KEYEXP128(key_schedule[9], 0x36); + break; + } + case 32: { + /* 256 bit key setup */ + key_schedule[0] = _mm_loadu_si128((const __m128i*) enc_key); + key_schedule[1] = _mm_loadu_si128((const __m128i*) (enc_key+16)); + key_schedule[2] = KEYEXP256(key_schedule[0], key_schedule[1], 0x01); + key_schedule[3] = KEYEXP256_2(key_schedule[1], key_schedule[2]); + key_schedule[4] = KEYEXP256(key_schedule[2], key_schedule[3], 0x02); + key_schedule[5] = KEYEXP256_2(key_schedule[3], key_schedule[4]); + key_schedule[6] = KEYEXP256(key_schedule[4], key_schedule[5], 0x04); + key_schedule[7] = KEYEXP256_2(key_schedule[5], key_schedule[6]); + key_schedule[8] = KEYEXP256(key_schedule[6], key_schedule[7], 0x08); + key_schedule[9] = KEYEXP256_2(key_schedule[7], key_schedule[8]); + key_schedule[10] = KEYEXP256(key_schedule[8], key_schedule[9], 0x10); + key_schedule[11] = KEYEXP256_2(key_schedule[9], key_schedule[10]); + key_schedule[12] = KEYEXP256(key_schedule[10], key_schedule[11], 0x20); + key_schedule[13] = KEYEXP256_2(key_schedule[11], key_schedule[12]); + key_schedule[14] = KEYEXP256(key_schedule[12], key_schedule[13], 0x40); + break; + } + } +} + +// Declares a global variable for efficiency. +__m128i m_global; + +/* + * Encrypts a plaintext using AES256. + */ +static inline void ni_aes256_enc(const uint8_t *plainText, uint8_t *cipherText) { + m_global = _mm_loadu_si128(reinterpret_cast(plainText)); + + DO_ENC_BLOCK_256(m_global, key_schedule); + + _mm_storeu_si128(reinterpret_cast<__m128i *>(cipherText), m_global); +} + +/* + * Encrypts a plaintext using AES128 with 2 rounds. + */ +static inline void ni_aes128_enc(const uint8_t *plainText, uint8_t *cipherText) { + m_global = _mm_loadu_si128(reinterpret_cast(plainText)); + + // Uses the 2 round encryption innstead of the full 10 round encryption + DO_ENC_BLOCK_2ROUND(m_global, key_schedule); + + _mm_storeu_si128(reinterpret_cast<__m128i *>(cipherText), m_global); +} + +/* + * Encrypts an integer using AES128 with 2 rounds. + */ +static inline __m128i ni_aes128_enc_int(__m128i plainText) { + // Uses the 2 round encryption innstead of the full 10 round encryption + DO_ENC_BLOCK_2ROUND(plainText, key_schedule); + return plainText; +} + +__m128i m1; +__m128i m2; +__m128i m3; +__m128i m4; + +/* + * Uses AES cache mode to map a 2 block ciphertext into 128 bit result. + */ +static inline void ni_aes128_2b(uint8_t *block1, uint8_t *block2, uint8_t *res) { + m1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block1)); + m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block2)); + m3 = ni_aes128_enc_int(m1); // E(L) + m3 = ni_aes128_enc_int(_mm_xor_si128(m3, m2)); + _mm_storeu_si128(reinterpret_cast<__m128i *>(res), m3); +} + +/* + * Uses AES cache mode to map a 3 block ciphertext into 128 bit result. + */ +static inline void ni_aes128_3b(uint8_t *block1, uint8_t* block2, uint8_t *block3, uint8_t* res) { + m1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block1)); + m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block2)); + + m1 = ni_aes128_enc_int(m1); // E(La) + m2 = ni_aes128_enc_int(m2); // E(Ra) + + m1 = _mm_xor_si128(m1, m2); + m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block3)); + + m2 = ni_aes128_enc_int(m2); + m1 = _mm_xor_si128(m1, m2); + m3 = ni_aes128_enc_int(m1); + _mm_storeu_si128(reinterpret_cast<__m128i *>(res), m3); +} + +/* + * Uses AES cache mode to map a 4 block ciphertext into 128 bit result. + */ +static inline void ni_aes128_4b(uint8_t *block1, uint8_t* block2, uint8_t *block3, uint8_t* block4, uint8_t* res) { + m1 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block1)); + m2 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block3)); + m3 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block2)); + m4 = _mm_loadu_si128(reinterpret_cast<__m128i *>(block4)); + + m1 = ni_aes128_enc_int(m1); // E(La) + m1 = _mm_xor_si128(m1, m3); + m1 = ni_aes128_enc_int(m1); // E(E(La) ^ Lb) + m2 = ni_aes128_enc_int(m2); // E(Ra) + + m1 = _mm_xor_si128(m1, m2); // xor e(Ra) + m1 = _mm_xor_si128(m1, m4); // xor Rb + + m3 = ni_aes128_enc_int(m1); + _mm_storeu_si128(reinterpret_cast<__m128i *>(res), m3); +} + +#endif // SRC_CPP_AES_HPP_ diff --git a/lib/chiapos/src/cmake_aesni_test.cpp b/lib/chiapos/src/cmake_aesni_test.cpp new file mode 100644 index 000000000000..daed6189dfb3 --- /dev/null +++ b/lib/chiapos/src/cmake_aesni_test.cpp @@ -0,0 +1,16 @@ +#include "aes.hpp" + +int main() { + uint8_t enc_key[32]; + uint8_t in[16]; + uint8_t out[16]; + + memset(enc_key,0x00,sizeof(enc_key)); + memset(in,0x00,sizeof(in)); + + aes_load_key(enc_key, sizeof(enc_key)); + aes256_enc(in, out); + + return 0; +} + From 5676e5ca9d788275565a09a4c2f1b87faa27bf0c Mon Sep 17 00:00:00 2001 From: Mariano Sorgente Date: Fri, 7 Feb 2020 14:55:52 -0500 Subject: [PATCH 11/38] Don't allow farming plots with wrong key --- src/harvester.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/harvester.py b/src/harvester.py index 6b92bd7860b9..701f93733272 100644 --- a/src/harvester.py +++ b/src/harvester.py @@ -76,6 +76,7 @@ class Harvester: log.warning( f"Plot {partial_filename} has a pool key that is not in the farmer's pool_pk list." ) + continue found = False for filename in potential_filenames: From 8daa3c1f2ef5f62af8b9e465bb2cbaaaa4e05708 Mon Sep 17 00:00:00 2001 From: Antonio Caccese <34692225+A-Caccese@users.noreply.github.com> Date: Tue, 11 Feb 2020 19:01:17 +0100 Subject: [PATCH 12/38] Update README.md In "Each line that starts with `pip ...` becomes `python -m pip ...`" changed "python -m pip" with "python3.7 -m pip" (paragraph "Alternate method for Ubuntu 18.04 LTS"). The command "python -m pip" calls the wrong version of python. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c3e4f4ca381d..1face264c568 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ sudo sh install.sh #### Alternate method for Ubuntu 18.04 LTS In `./install.sh`: Change `python3` to `python3.7` -Each line that starts with `pip ...` becomes `python -m pip ...` +Each line that starts with `pip ...` becomes `python3.7 -m pip ...` ```bash sudo apt-get -y update From b0f48c1a252ce5d42e4fafebd19b49636c9031ee Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Fri, 14 Feb 2020 11:59:38 -0600 Subject: [PATCH 13/38] fix signed-unsigned comparison warning (#98) --- lib/chiapos/src/aes.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/chiapos/src/aes.hpp b/lib/chiapos/src/aes.hpp index 129782be3c43..db91ad7c64ff 100644 --- a/lib/chiapos/src/aes.hpp +++ b/lib/chiapos/src/aes.hpp @@ -226,7 +226,7 @@ uint8_t RoundKey256[240]; // AES_KEYLEN 32 Key length in bytes // This function produces Nb(Nr+1) round keys. The round keys are used in each round to decrypt the states. static void KeyExpansion(uint8_t* RoundKey, const uint8_t* Key, int keyNr, int keyNk) { - unsigned i, j, k; + int i, j, k; uint8_t tempa[4]; // Used for the column/row operations // The first round key is the key itself. From 248cbbfe45c3c43b7103f3134ecea4beddc55933 Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Wed, 19 Feb 2020 15:35:17 -0800 Subject: [PATCH 14/38] Update CentOS 7.7 - lib dir Add LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib for CentOS 7.7 install --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6f9ab885b66d..0955368f8510 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,7 @@ tar -zxvf boost_1_72_0.tar.gz cd boost_1_72_0 ./bootstrap.sh --prefix=/usr/local sudo ./b2 install --prefix=/usr/local --with=all; cd .. +LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib cd chia-blockchain @@ -279,4 +280,4 @@ You can also use the [HTTP RPC](https://github.com/Chia-Network/chia-blockchain/ ```bash curl -X POST http://localhost:8555/get_blockchain_state -``` \ No newline at end of file +``` From 5847b53c26bf993bd225cd9998f31e9adb90bcb2 Mon Sep 17 00:00:00 2001 From: Yostra Date: Wed, 19 Feb 2020 15:21:45 -0800 Subject: [PATCH 15/38] build bip158 --- lib/bip158/CMakeLists.txt | 12 ++++++++---- requirements.txt | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/bip158/CMakeLists.txt b/lib/bip158/CMakeLists.txt index 70f85ac5935c..9effa44cbeab 100644 --- a/lib/bip158/CMakeLists.txt +++ b/lib/bip158/CMakeLists.txt @@ -8,20 +8,23 @@ ENDIF() project(chiabip158) +link_directories(/usr/local/opt/openssl/lib) + include_directories( ${INCLUDE_DIRECTORIES} ${CMAKE_CURRENT_SOURCE_DIR}/src /usr/local/opt/openssl/include + /usr/local/opt/boost/include ) set (CMAKE_CXX_FLAGS "-DHAVE_WORKING_BOOST_SLEEP -g -O3 -Wall -msse2 -msse -march=native -std=c++14 -maes") FILE(GLOB_RECURSE MyCSources src/*.cpp) -ADD_LIBRARY(biplib ${MyCSources}) +ADD_LIBRARY(biplib ${MyCSources}) add_subdirectory(lib/pybind11) -pybind11_add_module(chiabip158 +pybind11_add_module(chiabip158 ${CMAKE_CURRENT_SOURCE_DIR}/python-bindings/chiabip158.cpp ${CMAKE_CURRENT_SOURCE_DIR}/python-bindings/PyBIP158.cpp) @@ -29,6 +32,7 @@ add_executable(bip158 main.cpp ) -target_link_libraries(bip158 biplib -lboost_system -lpthread -lboost_thread -lboost_filesystem -lssl -lcrypto) -target_link_libraries(chiabip158 PRIVATE biplib -lboost_system -lpthread -lboost_thread -lboost_filesystem -lssl -lcrypto) +find_package(Boost COMPONENTS system filesystem thread REQUIRED) +target_link_libraries(bip158 biplib ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} -lpthread -lssl -lcrypto) +target_link_libraries(chiabip158 PRIVATE biplib ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} -lpthread -lssl -lcrypto) diff --git a/requirements.txt b/requirements.txt index d0803f561083..9451eedf03d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,6 +48,7 @@ yarl==1.4.2 zipp==2.0.0 sortedcontainers==2.1.0 -e lib/chiapos +-e lib/bip158 -e lib/py-setproctitle -e lib/chiavdf/fast_vdf -e git+https://github.com/Chia-Network/clvm.git@134c2f92d575a542272ce0cbe59c167b255e50ae#egg=clvm From 5115577da222e167db26b55792f27dddca33c482 Mon Sep 17 00:00:00 2001 From: Yostra Date: Fri, 21 Feb 2020 14:04:13 -0800 Subject: [PATCH 16/38] expose api to bip158 binding --- .gitignore | 6 ++- lib/bip158/python-bindings/PyBIP158.cpp | 10 ++++ lib/bip158/python-bindings/PyBIP158.h | 2 + lib/bip158/python-bindings/chiabip158.cpp | 2 + tests/test_filter.py | 56 +++++++++++++++++++++++ 5 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 tests/test_filter.py diff --git a/.gitignore b/.gitignore index fadb6636bb5a..74461410e851 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,8 @@ pip-delete-this-directory.txt # Packaging chia-blockchain.tar.gz -src/wallet/electron/node_modules/ \ No newline at end of file +# Electron wallet node modules +src/wallet/electron/node_modules/ + +# Bip158 build dir +lib/bip158/build/ \ No newline at end of file diff --git a/lib/bip158/python-bindings/PyBIP158.cpp b/lib/bip158/python-bindings/PyBIP158.cpp index c04dc609cf2b..826c64015aa4 100644 --- a/lib/bip158/python-bindings/PyBIP158.cpp +++ b/lib/bip158/python-bindings/PyBIP158.cpp @@ -29,6 +29,16 @@ PyBIP158::PyBIP158(std::vector< std::vector< unsigned char > >& hashes) filter=new GCSFilter({0, 0, 20, 1 << 20},elements); } +PyBIP158::PyBIP158(std::vector< unsigned char > & encoded_filter) +{ + filter=new GCSFilter({0, 0, 20, 1 << 20}, encoded_filter); +} + +const std::vector& PyBIP158::GetEncoded() +{ + return filter->GetEncoded(); +} + PyBIP158::~PyBIP158() { delete filter; diff --git a/lib/bip158/python-bindings/PyBIP158.h b/lib/bip158/python-bindings/PyBIP158.h index db41a91cd400..baaf4becf145 100644 --- a/lib/bip158/python-bindings/PyBIP158.h +++ b/lib/bip158/python-bindings/PyBIP158.h @@ -25,6 +25,8 @@ public: public: PyBIP158(std::vector< std::vector< unsigned char > >& hashes); + PyBIP158(std::vector< unsigned char > & encoded_filter); + const std::vector& GetEncoded(); ~PyBIP158(); bool Match(std::vector< unsigned char >& hash); diff --git a/lib/bip158/python-bindings/chiabip158.cpp b/lib/bip158/python-bindings/chiabip158.cpp index 70b67fc5a4ab..a43b034a63d5 100644 --- a/lib/bip158/python-bindings/chiabip158.cpp +++ b/lib/bip158/python-bindings/chiabip158.cpp @@ -24,6 +24,8 @@ PYBIND11_MODULE(chiabip158, mod) { py::class_> clsPyBIP158(mod, "PyBIP158"); clsPyBIP158.def(py::init >&>()); + clsPyBIP158.def(py::init< std::vector< unsigned char > &>()); + clsPyBIP158.def("GetEncoded",(const std::vector< unsigned char >& (PyBIP158::*)()) &PyBIP158::GetEncoded); clsPyBIP158.def("Match", (bool (PyBIP158::*)(std::vector< unsigned char >&)) &PyBIP158::Match); clsPyBIP158.def("MatchAny", (bool (PyBIP158::*)(std::vector< std::vector< unsigned char > >&)) &PyBIP158::MatchAny); } diff --git a/tests/test_filter.py b/tests/test_filter.py new file mode 100644 index 000000000000..6e4c22d89e56 --- /dev/null +++ b/tests/test_filter.py @@ -0,0 +1,56 @@ +import asyncio +from typing import List + +import pytest +from blspy import ExtendedPrivateKey +from chiabip158 import PyBIP158 + +from src.wallet.wallet import Wallet +from tests.setup_nodes import setup_two_nodes, test_constants, bt + + +@pytest.fixture(scope="module") +def event_loop(): + loop = asyncio.get_event_loop() + yield loop + + +class TestFilter: + @pytest.fixture(scope="function") + async def two_nodes(self): + async for _ in setup_two_nodes({"COINBASE_FREEZE_PERIOD": 0}): + yield _ + + @pytest.mark.asyncio + async def test_basic_filter_test(self, two_nodes): + sk = bytes(ExtendedPrivateKey.from_seed(b"")).hex() + key_config = {"wallet_sk": sk} + + wallet = await Wallet.create({}, key_config) + wallet.wallet_store._clear_database() + + num_blocks = 2 + blocks = bt.get_consecutive_blocks( + test_constants, + num_blocks, + [], + 10, + reward_puzzlehash=wallet.get_new_puzzlehash(), + ) + + for i in range(1, num_blocks): + byte_array_tx: List[bytes] = [] + block = blocks[i] + coinbase = bytearray(block.body.coinbase.puzzle_hash) + fee = bytearray(block.body.fees_coin.puzzle_hash) + byte_array_tx.append(coinbase) + byte_array_tx.append(fee) + + pl = PyBIP158(byte_array_tx) + present = pl.Match(coinbase) + fee_present = pl.Match(fee) + + assert present + assert fee_present + + await wallet.wallet_store.close() \ No newline at end of file From f02ea59d5de9ebb6f92c326419b0e35c2be2c6d6 Mon Sep 17 00:00:00 2001 From: Yostra Date: Sat, 22 Feb 2020 15:45:19 -0800 Subject: [PATCH 17/38] include filter in header data --- src/blockchain.py | 2 +- src/consensus/constants.py | 2 +- src/full_node.py | 20 +++++++++++++++++--- src/mempool_manager.py | 2 +- src/types/full_block.py | 2 +- src/types/header.py | 2 +- src/util/mempool_check_conditions.py | 2 +- src/wallet/wallet.py | 2 +- tests/block_tools.py | 23 +++++++++++++++++++++-- tests/test_blockchain.py | 8 ++++---- tests/test_filter.py | 2 +- 11 files changed, 50 insertions(+), 17 deletions(-) diff --git a/src/blockchain.py b/src/blockchain.py index 5f80879ee3c8..8548e86752d8 100644 --- a/src/blockchain.py +++ b/src/blockchain.py @@ -927,7 +927,7 @@ class Blockchain: if not block.body.transactions: return Err.UNKNOWN # Get List of names removed, puzzles hashes for removed coins and conditions crated - error, npc_list, cost = await get_name_puzzle_conditions( + error, npc_list, cost = get_name_puzzle_conditions( block.body.transactions ) diff --git a/src/consensus/constants.py b/src/consensus/constants.py index 2ba4ea956ff4..755b63e225c9 100644 --- a/src/consensus/constants.py +++ b/src/consensus/constants.py @@ -25,7 +25,7 @@ constants: Dict[str, Any] = { "PROPAGATION_DELAY_THRESHOLD": 1500, # Hardcoded genesis block, generated using tests/block_tools.py # Replace this any time the above constants change. - "GENESIS_BLOCK": b'\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x15N3\xd3\xf9H\xc2K\x96\xfe\xf2f\xa2\xbf\x87\x0e\x0f,\xd0\xd4\x0f6s\xb1".\\\xf5\x8a\xb4\x03\x84\x8e\xf9\xbb\xa1\xca\xdef3:\xe4?\x0c\xe5\xc6\x12\x80\x17\xd2\xcc\xd7\xb4m\x94\xb7V\x959\xed4\x89\x04b\x08\x07^\xca`\x8f#%\xe9\x9c\x9d\x86y\x10\x96W\x9d\xce\xc1\x15r\x97\x91U\n\x11<\xdf\xb2\xfc\xfb<\x13\x00\x00\x00\x98\x97N\xe3py\xb5*\xe8\xbe_\xa5\xa5mq\xceY\x08$;\xe1\xef;:\xc0\x93\x84\xa6(`\xe7\xf1\x0b\xb6\x08\xb4\xa1\xc7\x03\xe5v\xb9)\xd2\xc9|\t\xcc\xb3\xd4\x05\xd9-\xa2;\xeb\xdd!\xd1\\Oh<\x01+\x9e\xbfK\xe3y\xd8\xd0T\xac\xb52+Zj+)\xad\xabIg\x82\x998\xe5\x86\xa3[\xd1\xc1\xe9\x9dD\xbb\xe2IN\x01\xdc\x1e\xcc\t\x1c\xa7\xe4\xb92\x98^\xd5\xd6\xce\xe1Z\xcd\x9e\xee\xba\x9f,\x94B?}\xf7\xeb\x8f,\xbds\xc4g\x02"}\x88\x17.\xf7\xd3\x89\x15Vi\xbe2\xdct\xc4\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x02JW\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\xf6\xcb\xc9@\x18<\xf3NgW\xe1v\xc5ag]\x1e\xb65\x15\xd6X\xb7W\xa3\xe4\xb1\xea\xf4!\xaf\xa6\x9b0v,PR\xe4L\x87\xff\xc9\xe3\x13\xf2\xf0\xcb*3R\x91dE8\xf6S<=\xfd\x87u\xf7\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x86\x91#\xeb)\xc2\xae\x04Q\x041&\xa32\x02\xd4\xdbge\x9b\tj\x02\xa8\x9aH\x15?\xa5\xcc\xdd\x84\xcf\x10:\xb5Ki"\xd2\x07\xf5\xd3\xc8\xb1U_P\xcb^\xd8"\x95?\x18%\xb8\xdf\xa7\x82\xaa\xb2\xbb\x03\x00\x00\x03\xa6\x00O\x90+E\x9a#\xf2\xb1D\x88|\x8f\nx\xe4\xb6\xe7\x02\x10\x11\xb1\x13[u)\xfe*\x94\x9c\r\x91`\xfc\xa8\x0c\xb1\x00,?p\xba\xb2\x82\xf2n&\x84\x1f\xe0U\xe9\xc2\xe4\xc2Y\xc8\x9bg\xf9\xd1\x8b\x02\xdf\x0c\xff\xbf\xd4n\x11\tU\x97\x8a\xe2\x02^K&\'\x07\xe1\xf2\xa7\xf6nn\xdf\x9a\xd6uj6\xa2\xb0\xe4B\xde\xdf\xd8CMQJ%[\x83\xdd)\xef(\xce\x03\xb1DD&\x96i\x16\xc8\xbd\xe5cX\x86\x82P_\xa1\x00\x00\x00\x00\x00\x00\x0c\x90\x00\x17i6M\xe98n\xb9\r\x14O#\xc92\xc67\xd1Ta\x1f\t\x1c\xf6\x12\x97\xa8\xd5\xe8V%c\xd4\xb1\xae\xa7\x15%-U\x17mu\x058\xb4\x91J\xc7J\xca\x9b\xbf\\\x0fJ3\x18+!\x8a5"\xcbP\x00\x0c\xb7\x07L\xff\xff\x08\xe6/M\x8c\x05%X@\xa2\xed1\xe9\x08\xac\xc4Z\x93\x07\x16\xd9\xb1\xd7x\xa5\xea\x9br\x1aJ\xd5~a[\r\x03\xe2\xe7\xda\x08\xa3\x9cF\x1d\x9d\xa8\x90\xe2\xcd\x10P\x9d\x0f\x19#hR\xe1\x00\x18\x8e\xf6\xc6H\x93\x00\\\x02\xd8\xdf\xb3h\xb1\x02R\xa6\x0b\xd05\xe6\xb8\xd9\x07\x95\x81 EP\xb1\x9fb\xdb"\xac\x06\xc6\x03\xc9\x0e\x13\xc9\x93\x9f\x8f\x83\xbfy\x85\xc0\xa8$\xa1\xde\xae\x0e?\xdd\xc2\xab"\xb5I\x1a\xff\xf0\xea\x1c\xc8\xeb\xc9\x18\xb8:\x0f`s\xc2CiSIxk\xa8]\xab\xbb\xf4\xef\xf4\xaf\x81\x17\x9aenFg\xbf\xe4\xeb\xe9j\x92\xf0{\xda\xae\xd5\x16\xecX\xdf3\x9b~%Y\xde*L1\xc2\xfeO\xa8\x02\xd5\x00\x00\x00\x00\x00\x00C\x03\x00:sP\xf8\xb3\x1a%\x98K&\xde\x1dQi\x83\x97[K;CbJ\xba\x94M\xf1\xa5\x1c\xfb\xc3\xe5!\x16\xa8\x98}\x7f\xa1\xb0\x8b\xc8\x97~\x97\xc5\xb0\xe4\x13\xd8\x89\x14\x9d\x7f7_9\x1f\x93\x89\xc4\n\x02\xa8\x94\x00(\x99\xce\xbf\xe3\xd9Gu\xa8\xa8\xf1\xc7\xe4\x90\xff)O\xac,\xa9C;\xaf\x8a\x94\xa9\xcc\xcc|S\xb4\x7f*\xc2\xa2\xd9\x06\xfbA\xab\xbd\xfd\x91\x13\x18\xb4z\x16T6M\xed\xecYd)8j\n\x84\xaea\x82\xa9\x00[\xad\xb8K\xa5\x86\xd7\x0c\xa1\xc9\x06o/wF\xb7\xd9\x0c\xc2n\xfd7f8?\xbf\xc3\xa6U\xf8\xde\x81\x92\xa2y\x89\x88-\xfb\xea\x91\x9e\xf2\t\xf3\x14!R\xf70L\x9b\x84\x83W\xb5~\xd5R\xcf\x89\x7f\xeb\x84\xff\xf7\x9c\x12\xe7\xc0\xd2\xb2T=\x9c\xcb\x8a\x02z\xd7\xa8w\xc5C\r>\xe9"h\xf431\xdcAg\x16FP\xfcZ(\x1et\xb0\xe8Hx\xf079\xd0\xac\xd1F6\xa5\xc5\xaa\xa4\x05\xea\xc3RON[ \xaf7\x00\x00\x00\x00\x00\x01\xf6\x93\x00!f\xc3XQ\xaa\'\xad\xbf\xf3;\xe6DAy\xce\'\xd9ff}l`K\xcd|\x8c\x90\r\x1a>D\xecSg1\xb9\n\xab:+_Ag\x80\xa6J\xc7\xe6\xe2\xbb1EG\x01$\xdd\x8a\xeeV<\xc3\xba\x18\xff\xdf{\xe1\x8c\xc5\x7f\xe9\xd2\x07\x8f*\xbb\xf8\x90J\xf9\xda\xe3o\x14\xcer\x97"\xffe\xb1\xf1"\x10\x1d\x01\'b\xa0k\xaag)\xd4\x0c\xda\x1f_\x13\x1d;G\xc4D\xcd\x17\x08\xa0\xd5\x14\xc3V\rS9\xdd\xf8/\x00^\x83C\xa2\x88\x83\x1dg\xd7 \'U,\x95\x84vh\xe1\xfc\xcd\xc8c\x1d\xde\x91\xad\xf6o&f\xf5\xee\x17\xff\xaeM\x0c&\xda\x86\x84z`\x18\x10\xaf4Z#\x8f\x13GY\x18\xa0f\xed\xdf~\xdd\xbf\x07\xbdA\x00\x1a\xc9k\x8egp\xb8\xbd\xb5\xcf\r\x13\t=\x14~\x9a\xd3y\xe4X1\x127q\xa9-\x88:\xc0\xecH\x8f.\t\xca\xc9-\xb4ai\xe6\x9a\xa2\x19\xfc-\xb3\x10\xe1J\x10;\xda\xfa\xeb\xddB\x11\\2\xce\xff9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00^K\x05a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa7\xb0\x1a\xba$sST\x96\xa9[\xc1\xb2\x08\xbe5\xb41\x89\x8f\x19<\x18\x1db\x01!wG\x82\xc2\xb8\x0cN\x8b\x9f\xf7\x1fs\x80\x91\x15\x9f\xce\xfcv\xdf\x173&q\x8c)\x9b\x16u\x15\xb2x\xf4\xd9r~\xf0\x00\x00\x00\x00\x00\x00\x01\xf4\x00\x00\x00\x00\x00\x02JW\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xd9\xef\x0b\x83h`1\x94\x92\x915$\xf7Hh\xaeV\xa1\x91\xff\x89\xd33\xdd\x96\xb1\xa6\x01\xa4w\xc1T\x8f\xdb\xd6u\x1a\xd3s\xc1\xb2\xe1\x80\x9d\xa8\xc2X\x98\x10\xb7L\xc8\x8c\xf7M\\\xb2n\x9di\tmV\xf5\x9f\x00wE%\xa8GG\x03\x91\xac6p\xec\x8e\xef\x7f\x8f\xa7\x8a\xc6W-0\x90R\tH\xc7\xd1C8\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x8b)\xaa\x96x8\xd76J\xa6\x8b[\x98\t\xe0\\\xe3^7qD\x8c\xf5q\x08\xf2\xa2\xc9\xb03mv\x00\x00\x0c\xbb\xa1\x06\xe0\x00N\x1f\xe8;}6F\xd7\xec\xc7\x83\x16T\x96\x1f\xe6\x88,\xa4\x9b\xa3Lo\xd0\xe6\x89jW\xac\xba\xae)\xe9\x91?\x97\x0fU\xf5\xd8\xdc\x9e\xce\xbf~\xad\xc2\xbc\x17v|\x947N\x0e\xfa\xff\xe6;\xce@|\xe9{\xe2:\xa8H\xb4\xb9\xde;<;-\x9a\x03\xbf\xa3\xff\xed\x81\x0cd\x80|(I\x9e\x8c\xa5\x83\xdf\x8a\x1aX\xc1#\x19uE`)\xeblV\x1d\x8f\xe6\x1f\xfa\x03\xe2\xf4b\xdfO\x9c\x11\x1fHJ2\xbfvC\x8b^\x8b)\xaa\x96x8\xd76J\xa6\x8b[\x98\t\xe0\\\xe3^7qD\x8c\xf5q\x08\xf2\xa2\xc9\xb03mv\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', # noqa: E501 + "GENESIS_BLOCK": b'\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x15N3\xd3\xf9H\xc2K\x96\xfe\xf2f\xa2\xbf\x87\x0e\x0f,\xd0\xd4\x0f6s\xb1".\\\xf5\x8a\xb4\x03\x84\x8e\xf9\xbb\xa1\xca\xdef3:\xe4?\x0c\xe5\xc6\x12\x80\x17\xd2\xcc\xd7\xb4m\x94\xb7V\x959\xed4\x89\x04b\x08\x07^\xca`\x8f#%\xe9\x9c\x9d\x86y\x10\x96W\x9d\xce\xc1\x15r\x97\x91U\n\x11<\xdf\xb2\xfc\xfb<\x13\x00\x00\x00\x98\x97N\xe3py\xb5*\xe8\xbe_\xa5\xa5mq\xceY\x08$;\xe1\xef;:\xc0\x93\x84\xa6(`\xe7\xf1\x0b\xb6\x08\xb4\xa1\xc7\x03\xe5v\xb9)\xd2\xc9|\t\xcc\xb3\xd4\x05\xd9-\xa2;\xeb\xdd!\xd1\\Oh<\x01+\x9e\xbfK\xe3y\xd8\xd0T\xac\xb52+Zj+)\xad\xabIg\x82\x998\xe5\x86\xa3[\xd1\xc1\xe9\x9dD\xbb\xe2IN\x01\xdc\x1e\xcc\t\x1c\xa7\xe4\xb92\x98^\xd5\xd6\xce\xe1Z\xcd\x9e\xee\xba\x9f,\x94B?}\xf7\xeb\x8f,\xbds\xc4g\x02"}\x88\x17.\xf7\xd3\x89\x15Vi\xbe2\xdct\xc4\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x02JW\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\xf6\xcb\xc9@\x18<\xf3NgW\xe1v\xc5ag]\x1e\xb65\x15\xd6X\xb7W\xa3\xe4\xb1\xea\xf4!\xaf\xa6\x9b0v,PR\xe4L\x87\xff\xc9\xe3\x13\xf2\xf0\xcb*3R\x91dE8\xf6S<=\xfd\x87u\xf7\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x86\x91#\xeb)\xc2\xae\x04Q\x041&\xa32\x02\xd4\xdbge\x9b\tj\x02\xa8\x9aH\x15?\xa5\xcc\xdd\x84\xcf\x10:\xb5Ki"\xd2\x07\xf5\xd3\xc8\xb1U_P\xcb^\xd8"\x95?\x18%\xb8\xdf\xa7\x82\xaa\xb2\xbb\x03\x00\x00\x03\xa6\x00O\x90+E\x9a#\xf2\xb1D\x88|\x8f\nx\xe4\xb6\xe7\x02\x10\x11\xb1\x13[u)\xfe*\x94\x9c\r\x91`\xfc\xa8\x0c\xb1\x00,?p\xba\xb2\x82\xf2n&\x84\x1f\xe0U\xe9\xc2\xe4\xc2Y\xc8\x9bg\xf9\xd1\x8b\x02\xdf\x0c\xff\xbf\xd4n\x11\tU\x97\x8a\xe2\x02^K&\'\x07\xe1\xf2\xa7\xf6nn\xdf\x9a\xd6uj6\xa2\xb0\xe4B\xde\xdf\xd8CMQJ%[\x83\xdd)\xef(\xce\x03\xb1DD&\x96i\x16\xc8\xbd\xe5cX\x86\x82P_\xa1\x00\x00\x00\x00\x00\x00\x0c\x90\x00\x17i6M\xe98n\xb9\r\x14O#\xc92\xc67\xd1Ta\x1f\t\x1c\xf6\x12\x97\xa8\xd5\xe8V%c\xd4\xb1\xae\xa7\x15%-U\x17mu\x058\xb4\x91J\xc7J\xca\x9b\xbf\\\x0fJ3\x18+!\x8a5"\xcbP\x00\x0c\xb7\x07L\xff\xff\x08\xe6/M\x8c\x05%X@\xa2\xed1\xe9\x08\xac\xc4Z\x93\x07\x16\xd9\xb1\xd7x\xa5\xea\x9br\x1aJ\xd5~a[\r\x03\xe2\xe7\xda\x08\xa3\x9cF\x1d\x9d\xa8\x90\xe2\xcd\x10P\x9d\x0f\x19#hR\xe1\x00\x18\x8e\xf6\xc6H\x93\x00\\\x02\xd8\xdf\xb3h\xb1\x02R\xa6\x0b\xd05\xe6\xb8\xd9\x07\x95\x81 EP\xb1\x9fb\xdb"\xac\x06\xc6\x03\xc9\x0e\x13\xc9\x93\x9f\x8f\x83\xbfy\x85\xc0\xa8$\xa1\xde\xae\x0e?\xdd\xc2\xab"\xb5I\x1a\xff\xf0\xea\x1c\xc8\xeb\xc9\x18\xb8:\x0f`s\xc2CiSIxk\xa8]\xab\xbb\xf4\xef\xf4\xaf\x81\x17\x9aenFg\xbf\xe4\xeb\xe9j\x92\xf0{\xda\xae\xd5\x16\xecX\xdf3\x9b~%Y\xde*L1\xc2\xfeO\xa8\x02\xd5\x00\x00\x00\x00\x00\x00C\x03\x00:sP\xf8\xb3\x1a%\x98K&\xde\x1dQi\x83\x97[K;CbJ\xba\x94M\xf1\xa5\x1c\xfb\xc3\xe5!\x16\xa8\x98}\x7f\xa1\xb0\x8b\xc8\x97~\x97\xc5\xb0\xe4\x13\xd8\x89\x14\x9d\x7f7_9\x1f\x93\x89\xc4\n\x02\xa8\x94\x00(\x99\xce\xbf\xe3\xd9Gu\xa8\xa8\xf1\xc7\xe4\x90\xff)O\xac,\xa9C;\xaf\x8a\x94\xa9\xcc\xcc|S\xb4\x7f*\xc2\xa2\xd9\x06\xfbA\xab\xbd\xfd\x91\x13\x18\xb4z\x16T6M\xed\xecYd)8j\n\x84\xaea\x82\xa9\x00[\xad\xb8K\xa5\x86\xd7\x0c\xa1\xc9\x06o/wF\xb7\xd9\x0c\xc2n\xfd7f8?\xbf\xc3\xa6U\xf8\xde\x81\x92\xa2y\x89\x88-\xfb\xea\x91\x9e\xf2\t\xf3\x14!R\xf70L\x9b\x84\x83W\xb5~\xd5R\xcf\x89\x7f\xeb\x84\xff\xf7\x9c\x12\xe7\xc0\xd2\xb2T=\x9c\xcb\x8a\x02z\xd7\xa8w\xc5C\r>\xe9"h\xf431\xdcAg\x16FP\xfcZ(\x1et\xb0\xe8Hx\xf079\xd0\xac\xd1F6\xa5\xc5\xaa\xa4\x05\xea\xc3RON[ \xaf7\x00\x00\x00\x00\x00\x01\xf6\x93\x00!f\xc3XQ\xaa\'\xad\xbf\xf3;\xe6DAy\xce\'\xd9ff}l`K\xcd|\x8c\x90\r\x1a>D\xecSg1\xb9\n\xab:+_Ag\x80\xa6J\xc7\xe6\xe2\xbb1EG\x01$\xdd\x8a\xeeV<\xc3\xba\x18\xff\xdf{\xe1\x8c\xc5\x7f\xe9\xd2\x07\x8f*\xbb\xf8\x90J\xf9\xda\xe3o\x14\xcer\x97"\xffe\xb1\xf1"\x10\x1d\x01\'b\xa0k\xaag)\xd4\x0c\xda\x1f_\x13\x1d;G\xc4D\xcd\x17\x08\xa0\xd5\x14\xc3V\rS9\xdd\xf8/\x00^\x83C\xa2\x88\x83\x1dg\xd7 \'U,\x95\x84vh\xe1\xfc\xcd\xc8c\x1d\xde\x91\xad\xf6o&f\xf5\xee\x17\xff\xaeM\x0c&\xda\x86\x84z`\x18\x10\xaf4Z#\x8f\x13GY\x18\xa0f\xed\xdf~\xdd\xbf\x07\xbdA\x00\x1a\xc9k\x8egp\xb8\xbd\xb5\xcf\r\x13\t=\x14~\x9a\xd3y\xe4X1\x127q\xa9-\x88:\xc0\xecH\x8f.\t\xca\xc9-\xb4ai\xe6\x9a\xa2\x19\xfc-\xb3\x10\xe1J\x10;\xda\xfa\xeb\xddB\x11\\2\xce\xff9\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00^P\xc3\x17\x00\x00\x00\x04\x01\x1f\xa6H\xa7\xb0\x1a\xba$sST\x96\xa9[\xc1\xb2\x08\xbe5\xb41\x89\x8f\x19<\x18\x1db\x01!wG\x82\xc2\xb8\x0cN\x8b\x9f\xf7\x1fs\x80\x91\x15\x9f\xce\xfcv\xdf\x173&q\x8c)\x9b\x16u\x15\xb2x\xf4\xd9r~\xf0\x00\x00\x00\x00\x00\x00\x01\xf4\x00\x00\x00\x00\x00\x02JW\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00Y\xb5U\xa2\x94M\x93\xa1\xc3\xfbV-\x87E\xaa\x15\x8a>\xd5\xbd\xf9\xb9\x81#ZKUn\xfa\xe5\xcc\xe0vGE\xf3z\xbe Tuple[Optional[Err], List[NPC], int]: """ diff --git a/src/wallet/wallet.py b/src/wallet/wallet.py index 7898288f145d..2f2a4932fc39 100644 --- a/src/wallet/wallet.py +++ b/src/wallet/wallet.py @@ -357,7 +357,7 @@ class Wallet: npc_list: List[NPC] if response.body.transactions: - error, npc_list, cost = await get_name_puzzle_conditions( + error, npc_list, cost = get_name_puzzle_conditions( response.body.transactions ) diff --git a/tests/block_tools.py b/tests/block_tools.py index eedcb36acb18..e51b92cda685 100644 --- a/tests/block_tools.py +++ b/tests/block_tools.py @@ -5,6 +5,7 @@ from pathlib import Path import blspy from blspy import PrependSignature, PrivateKey, PublicKey +from chiabip158 import PyBIP158 from chiapos import DiskPlotter, DiskProver from lib.chiavdf.inkfish.classgroup import ClassGroup @@ -17,7 +18,7 @@ from src.pool import create_coinbase_coin_and_signature from src.types.body import Body from src.types.challenge import Challenge from src.types.classgroup import ClassgroupElement -from src.types.full_block import FullBlock +from src.types.full_block import FullBlock, additions_for_npc from src.types.hashable.BLSSignature import BLSSignature from src.types.hashable.Coin import Coin from src.types.hashable.Program import Program @@ -30,6 +31,8 @@ from src.util.ints import uint8, uint32, uint64 from src.util.hash import std_hash # Can't go much lower than 19, since plots start having no solutions +from src.util.mempool_check_conditions import get_name_puzzle_conditions + k: uint8 = uint8(19) # Uses many plots for testing, in order to guarantee proofs of space at every height num_plots = 40 @@ -442,11 +445,27 @@ class BlockTools: extension_data, ) + # Create filter + byte_array_tx: List[bytes32] = [] + if transactions: + error, npc_list, _ = get_name_puzzle_conditions(transactions) + additions: List[Coin] = additions_for_npc(npc_list) + for coin in additions: + byte_array_tx.append(bytearray(coin.puzzle_hash)) + for npc in npc_list: + byte_array_tx.append(bytearray(npc.coin_name)) + + byte_array_tx.append(bytearray(coinbase_coin.puzzle_hash)) + byte_array_tx.append(bytearray(fees_coin.puzzle_hash)) + + bip158: PyBIP158 = PyBIP158(byte_array_tx) + encoded = bytes(bip158.GetEncoded()) + header_data: HeaderData = HeaderData( height, prev_header_hash, timestamp, - bytes([0] * 32), + encoded, proof_of_space.get_hash(), body.get_hash(), uint64(prev_weight + difficulty), diff --git a/tests/test_blockchain.py b/tests/test_blockchain.py index 03de38d6a363..fbd487d5ffa4 100644 --- a/tests/test_blockchain.py +++ b/tests/test_blockchain.py @@ -90,7 +90,7 @@ class TestBlockValidation: blocks[9].header.data.height, bytes([1] * 32), blocks[9].header.data.timestamp, - blocks[9].header.data.filter_hash, + blocks[9].header.data.filter, blocks[9].header.data.proof_of_space_hash, blocks[9].header.data.body_hash, blocks[9].header.data.weight, @@ -117,7 +117,7 @@ class TestBlockValidation: blocks[9].header.data.height, blocks[9].header.data.prev_header_hash, blocks[9].header.data.timestamp - 1000, - blocks[9].header.data.filter_hash, + blocks[9].header.data.filter, blocks[9].header.data.proof_of_space_hash, blocks[9].header.data.body_hash, blocks[9].header.data.weight, @@ -141,7 +141,7 @@ class TestBlockValidation: blocks[9].header.data.height, blocks[9].header.data.prev_header_hash, uint64(int(time.time() + 3600 * 3)), - blocks[9].header.data.filter_hash, + blocks[9].header.data.filter, blocks[9].header.data.proof_of_space_hash, blocks[9].header.data.body_hash, blocks[9].header.data.weight, @@ -167,7 +167,7 @@ class TestBlockValidation: blocks[9].header.data.height, blocks[9].header.data.prev_header_hash, blocks[9].header.data.timestamp, - blocks[9].header.data.filter_hash, + blocks[9].header.data.filter, blocks[9].header.data.proof_of_space_hash, bytes([1] * 32), blocks[9].header.data.weight, diff --git a/tests/test_filter.py b/tests/test_filter.py index 6e4c22d89e56..a7beae07d984 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -27,7 +27,7 @@ class TestFilter: key_config = {"wallet_sk": sk} wallet = await Wallet.create({}, key_config) - wallet.wallet_store._clear_database() + await wallet.wallet_store._clear_database() num_blocks = 2 blocks = bt.get_consecutive_blocks( From 7091834176f3da05bd418f3803567f4686f42c70 Mon Sep 17 00:00:00 2001 From: Yostra Date: Sun, 23 Feb 2020 00:11:31 -0800 Subject: [PATCH 18/38] request addition protocol --- src/blockchain.py | 4 +--- src/full_node.py | 22 ++++++++++++++++++++++ src/protocols/wallet_protocol.py | 31 +++++++++++++++++++++++++++++++ src/types/full_block.py | 20 ++++++++++++++++---- src/wallet/wallet.py | 19 +++++++++++++++++++ tests/test_filter.py | 2 +- 6 files changed, 90 insertions(+), 8 deletions(-) diff --git a/src/blockchain.py b/src/blockchain.py index 8548e86752d8..888d408797d3 100644 --- a/src/blockchain.py +++ b/src/blockchain.py @@ -927,9 +927,7 @@ class Blockchain: if not block.body.transactions: return Err.UNKNOWN # Get List of names removed, puzzles hashes for removed coins and conditions crated - error, npc_list, cost = get_name_puzzle_conditions( - block.body.transactions - ) + error, npc_list, cost = get_name_puzzle_conditions(block.body.transactions) if cost > 6000: return Err.BLOCK_COST_EXCEEDS_MAX diff --git a/src/full_node.py b/src/full_node.py index b77ee9014ee3..66eab68132dd 100644 --- a/src/full_node.py +++ b/src/full_node.py @@ -1665,3 +1665,25 @@ class FullNode: Message("maybe_transaction", spend_bundle.name()), Delivery.BROADCAST, ) + + @api_request + async def request_additions( + self, request: src.protocols.wallet_protocol.RequestAdditions + ) -> OutboundMessageGenerator: + block: Optional[FullBlock] = await self.store.get_block(request.header_hash) + if block: + additions = block.additions() + response = src.protocols.wallet_protocol.Additions( + block.height, block.header_hash, additions + ) + yield OutboundMessage( + NodeType.WALLET, + Message("response_additions", response), + Delivery.BROADCAST, + ) + else: + yield OutboundMessage( + NodeType.WALLET, + Message("response_reject_additions", request), + Delivery.BROADCAST, + ) diff --git a/src/protocols/wallet_protocol.py b/src/protocols/wallet_protocol.py index f849ffc3dbc4..a8f133ef3386 100644 --- a/src/protocols/wallet_protocol.py +++ b/src/protocols/wallet_protocol.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from typing import List, Tuple from src.types.body import Body +from src.types.hashable.Coin import Coin from src.types.hashable.SpendBundle import SpendBundle from src.types.header_block import HeaderBlock from src.types.sized_bytes import bytes32 @@ -86,3 +87,33 @@ class FullProofForHash: @cbor_message class ProofHash: proof_hash: bytes32 + + +@dataclass(frozen=True) +@cbor_message +class RequestAdditions: + height: uint32 + header_hash: bytes32 + + +@dataclass(frozen=True) +@cbor_message +class RequestRemovals: + height: uint32 + header_hash: bytes32 + + +@dataclass(frozen=True) +@cbor_message +class Additions: + height: uint32 + header_hash: bytes32 + coins: List[Coin] + + +@dataclass(frozen=True) +@cbor_message +class Removals: + height: uint32 + header_hash: bytes32 + coins: List[Coin] diff --git a/src/types/full_block.py b/src/types/full_block.py index 315ed4595e61..b57351917733 100644 --- a/src/types/full_block.py +++ b/src/types/full_block.py @@ -50,6 +50,21 @@ class FullBlock(Streamable): def header_hash(self) -> bytes32: return self.header.header_hash + def additions(self) -> List[Coin]: + additions: List[Coin] = [] + + if self.body.transactions is not None: + # This should never throw here, block must be valid if it comes to here + err, npc_list, cost = get_name_puzzle_conditions(self.body.transactions) + # created coins + if npc_list is not None: + additions.extend(additions_for_npc(npc_list)) + + additions.append(self.body.coinbase) + additions.append(self.body.fees_coin) + + return additions + async def tx_removals_and_additions(self) -> Tuple[List[bytes32], List[Coin]]: """ Doesn't return coinbase and fee reward. @@ -60,11 +75,8 @@ class FullBlock(Streamable): additions: List[Coin] = [] if self.body.transactions is not None: - # ensure block program generates solutions # This should never throw here, block must be valid if it comes to here - err, npc_list, cost = get_name_puzzle_conditions( - self.body.transactions - ) + err, npc_list, cost = get_name_puzzle_conditions(self.body.transactions) # build removals list if npc_list is None: return [], [] diff --git a/src/wallet/wallet.py b/src/wallet/wallet.py index 2f2a4932fc39..baadcc18597b 100644 --- a/src/wallet/wallet.py +++ b/src/wallet/wallet.py @@ -421,6 +421,25 @@ class Wallet: async for reply in self.server.push_message(msg): self.log.info(reply) + async def _request_add_list(self, height: uint32, header_hash: bytes32): + obj = src.protocols.wallet_protocol.RequestAdditions(height, header_hash) + msg = OutboundMessage( + NodeType.FULL_NODE, + Message("request_additions", obj), + Delivery.BROADCAST, + ) + if self.server: + async for reply in self.server.push_message(msg): + self.log.info(reply) + + @api_request + async def response_additions(self, response: src.protocols.wallet_protocol.Additions): + print(response) + + @api_request + async def response_additions_rejected(self, response: src.protocols.wallet_protocol.RequestAdditions): + print(f"request rejected {response}") + @api_request async def transaction_ack(self, ack: src.protocols.wallet_protocol.TransactionAck): if ack.status: diff --git a/tests/test_filter.py b/tests/test_filter.py index a7beae07d984..bcfe0c99e74d 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -53,4 +53,4 @@ class TestFilter: assert present assert fee_present - await wallet.wallet_store.close() \ No newline at end of file + await wallet.wallet_store.close() From d656eaff9df88e25e578f53acd2b03d37241fa70 Mon Sep 17 00:00:00 2001 From: Yostra Date: Sun, 23 Feb 2020 13:11:05 -0800 Subject: [PATCH 19/38] wallet formatting --- src/wallet/electron/wallet_rpc/rpc_wallet.py | 29 ++++++------------- src/wallet/puzzles/load_clvm.py | 4 ++- src/wallet/puzzles/p2_conditions.py | 2 -- src/wallet/puzzles/p2_delegated_conditions.py | 2 -- src/wallet/puzzles/p2_delegated_puzzle.py | 6 ++-- .../puzzles/p2_m_of_n_delegate_direct.py | 3 +- src/wallet/puzzles/p2_puzzle_hash.py | 2 -- src/wallet/puzzles/puzzle_utils.py | 4 +-- src/wallet/wallet.py | 28 +++++++++++------- 9 files changed, 38 insertions(+), 42 deletions(-) diff --git a/src/wallet/electron/wallet_rpc/rpc_wallet.py b/src/wallet/electron/wallet_rpc/rpc_wallet.py index b104b5e272cf..44adbeac4197 100644 --- a/src/wallet/electron/wallet_rpc/rpc_wallet.py +++ b/src/wallet/electron/wallet_rpc/rpc_wallet.py @@ -5,13 +5,11 @@ import json from typing import Any from aiohttp import web -from blspy import ExtendedPrivateKey -from setproctitle import setproctitle from src.server.outbound_message import NodeType from src.server.server import ChiaServer from src.types.peer_info import PeerInfo -from src.util.config import load_config_cli, load_config +from src.util.config import load_config from src.wallet.wallet import Wallet @@ -65,35 +63,25 @@ class RpcWalletApiHandler: tx = await self.wallet.generate_signed_transaction(amount, puzzlehash) if tx is None: - response = { - "success": False - } + response = {"success": False} return obj_to_response(response) await self.wallet.push_transaction(tx) - response = { - "success": True - } + response = {"success": True} return obj_to_response(response) - response = { - "success": False - } + response = {"success": False} return obj_to_response(response) async def get_server_ready(self, request) -> web.Response: - response = { - "success": True - } + response = {"success": True} return obj_to_response(response) async def get_transactions(self, request) -> web.Response: - response = { - "success": True - } + response = {"success": True} return obj_to_response(response) async def get_wallet_balance(self, request) -> web.Response: @@ -155,8 +143,9 @@ async def start_rpc_server(): async def main(): cleanup = await start_rpc_server() - print('start running on {}') + print("start running on {}") await cleanup() -if __name__ == '__main__': + +if __name__ == "__main__": asyncio.run(main()) diff --git a/src/wallet/puzzles/load_clvm.py b/src/wallet/puzzles/load_clvm.py index 468f86683a98..a9ef0164a59d 100644 --- a/src/wallet/puzzles/load_clvm.py +++ b/src/wallet/puzzles/load_clvm.py @@ -4,6 +4,8 @@ from src.types.hashable.Program import Program def load_clvm(filename): - clvm_hex = pkg_resources.resource_string(__name__, "%s.hex" % filename).decode("utf8") + clvm_hex = pkg_resources.resource_string(__name__, "%s.hex" % filename).decode( + "utf8" + ) clvm_blob = bytes.fromhex(clvm_hex) return Program.from_bytes(clvm_blob) diff --git a/src/wallet/puzzles/p2_conditions.py b/src/wallet/puzzles/p2_conditions.py index e5b839b2949b..62b297b45e8c 100644 --- a/src/wallet/puzzles/p2_conditions.py +++ b/src/wallet/puzzles/p2_conditions.py @@ -10,8 +10,6 @@ require a delegated puzzle program, so in those cases, this is just what the doctor ordered. """ -import clvm - from clvm_tools import binutils from src.types.hashable.Program import Program diff --git a/src/wallet/puzzles/p2_delegated_conditions.py b/src/wallet/puzzles/p2_delegated_conditions.py index 2d5ca2a96cc4..40a63f4541e1 100644 --- a/src/wallet/puzzles/p2_delegated_conditions.py +++ b/src/wallet/puzzles/p2_delegated_conditions.py @@ -10,8 +10,6 @@ the doctor ordered. """ -import clvm - from clvm_tools import binutils from src.types.condition_opcodes import ConditionOpcode diff --git a/src/wallet/puzzles/p2_delegated_puzzle.py b/src/wallet/puzzles/p2_delegated_puzzle.py index a0f4d7b9c87b..0c6d7489e9e4 100644 --- a/src/wallet/puzzles/p2_delegated_puzzle.py +++ b/src/wallet/puzzles/p2_delegated_puzzle.py @@ -24,8 +24,10 @@ from . import p2_conditions def puzzle_for_pk(public_key) -> Program: aggsig = ConditionOpcode.AGG_SIG[0] - TEMPLATE = (f"(c (c (q {aggsig}) (c (q 0x%s) (c (sha256tree (f (a))) (q ())))) " - f"((c (f (a)) (f (r (a))))))") + TEMPLATE = ( + f"(c (c (q {aggsig}) (c (q 0x%s) (c (sha256tree (f (a))) (q ())))) " + f"((c (f (a)) (f (r (a))))))" + ) return Program.to(binutils.assemble(TEMPLATE % public_key.hex())) diff --git a/src/wallet/puzzles/p2_m_of_n_delegate_direct.py b/src/wallet/puzzles/p2_m_of_n_delegate_direct.py index 5d0967d0375c..8cde02025859 100644 --- a/src/wallet/puzzles/p2_m_of_n_delegate_direct.py +++ b/src/wallet/puzzles/p2_m_of_n_delegate_direct.py @@ -17,7 +17,8 @@ puzzle_prog_template = load_clvm("make_puzzle_m_of_n_direct.clvm") def puzzle_for_m_of_public_key_list(m, public_key_list): format_tuple = tuple( binutils.disassemble(Program.to(_)) - for _ in (puzzle_prog_template, m, public_key_list)) + for _ in (puzzle_prog_template, m, public_key_list) + ) puzzle_src = "((c (q %s) (c (q %s) (c (q %s) (a)))))" % format_tuple puzzle_prog = binutils.assemble(puzzle_src) return Program.to(puzzle_prog) diff --git a/src/wallet/puzzles/p2_puzzle_hash.py b/src/wallet/puzzles/p2_puzzle_hash.py index 4905034743c0..aec1b3763d34 100644 --- a/src/wallet/puzzles/p2_puzzle_hash.py +++ b/src/wallet/puzzles/p2_puzzle_hash.py @@ -5,8 +5,6 @@ In this puzzle program, the solution must be a reveal of the puzzle with the giv hash along with its solution. """ -import clvm - from clvm_tools import binutils from src.types.hashable.Program import Program diff --git a/src/wallet/puzzles/puzzle_utils.py b/src/wallet/puzzles/puzzle_utils.py index 7a526b7e207a..9276f1829ef5 100644 --- a/src/wallet/puzzles/puzzle_utils.py +++ b/src/wallet/puzzles/puzzle_utils.py @@ -1,7 +1,6 @@ from src.util.condition_tools import ConditionOpcode - def make_create_coin_condition(puzzle_hash, amount): return [ConditionOpcode.CREATE_COIN, puzzle_hash, amount] @@ -17,6 +16,7 @@ def make_assert_coin_consumed_condition(coin_name): def make_assert_my_coin_id_condition(coin_name): return [ConditionOpcode.ASSERT_MY_COIN_ID, coin_name] + def make_assert_block_index_exceeds_condition(block_index): return [ConditionOpcode.ASSERT_BLOCK_INDEX_EXCEEDS, block_index] @@ -26,4 +26,4 @@ def make_assert_block_age_exceeds_condition(block_index): def make_assert_time_exceeds_condition(time): - return [ConditionOpcode.ASSERT_TIME_EXCEEDS, time] \ No newline at end of file + return [ConditionOpcode.ASSERT_TIME_EXCEEDS, time] diff --git a/src/wallet/wallet.py b/src/wallet/wallet.py index baadcc18597b..1004899595a3 100644 --- a/src/wallet/wallet.py +++ b/src/wallet/wallet.py @@ -121,7 +121,11 @@ class Wallet: async def get_unconfirmed_balance(self) -> uint64: confirmed = await self.get_confirmed_balance() - result = confirmed - self.unconfirmed_removal_amount + self.unconfirmed_addition_amount + result = ( + confirmed + - self.unconfirmed_removal_amount + + self.unconfirmed_addition_amount + ) return uint64(result) def can_generate_puzzle_hash(self, hash: bytes32) -> bool: @@ -153,9 +157,9 @@ class Wallet: if amount > await self.get_unconfirmed_balance(): return None - unspent: Set[ - CoinRecord - ] = await self.wallet_store.get_coin_records_by_spent(False) + unspent: Set[CoinRecord] = await self.wallet_store.get_coin_records_by_spent( + False + ) sum = 0 used_coins: Set = set() @@ -246,7 +250,9 @@ class Wallet: changepuzzlehash = self.get_new_puzzlehash() primaries.append({"puzzlehash": changepuzzlehash, "amount": change}) # add change coin into temp_utxo set - self.tmp_coins.add(Coin(coin.name(), changepuzzlehash, uint64(change))) + self.tmp_coins.add( + Coin(coin.name(), changepuzzlehash, uint64(change)) + ) solution = self.make_solution(primaries=primaries) output_created = True else: @@ -424,20 +430,22 @@ class Wallet: async def _request_add_list(self, height: uint32, header_hash: bytes32): obj = src.protocols.wallet_protocol.RequestAdditions(height, header_hash) msg = OutboundMessage( - NodeType.FULL_NODE, - Message("request_additions", obj), - Delivery.BROADCAST, + NodeType.FULL_NODE, Message("request_additions", obj), Delivery.BROADCAST, ) if self.server: async for reply in self.server.push_message(msg): self.log.info(reply) @api_request - async def response_additions(self, response: src.protocols.wallet_protocol.Additions): + async def response_additions( + self, response: src.protocols.wallet_protocol.Additions + ): print(response) @api_request - async def response_additions_rejected(self, response: src.protocols.wallet_protocol.RequestAdditions): + async def response_additions_rejected( + self, response: src.protocols.wallet_protocol.RequestAdditions + ): print(f"request rejected {response}") @api_request From 603778813f3535f67737af0cba408bf43309f45c Mon Sep 17 00:00:00 2001 From: Yostra Date: Sun, 23 Feb 2020 13:20:08 -0800 Subject: [PATCH 20/38] proper exclude now --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index fae60ef8199f..edb53fcd235a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -38,7 +38,7 @@ jobs: cd ../../../ - name: Lint source with flake8 run: | - ./.venv/bin/flake8 src --exclude src/wallet + ./.venv/bin/flake8 src --exclude src/wallet/electron/node_modules - name: Lint source with mypy run: | ./.venv/bin/mypy src tests From ae2e9220a1e7e9e8b0006d463d67e35fa62fb2b3 Mon Sep 17 00:00:00 2001 From: Yostra Date: Mon, 24 Feb 2020 09:16:47 -0800 Subject: [PATCH 21/38] refactor wallet --- src/wallet/electron/wallet_rpc/rpc_wallet.py | 22 +- src/wallet/transaction_record.py | 25 ++ src/wallet/wallet.py | 277 +------------------ src/wallet/wallet_node.py | 192 +++++++++++++ src/wallet/wallet_state_manager.py | 161 +++++++++++ src/wallet/wallet_transaction_store.py | 150 ++++++++++ 6 files changed, 552 insertions(+), 275 deletions(-) create mode 100644 src/wallet/transaction_record.py create mode 100644 src/wallet/wallet_node.py create mode 100644 src/wallet/wallet_state_manager.py create mode 100644 src/wallet/wallet_transaction_store.py diff --git a/src/wallet/electron/wallet_rpc/rpc_wallet.py b/src/wallet/electron/wallet_rpc/rpc_wallet.py index 44adbeac4197..57f5733bfbdf 100644 --- a/src/wallet/electron/wallet_rpc/rpc_wallet.py +++ b/src/wallet/electron/wallet_rpc/rpc_wallet.py @@ -10,7 +10,7 @@ from src.server.outbound_message import NodeType from src.server.server import ChiaServer from src.types.peer_info import PeerInfo from src.util.config import load_config -from src.wallet.wallet import Wallet +from src.wallet.wallet_node import WalletNode class EnhancedJSONEncoder(json.JSONEncoder): @@ -42,14 +42,14 @@ class RpcWalletApiHandler: to the full node. """ - def __init__(self, wallet: Wallet): - self.wallet = wallet + def __init__(self, wallet_node: WalletNode): + self.wallet_node = wallet_node async def get_next_puzzle_hash(self, request) -> web.Response: """ Returns a new puzzlehash """ - puzzlehash = self.wallet.get_new_puzzlehash().hex() + puzzlehash = self.wallet_node.wallet.get_new_puzzlehash().hex() response = { "puzzlehash": puzzlehash, } @@ -60,13 +60,13 @@ class RpcWalletApiHandler: if "amount" in request_data and "puzzlehash" in request_data: amount = int(request_data["amount"]) puzzlehash = request_data["puzzlehash"] - tx = await self.wallet.generate_signed_transaction(amount, puzzlehash) + tx = await self.wallet_node.wallet.generate_signed_transaction(amount, puzzlehash) if tx is None: response = {"success": False} return obj_to_response(response) - await self.wallet.push_transaction(tx) + await self.wallet_node.wallet.push_transaction(tx) response = {"success": True} return obj_to_response(response) @@ -106,19 +106,19 @@ async def start_rpc_server(): raise RuntimeError( "Keys not generated. Run python3 ./scripts/regenerate_keys.py." ) - wallet = await Wallet.create(config, key_config) + wallet_node = await WalletNode.create(config, key_config) - server = ChiaServer(9257, wallet, NodeType.WALLET) - wallet.set_server(server) + server = ChiaServer(9257, wallet_node, NodeType.WALLET) + wallet_node.set_server(server) full_node_peer = PeerInfo( config["full_node_peer"]["host"], config["full_node_peer"]["port"] ) - _ = await server.start_server("127.0.0.1", wallet._on_connect) + _ = await server.start_server("127.0.0.1", wallet_node._on_connect) await asyncio.sleep(1) _ = await server.start_client(full_node_peer, None) - handler = RpcWalletApiHandler(wallet) + handler = RpcWalletApiHandler(wallet_node) app = web.Application() app.add_routes( [ diff --git a/src/wallet/transaction_record.py b/src/wallet/transaction_record.py new file mode 100644 index 000000000000..3b254cb15511 --- /dev/null +++ b/src/wallet/transaction_record.py @@ -0,0 +1,25 @@ +from dataclasses import dataclass + +from src.types.hashable.SpendBundle import SpendBundle +from src.types.sized_bytes import bytes32 +from src.util.streamable import Streamable, streamable +from src.util.ints import uint32, uint64 + + +@dataclass(frozen=True) +@streamable +class TransactionRecord(Streamable): + """ + Used for storing transaction data and status in wallets + """ + + confirmed_block_index: uint32 + created_at_index: uint32 + confirmed: bool + sent: bool + created_at_time: uint64 + spend_bundle: SpendBundle + + @property + def name(self) -> bytes32: + return self.spend_bundle.name() diff --git a/src/wallet/wallet.py b/src/wallet/wallet.py index 1004899595a3..9798d7579d15 100644 --- a/src/wallet/wallet.py +++ b/src/wallet/wallet.py @@ -1,31 +1,20 @@ -from pathlib import Path -from typing import Dict, Optional, List, Set, Tuple +from typing import Dict, Optional, List, Tuple import clvm from blspy import ExtendedPrivateKey, PublicKey import logging -import src.protocols.wallet_protocol -from src.full_node import OutboundMessageGenerator -from src.protocols.wallet_protocol import ProofHash -from src.server.outbound_message import OutboundMessage, NodeType, Message, Delivery + from src.server.server import ChiaServer -from src.types.full_block import additions_for_npc from src.types.hashable.BLSSignature import BLSSignature -from src.types.hashable.Coin import Coin -from src.types.hashable.CoinRecord import CoinRecord from src.types.hashable.CoinSolution import CoinSolution from src.types.hashable.Program import Program from src.types.hashable.SpendBundle import SpendBundle -from src.types.name_puzzle_condition import NPC from src.types.sized_bytes import bytes32 -from src.util.hash import std_hash -from src.util.api_decorators import api_request from src.util.condition_tools import ( conditions_for_solution, conditions_by_opcode, hash_key_pairs_for_conditions_dict, ) -from src.util.ints import uint32, uint64 -from src.util.mempool_check_conditions import get_name_puzzle_conditions +from src.util.ints import uint64 from src.wallet.BLSPrivateKey import BLSPrivateKey from src.wallet.puzzles.p2_conditions import puzzle_for_conditions from src.wallet.puzzles.p2_delegated_puzzle import puzzle_for_pk @@ -35,6 +24,8 @@ from src.wallet.puzzles.puzzle_utils import ( make_assert_coin_consumed_condition, make_create_coin_condition, ) + +from src.wallet.wallet_state_manager import WalletStateManager from src.wallet.wallet_store import WalletStore @@ -45,22 +36,9 @@ class Wallet: server: Optional[ChiaServer] next_address: int = 0 pubkey_num_lookup: Dict[bytes, int] - tmp_coins: Set[Coin] wallet_store: WalletStore - header_hash: List[bytes32] - start_index: int + wallet_state_manager: WalletStateManager - unconfirmed_removals: Set[Coin] - unconfirmed_removal_amount: int - - unconfirmed_additions: Set[Coin] - unconfirmed_addition_amount: int - - # This dict maps coin_id to SpendBundle, it will contain duplicate values by design - coin_spend_bundle_map: Dict[bytes32, SpendBundle] - - # Spendbundle_ID : Spendbundle - pending_spend_bundles: Dict[bytes32, SpendBundle] log: logging.Logger # TODO Don't allow user to send tx until wallet is synced @@ -70,7 +48,7 @@ class Wallet: send_queue: Dict[bytes32, SpendBundle] @staticmethod - async def create(config: Dict, key_config: Dict, name: str = None): + async def create(config: Dict, key_config: Dict, wallet_state_manager: WalletStateManager, name: str = None): self = Wallet() print("init wallet") self.config = config @@ -82,23 +60,8 @@ class Wallet: else: self.log = logging.getLogger(__name__) + self.wallet_state_manager = wallet_state_manager self.pubkey_num_lookup = {} - self.tmp_coins = set() - pub_hex = self.private_key.get_public_key().serialize().hex() - path = Path(f"wallet_db_{pub_hex}.db") - self.wallet_store = await WalletStore.create(path) - self.header_hash = [] - self.unconfirmed_additions = set() - self.unconfirmed_removals = set() - self.pending_spend_bundles = {} - self.coin_spend_bundle_map = {} - self.unconfirmed_addition_amount = 0 - self.unconfirmed_removal_amount = 0 - - self.synced = False - - self.send_queue = {} - self.server = None return self @@ -109,24 +72,10 @@ class Wallet: return pubkey async def get_confirmed_balance(self) -> uint64: - record_list: Set[ - CoinRecord - ] = await self.wallet_store.get_coin_records_by_spent(False) - amount: uint64 = uint64(0) - - for record in record_list: - amount = uint64(amount + record.coin.amount) - - return uint64(amount) + return await self.wallet_state_manager.get_confirmed_balance() async def get_unconfirmed_balance(self) -> uint64: - confirmed = await self.get_confirmed_balance() - result = ( - confirmed - - self.unconfirmed_removal_amount - + self.unconfirmed_addition_amount - ) - return uint64(result) + return await self.wallet_state_manager.get_unconfirmed_balance() def can_generate_puzzle_hash(self, hash: bytes32) -> bool: return any( @@ -152,48 +101,6 @@ class Wallet: puzzlehash: bytes32 = puzzle.get_hash() return puzzlehash - async def select_coins(self, amount) -> Optional[Set[Coin]]: - - if amount > await self.get_unconfirmed_balance(): - return None - - unspent: Set[CoinRecord] = await self.wallet_store.get_coin_records_by_spent( - False - ) - sum = 0 - used_coins: Set = set() - - """ - Try to use coins from the store, if there isn't enough of "unused" - coins use change coins that are not confirmed yet - """ - for coinrecord in unspent: - if sum >= amount: - break - if coinrecord.coin.name in self.unconfirmed_removals: - continue - sum += coinrecord.coin.amount - used_coins.add(coinrecord.coin) - - """ - This happens when we couldn't use one of the coins because it's already used - but unconfirmed, and we are waiting for the change. (unconfirmed_additions) - """ - if sum < amount: - for coin in self.unconfirmed_additions: - if sum > amount: - break - if coin.name in self.unconfirmed_removals: - continue - sum += coin.amount - used_coins.add(coin) - - if sum >= amount: - return used_coins - else: - # This shouldn't happen because of: if amount > self.get_unconfirmed_balance(): - return None - def set_server(self, server: ChiaServer): self.server = server @@ -230,7 +137,7 @@ class Wallet: async def generate_unsigned_transaction( self, amount: int, newpuzzlehash: bytes32, fee: int = 0 ) -> List[Tuple[Program, CoinSolution]]: - utxos = await self.select_coins(amount + fee) + utxos = await self.wallet_state_manager.select_coins(amount + fee) if utxos is None: return [] spends: List[Tuple[Program, CoinSolution]] = [] @@ -249,10 +156,7 @@ class Wallet: if change > 0: changepuzzlehash = self.get_new_puzzlehash() primaries.append({"puzzlehash": changepuzzlehash, "amount": change}) - # add change coin into temp_utxo set - self.tmp_coins.add( - Coin(coin.name(), changepuzzlehash, uint64(change)) - ) + solution = self.make_solution(primaries=primaries) output_created = True else: @@ -298,167 +202,12 @@ class Wallet: return None return self.sign_transaction(transaction) - async def coin_removed(self, coin_name: bytes32, index: uint32): - self.log.info("remove coin") - await self.wallet_store.set_spent(coin_name, index) - - async def coin_added(self, coin: Coin, index: uint32, coinbase: bool): - self.log.info("add coin") - coin_record: CoinRecord = CoinRecord(coin, index, uint32(0), False, coinbase) - await self.wallet_store.add_coin_record(coin_record) - - async def _on_connect(self) -> OutboundMessageGenerator: - """ - Whenever we connect to a FullNode we request new proof_hashes by sending last proof hash we have - """ - self.log.info(f"Requesting proof hashes") - request = ProofHash(std_hash(b"deadbeef")) - yield OutboundMessage( - NodeType.FULL_NODE, - Message("request_proof_hashes", request), - Delivery.BROADCAST, - ) - - @api_request - async def proof_hash( - self, request: src.protocols.wallet_protocol.ProofHash - ) -> OutboundMessageGenerator: - """ - Received a proof hash from the FullNode - """ - self.log.info(f"Received a new proof hash: {request}") - reply_request = ProofHash(std_hash(b"a")) - # TODO Store and decide if we want full proof for this proof hash - yield OutboundMessage( - NodeType.FULL_NODE, - Message("request_full_proof_for_hash", reply_request), - Delivery.RESPOND, - ) - - @api_request - async def full_proof_for_hash( - self, request: src.protocols.wallet_protocol.FullProofForHash - ): - """ - We've received a full proof for hash we requested - """ - # TODO Validate full proof - self.log.info(f"Received new proof: {request}") - - @api_request - async def received_body(self, response: src.protocols.wallet_protocol.RespondBody): - """ - Called when body is received from the FullNode - """ - - # Retry sending queued up transactions - await self.retry_send_queue() - - additions: List[Coin] = [] - - if self.can_generate_puzzle_hash(response.body.coinbase.puzzle_hash): - await self.coin_added(response.body.coinbase, response.height, True) - if self.can_generate_puzzle_hash(response.body.fees_coin.puzzle_hash): - await self.coin_added(response.body.fees_coin, response.height, True) - - npc_list: List[NPC] - if response.body.transactions: - error, npc_list, cost = get_name_puzzle_conditions( - response.body.transactions - ) - - additions.extend(additions_for_npc(npc_list)) - - for added_coin in additions: - if self.can_generate_puzzle_hash(added_coin.puzzle_hash): - await self.coin_added(added_coin, response.height, False) - - for npc in npc_list: - if self.can_generate_puzzle_hash(npc.puzzle_hash): - await self.coin_removed(npc.coin_name, response.height) - - @api_request - async def new_tip(self, header: src.protocols.wallet_protocol.Header): - self.log.info("new tip received") - - async def retry_send_queue(self): - for key, val in self.send_queue: - await self._send_transaction(val) - - def remove_from_queue(self, spendbundle_id: bytes32): - if spendbundle_id in self.send_queue: - del self.send_queue[spendbundle_id] - async def push_transaction(self, spend_bundle: SpendBundle): """ Use this API to make transactions. """ - self.send_queue[spend_bundle.name()] = spend_bundle - additions: List[Coin] = spend_bundle.additions() - removals: List[Coin] = spend_bundle.removals() - - addition_amount = 0 - for coin in additions: - if self.can_generate_puzzle_hash(coin.puzzle_hash): - self.unconfirmed_additions.add(coin) - self.coin_spend_bundle_map[coin.name()] = spend_bundle - addition_amount += coin.amount - - removal_amount = 0 - for coin in removals: - self.unconfirmed_removals.add(coin) - self.coin_spend_bundle_map[coin.name()] = spend_bundle - removal_amount += coin.amount - - # Update unconfirmed state - self.unconfirmed_removal_amount += removal_amount - self.unconfirmed_addition_amount += addition_amount - - self.pending_spend_bundles[spend_bundle.name()] = spend_bundle + await self.wallet_state_manager.add_pending_transaction(spend_bundle) await self._send_transaction(spend_bundle) async def _send_transaction(self, spend_bundle: SpendBundle): - """ Sends spendbundle to connected full Nodes.""" - - msg = OutboundMessage( - NodeType.FULL_NODE, - Message("wallet_transaction", spend_bundle), - Delivery.BROADCAST, - ) if self.server: async for reply in self.server.push_message(msg): self.log.info(reply) - - async def _request_add_list(self, height: uint32, header_hash: bytes32): - obj = src.protocols.wallet_protocol.RequestAdditions(height, header_hash) - msg = OutboundMessage( - NodeType.FULL_NODE, Message("request_additions", obj), Delivery.BROADCAST, - ) - if self.server: - async for reply in self.server.push_message(msg): - self.log.info(reply) - - @api_request - async def response_additions( - self, response: src.protocols.wallet_protocol.Additions - ): - print(response) - - @api_request - async def response_additions_rejected( - self, response: src.protocols.wallet_protocol.RequestAdditions - ): - print(f"request rejected {response}") - - @api_request - async def transaction_ack(self, ack: src.protocols.wallet_protocol.TransactionAck): - if ack.status: - self.remove_from_queue(ack.txid) - self.log.info(f"SpendBundle has been received by the FullNode. id: {id}") - else: - self.log.info(f"SpendBundle has been rejected by the FullNode. id: {id}") - - async def requestLCA(self): - msg = OutboundMessage( - NodeType.FULL_NODE, Message("request_lca", None), Delivery.BROADCAST, - ) - async for reply in self.server.push_message(msg): - self.log.info(reply) diff --git a/src/wallet/wallet_node.py b/src/wallet/wallet_node.py new file mode 100644 index 000000000000..c7c2a782472e --- /dev/null +++ b/src/wallet/wallet_node.py @@ -0,0 +1,192 @@ +from pathlib import Path +from typing import Dict, Optional, List +from blspy import ExtendedPrivateKey +import logging +import src.protocols.wallet_protocol +from src.full_node import OutboundMessageGenerator +from src.protocols.wallet_protocol import ProofHash +from src.server.outbound_message import OutboundMessage, NodeType, Message, Delivery +from src.server.server import ChiaServer +from src.types.full_block import additions_for_npc +from src.types.hashable.Coin import Coin +from src.types.hashable.SpendBundle import SpendBundle +from src.types.name_puzzle_condition import NPC +from src.types.sized_bytes import bytes32 +from src.util.hash import std_hash +from src.util.api_decorators import api_request +from src.util.ints import uint32 +from src.util.mempool_check_conditions import get_name_puzzle_conditions +from src.wallet.wallet import Wallet +from src.wallet.wallet_state_manager import WalletStateManager +from src.wallet.wallet_store import WalletStore +from src.wallet.wallet_transaction_store import WalletTransactionStore + + +class WalletNode: + private_key: ExtendedPrivateKey + key_config: Dict + config: Dict + server: Optional[ChiaServer] + wallet_store: WalletStore + wallet_state_manager: WalletStateManager + header_hash: List[bytes32] + start_index: int + log: logging.Logger + wallet: Wallet + + @staticmethod + async def create(config: Dict, key_config: Dict, name: str = None): + self = WalletNode() + print("init wallet node") + self.config = config + self.key_config = key_config + sk_hex = self.key_config["wallet_sk"] + self.private_key = ExtendedPrivateKey.from_bytes(bytes.fromhex(sk_hex)) + if name: + self.log = logging.getLogger(name) + else: + self.log = logging.getLogger(__name__) + + pub_hex = self.private_key.get_public_key().serialize().hex() + path = Path(f"wallet_db_{pub_hex}.db") + self.wallet_store = await WalletStore.create(path) + self.tx_store = await WalletTransactionStore.create(path) + + self.wallet_state_manager = await WalletStateManager.create(config, self.wallet_store, self.tx_store) + self.wallet = await Wallet.create(config, key_config, self.wallet_state_manager) + + self.server = None + + return self + + def set_server(self, server: ChiaServer): + self.server = server + self.wallet.set_server(server) + + async def _on_connect(self) -> OutboundMessageGenerator: + """ + Whenever we connect to a FullNode we request new proof_hashes by sending last proof hash we have + """ + self.log.info(f"Requesting proof hashes") + request = ProofHash(std_hash(b"deadbeef")) + yield OutboundMessage( + NodeType.FULL_NODE, + Message("request_proof_hashes", request), + Delivery.BROADCAST, + ) + + @api_request + async def proof_hash( + self, request: src.protocols.wallet_protocol.ProofHash + ) -> OutboundMessageGenerator: + """ + Received a proof hash from the FullNode + """ + self.log.info(f"Received a new proof hash: {request}") + reply_request = ProofHash(std_hash(b"a")) + # TODO Store and decide if we want full proof for this proof hash + yield OutboundMessage( + NodeType.FULL_NODE, + Message("request_full_proof_for_hash", reply_request), + Delivery.RESPOND, + ) + + @api_request + async def full_proof_for_hash( + self, request: src.protocols.wallet_protocol.FullProofForHash + ): + """ + We've received a full proof for hash we requested + """ + # TODO Validate full proof + self.log.info(f"Received new proof: {request}") + + @api_request + async def received_body(self, response: src.protocols.wallet_protocol.RespondBody): + """ + Called when body is received from the FullNode + """ + + # Retry sending queued up transactions + await self.retry_send_queue() + + additions: List[Coin] = [] + + if self.wallet.can_generate_puzzle_hash(response.body.coinbase.puzzle_hash): + await self.wallet_state_manager.coin_added(response.body.coinbase, response.height, True) + if self.wallet.can_generate_puzzle_hash(response.body.fees_coin.puzzle_hash): + await self.wallet_state_manager.coin_added(response.body.fees_coin, response.height, True) + + npc_list: List[NPC] + if response.body.transactions: + error, npc_list, cost = get_name_puzzle_conditions( + response.body.transactions + ) + + additions.extend(additions_for_npc(npc_list)) + + for added_coin in additions: + if self.wallet.can_generate_puzzle_hash(added_coin.puzzle_hash): + await self.wallet_state_manager.coin_added(added_coin, response.height, False) + + for npc in npc_list: + if self.wallet.can_generate_puzzle_hash(npc.puzzle_hash): + await self.wallet_state_manager.coin_removed(npc.coin_name, response.height) + + @api_request + async def new_lca(self, header: src.protocols.wallet_protocol.Header): + self.log.info("new tip received") + + async def retry_send_queue(self): + records = await self.wallet_state_manager.get_send_queue() + for record in records: + await self._send_transaction(record.spend_bundle) + + async def _send_transaction(self, spend_bundle: SpendBundle): + """ Sends spendbundle to connected full Nodes.""" + await self.wallet_state_manager.add_pending_transaction(spend_bundle) + + msg = OutboundMessage( + NodeType.FULL_NODE, + Message("wallet_transaction", spend_bundle), + Delivery.BROADCAST, + ) + if self.server: + async for reply in self.server.push_message(msg): + self.log.info(reply) + + async def _request_add_list(self, height: uint32, header_hash: bytes32): + obj = src.protocols.wallet_protocol.RequestAdditions(height, header_hash) + msg = OutboundMessage( + NodeType.FULL_NODE, Message("request_additions", obj), Delivery.BROADCAST, + ) + if self.server: + async for reply in self.server.push_message(msg): + self.log.info(reply) + + @api_request + async def response_additions( + self, response: src.protocols.wallet_protocol.Additions + ): + print(response) + + @api_request + async def response_additions_rejected( + self, response: src.protocols.wallet_protocol.RequestAdditions + ): + print(f"request rejected {response}") + + @api_request + async def transaction_ack(self, ack: src.protocols.wallet_protocol.TransactionAck): + if ack.status: + await self.wallet_state_manager.remove_from_queue(ack.txid) + self.log.info(f"SpendBundle has been received by the FullNode. id: {id}") + else: + self.log.info(f"SpendBundle has been rejected by the FullNode. id: {id}") + + async def requestLCA(self): + msg = OutboundMessage( + NodeType.FULL_NODE, Message("request_lca", None), Delivery.BROADCAST, + ) + async for reply in self.server.push_message(msg): + self.log.info(reply) diff --git a/src/wallet/wallet_state_manager.py b/src/wallet/wallet_state_manager.py new file mode 100644 index 000000000000..7ece925d8b58 --- /dev/null +++ b/src/wallet/wallet_state_manager.py @@ -0,0 +1,161 @@ +import time +from typing import Dict, Optional, List, Set +import logging +from src.types.hashable.Coin import Coin +from src.types.hashable.CoinRecord import CoinRecord +from src.types.hashable.SpendBundle import SpendBundle +from src.types.sized_bytes import bytes32 +from src.util.ints import uint32, uint64 +from src.wallet.transaction_record import TransactionRecord +from src.wallet.wallet_store import WalletStore +from src.wallet.wallet_transaction_store import WalletTransactionStore + + +class WalletStateManager: + key_config: Dict + config: Dict + next_address: int = 0 + pubkey_num_lookup: Dict[bytes, int] + tmp_coins: Set[Coin] + wallet_store: WalletStore + tx_store: WalletTransactionStore + header_hash: List[bytes32] + start_index: int + + unconfirmed_additions: Dict[bytes32, Coin] + unconfirmed_removals: Dict[bytes32, Coin] + + log: logging.Logger + + # TODO Don't allow user to send tx until wallet is synced + synced: bool + + @staticmethod + async def create(config: Dict, wallet_store: WalletStore, tx_store: WalletTransactionStore, name: str = None): + self = WalletStateManager() + print("init wallet") + self.config = config + + if name: + self.log = logging.getLogger(name) + else: + self.log = logging.getLogger(__name__) + + self.header_hash = [] + self.wallet_store = wallet_store + self.tx_store = tx_store + self.synced = False + self.unconfirmed_additions = {} + self.unconfirmed_removals = {} + + return self + + async def get_confirmed_balance(self) -> uint64: + record_list: Set[ + CoinRecord + ] = await self.wallet_store.get_coin_records_by_spent(False) + amount: uint64 = uint64(0) + + for record in record_list: + amount = uint64(amount + record.coin.amount) + + return uint64(amount) + + async def get_unconfirmed_balance(self) -> uint64: + confirmed = await self.get_confirmed_balance() + addition_amount = 0 + removal_amount = 0 + for key, addition in self.unconfirmed_additions.items(): + addition_amount += addition.amount + for key, removal in self.unconfirmed_removals.items(): + removal_amount += removal.amount + result = ( + confirmed + - removal_amount + + addition_amount + ) + return uint64(result) + + async def select_coins(self, amount) -> Optional[Set[Coin]]: + + if amount > await self.get_unconfirmed_balance(): + return None + + unspent: Set[CoinRecord] = await self.wallet_store.get_coin_records_by_spent( + False + ) + sum = 0 + used_coins: Set = set() + + """ + Try to use coins from the store, if there isn't enough of "unused" + coins use change coins that are not confirmed yet + """ + for coinrecord in unspent: + if sum >= amount: + break + if coinrecord.coin.name in self.unconfirmed_removals: + continue + sum += coinrecord.coin.amount + used_coins.add(coinrecord.coin) + + """ + This happens when we couldn't use one of the coins because it's already used + but unconfirmed, and we are waiting for the change. (unconfirmed_additions) + """ + if sum < amount: + for coin in self.unconfirmed_additions: + if sum > amount: + break + if coin.name in self.unconfirmed_removals: + continue + sum += coin.amount + used_coins.add(coin) + + if sum >= amount: + return used_coins + else: + # This shouldn't happen because of: if amount > self.get_unconfirmed_balance(): + return None + + async def coin_removed(self, coin_name: bytes32, index: uint32): + """ + Called when coin gets spent + """ + await self.wallet_store.set_spent(coin_name, index) + + async def coin_added(self, coin: Coin, index: uint32, coinbase: bool): + """ + Adding coin to the db + """ + coin_record: CoinRecord = CoinRecord(coin, index, uint32(0), False, coinbase) + await self.wallet_store.add_coin_record(coin_record) + + async def add_pending_transaction(self, spend_bundle: SpendBundle): + """ + Called from wallet_node before new transaction is sent to the full_node + """ + additions = spend_bundle.additions() + removals = spend_bundle.removals() + for add in additions: + self.unconfirmed_additions[add.name()] = add + for removal in removals: + self.unconfirmed_removals[removal.name()] = removal + + # Wallet node will use this queue to retry sending this transaction until full nodes receives it + now = uint64(int(time.time())) + tx_record = TransactionRecord(0, 0, False, False, now, spend_bundle) + await self.tx_store.add_transaction_record(tx_record) + + async def remove_from_queue(self, spendbundle_id: bytes32): + """ + Full node received our transaction, no need to keep it in queue anymore + """ + await self.tx_store.set_sent(spendbundle_id) + + async def get_send_queue(self) -> List[TransactionRecord]: + """ + Wallet Node uses this to retry sending transactions + """ + records = await self.tx_store.get_not_sent() + return records diff --git a/src/wallet/wallet_transaction_store.py b/src/wallet/wallet_transaction_store.py new file mode 100644 index 000000000000..ef52e0323181 --- /dev/null +++ b/src/wallet/wallet_transaction_store.py @@ -0,0 +1,150 @@ +import asyncio +from typing import Dict, Optional, List +from pathlib import Path +import aiosqlite +from src.types.hashable.SpendBundle import SpendBundle +from src.types.sized_bytes import bytes32 +from src.util.ints import uint32 +from src.wallet.transaction_record import TransactionRecord + + +class WalletTransactionStore: + """ + This object handles CoinRecords in DB used by wallet. + """ + + transaction_db: aiosqlite.Connection + # Whether or not we are syncing + sync_mode: bool = False + lock: asyncio.Lock + cache_size: uint32 + tx_record_cache: Dict[bytes32, TransactionRecord] + + @classmethod + async def create(cls, db_path: Path, cache_size: uint32 = uint32(600000)): + self = cls() + + self.cache_size = cache_size + + self.transaction_db = await aiosqlite.connect(db_path) + await self.transaction_db.execute( + ( + f"CREATE TABLE IF NOT EXISTS transaction_record(" + f"bundle_id text PRIMARY KEY," + f" confirmed_index bigint," + f" created_at_index bigint," + f" confirmed int," + f" sent int," + f" created_at_time bigint," + f" spend_bundle blob)" + ) + ) + + # Useful for reorg lookups + await self.transaction_db.execute( + "CREATE INDEX IF NOT EXISTS tx_confirmed_index on transaction_record(confirmed_index)" + ) + + await self.transaction_db.execute( + "CREATE INDEX IF NOT EXISTS tx_created_index on transaction_record(created_at_index)" + ) + + await self.transaction_db.execute( + "CREATE INDEX IF NOT EXISTS tx_confirmed on transaction_record(confirmed)" + ) + + await self.transaction_db.execute( + "CREATE INDEX IF NOT EXISTS tx_sent on transaction_record(sent)" + ) + + await self.transaction_db.execute( + "CREATE INDEX IF NOT EXISTS tx_created_time on transaction_record(created_at_time)" + ) + + await self.transaction_db.commit() + # Lock + self.lock = asyncio.Lock() # external + self.tx_record_cache = dict() + return self + + async def close(self): + await self.transaction_db.close() + + async def _init_cache(self): + print("init cache here") + + async def _clear_database(self): + cursor = await self.transaction_db.execute("DELETE FROM transaction_record") + await cursor.close() + await self.transaction_db.commit() + + # Store TransactionRecord in DB and Cache + async def add_transaction_record(self, record: TransactionRecord) -> None: + cursor = await self.transaction_db.execute( + "INSERT OR REPLACE INTO transaction_record VALUES(?, ?, ?, ?, ?, ?, ?)", + ( + record.name().hex(), + record.confirmed_block_index, + record.created_at_index, + int(record.confirmed), + int(record.sent), + record.created_at_time, + bytes(record.spend_bundle), + ), + ) + await cursor.close() + await self.transaction_db.commit() + self.tx_record_cache[record.name().hex()] = record + if len(self.tx_record_cache) > self.cache_size: + while len(self.tx_record_cache) > self.cache_size: + first_in = list(self.tx_record_cache.keys())[0] + self.tx_record_cache.pop(first_in) + + # Update transaction_record to be confirmed in DB + async def set_confirmed(self, id: bytes32, index: uint32): + current: Optional[TransactionRecord] = await self.get_transaction_record(id) + if current is None: + return + tx: TransactionRecord = TransactionRecord( + index, current.created_at_index, True, current.sent, current.created_at_time, current.spend_bundle, + ) # type: ignore # noqa + await self.add_transaction_record(tx) + + # Update transaction_record to be sent in DB + async def set_sent(self, id: bytes32): + current: Optional[TransactionRecord] = await self.get_transaction_record(id) + if current is None: + return + tx: TransactionRecord = TransactionRecord( + index, current.created_at_index, current.confirmed, True, current.created_at_time, current.spend_bundle, + ) # type: ignore # noqa + await self.add_transaction_record(tx) + + # Checks DB and cache for TransactionRecord with id: id and returns it + async def get_transaction_record(self, id: bytes32) -> Optional[TransactionRecord]: + if id.hex() in self.tx_record_cache: + return self.tx_record_cache[id.hex()] + cursor = await self.transaction_db.execute( + "SELECT * from transaction_record WHERE bundle_id=?", (id.hex(),) + ) + row = await cursor.fetchone() + await cursor.close() + if row is not None: + spend_bundle = SpendBundle.from_bytes(row[6]) + record = TransactionRecord(row[1], row[2], row[3], row[4], row[4], spend_bundle) + return record + return None + + async def get_not_sent(self) -> List[TransactionRecord]: + cursor = await self.transaction_db.execute( + "SELECT * from transaction_record WHERE sent=?", (0,) + ) + rows = await cursor.fetchall() + await cursor.close() + records = [] + for row in rows: + spend_bundle = SpendBundle.from_bytes(row[6]) + record = TransactionRecord(row[1], row[2], row[3], row[4], row[4], spend_bundle) + records.append(record) + + return records From b2baaa0fb9286f7e44b77d6f7b2d87c0f737d836 Mon Sep 17 00:00:00 2001 From: Yostra Date: Mon, 24 Feb 2020 10:03:16 -0800 Subject: [PATCH 22/38] more wallet stuff --- src/wallet/transaction_record.py | 10 ++++- src/wallet/wallet_node.py | 3 +- src/wallet/wallet_state_manager.py | 56 ++++++++++++++++---------- src/wallet/wallet_transaction_store.py | 33 ++++++++++----- 4 files changed, 67 insertions(+), 35 deletions(-) diff --git a/src/wallet/transaction_record.py b/src/wallet/transaction_record.py index 3b254cb15511..c2a824919a07 100644 --- a/src/wallet/transaction_record.py +++ b/src/wallet/transaction_record.py @@ -1,5 +1,7 @@ from dataclasses import dataclass +from typing import Dict, Optional +from src.types.hashable.Coin import Coin from src.types.hashable.SpendBundle import SpendBundle from src.types.sized_bytes import bytes32 from src.util.streamable import Streamable, streamable @@ -18,8 +20,12 @@ class TransactionRecord(Streamable): confirmed: bool sent: bool created_at_time: uint64 - spend_bundle: SpendBundle + spend_bundle: Optional[SpendBundle] + additions: Dict[bytes32, Coin] + removals: Dict[bytes32, Coin] @property def name(self) -> bytes32: - return self.spend_bundle.name() + if self.spend_bundle: + return self.spend_bundle.name() + return self.get_hash() diff --git a/src/wallet/wallet_node.py b/src/wallet/wallet_node.py index c7c2a782472e..98c5a2a0a3dc 100644 --- a/src/wallet/wallet_node.py +++ b/src/wallet/wallet_node.py @@ -140,7 +140,8 @@ class WalletNode: async def retry_send_queue(self): records = await self.wallet_state_manager.get_send_queue() for record in records: - await self._send_transaction(record.spend_bundle) + if record.spend_bundle: + await self._send_transaction(record.spend_bundle) async def _send_transaction(self, spend_bundle: SpendBundle): """ Sends spendbundle to connected full Nodes.""" diff --git a/src/wallet/wallet_state_manager.py b/src/wallet/wallet_state_manager.py index 7ece925d8b58..62e60efdf19f 100644 --- a/src/wallet/wallet_state_manager.py +++ b/src/wallet/wallet_state_manager.py @@ -14,17 +14,11 @@ from src.wallet.wallet_transaction_store import WalletTransactionStore class WalletStateManager: key_config: Dict config: Dict - next_address: int = 0 - pubkey_num_lookup: Dict[bytes, int] - tmp_coins: Set[Coin] wallet_store: WalletStore tx_store: WalletTransactionStore header_hash: List[bytes32] start_index: int - unconfirmed_additions: Dict[bytes32, Coin] - unconfirmed_removals: Dict[bytes32, Coin] - log: logging.Logger # TODO Don't allow user to send tx until wallet is synced @@ -45,8 +39,6 @@ class WalletStateManager: self.wallet_store = wallet_store self.tx_store = tx_store self.synced = False - self.unconfirmed_additions = {} - self.unconfirmed_removals = {} return self @@ -63,12 +55,16 @@ class WalletStateManager: async def get_unconfirmed_balance(self) -> uint64: confirmed = await self.get_confirmed_balance() + unconfirmed_tx = await self.tx_store.get_not_confirmed() addition_amount = 0 removal_amount = 0 - for key, addition in self.unconfirmed_additions.items(): - addition_amount += addition.amount - for key, removal in self.unconfirmed_removals.items(): - removal_amount += removal.amount + + for record in unconfirmed_tx: + for name, coin in record.additions.items(): + addition_amount += coin.additions + for name, coin in record.removals.items(): + removal_amount += coin.amount + result = ( confirmed - removal_amount @@ -76,6 +72,22 @@ class WalletStateManager: ) return uint64(result) + async def unconfirmed_additions(self) -> Dict[bytes32, Coin]: + additions: Dict[Coin] = {} + unconfirmed_tx = await self.tx_store.get_not_confirmed() + for record in unconfirmed_tx: + for name, coin in record.additions.items(): + additions[name] = coin + return additions + + async def unconfirmed_removals(self) -> Dict[bytes32, Coin]: + removals: Dict[Coin] = {} + unconfirmed_tx = await self.tx_store.get_not_confirmed() + for record in unconfirmed_tx: + for name, coin in record.removals.items(): + removals[name] = coin + return removals + async def select_coins(self, amount) -> Optional[Set[Coin]]: if amount > await self.get_unconfirmed_balance(): @@ -94,7 +106,7 @@ class WalletStateManager: for coinrecord in unspent: if sum >= amount: break - if coinrecord.coin.name in self.unconfirmed_removals: + if coinrecord.coin.name in await self.unconfirmed_removals(): continue sum += coinrecord.coin.amount used_coins.add(coinrecord.coin) @@ -104,7 +116,7 @@ class WalletStateManager: but unconfirmed, and we are waiting for the change. (unconfirmed_additions) """ if sum < amount: - for coin in self.unconfirmed_additions: + for coin in (await self.unconfirmed_additions()).values(): if sum > amount: break if coin.name in self.unconfirmed_removals: @@ -135,16 +147,16 @@ class WalletStateManager: """ Called from wallet_node before new transaction is sent to the full_node """ - additions = spend_bundle.additions() - removals = spend_bundle.removals() - for add in additions: - self.unconfirmed_additions[add.name()] = add - for removal in removals: - self.unconfirmed_removals[removal.name()] = removal + now = uint64(int(time.time())) + add_dict: Dict[bytes32, Coin] = {} + rem_dict: Dict[bytes32, Coin] = {} + for add in spend_bundle.additions(): + add_dict[add.name()] = add + for rem in spend_bundle.removals(): + rem_dict[rem.name()] = rem # Wallet node will use this queue to retry sending this transaction until full nodes receives it - now = uint64(int(time.time())) - tx_record = TransactionRecord(0, 0, False, False, now, spend_bundle) + tx_record = TransactionRecord(0, 0, False, False, now, spend_bundle, add_dict, rem_dict) await self.tx_store.add_transaction_record(tx_record) async def remove_from_queue(self, spendbundle_id: bytes32): diff --git a/src/wallet/wallet_transaction_store.py b/src/wallet/wallet_transaction_store.py index ef52e0323181..e6de91032eb5 100644 --- a/src/wallet/wallet_transaction_store.py +++ b/src/wallet/wallet_transaction_store.py @@ -36,7 +36,7 @@ class WalletTransactionStore: f" confirmed int," f" sent int," f" created_at_time bigint," - f" spend_bundle blob)" + f" transaction_record blob)" ) ) @@ -89,7 +89,7 @@ class WalletTransactionStore: int(record.confirmed), int(record.sent), record.created_at_time, - bytes(record.spend_bundle), + bytes(record), ), ) await cursor.close() @@ -106,8 +106,9 @@ class WalletTransactionStore: if current is None: return tx: TransactionRecord = TransactionRecord( - index, current.created_at_index, True, current.sent, current.created_at_time, current.spend_bundle, - ) # type: ignore # noqa + index, current.created_at_index, True, current.sent, current.created_at_time, current.spend_bundle, current.additions, + current.removals + ) await self.add_transaction_record(tx) # Update transaction_record to be sent in DB @@ -116,8 +117,9 @@ class WalletTransactionStore: if current is None: return tx: TransactionRecord = TransactionRecord( - index, current.created_at_index, current.confirmed, True, current.created_at_time, current.spend_bundle, - ) # type: ignore # noqa + current.confirmed_block_index, current.created_at_index, current.confirmed, True, current.created_at_time, current.spend_bundle, + current.additions, current.removals + ) await self.add_transaction_record(tx) # Checks DB and cache for TransactionRecord with id: id and returns it @@ -130,8 +132,7 @@ class WalletTransactionStore: row = await cursor.fetchone() await cursor.close() if row is not None: - spend_bundle = SpendBundle.from_bytes(row[6]) - record = TransactionRecord(row[1], row[2], row[3], row[4], row[4], spend_bundle) + record = TransactionRecord.from_bytes(row[6]) return record return None @@ -143,8 +144,20 @@ class WalletTransactionStore: await cursor.close() records = [] for row in rows: - spend_bundle = SpendBundle.from_bytes(row[6]) - record = TransactionRecord(row[1], row[2], row[3], row[4], row[4], spend_bundle) + record = TransactionRecord.from_bytes(row[6]) + records.append(record) + + return records + + async def get_not_confirmed(self) -> List[TransactionRecord]: + cursor = await self.transaction_db.execute( + "SELECT * from transaction_record WHERE confirmed=?", (0,) + ) + rows = await cursor.fetchall() + await cursor.close() + records = [] + for row in rows: + record = TransactionRecord.from_bytes(row[6]) records.append(record) return records From 7ac368c78454f09dd5a3e1684424a9cda6a85cc8 Mon Sep 17 00:00:00 2001 From: Yostra Date: Mon, 24 Feb 2020 10:33:24 -0800 Subject: [PATCH 23/38] clean --- src/wallet/electron/README.md | 17 ++++++++++------- src/wallet/electron/wallet_rpc/.gitignore | 2 -- src/wallet/electron/wallet_rpc/requirements.txt | 7 ------- 3 files changed, 10 insertions(+), 16 deletions(-) delete mode 100644 src/wallet/electron/wallet_rpc/.gitignore delete mode 100644 src/wallet/electron/wallet_rpc/requirements.txt diff --git a/src/wallet/electron/README.md b/src/wallet/electron/README.md index c35242204d0e..755bb292731e 100644 --- a/src/wallet/electron/README.md +++ b/src/wallet/electron/README.md @@ -1,9 +1,12 @@ -# Install -python3 -m venv .venv -. .venv/bin/activate +# Electron Wallet +## Install -pip install zerorpc -pip install pyinstaller +```npm install --runtime=electron --target=1.7.6``` -npm install --runtime=electron --target=1.7.6 -npm install electron-rebuild && ./node_modules/.bin/electron-rebuild +## Run: +```npm start``` + +## Error +If run fails because of electron try doing this + +```npm install electron-rebuild && ./node_modules/.bin/electron-rebuild``` diff --git a/src/wallet/electron/wallet_rpc/.gitignore b/src/wallet/electron/wallet_rpc/.gitignore deleted file mode 100644 index d646835b4988..000000000000 --- a/src/wallet/electron/wallet_rpc/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.pyc -__pycache__/ diff --git a/src/wallet/electron/wallet_rpc/requirements.txt b/src/wallet/electron/wallet_rpc/requirements.txt deleted file mode 100644 index 37565d6775ba..000000000000 --- a/src/wallet/electron/wallet_rpc/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -zerorpc -pyzmq -future -msgpack-python -gevent -pyinstaller -pypiwin32 From ff9a083d2acc51ece0a0fe92173b9d0700cc949e Mon Sep 17 00:00:00 2001 From: Yostra Date: Mon, 24 Feb 2020 10:41:54 -0800 Subject: [PATCH 24/38] cleanup --- src/wallet/electron/wallet_rpc/rpc_wallet.py | 4 +++- src/wallet/wallet.py | 13 +++++++++- src/wallet/wallet_node.py | 21 ++++++++++++---- src/wallet/wallet_state_manager.py | 25 +++++++++++--------- src/wallet/wallet_transaction_store.py | 21 ++++++++++++---- 5 files changed, 61 insertions(+), 23 deletions(-) diff --git a/src/wallet/electron/wallet_rpc/rpc_wallet.py b/src/wallet/electron/wallet_rpc/rpc_wallet.py index 57f5733bfbdf..3e3c0a8fed69 100644 --- a/src/wallet/electron/wallet_rpc/rpc_wallet.py +++ b/src/wallet/electron/wallet_rpc/rpc_wallet.py @@ -60,7 +60,9 @@ class RpcWalletApiHandler: if "amount" in request_data and "puzzlehash" in request_data: amount = int(request_data["amount"]) puzzlehash = request_data["puzzlehash"] - tx = await self.wallet_node.wallet.generate_signed_transaction(amount, puzzlehash) + tx = await self.wallet_node.wallet.generate_signed_transaction( + amount, puzzlehash + ) if tx is None: response = {"success": False} diff --git a/src/wallet/wallet.py b/src/wallet/wallet.py index 9798d7579d15..6a6de9087fcd 100644 --- a/src/wallet/wallet.py +++ b/src/wallet/wallet.py @@ -3,6 +3,7 @@ import clvm from blspy import ExtendedPrivateKey, PublicKey import logging +from src.server.outbound_message import OutboundMessage, NodeType, Message, Delivery from src.server.server import ChiaServer from src.types.hashable.BLSSignature import BLSSignature from src.types.hashable.CoinSolution import CoinSolution @@ -48,7 +49,12 @@ class Wallet: send_queue: Dict[bytes32, SpendBundle] @staticmethod - async def create(config: Dict, key_config: Dict, wallet_state_manager: WalletStateManager, name: str = None): + async def create( + config: Dict, + key_config: Dict, + wallet_state_manager: WalletStateManager, + name: str = None, + ): self = Wallet() print("init wallet") self.config = config @@ -209,5 +215,10 @@ class Wallet: async def _send_transaction(self, spend_bundle: SpendBundle): if self.server: + msg = OutboundMessage( + NodeType.FULL_NODE, + Message("wallet_transaction", spend_bundle), + Delivery.BROADCAST, + ) async for reply in self.server.push_message(msg): self.log.info(reply) diff --git a/src/wallet/wallet_node.py b/src/wallet/wallet_node.py index 98c5a2a0a3dc..842f37028f1d 100644 --- a/src/wallet/wallet_node.py +++ b/src/wallet/wallet_node.py @@ -33,6 +33,7 @@ class WalletNode: start_index: int log: logging.Logger wallet: Wallet + tx_store: WalletTransactionStore @staticmethod async def create(config: Dict, key_config: Dict, name: str = None): @@ -52,7 +53,9 @@ class WalletNode: self.wallet_store = await WalletStore.create(path) self.tx_store = await WalletTransactionStore.create(path) - self.wallet_state_manager = await WalletStateManager.create(config, self.wallet_store, self.tx_store) + self.wallet_state_manager = await WalletStateManager.create( + config, self.wallet_store, self.tx_store + ) self.wallet = await Wallet.create(config, key_config, self.wallet_state_manager) self.server = None @@ -113,9 +116,13 @@ class WalletNode: additions: List[Coin] = [] if self.wallet.can_generate_puzzle_hash(response.body.coinbase.puzzle_hash): - await self.wallet_state_manager.coin_added(response.body.coinbase, response.height, True) + await self.wallet_state_manager.coin_added( + response.body.coinbase, response.height, True + ) if self.wallet.can_generate_puzzle_hash(response.body.fees_coin.puzzle_hash): - await self.wallet_state_manager.coin_added(response.body.fees_coin, response.height, True) + await self.wallet_state_manager.coin_added( + response.body.fees_coin, response.height, True + ) npc_list: List[NPC] if response.body.transactions: @@ -127,11 +134,15 @@ class WalletNode: for added_coin in additions: if self.wallet.can_generate_puzzle_hash(added_coin.puzzle_hash): - await self.wallet_state_manager.coin_added(added_coin, response.height, False) + await self.wallet_state_manager.coin_added( + added_coin, response.height, False + ) for npc in npc_list: if self.wallet.can_generate_puzzle_hash(npc.puzzle_hash): - await self.wallet_state_manager.coin_removed(npc.coin_name, response.height) + await self.wallet_state_manager.coin_removed( + npc.coin_name, response.height + ) @api_request async def new_lca(self, header: src.protocols.wallet_protocol.Header): diff --git a/src/wallet/wallet_state_manager.py b/src/wallet/wallet_state_manager.py index 62e60efdf19f..02d352258a30 100644 --- a/src/wallet/wallet_state_manager.py +++ b/src/wallet/wallet_state_manager.py @@ -25,7 +25,12 @@ class WalletStateManager: synced: bool @staticmethod - async def create(config: Dict, wallet_store: WalletStore, tx_store: WalletTransactionStore, name: str = None): + async def create( + config: Dict, + wallet_store: WalletStore, + tx_store: WalletTransactionStore, + name: str = None, + ): self = WalletStateManager() print("init wallet") self.config = config @@ -61,19 +66,15 @@ class WalletStateManager: for record in unconfirmed_tx: for name, coin in record.additions.items(): - addition_amount += coin.additions + addition_amount += coin.amount for name, coin in record.removals.items(): removal_amount += coin.amount - result = ( - confirmed - - removal_amount - + addition_amount - ) + result = confirmed - removal_amount + addition_amount return uint64(result) async def unconfirmed_additions(self) -> Dict[bytes32, Coin]: - additions: Dict[Coin] = {} + additions: Dict[bytes32, Coin] = {} unconfirmed_tx = await self.tx_store.get_not_confirmed() for record in unconfirmed_tx: for name, coin in record.additions.items(): @@ -81,7 +82,7 @@ class WalletStateManager: return additions async def unconfirmed_removals(self) -> Dict[bytes32, Coin]: - removals: Dict[Coin] = {} + removals: Dict[bytes32, Coin] = {} unconfirmed_tx = await self.tx_store.get_not_confirmed() for record in unconfirmed_tx: for name, coin in record.removals.items(): @@ -119,7 +120,7 @@ class WalletStateManager: for coin in (await self.unconfirmed_additions()).values(): if sum > amount: break - if coin.name in self.unconfirmed_removals: + if coin.name in (await self.unconfirmed_removals()).values(): continue sum += coin.amount used_coins.add(coin) @@ -156,7 +157,9 @@ class WalletStateManager: rem_dict[rem.name()] = rem # Wallet node will use this queue to retry sending this transaction until full nodes receives it - tx_record = TransactionRecord(0, 0, False, False, now, spend_bundle, add_dict, rem_dict) + tx_record = TransactionRecord( + uint32(0), uint32(0), False, False, now, spend_bundle, add_dict, rem_dict + ) await self.tx_store.add_transaction_record(tx_record) async def remove_from_queue(self, spendbundle_id: bytes32): diff --git a/src/wallet/wallet_transaction_store.py b/src/wallet/wallet_transaction_store.py index e6de91032eb5..6a3c66ba92d0 100644 --- a/src/wallet/wallet_transaction_store.py +++ b/src/wallet/wallet_transaction_store.py @@ -2,7 +2,6 @@ import asyncio from typing import Dict, Optional, List from pathlib import Path import aiosqlite -from src.types.hashable.SpendBundle import SpendBundle from src.types.sized_bytes import bytes32 from src.util.ints import uint32 from src.wallet.transaction_record import TransactionRecord @@ -106,8 +105,14 @@ class WalletTransactionStore: if current is None: return tx: TransactionRecord = TransactionRecord( - index, current.created_at_index, True, current.sent, current.created_at_time, current.spend_bundle, current.additions, - current.removals + index, + current.created_at_index, + True, + current.sent, + current.created_at_time, + current.spend_bundle, + current.additions, + current.removals, ) await self.add_transaction_record(tx) @@ -117,8 +122,14 @@ class WalletTransactionStore: if current is None: return tx: TransactionRecord = TransactionRecord( - current.confirmed_block_index, current.created_at_index, current.confirmed, True, current.created_at_time, current.spend_bundle, - current.additions, current.removals + current.confirmed_block_index, + current.created_at_index, + current.confirmed, + True, + current.created_at_time, + current.spend_bundle, + current.additions, + current.removals, ) await self.add_transaction_record(tx) From c8d32a5958ed47d480ed782585841e76b49db42f Mon Sep 17 00:00:00 2001 From: Yostra Date: Mon, 24 Feb 2020 10:54:45 -0800 Subject: [PATCH 25/38] fix filter test --- src/wallet/wallet.py | 2 -- tests/test_filter.py | 10 ++++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/wallet/wallet.py b/src/wallet/wallet.py index 6a6de9087fcd..9d4e2309b852 100644 --- a/src/wallet/wallet.py +++ b/src/wallet/wallet.py @@ -27,7 +27,6 @@ from src.wallet.puzzles.puzzle_utils import ( ) from src.wallet.wallet_state_manager import WalletStateManager -from src.wallet.wallet_store import WalletStore class Wallet: @@ -37,7 +36,6 @@ class Wallet: server: Optional[ChiaServer] next_address: int = 0 pubkey_num_lookup: Dict[bytes, int] - wallet_store: WalletStore wallet_state_manager: WalletStateManager log: logging.Logger diff --git a/tests/test_filter.py b/tests/test_filter.py index bcfe0c99e74d..3ed4738880a1 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -6,6 +6,7 @@ from blspy import ExtendedPrivateKey from chiabip158 import PyBIP158 from src.wallet.wallet import Wallet +from src.wallet.wallet_node import WalletNode from tests.setup_nodes import setup_two_nodes, test_constants, bt @@ -25,9 +26,9 @@ class TestFilter: async def test_basic_filter_test(self, two_nodes): sk = bytes(ExtendedPrivateKey.from_seed(b"")).hex() key_config = {"wallet_sk": sk} - - wallet = await Wallet.create({}, key_config) - await wallet.wallet_store._clear_database() + wallet_node = await WalletNode.create({}, key_config) + wallet = wallet_node.wallet + await wallet_node.wallet_store._clear_database() num_blocks = 2 blocks = bt.get_consecutive_blocks( @@ -53,4 +54,5 @@ class TestFilter: assert present assert fee_present - await wallet.wallet_store.close() + await wallet_node.wallet_store.close() + await wallet_node.tx_store.close() From d9b268541166d2e1f9d9430c27eabd08d4023168 Mon Sep 17 00:00:00 2001 From: Yostra Date: Mon, 24 Feb 2020 12:41:17 -0800 Subject: [PATCH 26/38] correct balance lookup, streamable fix --- src/util/streamable.py | 2 +- src/wallet/transaction_record.py | 7 ++-- src/wallet/wallet.py | 4 ++ src/wallet/wallet_node.py | 2 +- src/wallet/wallet_state_manager.py | 30 +++++++++------ tests/test_wallet.py | 34 +++++++++++------ tests/test_wallet_protocol.py | 59 ------------------------------ 7 files changed, 50 insertions(+), 88 deletions(-) delete mode 100644 tests/test_wallet_protocol.py diff --git a/src/util/streamable.py b/src/util/streamable.py index 97b33d4dfaac..b4c4a7e814ae 100644 --- a/src/util/streamable.py +++ b/src/util/streamable.py @@ -181,7 +181,7 @@ class Streamable: f.write(uint32(len(item)).to_bytes(4, "big")) f.write(item.encode("utf-8")) elif f_type is bool: - f.write(bytes(item)) + f.write(int(item).to_bytes(4, "big")) else: raise NotImplementedError(f"can't stream {item}, {f_type}") diff --git a/src/wallet/transaction_record.py b/src/wallet/transaction_record.py index c2a824919a07..92cebf0f2187 100644 --- a/src/wallet/transaction_record.py +++ b/src/wallet/transaction_record.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Dict, Optional +from typing import Optional, List from src.types.hashable.Coin import Coin from src.types.hashable.SpendBundle import SpendBundle @@ -21,10 +21,9 @@ class TransactionRecord(Streamable): sent: bool created_at_time: uint64 spend_bundle: Optional[SpendBundle] - additions: Dict[bytes32, Coin] - removals: Dict[bytes32, Coin] + additions: List[Coin] + removals: List[Coin] - @property def name(self) -> bytes32: if self.spend_bundle: return self.spend_bundle.name() diff --git a/src/wallet/wallet.py b/src/wallet/wallet.py index 9d4e2309b852..eedca6f71f82 100644 --- a/src/wallet/wallet.py +++ b/src/wallet/wallet.py @@ -67,12 +67,15 @@ class Wallet: self.wallet_state_manager = wallet_state_manager self.pubkey_num_lookup = {} + self.server = None + return self def get_next_public_key(self) -> PublicKey: pubkey = self.private_key.public_child(self.next_address).get_public_key() self.pubkey_num_lookup[pubkey.serialize()] = self.next_address self.next_address = self.next_address + 1 + self.wallet_state_manager.next_address = self.next_address return pubkey async def get_confirmed_balance(self) -> uint64: @@ -103,6 +106,7 @@ class Wallet: def get_new_puzzlehash(self) -> bytes32: puzzle: Program = self.get_new_puzzle() puzzlehash: bytes32 = puzzle.get_hash() + self.wallet_state_manager.puzzlehash_set.add(puzzlehash) return puzzlehash def set_server(self, server: ChiaServer): diff --git a/src/wallet/wallet_node.py b/src/wallet/wallet_node.py index 842f37028f1d..d17785293ae0 100644 --- a/src/wallet/wallet_node.py +++ b/src/wallet/wallet_node.py @@ -54,7 +54,7 @@ class WalletNode: self.tx_store = await WalletTransactionStore.create(path) self.wallet_state_manager = await WalletStateManager.create( - config, self.wallet_store, self.tx_store + config, key_config, self.wallet_store, self.tx_store ) self.wallet = await Wallet.create(config, key_config, self.wallet_state_manager) diff --git a/src/wallet/wallet_state_manager.py b/src/wallet/wallet_state_manager.py index 02d352258a30..fc2ce876e8fc 100644 --- a/src/wallet/wallet_state_manager.py +++ b/src/wallet/wallet_state_manager.py @@ -18,15 +18,18 @@ class WalletStateManager: tx_store: WalletTransactionStore header_hash: List[bytes32] start_index: int + next_address: int log: logging.Logger # TODO Don't allow user to send tx until wallet is synced synced: bool + puzzlehash_set: set @staticmethod async def create( config: Dict, + key_config: Dict, wallet_store: WalletStore, tx_store: WalletTransactionStore, name: str = None, @@ -44,7 +47,9 @@ class WalletStateManager: self.wallet_store = wallet_store self.tx_store = tx_store self.synced = False + self.next_address = 0 + self.puzzlehash_set = set() return self async def get_confirmed_balance(self) -> uint64: @@ -65,9 +70,10 @@ class WalletStateManager: removal_amount = 0 for record in unconfirmed_tx: - for name, coin in record.additions.items(): - addition_amount += coin.amount - for name, coin in record.removals.items(): + for coin in record.additions: + if coin.puzzle_hash in self.puzzlehash_set: + addition_amount += coin.amount + for coin in record.removals: removal_amount += coin.amount result = confirmed - removal_amount + addition_amount @@ -77,16 +83,16 @@ class WalletStateManager: additions: Dict[bytes32, Coin] = {} unconfirmed_tx = await self.tx_store.get_not_confirmed() for record in unconfirmed_tx: - for name, coin in record.additions.items(): - additions[name] = coin + for coin in record.additions: + additions[coin.name()] = coin return additions async def unconfirmed_removals(self) -> Dict[bytes32, Coin]: removals: Dict[bytes32, Coin] = {} unconfirmed_tx = await self.tx_store.get_not_confirmed() for record in unconfirmed_tx: - for name, coin in record.removals.items(): - removals[name] = coin + for coin in record.removals: + removals[coin.name()] = coin return removals async def select_coins(self, amount) -> Optional[Set[Coin]]: @@ -149,16 +155,16 @@ class WalletStateManager: Called from wallet_node before new transaction is sent to the full_node """ now = uint64(int(time.time())) - add_dict: Dict[bytes32, Coin] = {} - rem_dict: Dict[bytes32, Coin] = {} + add_list: List[Coin] = [] + rem_list: List[Coin] = [] for add in spend_bundle.additions(): - add_dict[add.name()] = add + add_list.append(add) for rem in spend_bundle.removals(): - rem_dict[rem.name()] = rem + rem_list.append(rem) # Wallet node will use this queue to retry sending this transaction until full nodes receives it tx_record = TransactionRecord( - uint32(0), uint32(0), False, False, now, spend_bundle, add_dict, rem_dict + uint32(0), uint32(0), False, False, now, spend_bundle, add_list, rem_list ) await self.tx_store.add_transaction_record(tx_record) diff --git a/tests/test_wallet.py b/tests/test_wallet.py index 78d14d205e80..7b264c2a53c1 100644 --- a/tests/test_wallet.py +++ b/tests/test_wallet.py @@ -5,6 +5,7 @@ from blspy import ExtendedPrivateKey from src.protocols.wallet_protocol import RespondBody from src.wallet.wallet import Wallet +from src.wallet.wallet_node import WalletNode from tests.setup_nodes import setup_two_nodes, test_constants, bt @@ -25,8 +26,10 @@ class TestWallet: sk = bytes(ExtendedPrivateKey.from_seed(b"")).hex() key_config = {"wallet_sk": sk} - wallet = await Wallet.create({}, key_config) - await wallet.wallet_store._clear_database() + wallet_node = await WalletNode.create({}, key_config) + wallet = wallet_node.wallet + await wallet_node.wallet_store._clear_database() + await wallet_node.tx_store._clear_database() num_blocks = 10 blocks = bt.get_consecutive_blocks( @@ -39,11 +42,12 @@ class TestWallet: for i in range(1, num_blocks): a = RespondBody(blocks[i].body, blocks[i].height) - await wallet.received_body(a) + await wallet_node.received_body(a) assert await wallet.get_confirmed_balance() == 144000000000000 - await wallet.wallet_store.close() + await wallet_node.wallet_store.close() + await wallet_node.tx_store.close() @pytest.mark.asyncio async def test_wallet_make_transaction(self, two_nodes): @@ -52,10 +56,15 @@ class TestWallet: key_config = {"wallet_sk": sk} key_config_b = {"wallet_sk": sk_b} - wallet = await Wallet.create({}, key_config) - await wallet.wallet_store._clear_database() - wallet_b = await Wallet.create({}, key_config_b) - await wallet_b.wallet_store._clear_database() + wallet_node = await WalletNode.create({}, key_config) + wallet = wallet_node.wallet + await wallet_node.wallet_store._clear_database() + await wallet_node.tx_store._clear_database() + + wallet_node_b = await WalletNode.create({}, key_config_b) + wallet_b = wallet_node_b.wallet + await wallet_node_b.wallet_store._clear_database() + await wallet_node_b.tx_store._clear_database() num_blocks = 10 blocks = bt.get_consecutive_blocks( @@ -68,7 +77,7 @@ class TestWallet: for i in range(1, num_blocks): a = RespondBody(blocks[i].body, blocks[i].height) - await wallet.received_body(a) + await wallet_node.received_body(a) assert await wallet.get_confirmed_balance() == 144000000000000 @@ -83,5 +92,8 @@ class TestWallet: assert confirmed_balance == 144000000000000 assert unconfirmed_balance == confirmed_balance - 10 - await wallet.wallet_store.close() - await wallet_b.wallet_store.close() + await wallet_node.wallet_store.close() + await wallet_node.tx_store.close() + + await wallet_node_b.wallet_store.close() + await wallet_node_b.tx_store.close() diff --git a/tests/test_wallet_protocol.py b/tests/test_wallet_protocol.py deleted file mode 100644 index 5e88afe892e8..000000000000 --- a/tests/test_wallet_protocol.py +++ /dev/null @@ -1,59 +0,0 @@ -import asyncio -import signal -import time - -import pytest -from blspy import ExtendedPrivateKey - -from src.protocols import full_node_protocol -from src.server.outbound_message import NodeType -from src.server.server import ChiaServer -from src.types.peer_info import PeerInfo -from src.wallet.wallet import Wallet -from tests.setup_nodes import setup_two_nodes, test_constants, bt - - -@pytest.fixture(scope="module") -def event_loop(): - loop = asyncio.get_event_loop() - yield loop - - -class TestWalletProtocol: - @pytest.fixture(scope="function") - async def two_nodes(self): - async for _ in setup_two_nodes({"COINBASE_FREEZE_PERIOD": 0}): - yield _ - - @pytest.mark.asyncio - async def test_wallet_connect(self, two_nodes): - num_blocks = 10 - blocks = bt.get_consecutive_blocks(test_constants, num_blocks, [], 10) - full_node_1, full_node_2, server_1, server_2 = two_nodes - - for i in range(1, num_blocks): - async for _ in full_node_1.respond_block( - full_node_protocol.RespondBlock(blocks[i]) - ): - pass - - sk = bytes(ExtendedPrivateKey.from_seed(b"")).hex() - key_config = {"wallet_sk": sk} - - wallet = await Wallet.create({}, key_config) - server = ChiaServer(8223, wallet, NodeType.WALLET) - - asyncio.get_running_loop().add_signal_handler(signal.SIGINT, server.close_all) - asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, server.close_all) - - _ = await server.start_server("127.0.0.1", wallet._on_connect) - await asyncio.sleep(2) - full_node_peer = PeerInfo(server_1._host, server_1._port) - _ = await server.start_client(full_node_peer, None) - - start_unf = time.time() - while time.time() - start_unf < 3: - # TODO check if we've synced proof hashes and verified number of proofs - await asyncio.sleep(0.1) - - await wallet.wallet_store.close() From 88311f851d855f344d15fdc27b6a02a7eda5ff5f Mon Sep 17 00:00:00 2001 From: Yostra Date: Mon, 24 Feb 2020 12:52:53 -0800 Subject: [PATCH 27/38] boost --- .github/workflows/pythonpackage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index edb53fcd235a..70dc3208636c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -21,6 +21,7 @@ jobs: - name: Install dependencies run: | brew update && brew install gmp boost || echo "" + sudo apt-get install libboost1.70 libboost1.70-dev | echo "" sh install.sh - name: Test proof of space run: | From 22736005949aa27e7db25f76d09ffdc9a52fbaec Mon Sep 17 00:00:00 2001 From: Yostra Date: Mon, 24 Feb 2020 12:56:46 -0800 Subject: [PATCH 28/38] Update INSTALL.md --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 616f90d2e453..cde1a2581bf0 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -5,7 +5,7 @@ To install the chia-blockchain node, follow the instructions according to your o Make sure [brew](https://brew.sh/) is available before starting the setup. ```bash brew upgrade python -brew install cmake gmp +brew install cmake gmp boost openssl git clone https://github.com/Chia-Network/chia-blockchain.git cd chia-blockchain From c45f08eb70cc9244430b81f67ea829e8e0b5ada5 Mon Sep 17 00:00:00 2001 From: Yostra Date: Mon, 24 Feb 2020 12:59:01 -0800 Subject: [PATCH 29/38] openssl --- .github/workflows/pythonpackage.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 70dc3208636c..e39c84db1000 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -20,8 +20,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - brew update && brew install gmp boost || echo "" + brew update && brew install gmp boost openssl || echo "" sudo apt-get install libboost1.70 libboost1.70-dev | echo "" + sudo apt-get install libssl-dev sh install.sh - name: Test proof of space run: | From f497700c205fb629042434b7611e0eba2ee55c05 Mon Sep 17 00:00:00 2001 From: Yostra Date: Mon, 24 Feb 2020 13:01:17 -0800 Subject: [PATCH 30/38] boost --- .github/workflows/pythonpackage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index e39c84db1000..79dee7a0dd5d 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -21,8 +21,8 @@ jobs: - name: Install dependencies run: | brew update && brew install gmp boost openssl || echo "" - sudo apt-get install libboost1.70 libboost1.70-dev | echo "" - sudo apt-get install libssl-dev + sudo apt-get install libboost-all-dev || echo "" + sudo apt-get install libssl-dev || echo "" sh install.sh - name: Test proof of space run: | From cc8095a48d546a4903a92526777dea626693767a Mon Sep 17 00:00:00 2001 From: Yostra Date: Mon, 24 Feb 2020 14:03:41 -0800 Subject: [PATCH 31/38] update dependencies --- requirements.txt | 4 +- src/wallet/electron/package-lock.json | 202 +------------------------- src/wallet/electron/package.json | 5 +- 3 files changed, 11 insertions(+), 200 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9451eedf03d3..7619a7cc7628 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,5 +51,5 @@ sortedcontainers==2.1.0 -e lib/bip158 -e lib/py-setproctitle -e lib/chiavdf/fast_vdf --e git+https://github.com/Chia-Network/clvm.git@134c2f92d575a542272ce0cbe59c167b255e50ae#egg=clvm --e git+https://github.com/Chia-Network/clvm_tools.git@208dc0bbf41e16d8e5dd8cbc1510e1528a48bbe1#egg=clvm_tools +-e git+https://github.com/Chia-Network/clvm.git@bb538804062c01f999a228c4fc1e17a6e2835851#egg=clvm +-e git+https://github.com/Chia-Network/clvm_tools.git@343a82f16f3da1d042283f9f8969315e0c71bcf5#egg=clvm_tools diff --git a/src/wallet/electron/package-lock.json b/src/wallet/electron/package-lock.json index c36be3dbf777..da026da0de8c 100644 --- a/src/wallet/electron/package-lock.json +++ b/src/wallet/electron/package-lock.json @@ -177,15 +177,6 @@ "chainsaw": "~0.1.0" } }, - "bl": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", - "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", - "requires": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, "bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", @@ -504,14 +495,6 @@ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" }, - "decompress-response": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", - "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", - "requires": { - "mimic-response": "^1.0.0" - } - }, "decompress-zip": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/decompress-zip/-/decompress-zip-0.3.2.tgz", @@ -556,7 +539,8 @@ "deep-extend": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true }, "deep-is": { "version": "0.1.3", @@ -858,14 +842,6 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==" }, - "end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "requires": { - "once": "^1.4.0" - } - }, "env-paths": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-1.0.0.tgz", @@ -998,16 +974,6 @@ "es5-ext": "~0.10.14" } }, - "event-lite": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/event-lite/-/event-lite-0.1.2.tgz", - "integrity": "sha512-HnSYx1BsJ87/p6swwzv+2v6B4X+uxUteoDfRxsAb1S1BePzQqOLevVmkdA15GHJVd9A9Ok6wygUR18Hu0YeV9g==" - }, - "expand-template": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-1.1.1.tgz", - "integrity": "sha512-cebqLtV8KOZfw0UI8TEFWxtczxxC1jvyUvx6H4fyp1K1FN7A4Q+uggVUlOsI1K8AGU0rwOGqP8nCapdrw8CYQg==" - }, "ext": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", @@ -1094,11 +1060,6 @@ "mime-types": "^2.1.12" } }, - "fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" - }, "fs-extra": { "version": "0.30.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.30.0.tgz", @@ -1240,11 +1201,6 @@ "assert-plus": "^1.0.0" } }, - "github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=" - }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -1356,18 +1312,14 @@ "ini": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "dev": true }, "insert-css": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/insert-css/-/insert-css-2.0.0.tgz", "integrity": "sha1-610Ql7dUL0x56jBg067gfQU4gPQ=" }, - "int64-buffer": { - "version": "0.1.10", - "resolved": "https://registry.npmjs.org/int64-buffer/-/int64-buffer-0.1.10.tgz", - "integrity": "sha1-J3siiofZWtd30HwTgyAiQGpHNCM=" - }, "is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -1611,11 +1563,6 @@ "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==" }, - "mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" - }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -1698,22 +1645,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, - "msgpack-lite": { - "version": "0.1.26", - "resolved": "https://registry.npmjs.org/msgpack-lite/-/msgpack-lite-0.1.26.tgz", - "integrity": "sha1-3TxQsm8FnyXn7e42REGDWOKprYk=", - "requires": { - "event-lite": "^0.1.1", - "ieee754": "^1.1.8", - "int64-buffer": "^0.1.9", - "isarray": "^1.0.0" - } - }, - "nan": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", - "integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==" - }, "next-tick": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", @@ -1761,11 +1692,6 @@ } } }, - "noop-logger": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", - "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=" - }, "nopt": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", @@ -2049,28 +1975,6 @@ "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==" }, - "prebuild-install": { - "version": "2.5.3", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-2.5.3.tgz", - "integrity": "sha512-/rI36cN2g7vDQnKWN8Uzupi++KjyqS9iS+/fpwG4Ea8d0Pip0PQ5bshUNzVwt+/D2MRfhVAplYMMvWLqWrCF/g==", - "requires": { - "detect-libc": "^1.0.3", - "expand-template": "^1.0.2", - "github-from-package": "0.0.0", - "minimist": "^1.2.0", - "mkdirp": "^0.5.1", - "node-abi": "^2.2.0", - "noop-logger": "^0.1.1", - "npmlog": "^4.0.1", - "os-homedir": "^1.0.1", - "pump": "^2.0.1", - "rc": "^1.1.6", - "simple-get": "^2.7.0", - "tar-fs": "^1.13.0", - "tunnel-agent": "^0.6.0", - "which-pm-runs": "^1.0.0" - } - }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -2137,15 +2041,6 @@ "resolved": "https://registry.npmjs.org/psl/-/psl-1.7.0.tgz", "integrity": "sha512-5NsSEDv8zY70ScRnOTn7bK7eanl2MvFrOrS/R6x+dBt5g1ghnj9Zv90kO8GwT8gxcu2ANyFprnFYB85IogIJOQ==" }, - "pump": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", - "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", @@ -2310,6 +2205,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, "requires": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -2505,21 +2401,6 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" }, - "simple-concat": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz", - "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=" - }, - "simple-get": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.1.tgz", - "integrity": "sha512-lSSHRSw3mQNUGPAYRqo7xy9dhKmxFXIjLjp4KHpf99GEH2VH7C3AM+Qfx6du6jhfUi6Vm7XnbEVEf7Wb6N8jRw==", - "requires": { - "decompress-response": "^3.3.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, "single-line-log": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/single-line-log/-/single-line-log-1.1.2.tgz", @@ -2705,7 +2586,8 @@ "strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "dev": true }, "sumchecker": { "version": "1.3.1", @@ -2739,42 +2621,6 @@ "yallist": "^3.0.3" } }, - "tar-fs": { - "version": "1.16.3", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz", - "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==", - "requires": { - "chownr": "^1.0.1", - "mkdirp": "^0.5.1", - "pump": "^1.0.0", - "tar-stream": "^1.1.2" - }, - "dependencies": { - "pump": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz", - "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==", - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - } - } - }, - "tar-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", - "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", - "requires": { - "bl": "^1.0.0", - "buffer-alloc": "^1.2.0", - "end-of-stream": "^1.0.0", - "fs-constants": "^1.0.0", - "readable-stream": "^2.3.0", - "to-buffer": "^1.1.1", - "xtend": "^4.0.0" - } - }, "throttleit": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-0.0.2.tgz", @@ -2835,11 +2681,6 @@ "os-tmpdir": "~1.0.1" } }, - "to-buffer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", - "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==" - }, "touch": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/touch/-/touch-0.0.3.tgz", @@ -2926,11 +2767,6 @@ "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, - "underscore": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.3.3.tgz", - "integrity": "sha1-R6xTaD2vgyv6lS4XdEF9pHgXrkI=" - }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", @@ -3006,11 +2842,6 @@ "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" }, - "which-pm-runs": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz", - "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=" - }, "wide-align": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", @@ -3211,25 +3042,6 @@ "requires": { "fd-slicer": "~1.0.1" } - }, - "zeromq": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/zeromq/-/zeromq-4.6.0.tgz", - "integrity": "sha512-sU7pQqQj7f/C6orJZAXls+NEKaVMZZtnZqpMPTq5d5dP78CmdC0g15XIviFAN6poPuKl9qlGt74vipOUUuNeWg==", - "requires": { - "nan": "^2.6.2", - "prebuild-install": "^2.2.2" - } - }, - "zerorpc": { - "version": "git+https://github.com/0rpc/zerorpc-node.git#45d2e510a6dc236863ac9a2b92da0a2511ef54d9", - "from": "git+https://github.com/0rpc/zerorpc-node.git", - "requires": { - "msgpack-lite": "^0.1.26", - "underscore": "1.3.3", - "uuid": "^3.0.0", - "zeromq": "^4.6.0" - } } } } diff --git a/src/wallet/electron/package.json b/src/wallet/electron/package.json index 58337333f096..c0ea657740bc 100644 --- a/src/wallet/electron/package.json +++ b/src/wallet/electron/package.json @@ -16,11 +16,10 @@ "dialogs": "^2.0.1", "electron-rebuild": "^1.10.0", "jquery": "^3.4.1", - "qrcode": "^1.4.4", - "zerorpc": "git+https://github.com/0rpc/zerorpc-node.git" + "qrcode": "^1.4.4" }, "devDependencies": { - "electron": "^1.7.6", + "electron": "^1.8.8", "electron-packager": "^9.0.1" } } From 6337ec79e7387ac4cad0190978234a122d277473 Mon Sep 17 00:00:00 2001 From: Yostra Date: Mon, 24 Feb 2020 17:05:53 -0800 Subject: [PATCH 32/38] rename --- src/wallet/electron/main.js | 2 +- src/wallet/electron/{wallet_rpc => rpc}/rpc_wallet.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/wallet/electron/{wallet_rpc => rpc}/rpc_wallet.py (100%) diff --git a/src/wallet/electron/main.js b/src/wallet/electron/main.js index e1e0889d9dc1..392aa52e3cbc 100644 --- a/src/wallet/electron/main.js +++ b/src/wallet/electron/main.js @@ -9,7 +9,7 @@ const path = require('path') *************************************************************/ const PY_DIST_FOLDER = 'pydist' -const PY_FOLDER = 'wallet_rpc' +const PY_FOLDER = 'rpc' const PY_MODULE = 'rpc_wallet' // without .py suffix let pyProc = null diff --git a/src/wallet/electron/wallet_rpc/rpc_wallet.py b/src/wallet/electron/rpc/rpc_wallet.py similarity index 100% rename from src/wallet/electron/wallet_rpc/rpc_wallet.py rename to src/wallet/electron/rpc/rpc_wallet.py From 25ec803cea460810ff2cdc68a033c8a06e2cc356 Mon Sep 17 00:00:00 2001 From: Yostra Date: Mon, 24 Feb 2020 17:19:50 -0800 Subject: [PATCH 33/38] get all transactions --- src/wallet/electron/rpc/rpc_wallet.py | 11 ++++++++--- src/wallet/wallet.py | 3 ++- src/wallet/wallet_state_manager.py | 7 +++++++ src/wallet/wallet_store.py | 26 +++++++++++++------------- src/wallet/wallet_transaction_store.py | 13 +++++++++++++ 5 files changed, 43 insertions(+), 17 deletions(-) diff --git a/src/wallet/electron/rpc/rpc_wallet.py b/src/wallet/electron/rpc/rpc_wallet.py index 3e3c0a8fed69..f07ca93e6f25 100644 --- a/src/wallet/electron/rpc/rpc_wallet.py +++ b/src/wallet/electron/rpc/rpc_wallet.py @@ -82,17 +82,22 @@ class RpcWalletApiHandler: return obj_to_response(response) async def get_transactions(self, request) -> web.Response: + transactions = await self.wallet_node.wallet_state_manager.get_all_transactions() - response = {"success": True} + response = {"success": True, "txs": transactions} return obj_to_response(response) async def get_wallet_balance(self, request) -> web.Response: + balance = await self.wallet_node.wallet.get_confirmed_balance() + pending_balance = await self.wallet_node.wallet.get_unconfirmed_balance() + response = { "success": True, - "confirmed_wallet_balance": 0, - "unconfirmed_wallet_balance": 0, + "confirmed_wallet_balance": balance, + "unconfirmed_wallet_balance": pending_balance, } + return obj_to_response(response) diff --git a/src/wallet/wallet.py b/src/wallet/wallet.py index eedca6f71f82..c37706a63afc 100644 --- a/src/wallet/wallet.py +++ b/src/wallet/wallet.py @@ -203,6 +203,7 @@ class Wallet: async def generate_signed_transaction( self, amount, newpuzzlehash, fee: int = 0 ) -> Optional[SpendBundle]: + """ Use this to generate transaction. """ transaction = await self.generate_unsigned_transaction( amount, newpuzzlehash, fee ) @@ -211,7 +212,7 @@ class Wallet: return self.sign_transaction(transaction) async def push_transaction(self, spend_bundle: SpendBundle): - """ Use this API to make transactions. """ + """ Use this API to send transactions. """ await self.wallet_state_manager.add_pending_transaction(spend_bundle) await self._send_transaction(spend_bundle) diff --git a/src/wallet/wallet_state_manager.py b/src/wallet/wallet_state_manager.py index fc2ce876e8fc..0219f88dcd31 100644 --- a/src/wallet/wallet_state_manager.py +++ b/src/wallet/wallet_state_manager.py @@ -180,3 +180,10 @@ class WalletStateManager: """ records = await self.tx_store.get_not_sent() return records + + async def get_all_transactions(self) -> List[TransactionRecord]: + """ + Retrieves all confirmed and pending transactions + """ + records = await self.tx_store.get_all_transactions() + return records diff --git a/src/wallet/wallet_store.py b/src/wallet/wallet_store.py index 3a3354305d5d..44e6d7600328 100644 --- a/src/wallet/wallet_store.py +++ b/src/wallet/wallet_store.py @@ -17,7 +17,7 @@ class WalletStore: # Whether or not we are syncing sync_mode: bool = False lock: asyncio.Lock - lca_coin_records: Dict[str, CoinRecord] + coin_record_cache: Dict[str, CoinRecord] cache_size: uint32 @classmethod @@ -61,7 +61,7 @@ class WalletStore: await self.coin_record_db.commit() # Lock self.lock = asyncio.Lock() # external - self.lca_coin_records = dict() + self.coin_record_cache = dict() return self async def close(self): @@ -89,11 +89,11 @@ class WalletStore: ) await cursor.close() await self.coin_record_db.commit() - self.lca_coin_records[record.coin.name().hex()] = record - if len(self.lca_coin_records) > self.cache_size: - while len(self.lca_coin_records) > self.cache_size: - first_in = list(self.lca_coin_records.keys())[0] - del self.lca_coin_records[first_in] + self.coin_record_cache[record.coin.name().hex()] = record + if len(self.coin_record_cache) > self.cache_size: + while len(self.coin_record_cache) > self.cache_size: + first_in = list(self.coin_record_cache.keys())[0] + del self.coin_record_cache[first_in] # Update coin_record to be spent in DB async def set_spent(self, coin_name: bytes32, index: uint32): @@ -102,13 +102,13 @@ class WalletStore: return spent: CoinRecord = CoinRecord( current.coin, current.confirmed_block_index, index, True, current.coinbase, - ) # type: ignore # noqa + ) await self.add_coin_record(spent) # Checks DB and DiffStores for CoinRecord with coin_name and returns it async def get_coin_record(self, coin_name: bytes32) -> Optional[CoinRecord]: - if coin_name.hex() in self.lca_coin_records: - return self.lca_coin_records[coin_name.hex()] + if coin_name.hex() in self.coin_record_cache: + return self.coin_record_cache[coin_name.hex()] cursor = await self.coin_record_db.execute( "SELECT * from coin_record WHERE coin_name=?", (coin_name.hex(),) ) @@ -157,7 +157,7 @@ class WalletStore: async def rollback_lca_to_block(self, block_index): # Update memory cache delete_queue: bytes32 = [] - for coin_name, coin_record in self.lca_coin_records.items(): + for coin_name, coin_record in self.coin_record_cache.items(): if coin_record.spent_block_index > block_index: new_record = CoinRecord( coin_record.coin, @@ -166,12 +166,12 @@ class WalletStore: False, coin_record.coinbase, ) - self.lca_coin_records[coin_record.coin.name().hex()] = new_record + self.coin_record_cache[coin_record.coin.name().hex()] = new_record if coin_record.confirmed_block_index > block_index: delete_queue.append(coin_name) for coin_name in delete_queue: - del self.lca_coin_records[coin_name] + del self.coin_record_cache[coin_name] # Delete from storage c1 = await self.coin_record_db.execute( diff --git a/src/wallet/wallet_transaction_store.py b/src/wallet/wallet_transaction_store.py index 6a3c66ba92d0..02d7b601ffcf 100644 --- a/src/wallet/wallet_transaction_store.py +++ b/src/wallet/wallet_transaction_store.py @@ -172,3 +172,16 @@ class WalletTransactionStore: records.append(record) return records + + async def get_all_transactions(self) -> List[TransactionRecord]: + cursor = await self.transaction_db.execute( + "SELECT * from transaction_record" + ) + rows = await cursor.fetchall() + await cursor.close() + records = [] + for row in rows: + record = TransactionRecord.from_bytes(row[6]) + records.append(record) + + return records From 03078abaf6834654c5872bea796ef98c774883ef Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 25 Feb 2020 13:09:00 -0800 Subject: [PATCH 34/38] formatting --- src/wallet/electron/rpc/rpc_wallet.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wallet/electron/rpc/rpc_wallet.py b/src/wallet/electron/rpc/rpc_wallet.py index f07ca93e6f25..56f3a44774b1 100644 --- a/src/wallet/electron/rpc/rpc_wallet.py +++ b/src/wallet/electron/rpc/rpc_wallet.py @@ -82,7 +82,9 @@ class RpcWalletApiHandler: return obj_to_response(response) async def get_transactions(self, request) -> web.Response: - transactions = await self.wallet_node.wallet_state_manager.get_all_transactions() + transactions = ( + await self.wallet_node.wallet_state_manager.get_all_transactions() + ) response = {"success": True, "txs": transactions} return obj_to_response(response) From 26f4cf61886446c992b228ba4c9151fd503e473a Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 25 Feb 2020 13:09:33 -0800 Subject: [PATCH 35/38] formating --- src/wallet/wallet_transaction_store.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/wallet/wallet_transaction_store.py b/src/wallet/wallet_transaction_store.py index 02d7b601ffcf..24883b115d67 100644 --- a/src/wallet/wallet_transaction_store.py +++ b/src/wallet/wallet_transaction_store.py @@ -174,9 +174,7 @@ class WalletTransactionStore: return records async def get_all_transactions(self) -> List[TransactionRecord]: - cursor = await self.transaction_db.execute( - "SELECT * from transaction_record" - ) + cursor = await self.transaction_db.execute("SELECT * from transaction_record") rows = await cursor.fetchall() await cursor.close() records = [] From 17d128d2dee216b7959ce8216f7f090673933acf Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 25 Feb 2020 13:11:37 -0800 Subject: [PATCH 36/38] Bram's reference merkle set --- src/util/MerkleSet.py | 356 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 356 insertions(+) create mode 100644 src/util/MerkleSet.py diff --git a/src/util/MerkleSet.py b/src/util/MerkleSet.py new file mode 100644 index 000000000000..ad67d84e86ed --- /dev/null +++ b/src/util/MerkleSet.py @@ -0,0 +1,356 @@ +from hashlib import blake2b, sha256 + +""" +A simple, confidence-inspiring Merkle Set standard + +Advantages of this standard: +Low CPU requirements +Small proofs of inclusion/exclusion +Reasonably simple implementation + +The main tricks in this standard are: + +Uses blake2b because that has the best performance on 512 bit inputs +Skips repeated hashing of exactly two things even when they share prefix bits + + +Proofs support proving including/exclusion for a large number of values in +a single string. They're a serialization of a subset of the tree. + +Proof format: + +multiproof: subtree +subtree: middle or terminal or truncated or empty +middle: MIDDLE 1 subtree subtree +terminal: TERMINAL 1 hash 32 +# If the sibling is empty truncated implies more than two children. +truncated: TRUNCATED 1 hash 32 +empty: EMPTY 1 +EMPTY: \x00 +TERMINAL: \x01 +MIDDLE: \x02 +TRUNCATED: \x03 +""" + +EMPTY = bytes([0]) +TERMINAL = bytes([1]) +MIDDLE = bytes([2]) +TRUNCATED = bytes([3]) + +BLANK = bytes([0] * 32) + +prehashed = {} + + +def init_prehashed(): + for x in [EMPTY, TERMINAL, MIDDLE]: + for y in [EMPTY, TERMINAL, MIDDLE]: + prehashed[x + y] = blake2b(bytes([0] * 30) + x + y) + + +init_prehashed() + + +def hashdown(mystr): + assert len(mystr) == 66 + h = prehashed[bytes(mystr[0:1] + mystr[33:34])].copy() + h.update(mystr[1:33] + mystr[34:]) + return h.digest()[:32] + + +def compress_root(mystr): + assert len(mystr) == 33 + if mystr[0:1] == MIDDLE: + return mystr[1:] + if mystr[0:1] == EMPTY: + assert mystr[1:] == BLANK + return BLANK + return blake2b(mystr).digest()[:32] + + +def get_bit(mybytes, pos): + assert len(mybytes) == 32 + return (mybytes[pos // 8] >> (7 - (pos % 8))) & 1 + + +class ReferenceMerkleSet: + def __init__(self, root=None): + self.root = root + if root is None: + self.root = _empty + + def get_root(self): + return compress_root(self.root.get_hash()) + + def add_already_hashed(self, toadd): + self.root = self.root.add(toadd, 0) + + def remove_already_hashed(self, toremove): + self.root = self.root.remove(toremove, 0) + + def is_included_already_hashed(self, tocheck): + proof = [] + r = self.root.is_included(tocheck, 0, proof) + return r, b"".join(proof) + + def _audit(self, hashes): + newhashes = [] + self.root._audit(newhashes, []) + assert newhashes == sorted(newhashes) + + +class EmptyNode: + def __init__(self): + self.hash = BLANK + + def get_hash(self): + return EMPTY + BLANK + + def is_empty(self): + return True + + def is_terminal(self): + return False + + def is_double(self): + raise SetError() + + def add(self, toadd, depth): + return TerminalNode(toadd) + + def remove(self, toremove, depth): + return self + + def is_included(self, tocheck, depth, p): + p.append(EMPTY) + return False + + def other_included(self, tocheck, depth, p, collapse): + p.append(EMPTY) + + def _audit(self, hashes, bits): + pass + + +_empty = EmptyNode() + + +class TerminalNode: + def __init__(self, hash, bits=None): + assert len(hash) == 32 + self.hash = hash + if bits is not None: + self._audit([], bits) + + def get_hash(self): + return TERMINAL + self.hash + + def is_empty(self): + return False + + def is_terminal(self): + return True + + def is_double(self): + raise SetError() + + def add(self, toadd, depth): + if toadd == self.hash: + return self + if toadd > self.hash: + return self._make_middle([self, TerminalNode(toadd)], depth) + else: + return self._make_middle([TerminalNode(toadd), self], depth) + + def _make_middle(self, children, depth): + cbits = [get_bit(child.hash, depth) for child in children] + if cbits[0] != cbits[1]: + return MiddleNode(children) + nextvals = [None, None] + nextvals[cbits[0] ^ 1] = _empty + nextvals[cbits[0]] = self._make_middle(children, depth + 1) + return MiddleNode(nextvals) + + def remove(self, toremove, depth): + if toremove == self.hash: + return _empty + return self + + def is_included(self, tocheck, depth, proof): + proof.append(TERMINAL + self.hash) + return tocheck == self.hash + + def other_included(self, tocheck, depth, p, collapse): + p.append(TERMINAL + self.hash) + + def _audit(self, hashes, bits): + hashes.append(self.hash) + for pos, v in enumerate(bits): + assert get_bit(self.hash, pos) == v + + +class MiddleNode: + def __init__(self, children): + self.children = children + if children[0].is_empty() and children[1].is_double(): + self.hash = children[1].hash + elif children[1].is_empty() and children[0].is_double(): + self.hash = children[0].hash + else: + if children[0].is_empty() and ( + children[1].is_empty() or children[1].is_terminal() + ): + raise SetError() + if children[1].is_empty() and children[0].is_terminal(): + raise SetError + if ( + children[0].is_terminal() + and children[1].is_terminal() + and children[0].hash >= children[1].hash + ): + raise SetError + self.hash = hashdown(children[0].get_hash() + children[1].get_hash()) + + def get_hash(self): + return MIDDLE + self.hash + + def is_empty(self): + return False + + def is_terminal(self): + return False + + def is_double(self): + if self.children[0].is_empty(): + return self.children[1].is_double() + if self.children[1].is_empty(): + return self.children[0].is_double() + return self.children[0].is_terminal() and self.children[1].is_terminal() + + def add(self, toadd, depth): + bit = get_bit(toadd, depth) + child = self.children[bit] + newchild = child.add(toadd, depth + 1) + if newchild is child: + return self + newvals = [x for x in self.children] + newvals[bit] = newchild + return MiddleNode(newvals) + + def remove(self, toremove, depth): + bit = get_bit(toremove, depth) + child = self.children[bit] + newchild = child.remove(toremove, depth + 1) + if newchild is child: + return self + otherchild = self.children[bit ^ 1] + if newchild.is_empty() and otherchild.is_terminal(): + return otherchild + if newchild.is_terminal() and otherchild.is_empty(): + return newchild + newvals = [x for x in self.children] + newvals[bit] = newchild + return MiddleNode(newvals) + + def is_included(self, tocheck, depth, p): + p.append(MIDDLE) + if get_bit(tocheck, depth) == 0: + r = self.children[0].is_included(tocheck, depth + 1, p) + self.children[1].other_included( + tocheck, depth + 1, p, not self.children[0].is_empty() + ) + return r + else: + self.children[0].other_included( + tocheck, depth + 1, p, not self.children[1].is_empty() + ) + return self.children[1].is_included(tocheck, depth + 1, p) + + def other_included(self, tocheck, depth, p, collapse): + if collapse or not self.is_double(): + p.append(TRUNCATED + self.hash) + else: + self.is_included(tocheck, depth, p) + + def _audit(self, hashes, bits): + self.children[0]._audit(hashes, bits + [0]) + self.children[1]._audit(hashes, bits + [1]) + + +class TruncatedNode: + def __init__(self, hash): + self.hash = hash + + def get_hash(self): + return MIDDLE + self.hash + + def is_empty(self): + return False + + def is_terminal(self): + return False + + def is_double(self): + return False + + def is_included(self, tocheck, depth, p): + raise SetError() + + def other_included(self, tocheck, depth, p, collapse): + p.append(TRUNCATED + self.hash) + + +class SetError(BaseException): + pass + + +def confirm_included(root, val, proof): + return confirm_not_included_already_hashed(root, sha256(val).digest(), proof) + + +def confirm_included_already_hashed(root, val, proof): + return _confirm(root, val, proof, True) + + +def confirm_not_included(root, val, proof): + return confirm_not_included_already_hashed(root, sha256(val).digest(), proof) + + +def confirm_not_included_already_hashed(root, val, proof): + return _confirm(root, val, proof, False) + + +def _confirm(root, val, proof, expected): + try: + p = deserialize_proof(proof) + if p.get_root() != root: + return False + r, junk = p.is_included_already_hashed(val) + return r == expected + except SetError: + return False + + +def deserialize_proof(proof): + try: + r, pos = _deserialize(proof, 0, []) + if pos != len(proof): + raise SetError() + return ReferenceMerkleSet(r) + except IndexError: + raise SetError() + + +def _deserialize(proof, pos, bits): + t = proof[pos: pos + 1] + if t == EMPTY: + return _empty, pos + 1 + if t == TERMINAL: + return TerminalNode(proof[pos + 1: pos + 33], bits), pos + 33 + if t == TRUNCATED: + return TruncatedNode(proof[pos + 1: pos + 33]), pos + 33 + if t != MIDDLE: + raise SetError() + v0, pos = _deserialize(proof, pos + 1, bits + [0]) + v1, pos = _deserialize(proof, pos, bits + [1]) + return MiddleNode([v0, v1]), pos From d7558bc181e9fdf2ccd01abd8d24c9849b9176e4 Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 25 Feb 2020 14:13:26 -0800 Subject: [PATCH 37/38] test merkle set --- tests/test_merkle_set.py | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/test_merkle_set.py diff --git a/tests/test_merkle_set.py b/tests/test_merkle_set.py new file mode 100644 index 000000000000..911a9b5190fe --- /dev/null +++ b/tests/test_merkle_set.py @@ -0,0 +1,42 @@ +import asyncio + +import pytest + +from src.util.MerkleSet import ReferenceMerkleSet, confirm_included_already_hashed +from tests.setup_nodes import test_constants, bt +from tests.wallet_tools import WalletTool + + +@pytest.fixture(scope="module") +def event_loop(): + loop = asyncio.get_event_loop() + yield loop + + +class TestMerkleSet: + @pytest.mark.asyncio + async def test_basics(self): + wallet_tool = WalletTool() + + num_blocks = 10 + blocks = bt.get_consecutive_blocks( + test_constants, + num_blocks, + [], + 10, + reward_puzzlehash=wallet_tool.get_new_puzzlehash(), + ) + + merkle_set = ReferenceMerkleSet() + for block in blocks: + merkle_set.add_already_hashed(block.body.coinbase.name()) + + for block in blocks: + result, proof = merkle_set.is_included_already_hashed(block.body.coinbase.name()) + assert result is True + result_fee, proof_fee = merkle_set.is_included_already_hashed(block.body.fees_coin.name()) + assert result_fee is False + validate_proof = confirm_included_already_hashed(merkle_set.get_root(), block.body.coinbase.name(), proof) + validate_proof_fee = confirm_included_already_hashed(merkle_set.get_root(), block.body.fees_coin.name(), proof_fee) + assert validate_proof is True + assert validate_proof_fee is False From d53b4c9ca82c2e11fdede889dd935540d0ba2186 Mon Sep 17 00:00:00 2001 From: Yostra Date: Tue, 25 Feb 2020 16:15:23 -0800 Subject: [PATCH 38/38] add addition and removal root to header --- src/full_node.py | 39 ++++++++++++++++++++++++++++++++++---- src/types/hashable/Coin.py | 16 ++++++++++++++++ src/util/MerkleSet.py | 4 ++-- tests/block_tools.py | 37 +++++++++++++++++++++++++++++++++--- tests/test_merkle_set.py | 4 ++-- 5 files changed, 89 insertions(+), 11 deletions(-) diff --git a/src/full_node.py b/src/full_node.py index 66eab68132dd..bac4912552a2 100644 --- a/src/full_node.py +++ b/src/full_node.py @@ -3,7 +3,6 @@ import concurrent import logging import time from asyncio import Event -from secrets import token_bytes from typing import AsyncGenerator, List, Optional, Tuple, Dict from chiabip158 import PyBIP158 @@ -18,6 +17,7 @@ from src.consensus.weight_verifier import verify_weight from src.protocols.wallet_protocol import FullProofForHash, ProofHash from src.store import FullNodeStore from src.protocols import farmer_protocol, full_node_protocol, timelord_protocol +from src.util.MerkleSet import MerkleSet from src.util.bundle_tools import best_solution_program from src.mempool_manager import MempoolManager from src.server.outbound_message import Delivery, Message, NodeType, OutboundMessage @@ -25,7 +25,7 @@ from src.server.server import ChiaServer from src.types.body import Body from src.types.challenge import Challenge from src.types.full_block import FullBlock -from src.types.hashable.Coin import Coin +from src.types.hashable.Coin import Coin, hash_coin_list from src.types.hashable.BLSSignature import BLSSignature from src.util.hash import std_hash from src.types.hashable.SpendBundle import SpendBundle @@ -1271,8 +1271,39 @@ class FullNode: iterations_needed: uint64 = calculate_iterations( request.proof_of_space, difficulty, vdf_ips, constants["MIN_BLOCK_TIME"], ) - additions_root = token_bytes(32) # TODO(straya) - removal_root = token_bytes(32) # TODO(straya) + + removal_merkle_set = MerkleSet() + addition_merkle_set = MerkleSet() + + additions = [] + removals = [] + + if spend_bundle: + additions = spend_bundle.additions() + removals = spend_bundle.removals() + + additions.append(request.coinbase) + additions.append(fees_coin) + + # Create removal Merkle set + for coin in removals: + removal_merkle_set.add_already_hashed(coin.name()) + + # Create addition Merkle set + puzzlehash_coins_map: Dict[bytes32, List[Coin]] = {} + for coin in additions: + if coin.puzzle_hash in puzzlehash_coins_map: + puzzlehash_coins_map[coin.puzzle_hash].append(coin) + else: + puzzlehash_coins_map[coin.puzzle_hash] = [coin] + + # Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash + for puzzle, coins in puzzlehash_coins_map.items(): + addition_merkle_set.add_already_hashed(puzzle) + addition_merkle_set.add_already_hashed(hash_coin_list(coins)) + + additions_root = addition_merkle_set.get_root() + removal_root = removal_merkle_set.get_root() block_header_data: HeaderData = HeaderData( uint32(target_tip.height + 1), diff --git a/src/types/hashable/Coin.py b/src/types/hashable/Coin.py index 2ca6c952f6a8..69219065d37f 100644 --- a/src/types/hashable/Coin.py +++ b/src/types/hashable/Coin.py @@ -1,9 +1,11 @@ import io from dataclasses import dataclass +from typing import List from clvm.casts import int_to_bytes, int_from_bytes from src.types.sized_bytes import bytes32 +from src.util.hash import std_hash from src.util.ints import uint64 from src.util.streamable import streamable, Streamable @@ -22,6 +24,10 @@ class Coin(Streamable): def name(self) -> bytes32: return self.get_hash() + @property + def name_str(self) -> str: + return self.name().hex() + @classmethod def from_bytes(cls, blob): parent_coin_info = blob[:32] @@ -35,3 +41,13 @@ class Coin(Streamable): f.write(self.puzzle_hash) f.write(int_to_bytes(self.amount)) return f.getvalue() + + +def hash_coin_list(coin_list: List[Coin]) -> bytes32: + coin_list.sort(key=lambda x: x.name_str, reverse=True) + buffer = bytearray() + + for coin in coin_list: + buffer.extend(coin.name()) + + return std_hash(buffer) diff --git a/src/util/MerkleSet.py b/src/util/MerkleSet.py index ad67d84e86ed..8f2b010c83df 100644 --- a/src/util/MerkleSet.py +++ b/src/util/MerkleSet.py @@ -73,7 +73,7 @@ def get_bit(mybytes, pos): return (mybytes[pos // 8] >> (7 - (pos % 8))) & 1 -class ReferenceMerkleSet: +class MerkleSet: def __init__(self, root=None): self.root = root if root is None: @@ -336,7 +336,7 @@ def deserialize_proof(proof): r, pos = _deserialize(proof, 0, []) if pos != len(proof): raise SetError() - return ReferenceMerkleSet(r) + return MerkleSet(r) except IndexError: raise SetError() diff --git a/tests/block_tools.py b/tests/block_tools.py index e51b92cda685..084f260d71eb 100644 --- a/tests/block_tools.py +++ b/tests/block_tools.py @@ -20,12 +20,13 @@ from src.types.challenge import Challenge from src.types.classgroup import ClassgroupElement from src.types.full_block import FullBlock, additions_for_npc from src.types.hashable.BLSSignature import BLSSignature -from src.types.hashable.Coin import Coin +from src.types.hashable.Coin import Coin, hash_coin_list from src.types.hashable.Program import Program from src.types.header import Header, HeaderData from src.types.proof_of_space import ProofOfSpace from src.types.proof_of_time import ProofOfTime from src.types.sized_bytes import bytes32 +from src.util.MerkleSet import MerkleSet from src.util.errors import NoProofsOfSpaceFound from src.util.ints import uint8, uint32, uint64 from src.util.hash import std_hash @@ -447,12 +448,16 @@ class BlockTools: # Create filter byte_array_tx: List[bytes32] = [] + tx_additions: List[Coin] = [] + tx_removals: List[bytes32] = [] if transactions: error, npc_list, _ = get_name_puzzle_conditions(transactions) additions: List[Coin] = additions_for_npc(npc_list) for coin in additions: + tx_additions.append(coin) byte_array_tx.append(bytearray(coin.puzzle_hash)) for npc in npc_list: + tx_removals.append(npc.coin_name) byte_array_tx.append(bytearray(npc.coin_name)) byte_array_tx.append(bytearray(coinbase_coin.puzzle_hash)) @@ -461,6 +466,32 @@ class BlockTools: bip158: PyBIP158 = PyBIP158(byte_array_tx) encoded = bytes(bip158.GetEncoded()) + removal_merkle_set = MerkleSet() + addition_merkle_set = MerkleSet() + + tx_additions.append(coinbase_coin) + tx_additions.append(fees_coin) + + # Create removal Merkle set + for coin_name in tx_removals: + removal_merkle_set.add_already_hashed(coin_name) + + # Create addition Merkle set + puzzlehash_coin_map: Dict[bytes32, List[Coin]] = {} + for coin in tx_additions: + if coin.puzzle_hash in puzzlehash_coin_map: + puzzlehash_coin_map[coin.puzzle_hash].append(coin) + else: + puzzlehash_coin_map[coin.puzzle_hash] = [coin] + + # Addition Merkle set contains puzzlehash and hash of all coins with that puzzlehash + for puzzle, coins in puzzlehash_coin_map.items(): + addition_merkle_set.add_already_hashed(puzzle) + addition_merkle_set.add_already_hashed(hash_coin_list(coins)) + + additions_root = addition_merkle_set.get_root() + removal_root = removal_merkle_set.get_root() + header_data: HeaderData = HeaderData( height, prev_header_hash, @@ -470,8 +501,8 @@ class BlockTools: body.get_hash(), uint64(prev_weight + difficulty), uint64(prev_iters + number_iters), - bytes([0] * 32), - bytes([0] * 32), + additions_root, + removal_root, ) header_hash_sig: PrependSignature = plot_sk.sign_prepend(header_data.get_hash()) diff --git a/tests/test_merkle_set.py b/tests/test_merkle_set.py index 911a9b5190fe..d49e3b3dd6bf 100644 --- a/tests/test_merkle_set.py +++ b/tests/test_merkle_set.py @@ -2,7 +2,7 @@ import asyncio import pytest -from src.util.MerkleSet import ReferenceMerkleSet, confirm_included_already_hashed +from src.util.MerkleSet import MerkleSet, confirm_included_already_hashed from tests.setup_nodes import test_constants, bt from tests.wallet_tools import WalletTool @@ -27,7 +27,7 @@ class TestMerkleSet: reward_puzzlehash=wallet_tool.get_new_puzzlehash(), ) - merkle_set = ReferenceMerkleSet() + merkle_set = MerkleSet() for block in blocks: merkle_set.add_already_hashed(block.body.coinbase.name())