Chia-Network/chia-blockchain
1
0
Fork 0
mirror of https://github.com/Chia-Network/chia-blockchain.git synced 2024-09-21 00:24:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

Allow specifying temporary and final directory for proof of space

Browse Source
This commit is contained in:
Mariano Sorgente 2020-01-31 17:16:40 +09:00
parent 9a04b6a9e2
commit a7ebba5b89
No known key found for this signature in database
GPG Key ID: 0F866338C369278C
14 changed files with 128 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@ -189,7 +189,9 @@ python -m scripts.regenerate_keys
## Step 4a: Run a full node
To run a full node on port 8444, and connect to the testnet, run the following command.
This will also start an ssh server in port 8222 for the UI, which you can connect to
to see the state of the node.
to see the state of the node. If you want to see std::out log output, modify the logging.std_out
variable in ./config/config.yaml.
```bash
./scripts/run_full_node.sh
ssh -p 8222 localhost
@ -201,7 +203,8 @@ Farmers are entities in the network who use their hard drive space to try to cre
blocks (like Bitcoin's miners), and earn block rewards. First, you must generate some hard drive plots, which
can take a long time depending on the [size of the plots](https://github.com/Chia-Network/chia-blockchain/wiki/k-sizes)
(the k variable). Then, run the farmer + full node with the following script. A full node is also started,
which you can ssh into to view the node UI (previous ssh command).
which you can ssh into to view the node UI (previous ssh command). You can also change the working directory and
final directory for plotting, with the "-t" and "-d" arguments to the create_plots script.
```bash
python -m scripts.create_plots -k 20 -n 10
sh ./scripts/run_farming.sh

2
config/config.yaml
View File

@ -14,7 +14,7 @@ harvester:
farmer_peer:
host: 127.0.0.1
port: 8447
# Location of all the plots, default ./plots
# Location of all the plots, default ./plots, for relative paths in plots.yaml.
# plot_root: "/mnt/pos"
logging: *logging

2
lib/chiapos/CMakeLists.txt
View File

@ -31,7 +31,7 @@ add_subdirectory(lib/pybind11)
pybind11_add_module(chiapos ${CMAKE_CURRENT_SOURCE_DIR}/python-bindings/chiapos.cpp)
set (CMAKE_CXX_FLAGS "-g -O3 -Wall -msse2 -msse -march=native -std=c++11 -maes")
set (CMAKE_CXX_FLAGS "-g -O3 -Wall -msse2 -msse -march=native -std=c++1z -maes")
add_executable(ProofOfSpace
src/cli.cpp

3
lib/chiapos/README.md
View File

@ -8,8 +8,7 @@ Only runs on 64 bit architectures with AES-NI support. Read the [Proof of Space
### Compile
```bash
git submodule update --init --recursive
mkdir build && cd build
mkdir -p build && cd build
cmake ../
cmake --build . -- -j 6
```

5
lib/chiapos/python-bindings/chiapos.cpp
View File

@ -34,13 +34,14 @@ PYBIND11_MODULE(chiapos, m) {
py::class_<DiskPlotter>(m, "DiskPlotter")
.def(py::init<>())
.def("create_plot_disk", [](DiskPlotter &dp, const std::string filename, uint8_t k,
.def("create_plot_disk", [](DiskPlotter &dp, const std::string tmp_dir, const std::string final_dir,
const std::string filename, uint8_t k,
const py::bytes &memo, const py::bytes &id) {
std::string memo_str(memo);
const uint8_t* memo_ptr = reinterpret_cast<const uint8_t*>(memo_str.data());
std::string id_str(id);
const uint8_t* id_ptr = reinterpret_cast<const uint8_t*>(id_str.data());
dp.CreatePlotDisk(filename, k, memo_ptr, len(memo), id_ptr, len(id));
dp.CreatePlotDisk(tmp_dir, final_dir, filename, k, memo_ptr, len(memo), id_ptr, len(id));
});
py::class_<DiskProver>(m, "DiskProver")

32
lib/chiapos/python-bindings/test.py
View File

@ -1,32 +0,0 @@
from chiapos import DiskProver, DiskPlotter, Verifier
from hashlib import sha256
import secrets
import os
challenge: bytes = bytes([i for i in range(0, 32)])
plot_id: bytes = bytes([5, 104, 52, 4, 51, 55, 23, 84, 91, 10, 111, 12, 13,
222, 151, 16, 228, 211, 254, 45, 92, 198, 204, 10, 9,
10, 11, 129, 139, 171, 15, 23])
filename = "./myplot.dat"
pl = DiskPlotter()
pl.create_plot_disk(filename, 21, bytes([1, 2, 3, 4, 5]), plot_id)
pr = DiskProver(filename)
total_proofs: int = 0
iterations: int = 5000
v = Verifier()
for i in range(iterations):
challenge = sha256(i.to_bytes(4, "big")).digest()
for index, quality in enumerate(pr.get_qualities_for_challenge(challenge)):
proof = pr.get_full_proof(challenge, index)
total_proofs += 1
ver_quality = v.validate_proof(plot_id, 21, challenge, proof)
assert(quality == ver_quality)
os.remove(filename)
print(f"total proofs {total_proofs} out of {iterations}\
{total_proofs / iterations}")

2
lib/chiapos/setup.py
View File

@ -66,7 +66,7 @@ class CMakeBuild(build_ext):
setup(
name='chiapos',
version='0.2.2',
version='0.2.3',
author='Mariano Sorgente',
author_email='mariano@chia.net',
description='Chia proof of space plotting, proving, and verifying (wraps C++)',

7
lib/chiapos/src/cli.cpp
View File

@ -64,6 +64,8 @@ int main(int argc, char *argv[]) {
// Default values
uint8_t k = 20;
string filename = "plot.dat";
string tempdir = ".";
string finaldir = ".";
string operation = "help";
string memo = "0102030405";
string id = "022fb42c08c12de3a6af053880199806532e79515f94e83461612101f9412f9e";
@ -71,6 +73,8 @@ int main(int argc, char *argv[]) {
options.allow_unrecognised_options()
.add_options()
("k, size", "Plot size", cxxopts::value<uint8_t>(k))
("t, tempdir", "Temporary directory", cxxopts::value<string>(tempdir))
("d, finaldir", "Final directory", cxxopts::value<string>(finaldir))
("f, file", "Filename", cxxopts::value<string>(filename))
("m, memo", "Memo to insert into the plot", cxxopts::value<string>(memo))
("i, id", "Unique 32-byte seed for the plot", cxxopts::value<string>(id))
@ -82,6 +86,7 @@ int main(int argc, char *argv[]) {
HelpAndQuit(options);
}
operation = argv[1];
std::cout << "operation" << operation << std::endl;
if (operation == "help") {
HelpAndQuit(options);
@ -101,7 +106,7 @@ int main(int argc, char *argv[]) {
HexToBytes(id, id_bytes);
DiskPlotter plotter = DiskPlotter();
plotter.CreatePlotDisk(filename, k, memo_bytes, 5, id_bytes, 32);
plotter.CreatePlotDisk(tempdir, finaldir, filename, k, memo_bytes, 5, id_bytes, 32);
} else if (operation == "prove") {
if (argc < 3) {
HelpAndQuit(options);

46
lib/chiapos/src/plotter_disk.hpp
View File

@ -25,6 +25,8 @@
#include <vector>
#include <string>
#include <utility>
#include <filesystem>
#include "util.hpp"
#include "encoding.hpp"
@ -71,13 +73,32 @@ class DiskPlotter {
// This method creates a plot on disk with the filename. A temporary file, "plotting" + filename,
// is created and will be larger than the final plot file. This file is deleted at the end of
// the process.
void CreatePlotDisk(std::string filename, uint8_t k, const uint8_t* memo,
void CreatePlotDisk(std::string tmp_dirname, std::string final_dirname, std::string filename,
uint8_t k, const uint8_t* memo,
uint32_t memo_len, const uint8_t* id, uint32_t id_len) {
std::cout << std::endl << "Starting plotting progress into file " << filename << "." << std::endl;
std::cout << std::endl << "Starting plotting progress into temporary dir " << tmp_dirname << "." << std::endl;
std::cout << "Memo: " << Util::HexStr(memo, memo_len) << std::endl;
std::cout << "ID: " << Util::HexStr(id, id_len) << std::endl;
std::cout << "Plot size is: " << static_cast<int>(k) << std::endl;
// Cross platform way to concatenate paths, c++17.
std::filesystem::path tmp_1_filename = std::filesystem::path(tmp_dirname) / std::filesystem::path(filename + ".tmp");
std::filesystem::path tmp_2_filename = std::filesystem::path(tmp_dirname) / std::filesystem::path(filename + ".2.tmp");
std::filesystem::path final_filename = std::filesystem::path(final_dirname) / std::filesystem::path(filename);
// Check if the paths exist
if (!std::filesystem::directory_entry(tmp_dirname).exists()) {
std::string err_string = "Directory " + tmp_dirname + " does not exist";
std::cerr << err_string << std::endl;
throw err_string;
}
if (!std::filesystem::directory_entry(final_dirname).exists()) {
std::string err_string = "Directory " + final_dirname + " does not exist";
std::cerr << err_string << std::endl;
throw err_string;
}
// These variables are used in the WriteParkToFile method. They are preallocatted here
// to save time.
first_line_point_bytes = new uint8_t[CalculateLinePointSize(k)];
@ -88,27 +109,25 @@ class DiskPlotter {
assert(k >= kMinPlotSize);
assert(k <= kMaxPlotSize);
std::string plot_filename = filename + ".tmp";
std::cout << std::endl << "Starting phase 1/4: Forward Propagation..." << std::endl;
Timer p1;
Timer all_phases;
std::vector<uint64_t> results = WritePlotFile(plot_filename, k, id, memo, memo_len);
std::vector<uint64_t> results = WritePlotFile(tmp_1_filename, k, id, memo, memo_len);
p1.PrintElapsed("Time for phase 1 =");
std::cout << std::endl << "Starting phase 2/4: Backpropagation..." << std::endl;
std::cout << std::endl << "Starting phase 2/4: Backpropagation into " << tmp_1_filename << " and " << tmp_2_filename << " ..." << std::endl;
Timer p2;
Backpropagate(filename, plot_filename, k, id, memo, memo_len, results);
Backpropagate(tmp_2_filename, tmp_1_filename, k, id, memo, memo_len, results);
p2.PrintElapsed("Time for phase 2 =");
std::cout << std::endl << "Starting phase 3/4: Compression..." << std::endl;
Timer p3;
Phase3Results res = CompressTables(k, results, filename, plot_filename, id, memo, memo_len);
Phase3Results res = CompressTables(k, results, tmp_2_filename, tmp_1_filename, id, memo, memo_len);
p3.PrintElapsed("Time for phase 3 =");
std::cout << std::endl << "Starting phase 4/4: Write Checkpoint tables..." << std::endl;
Timer p4;
WriteCTables(k, k + 1, filename, plot_filename, res);
WriteCTables(k, k + 1, tmp_2_filename, tmp_1_filename, res);
p4.PrintElapsed("Time for phase 4 =");
std::cout << "Approximate working space used: " <<
@ -117,7 +136,14 @@ class DiskPlotter {
static_cast<double>(res.final_table_begin_pointers[11])/(1024*1024*1024) << " GB" << std::endl;
all_phases.PrintElapsed("Total time =");
remove(plot_filename.c_str());
bool removed_1 = std::filesystem::remove(tmp_1_filename);
std::filesystem::copy(tmp_2_filename, final_filename, std::filesystem::copy_options::overwrite_existing);
bool removed_2 = std::filesystem::remove(tmp_2_filename);
std::cout << "Removed " << tmp_1_filename << "? " << removed_1 << std::endl;
std::cout << "Removed " << tmp_2_filename << "? " << removed_2 << std::endl;
std::cout << "Copied final file to " << final_filename << std::endl;
delete[] first_line_point_bytes;
delete[] park_stubs_bytes;

4
lib/chiapos/tests/test.cpp
View File

@ -364,7 +364,7 @@ void PlotAndTestProofOfSpace(std::string filename, uint32_t iterations, uint8_t
uint32_t expected_success) {
DiskPlotter plotter = DiskPlotter();
uint8_t memo[5] = {1, 2, 3, 4, 5};
plotter.CreatePlotDisk(filename, k, memo, 5, plot_id, 32);
plotter.CreatePlotDisk(".", ".", filename, k, memo, 5, plot_id, 32);
TestProofOfSpace(filename, iterations, k, plot_id, expected_success);
REQUIRE(remove(filename.c_str()) == 0);
}
@ -388,7 +388,7 @@ TEST_CASE("Invalid plot") {
uint8_t memo[5] = {1, 2, 3, 4, 5};
string filename = "invalid-plot.dat";
uint8_t k = 22;
plotter.CreatePlotDisk(filename, k, memo, 5, plot_id_1, 32);
plotter.CreatePlotDisk(".", ".", filename, k, memo, 5, plot_id_1, 32);
DiskProver prover(filename);
uint8_t* proof_data = new uint8_t[8 * k];
uint8_t challenge[32];

4
lib/chiapos/tests/test_python_bindings.py
View File

@ -13,7 +13,7 @@ class TestPythonBindings(unittest.TestCase):
10, 11, 129, 139, 171, 15, 23])
pl = DiskPlotter()
pl.create_plot_disk("./myplot.dat", 21, bytes([1, 2, 3, 4, 5]), plot_seed)
pl.create_plot_disk(".", ".", "myplot.dat", 21, bytes([1, 2, 3, 4, 5]), plot_seed)
pr = DiskProver("./myplot.dat")
total_proofs: int = 0
@ -32,7 +32,7 @@ class TestPythonBindings(unittest.TestCase):
print(f"total proofs {total_proofs} out of {iterations}\
{total_proofs / iterations}")
assert total_proofs == 4647
os.remove("./myplot.dat")
os.remove("myplot.dat")
if __name__ == '__main__':

27
scripts/check_plots.py
View File

@ -20,7 +20,9 @@ def main():
"""
parser = argparse.ArgumentParser(description="Chia plot checking script.")
parser.add_argument("-n", "--num", help="Number of challenges", type=int, default=1000)
parser.add_argument(
"-n", "--num", help="Number of challenges", type=int, default=1000
)
args = parser.parse_args()
v = Verifier()
@ -29,8 +31,9 @@ def main():
for plot_filename, plot_info in plot_config["plots"].items():
plot_seed: bytes32 = ProofOfSpace.calculate_plot_seed(
PublicKey.from_bytes(bytes.fromhex(plot_info["pool_pk"])),
PrivateKey.from_bytes(bytes.fromhex(plot_info["sk"])).get_public_key()
PrivateKey.from_bytes(bytes.fromhex(plot_info["sk"])).get_public_key(),
)
if not os.path.isfile(plot_filename):
# Tries relative path
full_path: str = os.path.join(plot_root, plot_filename)
if not os.path.isfile(full_path):
@ -40,19 +43,29 @@ def main():
print(f"Plot file {full_path} not found.")
continue
pr = DiskProver(full_path)
else:
pr = DiskProver(plot_filename)
total_proofs = 0
try:
for i in range(args.num):
challenge = sha256(i.to_bytes(32, "big")).digest()
for index, quality in enumerate(pr.get_qualities_for_challenge(challenge)):
for index, quality in enumerate(
pr.get_qualities_for_challenge(challenge)
):
proof = pr.get_full_proof(challenge, index)
total_proofs += 1
ver_quality = v.validate_proof(plot_seed, pr.get_size(), challenge, proof)
assert(quality == ver_quality)
ver_quality = v.validate_proof(
plot_seed, pr.get_size(), challenge, proof
)
assert quality == ver_quality
except BaseException as e:
print(f"{type(e)}: {e} error in proving/verifying for plot {plot_filename}")
print(f"{plot_filename}: Proofs {total_proofs} / {args.num}, {round(total_proofs/float(args.num), 4)}")
print(
f"{type(e)}: {e} error in proving/verifying for plot {plot_filename}"
)
print(
f"{plot_filename}: Proofs {total_proofs} / {args.num}, {round(total_proofs/float(args.num), 4)}"
)
else:
print(f"Not plot file found at {plot_config_filename}")

29
scripts/create_plots.py
View File

@ -10,7 +10,6 @@ from definitions import ROOT_DIR
from src.types.proof_of_space import ProofOfSpace
from src.types.sized_bytes import bytes32
plot_root = os.path.join(ROOT_DIR, "plots")
plot_config_filename = os.path.join(ROOT_DIR, "config", "plots.yaml")
key_config_filename = os.path.join(ROOT_DIR, "config", "keys.yaml")
@ -28,11 +27,27 @@ def main():
parser.add_argument(
"-p", "--pool_pub_key", help="Hex public key of pool", type=str, default=""
)
parser.add_argument(
"-t",
"--tmp_dir",
help="Temporary directory for plotting files (relative or absolute)",
type=str,
default="./plots",
)
parser.add_argument(
"-d",
"--final_dir",
help="Final directory for plots (relative or absolute)",
type=str,
default="./plots",
)
# We need the keys file, to access pool keys (if the exist), and the sk_seed.
args = parser.parse_args()
if not os.path.isfile(key_config_filename):
raise RuntimeError("Keys not generated. Run python3.7 ./scripts/regenerate_keys.py.")
raise RuntimeError(
"Keys not generated. Run python3 ./scripts/regenerate_keys.py."
)
# The seed is what will be used to generate a private key for each plot
key_config = safe_load(open(key_config_filename, "r"))
@ -62,13 +77,15 @@ def main():
pool_pk, sk.get_public_key()
)
filename: str = f"plot-{i}-{args.size}-{plot_seed}.dat"
full_path: str = os.path.join(plot_root, filename)
full_path: str = os.path.join(args.final_dir, filename)
if os.path.isfile(full_path):
print(f"Plot {filename} already exists")
else:
# Creates the plot. This will take a long time for larger plots.
plotter: DiskPlotter = DiskPlotter()
plotter.create_plot_disk(full_path, args.size, bytes([]), plot_seed)
plotter.create_plot_disk(
args.tmp_dir, args.final_dir, filename, args.size, bytes([]), plot_seed
)
# Updates the config if necessary.
if os.path.isfile(plot_config_filename):
@ -76,8 +93,8 @@ def main():
else:
plot_config = {"plots": {}}
plot_config_plots_new = deepcopy(plot_config["plots"])
if filename not in plot_config_plots_new:
plot_config_plots_new[filename] = {
if full_path not in plot_config_plots_new:
plot_config_plots_new[full_path] = {
"sk": bytes(sk).hex(),
"pool_pk": bytes(pool_pk).hex(),
}

28
src/harvester.py
View File

@ -60,26 +60,34 @@ class Harvester:
use any plots which don't have one of the pool keys.
"""
for partial_filename, plot_config in self.plot_config["plots"].items():
potential_filenames = [partial_filename]
if "plot_root" in self.config:
filename = os.path.join(self.config["plot_root"], partial_filename)
potential_filenames.append(
os.path.join(self.config["plot_root"], partial_filename)
)
else:
filename = os.path.join(ROOT_DIR, "plots", partial_filename)
potential_filenames.append(
os.path.join(ROOT_DIR, "plots", partial_filename)
)
pool_pubkey = PublicKey.from_bytes(bytes.fromhex(plot_config["pool_pk"]))
# Only use plots that correct pools associated with them
if pool_pubkey in harvester_handshake.pool_pubkeys:
if pool_pubkey not in harvester_handshake.pool_pubkeys:
log.warning(
f"Plot {partial_filename} has a pool key that is not in the farmer's pool_pk list."
)
found = False
for filename in potential_filenames:
if os.path.isfile(filename):
self.provers[partial_filename] = DiskProver(filename)
log.info(
f"Farming plot {filename} of size {self.provers[partial_filename].get_size()}"
)
else:
log.warning(f"Plot at {filename} does not exist.")
else:
log.warning(
f"Plot {filename} has a pool key that is not in the farmer's pool_pk list."
)
found = True
break
if not found:
log.warning(f"Plot at {potential_filenames} does not exist.")
@api_request
async def new_challenge(self, new_challenge: harvester_protocol.NewChallenge):