From 7811d5833faae0ddd33eb723ba7dddc51944d70c Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Wed, 10 Jul 2024 21:33:09 +0000 Subject: [PATCH 01/18] Add commitment timing to bench --- .../cpp/scripts/analyze_client_ivc_bench.py | 106 ++++++++++++++---- .../cpp/scripts/benchmark_client_ivc.sh | 1 + .../plonk_honk_shared/instance_inspector.hpp | 13 +++ .../barretenberg/ultra_honk/oink_prover.cpp | 86 ++++++++++---- 4 files changed, 163 insertions(+), 43 deletions(-) diff --git a/barretenberg/cpp/scripts/analyze_client_ivc_bench.py b/barretenberg/cpp/scripts/analyze_client_ivc_bench.py index 95dcbde7619..117beb21b33 100644 --- a/barretenberg/cpp/scripts/analyze_client_ivc_bench.py +++ b/barretenberg/cpp/scripts/analyze_client_ivc_bench.py @@ -4,6 +4,7 @@ PREFIX = Path("build-op-count-time") IVC_BENCH_JSON = Path("client_ivc_bench.json") BENCHMARK = "ClientIVCBench/Full/6" +# BENCHMARK = "ClientIVCBench/FullStructured/6" # Single out an independent set of functions accounting for most of BENCHMARK's real_time to_keep = [ @@ -71,10 +72,73 @@ print(f"{key:<{max_label_length}}{time_ms:>8.0f} {time_ms/total_time_ms:>8.2%}") -# Relations breakdown -# Note: The timings here are off likely because the tracking is occuring in a hot loop but -# they should be meaningful relative to one another -print('\nRelation contributions (times to be interpreted relatively):') +# # Relations breakdown +# # Note: The timings here are off likely because the tracking is occuring in a hot loop but +# # they should be meaningful relative to one another +# print('\nRelation contributions (times to be interpreted relatively):') +# relations = [ +# "Arithmetic::accumulate(t)", +# "Permutation::accumulate(t)", +# "Lookup::accumulate(t)", +# "DeltaRange::accumulate(t)", +# "Elliptic::accumulate(t)", +# "Auxiliary::accumulate(t)", +# "EccOp::accumulate(t)", +# "DatabusRead::accumulate(t)", +# "PoseidonExt::accumulate(t)", +# "PoseidonInt::accumulate(t)", +# ] +# with open(PREFIX/IVC_BENCH_JSON, "r") as read_file: +# read_result = json.load(read_file) +# for _bench in read_result["benchmarks"]: +# if _bench["name"] == BENCHMARK: +# bench = _bench +# bench_components = dict(filter(lambda x: x[0] in relations, bench.items())) + +# # For each kept time, get the proportion over all kept times. +# sum_of_kept_times_ms = sum(float(time) +# for _, time in bench_components.items())/1e6 +# max_label_length = max(len(label) for label in relations) +# column = {"function": "function", "ms": "ms", "%": "% sum"} +# print( +# f"{column['function']:<{max_label_length}}{column['ms']:>8} {column['%']:>8}") +# for key in relations: +# if key not in bench: +# time_ms = 0 +# else: +# time_ms = bench[key]/1e6 +# print(f"{key:<{max_label_length}}{time_ms:>8.0f} {time_ms/sum_of_kept_times_ms:>8.2%}") +# Print header + +def print_contributions(prefix, ivc_bench_json, benchmark, relations): + + # Read JSON file and extract benchmark + try: + with open(prefix / ivc_bench_json, "r") as read_file: + read_result = json.load(read_file) + bench = next((_bench for _bench in read_result["benchmarks"] if _bench["name"] == benchmark), None) + if not bench: + raise ValueError(f"Benchmark '{benchmark}' not found in the JSON file.") + except FileNotFoundError: + print(f"File not found: {prefix / ivc_bench_json}") + return + + # Filter and sum up kept times + bench_components = {key: bench[key] for key in relations if key in bench} + sum_of_kept_times_ms = sum(float(time) for time in bench_components.values()) / 1e6 + print(f"Total time accounted for (ms): {sum_of_kept_times_ms:>8.0f}") + + # Print results + max_label_length = max(len(label) for label in relations) + column_headers = {"operation": "operation", "ms": "ms", "%": "% sum"} + print(f"{column_headers['operation']:<{max_label_length}}{column_headers['ms']:>8} {column_headers['%']:>8}") + + for key in relations: + time_ms = bench_components.get(key, 0) / 1e6 + percentage = time_ms / sum_of_kept_times_ms if sum_of_kept_times_ms > 0 else 0 + print(f"{key:<{max_label_length}}{time_ms:>8.0f} {percentage:>8.2%}") + + relations = [ "Arithmetic::accumulate(t)", "Permutation::accumulate(t)", @@ -87,23 +151,19 @@ "PoseidonExt::accumulate(t)", "PoseidonInt::accumulate(t)", ] -with open(PREFIX/IVC_BENCH_JSON, "r") as read_file: - read_result = json.load(read_file) - for _bench in read_result["benchmarks"]: - if _bench["name"] == BENCHMARK: - bench = _bench -bench_components = dict(filter(lambda x: x[0] in relations, bench.items())) -# For each kept time, get the proportion over all kept times. -sum_of_kept_times_ms = sum(float(time) - for _, time in bench_components.items())/1e6 -max_label_length = max(len(label) for label in relations) -column = {"function": "function", "ms": "ms", "%": "% sum"} -print( - f"{column['function']:<{max_label_length}}{column['ms']:>8} {column['%']:>8}") -for key in relations: - if key not in bench: - time_ms = 0 - else: - time_ms = bench[key]/1e6 - print(f"{key:<{max_label_length}}{time_ms:>8.0f} {time_ms/sum_of_kept_times_ms:>8.2%}") \ No newline at end of file +print('\nRelation contributions (times to be interpreted relatively):') +print_contributions(PREFIX, IVC_BENCH_JSON, BENCHMARK, relations) + +commitments = [ + "COMMIT::wires(t)", + "COMMIT::z_perm(t)", + "COMMIT::databus(t)", + "COMMIT::ecc_op_wires(t)", + "COMMIT::lookup_inverses(t)", + "COMMIT::databus_inverses(t)", + "COMMIT::lookup_counts_tags(t)", +] + +print('\nCommitment contributions:') +print_contributions(PREFIX, IVC_BENCH_JSON, BENCHMARK, commitments) \ No newline at end of file diff --git a/barretenberg/cpp/scripts/benchmark_client_ivc.sh b/barretenberg/cpp/scripts/benchmark_client_ivc.sh index 7991ef87940..e30808e80d5 100755 --- a/barretenberg/cpp/scripts/benchmark_client_ivc.sh +++ b/barretenberg/cpp/scripts/benchmark_client_ivc.sh @@ -4,6 +4,7 @@ set -eu TARGET="client_ivc_bench" # Note: to run structured trace version, change "Full" to "FullStructured" here and in analyze script FILTER="ClientIVCBench/Full/6$" +# FILTER="ClientIVCBench/FullStructured/6$" BUILD_DIR=build-op-count-time # Move above script dir. diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/instance_inspector.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/instance_inspector.hpp index add3db82ec1..6eba9860987 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/instance_inspector.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/instance_inspector.hpp @@ -1,6 +1,7 @@ #pragma once #include "barretenberg/common/log.hpp" +#include namespace bb::instance_inspector { @@ -15,6 +16,18 @@ bool is_non_zero(auto& polynomial) return false; } +// Determine whether a polynomial has at least one non-zero coefficient +void count_non_zero(auto& polynomial) +{ + uint32_t num_non_zero = 0; + for (auto& coeff : polynomial) { + if (!coeff.is_zero()) { + num_non_zero++; + } + } + info("Poly of size ", polynomial.size(), " has ", num_non_zero, " non-zero coefficients."); +} + /** * @brief Utility for indicating which polynomials in a prover instance are identically zero * diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp index 6626b48623f..8dc81064db3 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp @@ -1,4 +1,5 @@ #include "barretenberg/ultra_honk/oink_prover.hpp" +#include "barretenberg/plonk_honk_shared/instance_inspector.hpp" #include "barretenberg/relations/logderiv_lookup_relation.hpp" namespace bb { @@ -66,9 +67,17 @@ template void OinkProver::execute_wire_commitment { // Commit to the first three wire polynomials of the instance // We only commit to the fourth wire polynomial after adding memory recordss - witness_commitments.w_l = commitment_key->commit(proving_key.polynomials.w_l); - witness_commitments.w_r = commitment_key->commit(proving_key.polynomials.w_r); - witness_commitments.w_o = commitment_key->commit(proving_key.polynomials.w_o); + { + BB_OP_COUNT_TIME_NAME("COMMIT::wires"); + witness_commitments.w_l = commitment_key->commit(proving_key.polynomials.w_l); + witness_commitments.w_r = commitment_key->commit(proving_key.polynomials.w_r); + witness_commitments.w_o = commitment_key->commit(proving_key.polynomials.w_o); + } + + info("Wires:"); + instance_inspector::count_non_zero(proving_key.polynomials.w_l); + instance_inspector::count_non_zero(proving_key.polynomials.w_r); + instance_inspector::count_non_zero(proving_key.polynomials.w_o); auto wire_comms = witness_commitments.get_wires(); auto wire_labels = commitment_labels.get_wires(); @@ -78,10 +87,19 @@ template void OinkProver::execute_wire_commitment if constexpr (IsGoblinFlavor) { // Commit to Goblin ECC op wires - witness_commitments.ecc_op_wire_1 = commitment_key->commit(proving_key.polynomials.ecc_op_wire_1); - witness_commitments.ecc_op_wire_2 = commitment_key->commit(proving_key.polynomials.ecc_op_wire_2); - witness_commitments.ecc_op_wire_3 = commitment_key->commit(proving_key.polynomials.ecc_op_wire_3); - witness_commitments.ecc_op_wire_4 = commitment_key->commit(proving_key.polynomials.ecc_op_wire_4); + { + BB_OP_COUNT_TIME_NAME("COMMIT::ecc_op_wires"); + witness_commitments.ecc_op_wire_1 = commitment_key->commit(proving_key.polynomials.ecc_op_wire_1); + witness_commitments.ecc_op_wire_2 = commitment_key->commit(proving_key.polynomials.ecc_op_wire_2); + witness_commitments.ecc_op_wire_3 = commitment_key->commit(proving_key.polynomials.ecc_op_wire_3); + witness_commitments.ecc_op_wire_4 = commitment_key->commit(proving_key.polynomials.ecc_op_wire_4); + } + + info("ECC op wires:"); + instance_inspector::count_non_zero(proving_key.polynomials.ecc_op_wire_1); + instance_inspector::count_non_zero(proving_key.polynomials.ecc_op_wire_2); + instance_inspector::count_non_zero(proving_key.polynomials.ecc_op_wire_3); + instance_inspector::count_non_zero(proving_key.polynomials.ecc_op_wire_4); auto op_wire_comms = witness_commitments.get_ecc_op_wires(); auto labels = commitment_labels.get_ecc_op_wires(); @@ -90,14 +108,19 @@ template void OinkProver::execute_wire_commitment } // Commit to DataBus columns and corresponding read counts - witness_commitments.calldata = commitment_key->commit(proving_key.polynomials.calldata); - witness_commitments.calldata_read_counts = commitment_key->commit(proving_key.polynomials.calldata_read_counts); + { + BB_OP_COUNT_TIME_NAME("COMMIT::databus"); + witness_commitments.calldata = commitment_key->commit(proving_key.polynomials.calldata); + witness_commitments.calldata_read_counts = + commitment_key->commit(proving_key.polynomials.calldata_read_counts); + witness_commitments.return_data = commitment_key->commit(proving_key.polynomials.return_data); + witness_commitments.return_data_read_counts = + commitment_key->commit(proving_key.polynomials.return_data_read_counts); + } + transcript->send_to_verifier(domain_separator + commitment_labels.calldata, witness_commitments.calldata); transcript->send_to_verifier(domain_separator + commitment_labels.calldata_read_counts, witness_commitments.calldata_read_counts); - witness_commitments.return_data = commitment_key->commit(proving_key.polynomials.return_data); - witness_commitments.return_data_read_counts = - commitment_key->commit(proving_key.polynomials.return_data_read_counts); transcript->send_to_verifier(domain_separator + commitment_labels.return_data, witness_commitments.return_data); transcript->send_to_verifier(domain_separator + commitment_labels.return_data_read_counts, witness_commitments.return_data_read_counts); @@ -121,9 +144,15 @@ template void OinkProver::execute_sorted_list_acc relation_parameters.eta, relation_parameters.eta_two, relation_parameters.eta_three); // Commit to lookup argument polynomials and the finalized (i.e. with memory records) fourth wire polynomial - witness_commitments.lookup_read_counts = commitment_key->commit(proving_key.polynomials.lookup_read_counts); - witness_commitments.lookup_read_tags = commitment_key->commit(proving_key.polynomials.lookup_read_tags); - witness_commitments.w_4 = commitment_key->commit(proving_key.polynomials.w_4); + { + BB_OP_COUNT_TIME_NAME("COMMIT::lookup_counts_tags"); + witness_commitments.lookup_read_counts = commitment_key->commit(proving_key.polynomials.lookup_read_counts); + witness_commitments.lookup_read_tags = commitment_key->commit(proving_key.polynomials.lookup_read_tags); + } + { + BB_OP_COUNT_TIME_NAME("COMMIT::wires"); + witness_commitments.w_4 = commitment_key->commit(proving_key.polynomials.w_4); + } transcript->send_to_verifier(domain_separator + commitment_labels.lookup_read_counts, witness_commitments.lookup_read_counts); @@ -145,18 +174,31 @@ template void OinkProver::execute_log_derivative_ // Compute the inverses used in log-derivative lookup relations proving_key.compute_logderivative_inverses(relation_parameters); - witness_commitments.lookup_inverses = commitment_key->commit(proving_key.polynomials.lookup_inverses); + { + BB_OP_COUNT_TIME_NAME("COMMIT::lookup_inverses"); + witness_commitments.lookup_inverses = commitment_key->commit(proving_key.polynomials.lookup_inverses); + } transcript->send_to_verifier(domain_separator + commitment_labels.lookup_inverses, witness_commitments.lookup_inverses); + info("Lookup inverses:"); + instance_inspector::count_non_zero(proving_key.polynomials.lookup_inverses); + // If Mega, commit to the databus inverse polynomials and send if constexpr (IsGoblinFlavor) { - witness_commitments.calldata_inverses = commitment_key->commit(proving_key.polynomials.calldata_inverses); - witness_commitments.return_data_inverses = commitment_key->commit(proving_key.polynomials.return_data_inverses); + { + BB_OP_COUNT_TIME_NAME("COMMIT::databus_inverses"); + witness_commitments.calldata_inverses = commitment_key->commit(proving_key.polynomials.calldata_inverses); + witness_commitments.return_data_inverses = + commitment_key->commit(proving_key.polynomials.return_data_inverses); + } transcript->send_to_verifier(domain_separator + commitment_labels.calldata_inverses, witness_commitments.calldata_inverses); transcript->send_to_verifier(domain_separator + commitment_labels.return_data_inverses, witness_commitments.return_data_inverses); + info("Databus inverses:"); + instance_inspector::count_non_zero(proving_key.polynomials.calldata_inverses); + instance_inspector::count_non_zero(proving_key.polynomials.return_data_inverses); } } @@ -168,8 +210,12 @@ template void OinkProver::execute_grand_product_c { proving_key.compute_grand_product_polynomials(relation_parameters); - witness_commitments.z_perm = commitment_key->commit(proving_key.polynomials.z_perm); - + { + BB_OP_COUNT_TIME_NAME("COMMIT::z_perm"); + witness_commitments.z_perm = commitment_key->commit(proving_key.polynomials.z_perm); + } + info("Z_Perm:"); + instance_inspector::count_non_zero(proving_key.polynomials.z_perm); transcript->send_to_verifier(domain_separator + commitment_labels.z_perm, witness_commitments.z_perm); } From 958a6820d0e051e77d0c74cd834cd544aedea7b8 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Thu, 11 Jul 2024 22:28:49 +0000 Subject: [PATCH 02/18] constructing an srs view and computing commitments on reduced inputs --- .../commitment_schemes/commitment_key.hpp | 28 +++ .../processed_commitment.test.cpp | 197 ++++++++++++++++++ .../srs/factories/file_crs_factory.hpp | 1 + 3 files changed, 226 insertions(+) create mode 100644 barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp index 934b6dae9de..9a8f29c42d6 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp @@ -18,6 +18,7 @@ #include #include +#include #include namespace bb { @@ -34,6 +35,13 @@ template class CommitmentKey { using Fr = typename Curve::ScalarField; using Commitment = typename Curve::AffineElement; + using G1 = typename Curve::AffineElement; + + using IndicesView = std::ranges::iota_view; + using EvenIndicesView = std::ranges::filter_view>; + using SrsViewType = std::ranges::transform_view>; + + SrsViewType srs_view; public: scalar_multiplication::pippenger_runtime_state pippenger_runtime_state; @@ -75,6 +83,26 @@ template class CommitmentKey { return scalar_multiplication::pippenger_unsafe( const_cast(polynomial.data()), srs->get_monomial_points(), degree, pippenger_runtime_state); }; + + auto get_srs_view() + { + size_t point_table_size = 2 * srs->get_monomial_size(); + G1* point_table = srs->get_monomial_points(); + + auto srs_view = + std::views::iota(static_cast(0), point_table_size) | // generate view of indices 0, ..., n-1 + std::views::filter([](int i) { return i % 2 == 0; }) | // create a view of the even indices only + std::views::transform([point_table](size_t i) { return point_table[i]; }); // extract even srs + + info("Internal points:"); + for (auto element : srs_view) { + info(element); + } + + return srs_view; + } + + // SrsViewType get_srs_view() const { return srs_view; } }; } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp new file mode 100644 index 00000000000..352837a59a9 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp @@ -0,0 +1,197 @@ +#include "barretenberg/commitment_schemes/commitment_key.hpp" +#include "barretenberg/commitment_schemes/verification_key.hpp" +#include "barretenberg/common/zip_view.hpp" +#include "barretenberg/ecc/curves/bn254/g1.hpp" +#include "barretenberg/polynomials/polynomial.hpp" +#include "barretenberg/srs/factories/file_crs_factory.hpp" +#include "claim.hpp" +#include +#include + +#include + +namespace bb { + +// constexpr size_t COMMITMENT_TEST_NUM_POINTS = 4096; + +template class CommitmentKeyTest : public ::testing::Test { + using CK = CommitmentKey; + + using Fr = typename Curve::ScalarField; + using Commitment = typename Curve::AffineElement; + using Polynomial = bb::Polynomial; + + public: + Polynomial random_polynomial(const size_t n) + { + Polynomial p(n); + for (size_t i = 0; i < n; ++i) { + p[i] = Fr::random_element(); + } + return p; + } + + template inline std::shared_ptr create_commitment_key(const size_t num_points); + + template <> + inline std::shared_ptr> create_commitment_key>( + const size_t num_points) + { + srs::init_crs_factory("../srs_db/ignition"); + return std::make_shared>(num_points); + } +}; + +using Curves = ::testing::Types; + +TYPED_TEST_SUITE(CommitmentKeyTest, Curves); + +TYPED_TEST(CommitmentKeyTest, CommitSimple) +{ + using Curve = TypeParam; + using CK = CommitmentKey; + using G1 = Curve::AffineElement; + using Fr = typename Curve::ScalarField; + using Polynomial = bb::Polynomial; + + const size_t num_points = 10; + + auto key = TestFixture::template create_commitment_key(num_points); + + const size_t point_table_size = 2 * num_points; + std::span point_table(key->srs->get_monomial_points(), point_table_size); + + std::vector srs; + + size_t idx = 0; + for (auto& element : point_table) { + if (idx % 2 == 0) { + srs.emplace_back(element); + } + idx++; + } + + const size_t num_nonzero = 2; + Polynomial poly{ num_points }; + for (size_t i = 0; i < num_nonzero; ++i) { + size_t idx = (i + 1) * (i + 1) % num_points; + poly[idx] = Fr::random_element(); + } + + G1 normal_result = key->commit(poly); + + std::vector scalars; + std::vector points; + + for (auto [scalar, point] : zip_view(poly, srs)) { + if (!scalar.is_zero()) { + scalars.emplace_back(scalar); + points.emplace_back(point); + } + } + + EXPECT_EQ(points.size(), num_nonzero); + + G1 reduced_result; + reduced_result.self_set_infinity(); + for (auto [scalar, point] : zip_view(scalars, points)) { + reduced_result = reduced_result + point * scalar; + } + + EXPECT_EQ(reduced_result, normal_result); + + G1 pippenger_result = scalar_multiplication::pippenger_without_endomorphism_basis_points( + scalars.data(), points.data(), num_nonzero, key->pippenger_runtime_state); + + EXPECT_EQ(pippenger_result, normal_result); +} + +TYPED_TEST(CommitmentKeyTest, SrsView) +{ + using Curve = TypeParam; + using CK = CommitmentKey; + using G1 = Curve::AffineElement; + + const size_t num_points = 10; + + auto key = TestFixture::template create_commitment_key(num_points); + + const size_t point_table_size = 2 * num_points; + std::span point_table(key->srs->get_monomial_points(), point_table_size); + + // Manually construct the SRS from the point table + std::vector srs; + size_t idx = 0; + for (auto& element : point_table) { + if (idx % 2 == 0) { + srs.emplace_back(element); + } + idx++; + } + + // Create a view of the even elements of the point table (i.e. the raw srs points) + auto srs_view = std::views::iota(static_cast(0), point_table_size) | // generate view of indices 0, ..., n-1 + std::views::filter([](int i) { return i % 2 == 0; }) | // create a view of the even indices only + std::views::transform([&point_table](size_t i) { return point_table[i]; }); // extract even srs + + // Check that the two agree + size_t point_idx = 0; + for (auto element : srs_view) { + EXPECT_EQ(srs[point_idx], element); + point_idx++; + } +} + +TYPED_TEST(CommitmentKeyTest, SrsViewCommit) +{ + using Curve = TypeParam; + using CK = CommitmentKey; + using G1 = Curve::AffineElement; + using Fr = Curve::ScalarField; + using Polynomial = bb::Polynomial; + + const size_t num_points = 10; + + auto key = TestFixture::template create_commitment_key(num_points); + + const size_t num_nonzero = 2; + Polynomial poly{ num_points }; + for (size_t i = 0; i < num_nonzero; ++i) { + size_t idx = (i + 1) * (i + 1) % num_points; + poly[idx] = Fr::random_element(); + } + + G1 normal_result = key->commit(poly); + + // Create a view of the even elements of the point table (i.e. the raw srs points) + auto srs_view = key->get_srs_view(); + + std::vector scalars; + std::vector points; + + size_t point_idx = 0; + for (auto point : srs_view) { + Fr& scalar = poly[point_idx++]; + if (!scalar.is_zero()) { + scalars.emplace_back(scalar); + points.emplace_back(point); + } + } + + EXPECT_EQ(points.size(), num_nonzero); + + G1 reduced_result; + reduced_result.self_set_infinity(); + for (auto [scalar, point] : zip_view(scalars, points)) { + reduced_result = reduced_result + point * scalar; + } + + EXPECT_EQ(reduced_result, normal_result); + + G1 pippenger_result = scalar_multiplication::pippenger_without_endomorphism_basis_points( + scalars.data(), points.data(), num_nonzero, key->pippenger_runtime_state); + + EXPECT_EQ(pippenger_result, normal_result); +} + +} // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/srs/factories/file_crs_factory.hpp b/barretenberg/cpp/src/barretenberg/srs/factories/file_crs_factory.hpp index 149d9794098..e769fc7980e 100644 --- a/barretenberg/cpp/src/barretenberg/srs/factories/file_crs_factory.hpp +++ b/barretenberg/cpp/src/barretenberg/srs/factories/file_crs_factory.hpp @@ -37,6 +37,7 @@ template class FileProverCrs : public ProverCrs { monomials_ = scalar_multiplication::point_table_alloc(num_points); srs::IO::read_transcript_g1(monomials_.get(), num_points, path); + // WORKTODO scalar_multiplication::generate_pippenger_point_table(monomials_.get(), monomials_.get(), num_points); }; From c62842b20b2b666cdcaebff79d34bd2779df6cc9 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Mon, 15 Jul 2024 18:12:16 +0000 Subject: [PATCH 03/18] add commit bench from my other branch --- .../commitment_schemes/commit.bench.cpp | 122 +++++++++++++++++- 1 file changed, 117 insertions(+), 5 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp index c37d893e125..70d8861605b 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp @@ -1,31 +1,143 @@ #include "barretenberg/commitment_schemes/commitment_key.hpp" +#include "barretenberg/common/zip_view.hpp" #include "barretenberg/srs/factories/mem_bn254_crs_factory.hpp" +#include #include +#include +#include +#include namespace bb { template std::shared_ptr> create_commitment_key(const size_t num_points) { + bb::srs::init_crs_factory("../srs_db/ignition"); std::string srs_path; return std::make_shared>(num_points); } -constexpr size_t MAX_LOG_NUM_POINTS = 24; +template +std::vector extract_srs(std::shared_ptr> commitment_key, + const size_t num_points) +{ + using G1 = typename Curve::AffineElement; + std::vector monomials; + size_t idx = 0; + std::span point_table(commitment_key->srs->get_monomial_points(), 2 * num_points); + + for (auto& element : point_table) { + if (idx % 2 == 0) { + monomials.emplace_back(element); + } + idx++; + if (monomials.size() == num_points) { + break; + } + } + return monomials; +} + +constexpr size_t MAX_LOG_NUM_POINTS = 18; constexpr size_t MAX_NUM_POINTS = 1 << MAX_LOG_NUM_POINTS; -auto key = create_commitment_key(MAX_NUM_POINTS); +// auto key = create_commitment_key(MAX_NUM_POINTS); -template void bench_commit(::benchmark::State& state) +auto key() +{ + return create_commitment_key(MAX_NUM_POINTS); +} + +template void bench_commit_zero(::benchmark::State& state) { const size_t num_points = 1 << state.range(0); const auto polynomial = Polynomial(num_points); for (auto _ : state) { - benchmark::DoNotOptimize(key->commit(polynomial)); + key()->commit(polynomial); + // benchmark::DoNotOptimize(key()->commit(polynomial)); + } +} + +template void bench_commit_sparse(::benchmark::State& state) +{ + using Fr = typename Curve::ScalarField; + const size_t num_points = 1 << state.range(0); + auto polynomial = Polynomial(num_points); + polynomial[25] = 1; + polynomial[22] = 1; + for (auto _ : state) { + key()->commit(polynomial); + // benchmark::DoNotOptimize(key()->commit(polynomial)); + } +} + +template void bench_commit_sparse_processed(::benchmark::State& state) +{ + using G1 = typename Curve::AffineElement; + using Fr = typename Curve::ScalarField; + const size_t num_points = 1 << state.range(0); + auto polynomial = Polynomial(num_points); + polynomial[25] = 1; + polynomial[22] = 1; + + auto srs = extract_srs(key(), num_points); + + ASSERT(polynomial.size() == srs.size()); + + // auto zipped = zip_view(polynomial, srs); + + std::vector scalars; + std::vector points; + + for (auto [scalar, point] : zip_view(polynomial, srs)) { + if (!scalar.is_zero()) { + scalars.emplace_back(scalar); + points.emplace_back(point); + } + } + + info(scalars.size()); + + for (auto _ : state) { + benchmark::DoNotOptimize(key()->commit(polynomial)); + } +} + +template void bench_commit_sparse_random(::benchmark::State& state) +{ + using Fr = typename Curve::ScalarField; + const size_t num_points = 1 << state.range(0); + auto polynomial = Polynomial(num_points); + polynomial[25] = Fr::random_element(); + polynomial[22] = Fr::random_element(); + polynomial[21] = Fr::random_element(); + for (auto _ : state) { + key()->commit(polynomial); + // benchmark::DoNotOptimize(key()->commit(polynomial)); + } +} + +template void bench_commit_random(::benchmark::State& state) +{ + using Fr = typename Curve::ScalarField; + const size_t num_points = 1 << state.range(0); + auto polynomial = Polynomial(num_points); + for (auto& coeff : polynomial) { + coeff = Fr::random_element(); + } + for (auto _ : state) { + key()->commit(polynomial); + // benchmark::DoNotOptimize(key()->commit(polynomial)); } } -BENCHMARK(bench_commit)->DenseRange(10, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); +BENCHMARK(bench_commit_zero)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); +BENCHMARK(bench_commit_sparse)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); +// BENCHMARK(bench_commit_sparse_processed) +// ->DenseRange(14, MAX_LOG_NUM_POINTS) +// ->Unit(benchmark::kMillisecond); +BENCHMARK(bench_commit_sparse_random)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); +BENCHMARK(bench_commit_random)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); } // namespace bb From 6e940fd29f63b16f6d50304ceb7f2053593d2e46 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Mon, 15 Jul 2024 18:36:40 +0000 Subject: [PATCH 04/18] clean up commit bench a bit --- .../commitment_schemes/commit.bench.cpp | 96 ++++++++++--------- 1 file changed, 53 insertions(+), 43 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp index 70d8861605b..b2429cdf16a 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp @@ -17,6 +17,17 @@ template std::shared_ptr> create_commitmen return std::make_shared>(num_points); } +template +Polynomial sparse_random_poly(const size_t size, const size_t num_nonzero) +{ + using Fr = typename Curve::ScalarField; + auto polynomial = Polynomial(size); + + for (size_t i = 0; i < num_nonzero; i++) { + polynomial[i] = Fr::random_element(); + } +} + template std::vector extract_srs(std::shared_ptr> commitment_key, const size_t num_points) @@ -41,93 +52,92 @@ std::vector extract_srs(std::shared_ptr(MAX_NUM_POINTS); - -auto key() -{ - return create_commitment_key(MAX_NUM_POINTS); -} - template void bench_commit_zero(::benchmark::State& state) { + auto key = create_commitment_key(MAX_NUM_POINTS); + const size_t num_points = 1 << state.range(0); const auto polynomial = Polynomial(num_points); for (auto _ : state) { - key()->commit(polynomial); - // benchmark::DoNotOptimize(key()->commit(polynomial)); + key->commit(polynomial); } } template void bench_commit_sparse(::benchmark::State& state) { using Fr = typename Curve::ScalarField; + auto key = create_commitment_key(MAX_NUM_POINTS); + const size_t num_points = 1 << state.range(0); auto polynomial = Polynomial(num_points); polynomial[25] = 1; polynomial[22] = 1; for (auto _ : state) { - key()->commit(polynomial); - // benchmark::DoNotOptimize(key()->commit(polynomial)); + key->commit(polynomial); } } -template void bench_commit_sparse_processed(::benchmark::State& state) -{ - using G1 = typename Curve::AffineElement; - using Fr = typename Curve::ScalarField; - const size_t num_points = 1 << state.range(0); - auto polynomial = Polynomial(num_points); - polynomial[25] = 1; - polynomial[22] = 1; +// template void bench_commit_sparse_processed(::benchmark::State& state) +// { +// using G1 = typename Curve::AffineElement; +// using Fr = typename Curve::ScalarField; +// auto key = create_commitment_key(MAX_NUM_POINTS); - auto srs = extract_srs(key(), num_points); +// const size_t num_points = 1 << state.range(0); +// const size_t num_nonzero = 4; - ASSERT(polynomial.size() == srs.size()); +// auto polynomial = sparse_random_poly(num_points, num_nonzero); - // auto zipped = zip_view(polynomial, srs); +// auto srs_view = key->get_srs_view(); - std::vector scalars; - std::vector points; +// ASSERT(polynomial.size() == srs_view.size()); - for (auto [scalar, point] : zip_view(polynomial, srs)) { - if (!scalar.is_zero()) { - scalars.emplace_back(scalar); - points.emplace_back(point); - } - } +// // auto zipped = zip_view(polynomial, srs); - info(scalars.size()); +// std::vector scalars; +// std::vector points; - for (auto _ : state) { - benchmark::DoNotOptimize(key()->commit(polynomial)); - } -} +// for (auto [scalar, point] : zip_view(polynomial, srs)) { +// if (!scalar.is_zero()) { +// scalars.emplace_back(scalar); +// points.emplace_back(point); +// } +// } + +// info(scalars.size()); + +// for (auto _ : state) { +// benchmark::DoNotOptimize(key->commit(polynomial)); +// } +// } template void bench_commit_sparse_random(::benchmark::State& state) { using Fr = typename Curve::ScalarField; + auto key = create_commitment_key(MAX_NUM_POINTS); + const size_t num_points = 1 << state.range(0); - auto polynomial = Polynomial(num_points); - polynomial[25] = Fr::random_element(); - polynomial[22] = Fr::random_element(); - polynomial[21] = Fr::random_element(); + const size_t num_nonzero = 4; + + auto polynomial = sparse_random_poly(num_points, num_nonzero); + for (auto _ : state) { - key()->commit(polynomial); - // benchmark::DoNotOptimize(key()->commit(polynomial)); + key->commit(polynomial); } } template void bench_commit_random(::benchmark::State& state) { using Fr = typename Curve::ScalarField; + auto key = create_commitment_key(MAX_NUM_POINTS); + const size_t num_points = 1 << state.range(0); auto polynomial = Polynomial(num_points); for (auto& coeff : polynomial) { coeff = Fr::random_element(); } for (auto _ : state) { - key()->commit(polynomial); - // benchmark::DoNotOptimize(key()->commit(polynomial)); + key->commit(polynomial); } } From 170ac6394b5e4a808890cfd6552431b6311d7f8f Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Mon, 15 Jul 2024 19:01:26 +0000 Subject: [PATCH 05/18] commit_sparse method with test and benchmark --- .../commitment_schemes/commit.bench.cpp | 65 +++++++------------ .../commitment_schemes/commitment_key.hpp | 29 +++++++-- .../processed_commitment.test.cpp | 26 ++++++++ 3 files changed, 72 insertions(+), 48 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp index b2429cdf16a..163ebea623a 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp @@ -17,15 +17,15 @@ template std::shared_ptr> create_commitmen return std::make_shared>(num_points); } -template -Polynomial sparse_random_poly(const size_t size, const size_t num_nonzero) +template Polynomial sparse_random_poly(const size_t size, const size_t num_nonzero) { - using Fr = typename Curve::ScalarField; - auto polynomial = Polynomial(size); + auto polynomial = Polynomial(size); for (size_t i = 0; i < num_nonzero; i++) { - polynomial[i] = Fr::random_element(); + polynomial[i] = FF::random_element(); } + + return polynomial; } template @@ -77,52 +77,33 @@ template void bench_commit_sparse(::benchmark::State& state) } } -// template void bench_commit_sparse_processed(::benchmark::State& state) -// { -// using G1 = typename Curve::AffineElement; -// using Fr = typename Curve::ScalarField; -// auto key = create_commitment_key(MAX_NUM_POINTS); - -// const size_t num_points = 1 << state.range(0); -// const size_t num_nonzero = 4; - -// auto polynomial = sparse_random_poly(num_points, num_nonzero); - -// auto srs_view = key->get_srs_view(); - -// ASSERT(polynomial.size() == srs_view.size()); - -// // auto zipped = zip_view(polynomial, srs); - -// std::vector scalars; -// std::vector points; +template void bench_commit_sparse_random(::benchmark::State& state) +{ + using Fr = typename Curve::ScalarField; + auto key = create_commitment_key(MAX_NUM_POINTS); -// for (auto [scalar, point] : zip_view(polynomial, srs)) { -// if (!scalar.is_zero()) { -// scalars.emplace_back(scalar); -// points.emplace_back(point); -// } -// } + const size_t num_points = 1 << state.range(0); + const size_t num_nonzero = 100; -// info(scalars.size()); + auto polynomial = sparse_random_poly(num_points, num_nonzero); -// for (auto _ : state) { -// benchmark::DoNotOptimize(key->commit(polynomial)); -// } -// } + for (auto _ : state) { + key->commit(polynomial); + } +} -template void bench_commit_sparse_random(::benchmark::State& state) +template void bench_commit_sparse_random_preprocessed(::benchmark::State& state) { using Fr = typename Curve::ScalarField; auto key = create_commitment_key(MAX_NUM_POINTS); const size_t num_points = 1 << state.range(0); - const size_t num_nonzero = 4; + const size_t num_nonzero = 100; - auto polynomial = sparse_random_poly(num_points, num_nonzero); + auto polynomial = sparse_random_poly(num_points, num_nonzero); for (auto _ : state) { - key->commit(polynomial); + key->commit_sparse(polynomial); } } @@ -143,10 +124,10 @@ template void bench_commit_random(::benchmark::State& state) BENCHMARK(bench_commit_zero)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); BENCHMARK(bench_commit_sparse)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); -// BENCHMARK(bench_commit_sparse_processed) -// ->DenseRange(14, MAX_LOG_NUM_POINTS) -// ->Unit(benchmark::kMillisecond); BENCHMARK(bench_commit_sparse_random)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); +BENCHMARK(bench_commit_sparse_random_preprocessed) + ->DenseRange(14, MAX_LOG_NUM_POINTS) + ->Unit(benchmark::kMillisecond); BENCHMARK(bench_commit_random)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp index 9a8f29c42d6..b2bf49f5b81 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp @@ -94,15 +94,32 @@ template class CommitmentKey { std::views::filter([](int i) { return i % 2 == 0; }) | // create a view of the even indices only std::views::transform([point_table](size_t i) { return point_table[i]; }); // extract even srs - info("Internal points:"); - for (auto element : srs_view) { - info(element); - } - return srs_view; } - // SrsViewType get_srs_view() const { return srs_view; } + Commitment commit_sparse(std::span polynomial) + { + // BB_OP_COUNT_TIME(); + const size_t degree = polynomial.size(); + ASSERT(degree <= srs->get_monomial_size()); + // Create a view of the even elements of the point table (i.e. the raw srs points) + auto srs_view = get_srs_view(); + + std::vector scalars; + std::vector points; + + size_t point_idx = 0; + for (auto point : srs_view) { + const Fr& scalar = polynomial[point_idx++]; + if (!scalar.is_zero()) { + scalars.emplace_back(scalar); + points.emplace_back(point); + } + } + + return scalar_multiplication::pippenger_without_endomorphism_basis_points( + scalars.data(), points.data(), scalars.size(), pippenger_runtime_state); + } }; } // namespace bb diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp index 352837a59a9..62e66d582f9 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp @@ -194,4 +194,30 @@ TYPED_TEST(CommitmentKeyTest, SrsViewCommit) EXPECT_EQ(pippenger_result, normal_result); } +TYPED_TEST(CommitmentKeyTest, SrsViewCommitSparse) +{ + using Curve = TypeParam; + using CK = CommitmentKey; + using G1 = Curve::AffineElement; + using Fr = Curve::ScalarField; + using Polynomial = bb::Polynomial; + + const size_t num_points = 10; + + auto key = TestFixture::template create_commitment_key(num_points); + + const size_t num_nonzero = 2; + Polynomial poly{ num_points }; + for (size_t i = 0; i < num_nonzero; ++i) { + size_t idx = (i + 1) * (i + 1) % num_points; + poly[idx] = Fr::random_element(); + } + + G1 normal_result = key->commit(poly); + + G1 pippenger_result = key->commit_sparse(poly); + + EXPECT_EQ(pippenger_result, normal_result); +} + } // namespace bb From c8055e0fd6957aa4e49bcf2f1bbb88f7925e521f Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Mon, 15 Jul 2024 22:57:14 +0000 Subject: [PATCH 06/18] multithreaded commit sparse --- .../commitment_schemes/commit.bench.cpp | 33 ++++++++++++++++--- .../commitment_schemes/commitment_key.hpp | 33 ++++++++++++++----- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp index 163ebea623a..92092cc44d2 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp @@ -69,21 +69,43 @@ template void bench_commit_sparse(::benchmark::State& state) auto key = create_commitment_key(MAX_NUM_POINTS); const size_t num_points = 1 << state.range(0); + const size_t num_nonzero = 2; + auto polynomial = Polynomial(num_points); - polynomial[25] = 1; - polynomial[22] = 1; + for (size_t i = 0; i < num_nonzero; i++) { + polynomial[i] = 1; + } + for (auto _ : state) { key->commit(polynomial); } } +template void bench_commit_sparse_preprocessed(::benchmark::State& state) +{ + using Fr = typename Curve::ScalarField; + auto key = create_commitment_key(MAX_NUM_POINTS); + + const size_t num_points = 1 << state.range(0); + const size_t num_nonzero = 2; + + auto polynomial = Polynomial(num_points); + for (size_t i = 0; i < num_nonzero; i++) { + polynomial[i] = 1; + } + + for (auto _ : state) { + key->commit_sparse(polynomial); + } +} + template void bench_commit_sparse_random(::benchmark::State& state) { using Fr = typename Curve::ScalarField; auto key = create_commitment_key(MAX_NUM_POINTS); const size_t num_points = 1 << state.range(0); - const size_t num_nonzero = 100; + const size_t num_nonzero = 5; auto polynomial = sparse_random_poly(num_points, num_nonzero); @@ -98,7 +120,7 @@ template void bench_commit_sparse_random_preprocessed(::benchma auto key = create_commitment_key(MAX_NUM_POINTS); const size_t num_points = 1 << state.range(0); - const size_t num_nonzero = 100; + const size_t num_nonzero = 5; auto polynomial = sparse_random_poly(num_points, num_nonzero); @@ -124,6 +146,9 @@ template void bench_commit_random(::benchmark::State& state) BENCHMARK(bench_commit_zero)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); BENCHMARK(bench_commit_sparse)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); +BENCHMARK(bench_commit_sparse_preprocessed) + ->DenseRange(14, MAX_LOG_NUM_POINTS) + ->Unit(benchmark::kMillisecond); BENCHMARK(bench_commit_sparse_random)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); BENCHMARK(bench_commit_sparse_random_preprocessed) ->DenseRange(14, MAX_LOG_NUM_POINTS) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp index b2bf49f5b81..7ba12890db0 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp @@ -102,19 +102,36 @@ template class CommitmentKey { // BB_OP_COUNT_TIME(); const size_t degree = polynomial.size(); ASSERT(degree <= srs->get_monomial_size()); - // Create a view of the even elements of the point table (i.e. the raw srs points) - auto srs_view = get_srs_view(); + + G1* point_table = srs->get_monomial_points(); std::vector scalars; std::vector points; - size_t point_idx = 0; - for (auto point : srs_view) { - const Fr& scalar = polynomial[point_idx++]; - if (!scalar.is_zero()) { - scalars.emplace_back(scalar); - points.emplace_back(point); + const size_t num_threads = degree >= get_num_cpus_pow2() ? get_num_cpus_pow2() : 1; + const size_t block_size = degree / num_threads; + + std::vector> thread_scalars(num_threads); + std::vector> thread_points(num_threads); + + parallel_for(num_threads, [&](size_t thread_idx) { + const size_t start = thread_idx * block_size; + const size_t end = (thread_idx + 1) * block_size; + + for (size_t idx = start; idx < end; ++idx) { + + const G1& point = point_table[idx * 2]; + const Fr& scalar = polynomial[idx]; + if (!scalar.is_zero()) { + thread_scalars[thread_idx].emplace_back(scalar); + thread_points[thread_idx].emplace_back(point); + } } + }); + + for (size_t idx = 0; idx < num_threads; ++idx) { + scalars.insert(scalars.end(), thread_scalars[idx].begin(), thread_scalars[idx].end()); + points.insert(points.end(), thread_points[idx].begin(), thread_points[idx].end()); } return scalar_multiplication::pippenger_without_endomorphism_basis_points( From 2cdd476fdee88315714d9d5cd829120679b6ab20 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Mon, 15 Jul 2024 23:25:42 +0000 Subject: [PATCH 07/18] use comit sparse in oink --- .../barretenberg/ultra_honk/oink_prover.cpp | 40 +++++-------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp index 8dc81064db3..44c01a5c6c5 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp @@ -74,11 +74,6 @@ template void OinkProver::execute_wire_commitment witness_commitments.w_o = commitment_key->commit(proving_key.polynomials.w_o); } - info("Wires:"); - instance_inspector::count_non_zero(proving_key.polynomials.w_l); - instance_inspector::count_non_zero(proving_key.polynomials.w_r); - instance_inspector::count_non_zero(proving_key.polynomials.w_o); - auto wire_comms = witness_commitments.get_wires(); auto wire_labels = commitment_labels.get_wires(); for (size_t idx = 0; idx < 3; ++idx) { @@ -89,18 +84,12 @@ template void OinkProver::execute_wire_commitment // Commit to Goblin ECC op wires { BB_OP_COUNT_TIME_NAME("COMMIT::ecc_op_wires"); - witness_commitments.ecc_op_wire_1 = commitment_key->commit(proving_key.polynomials.ecc_op_wire_1); - witness_commitments.ecc_op_wire_2 = commitment_key->commit(proving_key.polynomials.ecc_op_wire_2); - witness_commitments.ecc_op_wire_3 = commitment_key->commit(proving_key.polynomials.ecc_op_wire_3); - witness_commitments.ecc_op_wire_4 = commitment_key->commit(proving_key.polynomials.ecc_op_wire_4); + witness_commitments.ecc_op_wire_1 = commitment_key->commit_sparse(proving_key.polynomials.ecc_op_wire_1); + witness_commitments.ecc_op_wire_2 = commitment_key->commit_sparse(proving_key.polynomials.ecc_op_wire_2); + witness_commitments.ecc_op_wire_3 = commitment_key->commit_sparse(proving_key.polynomials.ecc_op_wire_3); + witness_commitments.ecc_op_wire_4 = commitment_key->commit_sparse(proving_key.polynomials.ecc_op_wire_4); } - info("ECC op wires:"); - instance_inspector::count_non_zero(proving_key.polynomials.ecc_op_wire_1); - instance_inspector::count_non_zero(proving_key.polynomials.ecc_op_wire_2); - instance_inspector::count_non_zero(proving_key.polynomials.ecc_op_wire_3); - instance_inspector::count_non_zero(proving_key.polynomials.ecc_op_wire_4); - auto op_wire_comms = witness_commitments.get_ecc_op_wires(); auto labels = commitment_labels.get_ecc_op_wires(); for (size_t idx = 0; idx < Flavor::NUM_WIRES; ++idx) { @@ -110,12 +99,12 @@ template void OinkProver::execute_wire_commitment // Commit to DataBus columns and corresponding read counts { BB_OP_COUNT_TIME_NAME("COMMIT::databus"); - witness_commitments.calldata = commitment_key->commit(proving_key.polynomials.calldata); + witness_commitments.calldata = commitment_key->commit_sparse(proving_key.polynomials.calldata); witness_commitments.calldata_read_counts = - commitment_key->commit(proving_key.polynomials.calldata_read_counts); - witness_commitments.return_data = commitment_key->commit(proving_key.polynomials.return_data); + commitment_key->commit_sparse(proving_key.polynomials.calldata_read_counts); + witness_commitments.return_data = commitment_key->commit_sparse(proving_key.polynomials.return_data); witness_commitments.return_data_read_counts = - commitment_key->commit(proving_key.polynomials.return_data_read_counts); + commitment_key->commit_sparse(proving_key.polynomials.return_data_read_counts); } transcript->send_to_verifier(domain_separator + commitment_labels.calldata, witness_commitments.calldata); @@ -181,24 +170,19 @@ template void OinkProver::execute_log_derivative_ transcript->send_to_verifier(domain_separator + commitment_labels.lookup_inverses, witness_commitments.lookup_inverses); - info("Lookup inverses:"); - instance_inspector::count_non_zero(proving_key.polynomials.lookup_inverses); - // If Mega, commit to the databus inverse polynomials and send if constexpr (IsGoblinFlavor) { { BB_OP_COUNT_TIME_NAME("COMMIT::databus_inverses"); - witness_commitments.calldata_inverses = commitment_key->commit(proving_key.polynomials.calldata_inverses); + witness_commitments.calldata_inverses = + commitment_key->commit_sparse(proving_key.polynomials.calldata_inverses); witness_commitments.return_data_inverses = - commitment_key->commit(proving_key.polynomials.return_data_inverses); + commitment_key->commit_sparse(proving_key.polynomials.return_data_inverses); } transcript->send_to_verifier(domain_separator + commitment_labels.calldata_inverses, witness_commitments.calldata_inverses); transcript->send_to_verifier(domain_separator + commitment_labels.return_data_inverses, witness_commitments.return_data_inverses); - info("Databus inverses:"); - instance_inspector::count_non_zero(proving_key.polynomials.calldata_inverses); - instance_inspector::count_non_zero(proving_key.polynomials.return_data_inverses); } } @@ -214,8 +198,6 @@ template void OinkProver::execute_grand_product_c BB_OP_COUNT_TIME_NAME("COMMIT::z_perm"); witness_commitments.z_perm = commitment_key->commit(proving_key.polynomials.z_perm); } - info("Z_Perm:"); - instance_inspector::count_non_zero(proving_key.polynomials.z_perm); transcript->send_to_verifier(domain_separator + commitment_labels.z_perm, witness_commitments.z_perm); } From 5b3926a21b2daa757f3385b278071962047c4d4d Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Tue, 16 Jul 2024 13:03:07 +0000 Subject: [PATCH 08/18] cleanup --- .../cpp/scripts/analyze_client_ivc_bench.py | 52 +++---------------- .../processed_commitment.test.cpp | 2 - .../plonk_honk_shared/instance_inspector.hpp | 13 ----- .../srs/factories/file_crs_factory.hpp | 1 - .../barretenberg/ultra_honk/oink_prover.cpp | 10 +++- 5 files changed, 15 insertions(+), 63 deletions(-) diff --git a/barretenberg/cpp/scripts/analyze_client_ivc_bench.py b/barretenberg/cpp/scripts/analyze_client_ivc_bench.py index 117beb21b33..2a19ef205ad 100644 --- a/barretenberg/cpp/scripts/analyze_client_ivc_bench.py +++ b/barretenberg/cpp/scripts/analyze_client_ivc_bench.py @@ -72,73 +72,35 @@ print(f"{key:<{max_label_length}}{time_ms:>8.0f} {time_ms/total_time_ms:>8.2%}") -# # Relations breakdown -# # Note: The timings here are off likely because the tracking is occuring in a hot loop but -# # they should be meaningful relative to one another -# print('\nRelation contributions (times to be interpreted relatively):') -# relations = [ -# "Arithmetic::accumulate(t)", -# "Permutation::accumulate(t)", -# "Lookup::accumulate(t)", -# "DeltaRange::accumulate(t)", -# "Elliptic::accumulate(t)", -# "Auxiliary::accumulate(t)", -# "EccOp::accumulate(t)", -# "DatabusRead::accumulate(t)", -# "PoseidonExt::accumulate(t)", -# "PoseidonInt::accumulate(t)", -# ] -# with open(PREFIX/IVC_BENCH_JSON, "r") as read_file: -# read_result = json.load(read_file) -# for _bench in read_result["benchmarks"]: -# if _bench["name"] == BENCHMARK: -# bench = _bench -# bench_components = dict(filter(lambda x: x[0] in relations, bench.items())) - -# # For each kept time, get the proportion over all kept times. -# sum_of_kept_times_ms = sum(float(time) -# for _, time in bench_components.items())/1e6 -# max_label_length = max(len(label) for label in relations) -# column = {"function": "function", "ms": "ms", "%": "% sum"} -# print( -# f"{column['function']:<{max_label_length}}{column['ms']:>8} {column['%']:>8}") -# for key in relations: -# if key not in bench: -# time_ms = 0 -# else: -# time_ms = bench[key]/1e6 -# print(f"{key:<{max_label_length}}{time_ms:>8.0f} {time_ms/sum_of_kept_times_ms:>8.2%}") -# Print header - -def print_contributions(prefix, ivc_bench_json, benchmark, relations): +# Extract a set of components from the benchmark data and display timings and relative percentages +def print_contributions(prefix, ivc_bench_json, bench_name, components): # Read JSON file and extract benchmark try: with open(prefix / ivc_bench_json, "r") as read_file: read_result = json.load(read_file) - bench = next((_bench for _bench in read_result["benchmarks"] if _bench["name"] == benchmark), None) + bench = next((_bench for _bench in read_result["benchmarks"] if _bench["name"] == bench_name), None) if not bench: - raise ValueError(f"Benchmark '{benchmark}' not found in the JSON file.") + raise ValueError(f"Benchmark '{bench_name}' not found in the JSON file.") except FileNotFoundError: print(f"File not found: {prefix / ivc_bench_json}") return # Filter and sum up kept times - bench_components = {key: bench[key] for key in relations if key in bench} + bench_components = {key: bench[key] for key in components if key in bench} sum_of_kept_times_ms = sum(float(time) for time in bench_components.values()) / 1e6 print(f"Total time accounted for (ms): {sum_of_kept_times_ms:>8.0f}") # Print results - max_label_length = max(len(label) for label in relations) + max_label_length = max(len(label) for label in components) column_headers = {"operation": "operation", "ms": "ms", "%": "% sum"} print(f"{column_headers['operation']:<{max_label_length}}{column_headers['ms']:>8} {column_headers['%']:>8}") - for key in relations: + for key in components: time_ms = bench_components.get(key, 0) / 1e6 percentage = time_ms / sum_of_kept_times_ms if sum_of_kept_times_ms > 0 else 0 print(f"{key:<{max_label_length}}{time_ms:>8.0f} {percentage:>8.2%}") - relations = [ "Arithmetic::accumulate(t)", "Permutation::accumulate(t)", diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp index 62e66d582f9..64a67da7edb 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp @@ -12,8 +12,6 @@ namespace bb { -// constexpr size_t COMMITMENT_TEST_NUM_POINTS = 4096; - template class CommitmentKeyTest : public ::testing::Test { using CK = CommitmentKey; diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/instance_inspector.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/instance_inspector.hpp index 6eba9860987..add3db82ec1 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/instance_inspector.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/instance_inspector.hpp @@ -1,7 +1,6 @@ #pragma once #include "barretenberg/common/log.hpp" -#include namespace bb::instance_inspector { @@ -16,18 +15,6 @@ bool is_non_zero(auto& polynomial) return false; } -// Determine whether a polynomial has at least one non-zero coefficient -void count_non_zero(auto& polynomial) -{ - uint32_t num_non_zero = 0; - for (auto& coeff : polynomial) { - if (!coeff.is_zero()) { - num_non_zero++; - } - } - info("Poly of size ", polynomial.size(), " has ", num_non_zero, " non-zero coefficients."); -} - /** * @brief Utility for indicating which polynomials in a prover instance are identically zero * diff --git a/barretenberg/cpp/src/barretenberg/srs/factories/file_crs_factory.hpp b/barretenberg/cpp/src/barretenberg/srs/factories/file_crs_factory.hpp index 641beff4aec..cf6b524d6f1 100644 --- a/barretenberg/cpp/src/barretenberg/srs/factories/file_crs_factory.hpp +++ b/barretenberg/cpp/src/barretenberg/srs/factories/file_crs_factory.hpp @@ -46,7 +46,6 @@ template class FileProverCrs : public ProverCrs { monomials_ = scalar_multiplication::point_table_alloc(num_points); srs::IO::read_transcript_g1(monomials_.get(), num_points, path); - // WORKTODO scalar_multiplication::generate_pippenger_point_table(monomials_.get(), monomials_.get(), num_points); }; diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp index 780327d3524..d6f5d913c6b 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/oink_prover.cpp @@ -86,7 +86,10 @@ template void OinkProver::execute_wire_commitment for (auto [commitment, polynomial, label] : zip_view(witness_commitments.get_ecc_op_wires(), proving_key.polynomials.get_ecc_op_wires(), commitment_labels.get_ecc_op_wires())) { - commitment = commitment_key->commit_sparse(polynomial); + { + BB_OP_COUNT_TIME_NAME("COMMIT::ecc_op_wires"); + commitment = commitment_key->commit_sparse(polynomial); + } transcript->send_to_verifier(domain_separator + label, commitment); } @@ -94,7 +97,10 @@ template void OinkProver::execute_wire_commitment for (auto [commitment, polynomial, label] : zip_view(witness_commitments.get_databus_entities(), proving_key.polynomials.get_databus_entities(), commitment_labels.get_databus_entities())) { - commitment = commitment_key->commit_sparse(polynomial); + { + BB_OP_COUNT_TIME_NAME("COMMIT::databus"); + commitment = commitment_key->commit_sparse(polynomial); + } transcript->send_to_verifier(domain_separator + label, commitment); } } From df99d1db736ecf21228048de554e692f2e385b91 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Tue, 16 Jul 2024 14:37:16 +0000 Subject: [PATCH 09/18] clean out comit key test suite --- .../commitment_schemes/commitment_key.hpp | 19 -- .../processed_commitment.test.cpp | 192 ++---------------- 2 files changed, 19 insertions(+), 192 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp index 4fbcf930251..2a1f885435a 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp @@ -37,12 +37,6 @@ template class CommitmentKey { using Commitment = typename Curve::AffineElement; using G1 = typename Curve::AffineElement; - using IndicesView = std::ranges::iota_view; - using EvenIndicesView = std::ranges::filter_view>; - using SrsViewType = std::ranges::transform_view>; - - SrsViewType srs_view; - public: scalar_multiplication::pippenger_runtime_state pippenger_runtime_state; std::shared_ptr> crs_factory; @@ -90,19 +84,6 @@ template class CommitmentKey { const_cast(polynomial.data()), srs->get_monomial_points(), degree, pippenger_runtime_state); }; - auto get_srs_view() - { - size_t point_table_size = 2 * srs->get_monomial_size(); - G1* point_table = srs->get_monomial_points(); - - auto srs_view = - std::views::iota(static_cast(0), point_table_size) | // generate view of indices 0, ..., n-1 - std::views::filter([](int i) { return i % 2 == 0; }) | // create a view of the even indices only - std::views::transform([point_table](size_t i) { return point_table[i]; }); // extract even srs - - return srs_view; - } - Commitment commit_sparse(std::span polynomial) { // BB_OP_COUNT_TIME(); diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp index 64a67da7edb..1bc1cd93d7f 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp @@ -1,12 +1,6 @@ #include "barretenberg/commitment_schemes/commitment_key.hpp" -#include "barretenberg/commitment_schemes/verification_key.hpp" -#include "barretenberg/common/zip_view.hpp" -#include "barretenberg/ecc/curves/bn254/g1.hpp" #include "barretenberg/polynomials/polynomial.hpp" #include "barretenberg/srs/factories/file_crs_factory.hpp" -#include "claim.hpp" -#include -#include #include @@ -20,16 +14,7 @@ template class CommitmentKeyTest : public ::testing::Test { using Polynomial = bb::Polynomial; public: - Polynomial random_polynomial(const size_t n) - { - Polynomial p(n); - for (size_t i = 0; i < n; ++i) { - p[i] = Fr::random_element(); - } - return p; - } - - template inline std::shared_ptr create_commitment_key(const size_t num_points); + template inline std::shared_ptr create_commitment_key(size_t num_points); template <> inline std::shared_ptr> create_commitment_key>( @@ -38,109 +23,22 @@ template class CommitmentKeyTest : public ::testing::Test { srs::init_crs_factory("../srs_db/ignition"); return std::make_shared>(num_points); } -}; - -using Curves = ::testing::Types; - -TYPED_TEST_SUITE(CommitmentKeyTest, Curves); - -TYPED_TEST(CommitmentKeyTest, CommitSimple) -{ - using Curve = TypeParam; - using CK = CommitmentKey; - using G1 = Curve::AffineElement; - using Fr = typename Curve::ScalarField; - using Polynomial = bb::Polynomial; - - const size_t num_points = 10; - - auto key = TestFixture::template create_commitment_key(num_points); - - const size_t point_table_size = 2 * num_points; - std::span point_table(key->srs->get_monomial_points(), point_table_size); - - std::vector srs; - - size_t idx = 0; - for (auto& element : point_table) { - if (idx % 2 == 0) { - srs.emplace_back(element); - } - idx++; - } - - const size_t num_nonzero = 2; - Polynomial poly{ num_points }; - for (size_t i = 0; i < num_nonzero; ++i) { - size_t idx = (i + 1) * (i + 1) % num_points; - poly[idx] = Fr::random_element(); - } - - G1 normal_result = key->commit(poly); - - std::vector scalars; - std::vector points; - - for (auto [scalar, point] : zip_view(poly, srs)) { - if (!scalar.is_zero()) { - scalars.emplace_back(scalar); - points.emplace_back(point); - } - } - - EXPECT_EQ(points.size(), num_nonzero); - - G1 reduced_result; - reduced_result.self_set_infinity(); - for (auto [scalar, point] : zip_view(scalars, points)) { - reduced_result = reduced_result + point * scalar; - } - - EXPECT_EQ(reduced_result, normal_result); - - G1 pippenger_result = scalar_multiplication::pippenger_without_endomorphism_basis_points( - scalars.data(), points.data(), num_nonzero, key->pippenger_runtime_state); - - EXPECT_EQ(pippenger_result, normal_result); -} - -TYPED_TEST(CommitmentKeyTest, SrsView) -{ - using Curve = TypeParam; - using CK = CommitmentKey; - using G1 = Curve::AffineElement; - - const size_t num_points = 10; - - auto key = TestFixture::template create_commitment_key(num_points); - - const size_t point_table_size = 2 * num_points; - std::span point_table(key->srs->get_monomial_points(), point_table_size); - // Manually construct the SRS from the point table - std::vector srs; - size_t idx = 0; - for (auto& element : point_table) { - if (idx % 2 == 0) { - srs.emplace_back(element); - } - idx++; + template <> + inline std::shared_ptr> create_commitment_key>( + const size_t num_points) + { + srs::init_grumpkin_crs_factory("../srs_db/grumpkin"); + return std::make_shared>(num_points); } +}; - // Create a view of the even elements of the point table (i.e. the raw srs points) - auto srs_view = std::views::iota(static_cast(0), point_table_size) | // generate view of indices 0, ..., n-1 - std::views::filter([](int i) { return i % 2 == 0; }) | // create a view of the even indices only - std::views::transform([&point_table](size_t i) { return point_table[i]; }); // extract even srs +using Curves = ::testing::Types; - // Check that the two agree - size_t point_idx = 0; - for (auto element : srs_view) { - EXPECT_EQ(srs[point_idx], element); - point_idx++; - } -} +TYPED_TEST_SUITE(CommitmentKeyTest, Curves); -TYPED_TEST(CommitmentKeyTest, SrsViewCommit) +// Check that commit and commit_sparse return the same result for a random sparse polynomial +TYPED_TEST(CommitmentKeyTest, CommitSparse) { using Curve = TypeParam; using CK = CommitmentKey; @@ -148,74 +46,22 @@ TYPED_TEST(CommitmentKeyTest, SrsViewCommit) using Fr = Curve::ScalarField; using Polynomial = bb::Polynomial; - const size_t num_points = 10; - - auto key = TestFixture::template create_commitment_key(num_points); + const size_t num_points = 1 << 10; + const size_t num_nonzero = 7; - const size_t num_nonzero = 2; + // Construct a sparse random polynomial Polynomial poly{ num_points }; for (size_t i = 0; i < num_nonzero; ++i) { size_t idx = (i + 1) * (i + 1) % num_points; poly[idx] = Fr::random_element(); } - G1 normal_result = key->commit(poly); - - // Create a view of the even elements of the point table (i.e. the raw srs points) - auto srs_view = key->get_srs_view(); - - std::vector scalars; - std::vector points; - - size_t point_idx = 0; - for (auto point : srs_view) { - Fr& scalar = poly[point_idx++]; - if (!scalar.is_zero()) { - scalars.emplace_back(scalar); - points.emplace_back(point); - } - } - - EXPECT_EQ(points.size(), num_nonzero); - - G1 reduced_result; - reduced_result.self_set_infinity(); - for (auto [scalar, point] : zip_view(scalars, points)) { - reduced_result = reduced_result + point * scalar; - } - - EXPECT_EQ(reduced_result, normal_result); - - G1 pippenger_result = scalar_multiplication::pippenger_without_endomorphism_basis_points( - scalars.data(), points.data(), num_nonzero, key->pippenger_runtime_state); - - EXPECT_EQ(pippenger_result, normal_result); -} - -TYPED_TEST(CommitmentKeyTest, SrsViewCommitSparse) -{ - using Curve = TypeParam; - using CK = CommitmentKey; - using G1 = Curve::AffineElement; - using Fr = Curve::ScalarField; - using Polynomial = bb::Polynomial; - - const size_t num_points = 10; - + // Commit to the polynomial using both the conventional commit method and the sparse commitment method auto key = TestFixture::template create_commitment_key(num_points); + G1 commit_result = key->commit(poly); + G1 sparse_commit_result = key->commit_sparse(poly); - const size_t num_nonzero = 2; - Polynomial poly{ num_points }; - for (size_t i = 0; i < num_nonzero; ++i) { - size_t idx = (i + 1) * (i + 1) % num_points; - poly[idx] = Fr::random_element(); - } - - G1 normal_result = key->commit(poly); - - G1 pippenger_result = key->commit_sparse(poly); - - EXPECT_EQ(pippenger_result, normal_result); + EXPECT_EQ(sparse_commit_result, commit_result); } } // namespace bb From 0696f0871d029faf92af1752d123af2b26ef2574 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Tue, 16 Jul 2024 14:51:51 +0000 Subject: [PATCH 10/18] fix gcc --- .../processed_commitment.test.cpp | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp index 1bc1cd93d7f..f973f8158be 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp @@ -15,23 +15,25 @@ template class CommitmentKeyTest : public ::testing::Test { public: template inline std::shared_ptr create_commitment_key(size_t num_points); +}; - template <> - inline std::shared_ptr> create_commitment_key>( - const size_t num_points) - { - srs::init_crs_factory("../srs_db/ignition"); - return std::make_shared>(num_points); - } +template <> +template <> +std::shared_ptr> CommitmentKeyTest::create_commitment_key< + CommitmentKey>(const size_t num_points) +{ + srs::init_crs_factory("../srs_db/ignition"); + return std::make_shared>(num_points); +} - template <> - inline std::shared_ptr> create_commitment_key>( - const size_t num_points) - { - srs::init_grumpkin_crs_factory("../srs_db/grumpkin"); - return std::make_shared>(num_points); - } -}; +template <> +template <> +std::shared_ptr> CommitmentKeyTest::create_commitment_key< + CommitmentKey>(const size_t num_points) +{ + srs::init_grumpkin_crs_factory("../srs_db/grumpkin"); + return std::make_shared>(num_points); +} using Curves = ::testing::Types; From 75fbea8f00cd9cc84819a7bfa6cb133f132bf369 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Tue, 16 Jul 2024 16:29:02 +0000 Subject: [PATCH 11/18] improve commit sparse and add comments --- .../cpp/scripts/analyze_client_ivc_bench.py | 1 - .../cpp/scripts/benchmark_client_ivc.sh | 1 - .../commitment_schemes/commitment_key.hpp | 30 +++++++++++++++---- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/barretenberg/cpp/scripts/analyze_client_ivc_bench.py b/barretenberg/cpp/scripts/analyze_client_ivc_bench.py index 2a19ef205ad..94c66435f46 100644 --- a/barretenberg/cpp/scripts/analyze_client_ivc_bench.py +++ b/barretenberg/cpp/scripts/analyze_client_ivc_bench.py @@ -4,7 +4,6 @@ PREFIX = Path("build-op-count-time") IVC_BENCH_JSON = Path("client_ivc_bench.json") BENCHMARK = "ClientIVCBench/Full/6" -# BENCHMARK = "ClientIVCBench/FullStructured/6" # Single out an independent set of functions accounting for most of BENCHMARK's real_time to_keep = [ diff --git a/barretenberg/cpp/scripts/benchmark_client_ivc.sh b/barretenberg/cpp/scripts/benchmark_client_ivc.sh index e30808e80d5..7991ef87940 100755 --- a/barretenberg/cpp/scripts/benchmark_client_ivc.sh +++ b/barretenberg/cpp/scripts/benchmark_client_ivc.sh @@ -4,7 +4,6 @@ set -eu TARGET="client_ivc_bench" # Note: to run structured trace version, change "Full" to "FullStructured" here and in analyze script FILTER="ClientIVCBench/Full/6$" -# FILTER="ClientIVCBench/FullStructured/6$" BUILD_DIR=build-op-count-time # Move above script dir. diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp index 2a1f885435a..88f42c5fbca 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp @@ -84,44 +84,62 @@ template class CommitmentKey { const_cast(polynomial.data()), srs->get_monomial_points(), degree, pippenger_runtime_state); }; + /** + * @brief Efficiently commit to a sparse polynomial + * @details Iterate through the {point, scalar} pairs that define the inputs to the commitment MSM, maintain (copy) + * only those for which the scalar is nonzero, then perform the MSM on the reduced inputs. + * @warning Method makes a copy of all {point, scalar} pairs that comprise the reduced input. Will not be efficient + * in terms of memory or computation for polynomials beyond a certain sparseness threeshold. + * + * @param polynomial + * @return Commitment + */ Commitment commit_sparse(std::span polynomial) { // BB_OP_COUNT_TIME(); const size_t degree = polynomial.size(); ASSERT(degree <= srs->get_monomial_size()); + // Extract the precomputed point table (contains raw SRS points at even indices and the corresponding + // endomorphism point (\beta*x, -y) at odd indices). G1* point_table = srs->get_monomial_points(); - std::vector scalars; - std::vector points; - + // Define structures needed to multithread the extraction of non-zero inputs const size_t num_threads = degree >= get_num_cpus_pow2() ? get_num_cpus_pow2() : 1; const size_t block_size = degree / num_threads; - std::vector> thread_scalars(num_threads); std::vector> thread_points(num_threads); + // Loop over all polynomial coefficients and keep {point, scalar} pairs for which scalar != 0 parallel_for(num_threads, [&](size_t thread_idx) { const size_t start = thread_idx * block_size; const size_t end = (thread_idx + 1) * block_size; for (size_t idx = start; idx < end; ++idx) { - const G1& point = point_table[idx * 2]; const Fr& scalar = polynomial[idx]; + if (!scalar.is_zero()) { thread_scalars[thread_idx].emplace_back(scalar); + // Save both the raw srs point and the precomputed endomorphism point from the point table + const G1& point = point_table[idx * 2]; + const G1& endo_point = point_table[idx * 2]; thread_points[thread_idx].emplace_back(point); + thread_points[thread_idx].emplace_back(endo_point); } } }); + // Reconstruct the full input to the pippenger from the individual threads + std::vector scalars; + std::vector points; for (size_t idx = 0; idx < num_threads; ++idx) { scalars.insert(scalars.end(), thread_scalars[idx].begin(), thread_scalars[idx].end()); points.insert(points.end(), thread_points[idx].begin(), thread_points[idx].end()); } - return scalar_multiplication::pippenger_without_endomorphism_basis_points( + // Call the version of pippenger which assumes all points are distinct + return scalar_multiplication::pippenger_unsafe( scalars.data(), points.data(), scalars.size(), pippenger_runtime_state); } }; From a339ff51051ece1bb75283a58c80ea6039561bb2 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Tue, 16 Jul 2024 16:49:42 +0000 Subject: [PATCH 12/18] clean up commit bench --- .../commitment_schemes/commit.bench.cpp | 41 +++++++------------ 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp index 92092cc44d2..631503ce30c 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp @@ -17,41 +17,25 @@ template std::shared_ptr> create_commitmen return std::make_shared>(num_points); } +// Generate a polynomial with a specified number of nonzero random coefficients template Polynomial sparse_random_poly(const size_t size, const size_t num_nonzero) { + auto& engine = numeric::get_debug_randomness(); auto polynomial = Polynomial(size); for (size_t i = 0; i < num_nonzero; i++) { - polynomial[i] = FF::random_element(); + size_t idx = engine.get_random_uint32() % size; + polynomial[idx] = FF::random_element(); } return polynomial; } -template -std::vector extract_srs(std::shared_ptr> commitment_key, - const size_t num_points) -{ - using G1 = typename Curve::AffineElement; - std::vector monomials; - size_t idx = 0; - std::span point_table(commitment_key->srs->get_monomial_points(), 2 * num_points); - - for (auto& element : point_table) { - if (idx % 2 == 0) { - monomials.emplace_back(element); - } - idx++; - if (monomials.size() == num_points) { - break; - } - } - return monomials; -} - constexpr size_t MAX_LOG_NUM_POINTS = 18; constexpr size_t MAX_NUM_POINTS = 1 << MAX_LOG_NUM_POINTS; +constexpr size_t SPARSE_NUM_NONZERO = 5; +// Commit to a zero polynomial template void bench_commit_zero(::benchmark::State& state) { auto key = create_commitment_key(MAX_NUM_POINTS); @@ -63,13 +47,14 @@ template void bench_commit_zero(::benchmark::State& state) } } +// Commit to a polynomial with sparse nonzero entries equal to 1 template void bench_commit_sparse(::benchmark::State& state) { using Fr = typename Curve::ScalarField; auto key = create_commitment_key(MAX_NUM_POINTS); const size_t num_points = 1 << state.range(0); - const size_t num_nonzero = 2; + const size_t num_nonzero = SPARSE_NUM_NONZERO; auto polynomial = Polynomial(num_points); for (size_t i = 0; i < num_nonzero; i++) { @@ -81,13 +66,14 @@ template void bench_commit_sparse(::benchmark::State& state) } } +// Commit to a polynomial with sparse nonzero entries equal to 1 using the commit_sparse method to preprocess the input template void bench_commit_sparse_preprocessed(::benchmark::State& state) { using Fr = typename Curve::ScalarField; auto key = create_commitment_key(MAX_NUM_POINTS); const size_t num_points = 1 << state.range(0); - const size_t num_nonzero = 2; + const size_t num_nonzero = SPARSE_NUM_NONZERO; auto polynomial = Polynomial(num_points); for (size_t i = 0; i < num_nonzero; i++) { @@ -99,13 +85,14 @@ template void bench_commit_sparse_preprocessed(::benchmark::Sta } } +// Commit to a polynomial with sparse random nonzero entries template void bench_commit_sparse_random(::benchmark::State& state) { using Fr = typename Curve::ScalarField; auto key = create_commitment_key(MAX_NUM_POINTS); const size_t num_points = 1 << state.range(0); - const size_t num_nonzero = 5; + const size_t num_nonzero = SPARSE_NUM_NONZERO; auto polynomial = sparse_random_poly(num_points, num_nonzero); @@ -114,13 +101,14 @@ template void bench_commit_sparse_random(::benchmark::State& st } } +// Commit to a polynomial with sparse random nonzero entries using the commit_sparse method to preprocess the input template void bench_commit_sparse_random_preprocessed(::benchmark::State& state) { using Fr = typename Curve::ScalarField; auto key = create_commitment_key(MAX_NUM_POINTS); const size_t num_points = 1 << state.range(0); - const size_t num_nonzero = 5; + const size_t num_nonzero = SPARSE_NUM_NONZERO; auto polynomial = sparse_random_poly(num_points, num_nonzero); @@ -129,6 +117,7 @@ template void bench_commit_sparse_random_preprocessed(::benchma } } +// Commit to a polynomial with dense random nonzero entries template void bench_commit_random(::benchmark::State& state) { using Fr = typename Curve::ScalarField; From d89771f7300d7646ed8f09139f0998f004b2a6bb Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Tue, 16 Jul 2024 17:36:27 +0000 Subject: [PATCH 13/18] clean and comment --- .../cpp/src/barretenberg/commitment_schemes/commit.bench.cpp | 5 ----- .../src/barretenberg/commitment_schemes/commitment_key.hpp | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp index 631503ce30c..dde63c95346 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp @@ -1,12 +1,7 @@ #include "barretenberg/commitment_schemes/commitment_key.hpp" -#include "barretenberg/common/zip_view.hpp" #include "barretenberg/srs/factories/mem_bn254_crs_factory.hpp" -#include #include -#include -#include -#include namespace bb { diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp index 88f42c5fbca..040db58a7e2 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp @@ -89,7 +89,7 @@ template class CommitmentKey { * @details Iterate through the {point, scalar} pairs that define the inputs to the commitment MSM, maintain (copy) * only those for which the scalar is nonzero, then perform the MSM on the reduced inputs. * @warning Method makes a copy of all {point, scalar} pairs that comprise the reduced input. Will not be efficient - * in terms of memory or computation for polynomials beyond a certain sparseness threeshold. + * in terms of memory or computation for polynomials beyond a certain sparseness threshold. * * @param polynomial * @return Commitment From ff801fe4cf86ba8300904f1ea07effe6cc3358e3 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Tue, 16 Jul 2024 19:57:10 +0000 Subject: [PATCH 14/18] fix endo point from debugging --- .../cpp/src/barretenberg/commitment_schemes/commitment_key.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp index 040db58a7e2..bd7258525ed 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp @@ -123,7 +123,7 @@ template class CommitmentKey { thread_scalars[thread_idx].emplace_back(scalar); // Save both the raw srs point and the precomputed endomorphism point from the point table const G1& point = point_table[idx * 2]; - const G1& endo_point = point_table[idx * 2]; + const G1& endo_point = point_table[idx * 2 + 1]; thread_points[thread_idx].emplace_back(point); thread_points[thread_idx].emplace_back(endo_point); } From 6139b4ad683494134e1b65fef48a2e822a49e07a Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Tue, 16 Jul 2024 20:24:45 +0000 Subject: [PATCH 15/18] update test --- .../commitment_schemes/processed_commitment.test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp index f973f8158be..f555d437d08 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp @@ -48,7 +48,7 @@ TYPED_TEST(CommitmentKeyTest, CommitSparse) using Fr = Curve::ScalarField; using Polynomial = bb::Polynomial; - const size_t num_points = 1 << 10; + const size_t num_points = 1 << 12; // large enough to ensure normal pippenger logic is used const size_t num_nonzero = 7; // Construct a sparse random polynomial From e503e5558244e147ea00120c76195bda7d1b6ca6 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Tue, 16 Jul 2024 21:06:09 +0000 Subject: [PATCH 16/18] cleanup and naming --- .../cpp/src/barretenberg/commitment_schemes/commitment_key.hpp | 3 +-- ...rocessed_commitment.test.cpp => sparse_commitment.test.cpp} | 0 2 files changed, 1 insertion(+), 2 deletions(-) rename barretenberg/cpp/src/barretenberg/commitment_schemes/{processed_commitment.test.cpp => sparse_commitment.test.cpp} (100%) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp index bd7258525ed..66f470448fa 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp @@ -18,7 +18,6 @@ #include #include -#include #include namespace bb { @@ -96,7 +95,7 @@ template class CommitmentKey { */ Commitment commit_sparse(std::span polynomial) { - // BB_OP_COUNT_TIME(); + BB_OP_COUNT_TIME(); const size_t degree = polynomial.size(); ASSERT(degree <= srs->get_monomial_size()); diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/sparse_commitment.test.cpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/commitment_schemes/processed_commitment.test.cpp rename to barretenberg/cpp/src/barretenberg/commitment_schemes/sparse_commitment.test.cpp From 982480d4fbd23e85060eddbacc26e974b5f811fd Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Thu, 18 Jul 2024 13:12:46 +0000 Subject: [PATCH 17/18] update commit bench with larger polys --- .../commitment_schemes/commit.bench.cpp | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp index dde63c95346..88a27cd1b6c 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commit.bench.cpp @@ -26,9 +26,10 @@ template Polynomial sparse_random_poly(const size_t size, cons return polynomial; } -constexpr size_t MAX_LOG_NUM_POINTS = 18; +constexpr size_t MIN_LOG_NUM_POINTS = 16; +constexpr size_t MAX_LOG_NUM_POINTS = 20; constexpr size_t MAX_NUM_POINTS = 1 << MAX_LOG_NUM_POINTS; -constexpr size_t SPARSE_NUM_NONZERO = 5; +constexpr size_t SPARSE_NUM_NONZERO = 100; // Commit to a zero polynomial template void bench_commit_zero(::benchmark::State& state) @@ -128,16 +129,24 @@ template void bench_commit_random(::benchmark::State& state) } } -BENCHMARK(bench_commit_zero)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); -BENCHMARK(bench_commit_sparse)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); +BENCHMARK(bench_commit_zero) + ->DenseRange(MIN_LOG_NUM_POINTS, MAX_LOG_NUM_POINTS) + ->Unit(benchmark::kMillisecond); +BENCHMARK(bench_commit_sparse) + ->DenseRange(MIN_LOG_NUM_POINTS, MAX_LOG_NUM_POINTS) + ->Unit(benchmark::kMillisecond); BENCHMARK(bench_commit_sparse_preprocessed) - ->DenseRange(14, MAX_LOG_NUM_POINTS) + ->DenseRange(MIN_LOG_NUM_POINTS, MAX_LOG_NUM_POINTS) + ->Unit(benchmark::kMillisecond); +BENCHMARK(bench_commit_sparse_random) + ->DenseRange(MIN_LOG_NUM_POINTS, MAX_LOG_NUM_POINTS) ->Unit(benchmark::kMillisecond); -BENCHMARK(bench_commit_sparse_random)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); BENCHMARK(bench_commit_sparse_random_preprocessed) - ->DenseRange(14, MAX_LOG_NUM_POINTS) + ->DenseRange(MIN_LOG_NUM_POINTS, MAX_LOG_NUM_POINTS) + ->Unit(benchmark::kMillisecond); +BENCHMARK(bench_commit_random) + ->DenseRange(MIN_LOG_NUM_POINTS, MAX_LOG_NUM_POINTS) ->Unit(benchmark::kMillisecond); -BENCHMARK(bench_commit_random)->DenseRange(14, MAX_LOG_NUM_POINTS)->Unit(benchmark::kMillisecond); } // namespace bb From 3ee1bab653ecb5b39398bb07403418e4a1d822a6 Mon Sep 17 00:00:00 2001 From: ledwards2225 Date: Mon, 22 Jul 2024 19:28:39 +0000 Subject: [PATCH 18/18] reserve space in vectors --- .../barretenberg/commitment_schemes/commitment_key.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp index 66f470448fa..c6b23b071e7 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/commitment_key.hpp @@ -129,9 +129,17 @@ template class CommitmentKey { } }); + // Compute total number of non-trivial input pairs + size_t num_nonzero_scalars = 0; + for (auto& scalars : thread_scalars) { + num_nonzero_scalars += scalars.size(); + } + // Reconstruct the full input to the pippenger from the individual threads std::vector scalars; std::vector points; + scalars.reserve(num_nonzero_scalars); + points.reserve(num_nonzero_scalars); for (size_t idx = 0; idx < num_threads; ++idx) { scalars.insert(scalars.end(), thread_scalars[idx].begin(), thread_scalars[idx].end()); points.insert(points.end(), thread_points[idx].begin(), thread_points[idx].end());