Merge pull request #55 from rbino/feat/generated-bindings

Generate bindings from Zig

.github/workflows/main.yml

@@ -19,8 +19,6 @@ jobs:
     steps:
       - name: Clone the repository
         uses: actions/checkout@v4
-        with:
-          submodules: recursive
 
       - name: Install OTP and Elixir
         uses: erlef/setup-beam@v1
@@ -102,23 +100,18 @@ jobs:
     steps:
       - name: Clone the repository
         uses: actions/checkout@v4
-        with:
-          submodules: recursive
 
       - name: Fetch TigerBeetle for Linux
         if: runner.os == 'Linux'
-        working-directory: ./src/tigerbeetle
         run: |
           curl -Lo tigerbeetle.zip https://linux.tigerbeetle.com && unzip tigerbeetle.zip
 
       - name: Fetch TigerBeetle for Mac
         if: runner.os == 'macOS'
-        working-directory: ./src/tigerbeetle
         run: |
           curl -Lo tigerbeetle.zip https://mac.tigerbeetle.com && unzip tigerbeetle.zip
 
       - name: Start TigerBeetle
-        working-directory: ./src/tigerbeetle
         run: |
           ./tigerbeetle format --cluster=0 --replica=0 --replica-count=1 --development 0_0.tigerbeetle
           ./tigerbeetle start --addresses=3000 --development 0_0.tigerbeetle &

.gitignore

@@ -27,6 +27,7 @@ tigerbeetlex-*.tar
 
 # Sometimes created in the root folder
 zig-cache/
+.zig-cache/
 
 # Ignore .elixir_ls generated folder
 .elixir_ls/

README.md

@@ -13,7 +13,7 @@ The package can be installed by adding `tigerbeetlex` to your list of dependenci
 ```elixir
 def deps do
   [
-    {:tigerbeetlex, github: "rbino/tigerbeetlex", submodules: true}
+    {:tigerbeetlex, github: "rbino/tigerbeetlex"}
   ]
 end
 ```
@@ -29,7 +29,7 @@ be found at <https://hexdocs.pm/tigerbeetlex>.
 Clone the repo and fetch dependencies:
 
 ```bash
-$ git clone --recurse-submodules https://github.com/rbino/tigerbeetlex.git
+$ git clone https://github.com/rbino/tigerbeetlex.git
 $ cd tigerbeetlex
 $ mix deps.get
 ```
@@ -64,7 +64,7 @@ $ mix test
 
 ## License
 
-Copyright 2023-2024 Riccardo Binetti
+Copyright 2023-2025 Riccardo Binetti
 
 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
 compliance with the License. You may obtain a copy of the License at

bench/benchmark.exs

@@ -40,9 +40,9 @@ bench = fn ->
 
       {elapsed, response} = :timer.tc(fn -> Connection.create_transfers(:tb, batch) end)
 
-      {:ok, stream} = response
+      {:ok, results} = response
 
-      if length(Enum.to_list(stream)) != length(chunk), do: raise("Invalid result")
+      if length(results) != length(chunk), do: raise("Invalid result")
 
       max = max(max_batch_us, elapsed)
       total = time_total_us + elapsed

build.zig

@@ -3,7 +3,7 @@ const std = @import("std");
 // Although this function looks imperative, note that its job is to
 // declaratively construct a build graph that will be executed by an external
 // runner.
-pub fn build(b: *std.Build) void {
+pub fn build(b: *std.Build) !void {
     // Standard target options allows the person running `zig build` to choose
     // what target to build for. Here we do not override the defaults, which
     // means any target is allowed, and the default is native. Other options
@@ -16,7 +16,7 @@ pub fn build(b: *std.Build) void {
     const optimize = b.standardOptimizeOption(.{});
 
     // Get ERTS_INCLUDE_DIR from env, which should be passed by :build_dot_zig
-    const erts_include_dir = std.process.getEnvVarOwned(b.allocator, "ERTS_INCLUDE_DIR") catch blk: {
+    const erts_include_dir = b.graph.env_map.get("ERTS_INCLUDE_DIR") orelse blk: {
         // Fallback to extracting it from the erlang shell so we can also execute zig build manually
         const argv = [_][]const u8{
             "erl",
@@ -31,6 +31,31 @@ pub fn build(b: *std.Build) void {
         break :blk b.run(&argv);
     };
 
+    const tigerbeetle_dep = b.dependency("tigerbeetle", .{});
+    const vsr_mod = b.createModule(.{
+        .root_source_file = tigerbeetle_dep.path("src/vsr.zig"),
+    });
+
+    const config_mod = b.createModule(.{
+        .root_source_file = b.path("src/config.zig"),
+    });
+
+    const elixir_bindings_generator = b.addExecutable(.{
+        .name = "elixir_bindings",
+        .root_source_file = b.path("tools/elixir_bindings.zig"),
+        .target = b.graph.host,
+    });
+    elixir_bindings_generator.root_module.addImport("config", config_mod);
+    elixir_bindings_generator.root_module.addImport("vsr", vsr_mod);
+
+    const elixir_bindings_generator_step = b.addRunArtifact(elixir_bindings_generator);
+
+    const elixir_bindings_formatting_step = b.addSystemCommand(&.{ "mix", "format" });
+    elixir_bindings_formatting_step.step.dependOn(&elixir_bindings_generator_step.step);
+
+    const generate = b.step("bindings", "Generates the Elixir bindings from TigerBeetle source");
+    generate.dependOn(&elixir_bindings_formatting_step.step);
+
     const lib = b.addSharedLibrary(.{
         .name = "tigerbeetlex",
         // In this case the main source file is merely a path, however, in more
@@ -41,6 +66,10 @@ pub fn build(b: *std.Build) void {
         .link_libc = true,
     });
     lib.addSystemIncludePath(.{ .cwd_relative = erts_include_dir });
+    // Config (vsr_config) imports
+    lib.root_module.addImport("config", config_mod);
+    // TigerBeetle imports
+    lib.root_module.addImport("vsr", vsr_mod);
     // This is needed to avoid errors on MacOS when loading the NIF
     lib.linker_allow_shlib_undefined = true;
 

build.zig.zon

@@ -0,0 +1,36 @@
+.{
+    // This is the default name used by packages depending on this one. For
+    // example, when a user runs `zig fetch --save <url>`, this field is used
+    // as the key in the `dependencies` table. Although the user can choose a
+    // different name, most users will stick with this provided value.
+    //
+    // It is redundant to include "zig" in this name because it is already
+    // within the Zig package namespace.
+    .name = "tigerbeetlex",
+
+    // This is a [Semantic Version](https://semver.org/).
+    // In a future version of Zig it will be used for package deduplication.
+    .version = "0.0.0",
+
+    // This field is optional.
+    // This is currently advisory only; Zig does not yet do anything
+    // with this value.
+    .minimum_zig_version = "0.13.0",
+
+    // This field is optional.
+    // Each dependency must either provide a `url` and `hash`, or a `path`.
+    // `zig build --fetch` can be used to fetch all dependencies of a package, recursively.
+    // Once all dependencies are fetched, `zig build` no longer requires
+    // internet connectivity.
+    .dependencies = .{
+        .tigerbeetle = .{
+            .url = "https://github.com/tigerbeetle/tigerbeetle/archive/refs/tags/0.16.27.tar.gz",
+            .hash = "1220799ff2ee1f14db1e01385a39d3c6668bfffb19cb5103907c4576272faf82eddf",
+        },
+    },
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+        "src",
+    },
+}

lib/tigerbeetlex.ex

@@ -65,15 +65,15 @@ defmodule TigerBeetlex do
       {:tigerbeetlex_response, request_ref, response}
 
   Where `request_ref` is the same `ref` returned when this function was called and `response` is
-  a response that can be decoded using `TigerBeetlex.Response.to_stream/1`.
+  a response that can be decoded using `TigerBeetlex.Response.decode/1`.
 
-  The value returned from `TigerBeetlex.Response.to_stream(response)` will either be
-  `{:error, reason}` or `{:ok, stream}`.
+  The value returned from `TigerBeetlex.Response.decode(response)` will either be
+  `{:error, reason}` or `{:ok, results}`.
 
-  `stream` is an enumerable that can lazily produce `%TigerBeetlex.CreateAccountError{}` structs
+  `results` is a list of `%TigerBeetlex.CreateAccountsResult{}` structs
   which contain the index of the account batch and the reason of the failure. An account has a
-  corresponding `%TigerBeetlex.CreateAccountError{}` only if it fails to be created, otherwise
-  the account has been created succesfully (so a successful request returns an empty stream).
+  corresponding `%TigerBeetlex.CreateAccountsResult{}` only if it fails to be created, otherwise
+  the account has been created succesfully (so a successful request returns an empty list).
 
   ## Examples
 
@@ -84,13 +84,11 @@ defmodule TigerBeetlex do
 
       {:ok, ref} = TigerBeetlex.create_accounts(client, batch)
 
-      {:ok, stream} =
-        receive do
-          {:tigerbeetlex_response, ^ref, response} -> TigerBeetlex.Response.to_stream(response)
-        end
+      receive do
+        {:tigerbeetlex_response, ^ref, response} -> TigerBeetlex.Response.decode(response)
+      end
 
-      Enum.to_list(stream)
-      #=> []
+      #=> {:ok, []}
 
       # Creation error
       batch =
@@ -99,13 +97,11 @@ defmodule TigerBeetlex do
 
       {:ok, ref} = TigerBeetlex.create_accounts(client, batch)
 
-      {:ok, stream} =
-        receive do
-          {:tigerbeetlex_response, ^ref, response} -> TigerBeetlex.Response.to_stream(response)
-        end
+      receive do
+        {:tigerbeetlex_response, ^ref, response} -> TigerBeetlex.Response.decode(response)
+      end
 
-      Enum.to_list(stream)
-      #=> [%TigerBeetlex.CreateAccountError{index: 0, reason: :id_must_not_be_zero}]
+      #=> {:ok, [%TigerBeetlex.CreateAccountError{index: 0, reason: :id_must_not_be_zero}]}
   """
   @spec create_accounts(client :: t(), account_batch :: TigerBeetlex.AccountBatch.t()) ::
           {:ok, reference()} | {:error, Types.create_accounts_error()}
@@ -128,15 +124,15 @@ defmodule TigerBeetlex do
       {:tigerbeetlex_response, request_ref, response}
 
   Where `request_ref` is the same `ref` returned when this function was called and `response` is
-  a response that can be decoded using `TigerBeetlex.Response.to_stream/1`.
+  a response that can be decoded using `TigerBeetlex.Response.decode/1`.
 
-  The value returned from `TigerBeetlex.Response.to_stream(response)` will either be
-  `{:error, reason}` or `{:ok, stream}`.
+  The value returned from `TigerBeetlex.Response.decode(response)` will either be
+  `{:error, reason}` or `{:ok, results}`.
 
-  `stream` is an enumerable that can lazily produce `%TigerBeetlex.CreateTransferError{}` structs
+  `results` is a list of `%TigerBeetlex.CreateTransfersResult{}` structs
   which contain the index of the transfer batch and the reason of the failure. An transfer has a
-  corresponding `%TigerBeetlex.CreateTransferError{}` only if it fails to be created, otherwise
-  the transfer has been created succesfully (so a successful request returns an empty stream).
+  corresponding `%TigerBeetlex.CreateTransfersResult{}` only if it fails to be created, otherwise
+  the transfer has been created succesfully (so a successful request returns an empty list).
 
   ## Examples
 
@@ -156,13 +152,11 @@ defmodule TigerBeetlex do
 
       {:ok, ref} = TigerBeetlex.create_transfers(client, batch)
 
-      {:ok, stream} =
-        receive do
-          {:tigerbeetlex_response, ^ref, response} -> TigerBeetlex.Response.to_stream(response)
-        end
+      receive do
+        {:tigerbeetlex_response, ^ref, response} -> TigerBeetlex.Response.decode(response)
+      end
 
-      Enum.to_list(stream)
-      #=> []
+      #=> {:ok, []}
 
       # Creation error
       batch =
@@ -180,13 +174,11 @@ defmodule TigerBeetlex do
 
       {:ok, ref} = TigerBeetlex.create_transfers(client, batch)
 
-      {:ok, stream} =
-        receive do
-          {:tigerbeetlex_response, ^ref, response} -> TigerBeetlex.Response.to_stream(response)
-        end
+      receive do
+        {:tigerbeetlex_response, ^ref, response} -> TigerBeetlex.Response.decode(response)
+      end
 
-      Enum.to_list(stream)
-      #=> [%TigerBeetlex.CreateTransferError{index: 0, reason: :id_must_not_be_zero}]
+      #=> {:ok, [%TigerBeetlex.CreateTransferError{index: 0, reason: :id_must_not_be_zero}]}
   """
   @spec create_transfers(client :: t(), transfer_batch :: TigerBeetlex.TransferBatch.t()) ::
           {:ok, reference()} | {:error, Types.create_transfers_error()}
@@ -209,14 +201,14 @@ defmodule TigerBeetlex do
       {:tigerbeetlex_response, request_ref, response}
 
   Where `request_ref` is the same `ref` returned when this function was called and `response` is
-  a response that can be decoded using `TigerBeetlex.Response.to_stream/1`.
+  a response that can be decoded using `TigerBeetlex.Response.decode/1`.
 
-  The value returned from `TigerBeetlex.Response.to_stream(response)` will either be
-  `{:error, reason}` or `{:ok, stream}`.
+  The value returned from `TigerBeetlex.Response.decode(response)` will either be
+  `{:error, reason}` or `{:ok, results}`.
 
-  `stream` is an enumerable that can lazily produce `%TigerBeetlex.Account{}` structs. If an id in
-  the batch does not correspond to an existing account, it will simply be skipped, so the result
-  could have less accounts then the provided ids in the id batch.
+  `results` is a list of `%TigerBeetlex.Account{}` structs. If an id in the batch does not
+  correspond to an existing account, it will simply be skipped, so the result could have less
+  accounts then the provided ids in the id batch.
 
   ## Examples
 
@@ -226,13 +218,11 @@ defmodule TigerBeetlex do
 
       {:ok, ref} = TigerBeetlex.lookup_accounts(client, batch)
 
-      {:ok, stream} =
-        receive do
-          {:tigerbeetlex_response, ^ref, response} -> TigerBeetlex.Response.to_stream(response)
-        end
+      receive do
+        {:tigerbeetlex_response, ^ref, response} -> TigerBeetlex.Response.decode(response)
+      end
 
-      Enum.to_list(stream)
-      #=> [%TigerBeetlex.Account{}]
+      #=> {:ok, [%TigerBeetlex.Account{}]}
   """
   @spec lookup_accounts(client :: t(), id_batch :: TigerBeetlex.IDBatch.t()) ::
           {:ok, reference()} | {:error, Types.lookup_accounts_error()}
@@ -255,14 +245,14 @@ defmodule TigerBeetlex do
       {:tigerbeetlex_response, request_ref, response}
 
   Where `request_ref` is the same `ref` returned when this function was called and `response` is
-  a response that can be decoded using `TigerBeetlex.Response.to_stream/1`.
+  a response that can be decoded using `TigerBeetlex.Response.decode/1`.
 
-  The value returned from `TigerBeetlex.Response.to_stream(response)` will either be
-  `{:error, reason}` or `{:ok, stream}`.
+  The value returned from `TigerBeetlex.Response.decode(response)` will either be
+  `{:error, reason}` or `{:ok, results}`.
 
-  `stream` is an enumerable that can lazily produce `%TigerBeetlex.Transfer{}` structs. If an id in
-  the batch does not correspond to an existing transfer, it will simply be skipped, so the result
-  could have less accounts then the provided ids in the id batch.
+  `results` is list of `%TigerBeetlex.Transfer{}` structs. If an id in the batch does not correspond
+  to an existing transfer, it will simply be skipped, so the result could have less accounts then
+  the provided ids in the id batch.
 
   ## Examples
 
@@ -272,13 +262,11 @@ defmodule TigerBeetlex do
 
       {:ok, ref} = TigerBeetlex.lookup_transfers(client, batch)
 
-      {:ok, stream} =
-        receive do
-          {:tigerbeetlex_response, ^ref, response} -> TigerBeetlex.Response.to_stream(response)
-        end
+      receive do
+        {:tigerbeetlex_response, ^ref, response} -> TigerBeetlex.Response.decode(response)
+      end
 
-      Enum.to_list(stream)
-      #=> [%TigerBeetlex.Transfer{}]
+      #=> {:ok, [%TigerBeetlex.Transfer{}]}
   """
   @spec lookup_transfers(client :: t(), id_batch :: TigerBeetlex.IDBatch.t()) ::
           {:ok, reference()} | {:error, Types.lookup_transfers_error()}