feat: make ShardingParams optional in sdk, required internally

adklempner · adklempner · commit 68d3229644f3 · 2024-01-24T07:36:00.000-08:00
diff --git a/packages/core/src/lib/base_protocol.ts b/packages/core/src/lib/base_protocol.ts
@@ -7,7 +7,11 @@ import type {
   PubsubTopic
 } from "@waku/interfaces";
 import { DefaultPubsubTopic } from "@waku/interfaces";
-import { Logger, shardInfoToPubsubTopics } from "@waku/utils";
+import {
+  ensureShardingConfigured,
+  Logger,
+  shardInfoToPubsubTopics
+} from "@waku/utils";
 import {
   getConnectedPeersForProtocolAndShard,
   getPeersForProtocol,
@@ -108,6 +112,8 @@ export class BaseProtocol implements IBaseProtocol {
         this.peerStore,
         [this.multicodec],
         this.options?.shardInfo
+          ? ensureShardingConfigured(this.options.shardInfo).shardInfo
+          : undefined
       );
 
     // Filter the peers based on discovery & number of peers requested
diff --git a/packages/core/src/lib/metadata/index.ts b/packages/core/src/lib/metadata/index.ts
@@ -132,7 +132,7 @@ class Metadata extends BaseProtocol implements IMetadata {
 }
 
 export function wakuMetadata(
-  shardInfo: ShardingParams
+  shardInfo: ShardInfo
 ): (components: Libp2pComponents) => IMetadata {
   return (components: Libp2pComponents) => new Metadata(shardInfo, components);
 }
diff --git a/packages/interfaces/src/constants.ts b/packages/interfaces/src/constants.ts
@@ -2,3 +2,8 @@
  * DefaultPubsubTopic is the default gossipsub topic to use for Waku.
  */
 export const DefaultPubsubTopic = "/waku/2/default-waku/proto";
+
+/**
+ * The default cluster ID for The Waku Network
+ */
+export const DEFAULT_CLUSTER_ID = 1;
diff --git a/packages/interfaces/src/message.ts b/packages/interfaces/src/message.ts
@@ -5,7 +5,7 @@ export interface SingleShardInfo {
   /**
    * Specifying this field indicates to the encoder/decoder that static sharding must be used.
    */
-  shard?: number;
+  shard: number;
 }
 
 export interface IRateLimitProof {
diff --git a/packages/interfaces/src/protocols.ts b/packages/interfaces/src/protocols.ts
@@ -60,7 +60,7 @@ export type ProtocolCreateOptions = {
    * See [Waku v2 Topic Usage Recommendations](https://rfc.vac.dev/spec/23/) for details.
    *
    */
-  shardInfo?: ShardingParams;
+  shardInfo?: Partial<ShardingParams>;
   /**
    * You can pass options to the `Libp2p` instance used by {@link @waku/core!WakuNode} using the `libp2p` property.
    * This property is the same type as the one passed to [`Libp2p.create`](https://github.com/libp2p/js-libp2p/blob/master/doc/API.md#create)
diff --git a/packages/sdk/src/create.ts b/packages/sdk/src/create.ts
@@ -16,18 +16,19 @@ import {
   wakuStore
 } from "@waku/core";
 import { enrTree, wakuDnsDiscovery } from "@waku/dns-discovery";
-import type {
-  CreateLibp2pOptions,
-  FullNode,
-  IMetadata,
-  Libp2p,
-  Libp2pComponents,
-  LightNode,
-  ProtocolCreateOptions,
-  ShardingParams
+import {
+  type CreateLibp2pOptions,
+  type FullNode,
+  type IMetadata,
+  type Libp2p,
+  type Libp2pComponents,
+  type LightNode,
+  type ProtocolCreateOptions,
+  type ShardInfo
 } from "@waku/interfaces";
 import { wakuPeerExchangeDiscovery } from "@waku/peer-exchange";
 import { RelayCreateOptions, wakuGossipSub, wakuRelay } from "@waku/relay";
+import { ensureShardingConfigured } from "@waku/utils";
 import { createLibp2p } from "libp2p";
 
 const DEFAULT_NODE_REQUIREMENTS = {
@@ -38,17 +39,6 @@ const DEFAULT_NODE_REQUIREMENTS = {
 
 export { Libp2pComponents };
 
-const ensureShardingConfigured = (shardInfo: ShardingParams): void => {
-  if (
-    ("shards" in shardInfo && shardInfo.shards.length < 1) ||
-    ("contentTopics" in shardInfo && shardInfo.contentTopics.length < 1)
-  ) {
-    throw new Error(
-      "Missing required configuration options for static sharding or autosharding."
-    );
-  }
-};
-
 /**
  * Create a Waku node configured to use autosharding or static sharding.
  */
@@ -61,7 +51,7 @@ export async function createNode(
     throw new Error("Shard info must be set");
   }
 
-  ensureShardingConfigured(options.shardInfo);
+  const shardInfo = ensureShardingConfigured(options.shardInfo);
 
   const libp2pOptions = options?.libp2p ?? {};
   const peerDiscovery = libp2pOptions.peerDiscovery ?? [];
@@ -71,7 +61,7 @@ export async function createNode(
   }
 
   const libp2p = await defaultLibp2p(
-    undefined,
+    shardInfo.shardInfo,
     wakuGossipSub(options),
     libp2pOptions,
     options?.userAgent
@@ -85,7 +75,7 @@ export async function createNode(
     options ?? {},
     [],
     libp2p,
-    options.shardInfo,
+    shardInfo.shardInfo,
     store,
     lightPush,
     filter
@@ -102,9 +92,9 @@ export async function createLightNode(
 ): Promise<LightNode> {
   options = options ?? {};
 
-  if (options.shardInfo) {
-    ensureShardingConfigured(options.shardInfo);
-  }
+  const shardInfo = options.shardInfo
+    ? ensureShardingConfigured(options.shardInfo)
+    : undefined;
 
   const libp2pOptions = options?.libp2p ?? {};
   const peerDiscovery = libp2pOptions.peerDiscovery ?? [];
@@ -114,7 +104,7 @@ export async function createLightNode(
   }
 
   const libp2p = await defaultLibp2p(
-    options.shardInfo,
+    shardInfo?.shardInfo,
     wakuGossipSub(options),
     libp2pOptions,
     options?.userAgent
@@ -128,7 +118,7 @@ export async function createLightNode(
     options ?? {},
     options.pubsubTopics,
     libp2p,
-    options.shardInfo,
+    shardInfo?.shardingParams,
     store,
     lightPush,
     filter
@@ -153,9 +143,9 @@ export async function createFullNode(
 ): Promise<FullNode> {
   options = options ?? {};
 
-  if (options.shardInfo) {
-    ensureShardingConfigured(options.shardInfo);
-  }
+  const shardInfo = options.shardInfo
+    ? ensureShardingConfigured(options.shardInfo)
+    : undefined;
 
   const libp2pOptions = options?.libp2p ?? {};
   const peerDiscovery = libp2pOptions.peerDiscovery ?? [];
@@ -165,7 +155,7 @@ export async function createFullNode(
   }
 
   const libp2p = await defaultLibp2p(
-    options.shardInfo,
+    shardInfo?.shardInfo,
     wakuGossipSub(options),
     libp2pOptions,
     options?.userAgent
@@ -180,7 +170,7 @@ export async function createFullNode(
     options ?? {},
     options.pubsubTopics,
     libp2p,
-    options.shardInfo,
+    shardInfo?.shardingParams,
     store,
     lightPush,
     filter,
@@ -207,7 +197,7 @@ type MetadataService = {
 };
 
 export async function defaultLibp2p(
-  shardInfo?: ShardingParams,
+  shardInfo?: ShardInfo,
   wakuGossipSub?: PubsubService["pubsub"],
   options?: Partial<CreateLibp2pOptions>,
   userAgent?: string
diff --git a/packages/sdk/src/relay/index.ts b/packages/sdk/src/relay/index.ts
@@ -1,6 +1,7 @@
 import { WakuNode, WakuOptions } from "@waku/core";
 import type { ProtocolCreateOptions, RelayNode } from "@waku/interfaces";
 import { RelayCreateOptions, wakuGossipSub, wakuRelay } from "@waku/relay";
+import { ensureShardingConfigured } from "@waku/utils";
 
 import { defaultLibp2p, defaultPeerDiscoveries } from "../create.js";
 
@@ -26,8 +27,12 @@ export async function createRelayNode(
     Object.assign(libp2pOptions, { peerDiscovery });
   }
 
+  const shardInfo = options.shardInfo
+    ? ensureShardingConfigured(options.shardInfo)
+    : undefined;
+
   const libp2p = await defaultLibp2p(
-    options.shardInfo,
+    shardInfo?.shardInfo,
     wakuGossipSub(options),
     libp2pOptions,
     options?.userAgent
@@ -39,7 +44,7 @@ export async function createRelayNode(
     options,
     options.pubsubTopics,
     libp2p,
-    options.shardInfo,
+    shardInfo?.shardingParams,
     undefined,
     undefined,
     undefined,
diff --git a/packages/tests/tests/filter/single_node/multiple_pubsub.node.spec.ts b/packages/tests/tests/filter/single_node/multiple_pubsub.node.spec.ts
@@ -9,6 +9,7 @@ import type {
 import { Protocols } from "@waku/interfaces";
 import {
   contentTopicToPubsubTopic,
+  contentTopicToShardIndex,
   pubsubTopicToSingleShardInfo,
   singleShardInfoToPubsubTopic
 } from "@waku/utils";
@@ -207,17 +208,25 @@ describe("Waku Filter V2 (Autosharding): Multiple PubsubTopics", function () {
   const customEncoder1 = createEncoder({
     contentTopic: customContentTopic1,
     pubsubTopicShardInfo: {
-      clusterId: 3
+      clusterId: 3,
+      shard: contentTopicToShardIndex(customContentTopic1)
     }
   });
-  const customDecoder1 = createDecoder(customContentTopic1, { clusterId: 3 });
+  const customDecoder1 = createDecoder(customContentTopic1, {
+    clusterId: 3,
+    shard: contentTopicToShardIndex(customContentTopic1)
+  });
   const customEncoder2 = createEncoder({
     contentTopic: customContentTopic2,
     pubsubTopicShardInfo: {
-      clusterId: 3
+      clusterId: 3,
+      shard: contentTopicToShardIndex(customContentTopic2)
     }
   });
-  const customDecoder2 = createDecoder(customContentTopic2, { clusterId: 3 });
+  const customDecoder2 = createDecoder(customContentTopic2, {
+    clusterId: 3,
+    shard: contentTopicToShardIndex(customContentTopic2)
+  });
 
   this.beforeEach(async function () {
     this.timeout(15000);
diff --git a/packages/tests/tests/getPeers.spec.ts b/packages/tests/tests/getPeers.spec.ts
@@ -11,7 +11,7 @@ import {
   Tags,
   utf8ToBytes
 } from "@waku/sdk";
-import { shardInfoToPubsubTopics } from "@waku/utils";
+import { ensureShardingConfigured, shardInfoToPubsubTopics } from "@waku/utils";
 import { getConnectedPeersForProtocolAndShard } from "@waku/utils/libp2p";
 import { expect } from "chai";
 import fc from "fast-check";
@@ -237,7 +237,7 @@ describe("getConnectedPeersForProtocolAndShard", function () {
       waku.libp2p.getConnections(),
       waku.libp2p.peerStore,
       waku.libp2p.getProtocols(),
-      shardInfo
+      ensureShardingConfigured(shardInfo).shardInfo
     );
     expect(peers.length).to.be.greaterThan(0);
   });
@@ -289,7 +289,7 @@ describe("getConnectedPeersForProtocolAndShard", function () {
       waku.libp2p.getConnections(),
       waku.libp2p.peerStore,
       waku.libp2p.getProtocols(),
-      shardInfo2
+      ensureShardingConfigured(shardInfo2).shardInfo
     );
     expect(peers.length).to.be.equal(1);
   });
@@ -341,7 +341,7 @@ describe("getConnectedPeersForProtocolAndShard", function () {
       waku.libp2p.getConnections(),
       waku.libp2p.peerStore,
       waku.libp2p.getProtocols(),
-      shardInfo2
+      ensureShardingConfigured(shardInfo2).shardInfo
     );
     expect(peers.length).to.be.equal(1);
   });
@@ -393,7 +393,7 @@ describe("getConnectedPeersForProtocolAndShard", function () {
       waku.libp2p.getConnections(),
       waku.libp2p.peerStore,
       waku.libp2p.getProtocols(),
-      shardInfo2
+      ensureShardingConfigured(shardInfo2).shardInfo
     );
     expect(peers.length).to.be.equal(1);
   });
diff --git a/packages/tests/tests/light-push/single_node/multiple_pubsub.node.spec.ts b/packages/tests/tests/light-push/single_node/multiple_pubsub.node.spec.ts
@@ -10,6 +10,8 @@ import {
 } from "@waku/interfaces";
 import {
   contentTopicToPubsubTopic,
+  contentTopicToShardIndex,
+  pubsubTopicToSingleShardInfo,
   singleShardInfoToPubsubTopic
 } from "@waku/utils";
 import { utf8ToBytes } from "@waku/utils/bytes";
@@ -202,11 +204,11 @@ describe("Waku Light Push (Autosharding): Multiple PubsubTopics", function () {
   };
   const customEncoder1 = createEncoder({
     contentTopic: customContentTopic1,
-    pubsubTopicShardInfo: shardInfo
+    pubsubTopicShardInfo: pubsubTopicToSingleShardInfo(autoshardingPubsubTopic1)
   });
   const customEncoder2 = createEncoder({
     contentTopic: customContentTopic2,
-    pubsubTopicShardInfo: shardInfo
+    pubsubTopicShardInfo: pubsubTopicToSingleShardInfo(autoshardingPubsubTopic2)
   });
 
   let nimPeerId: PeerId;
@@ -356,12 +358,16 @@ describe("Waku Light Push (named sharding): Multiple PubsubTopics", function ()
   const customEncoder1 = createEncoder({
     contentTopic: customContentTopic1,
     pubsubTopicShardInfo: {
-      clusterId
+      clusterId,
+      shard: contentTopicToShardIndex(customContentTopic1)
     }
   });
   const customEncoder2 = createEncoder({
     contentTopic: customContentTopic2,
-    pubsubTopicShardInfo: { clusterId }
+    pubsubTopicShardInfo: {
+      clusterId,
+      shard: contentTopicToShardIndex(customContentTopic2)
+    }
   });
 
   let nimPeerId: PeerId;
diff --git a/packages/tests/tests/relay/multiple_pubsub.node.spec.ts b/packages/tests/tests/relay/multiple_pubsub.node.spec.ts
@@ -14,6 +14,7 @@ import { Protocols } from "@waku/interfaces";
 import { createRelayNode } from "@waku/sdk/relay";
 import {
   contentTopicToPubsubTopic,
+  pubsubTopicToSingleShardInfo,
   singleShardInfoToPubsubTopic
 } from "@waku/utils";
 import { bytesToUtf8, utf8ToBytes } from "@waku/utils/bytes";
@@ -340,16 +341,20 @@ describe("Waku Relay (Autosharding), multiple pubsub topics", function () {
   };
   const customEncoder1 = createEncoder({
     contentTopic: customContentTopic1,
-    pubsubTopicShardInfo: {
-      clusterId: 3
-    }
+    pubsubTopicShardInfo: pubsubTopicToSingleShardInfo(autoshardingPubsubTopic1)
   });
-  const customDecoder1 = createDecoder(customContentTopic1, { clusterId: 3 });
+  const customDecoder1 = createDecoder(
+    customContentTopic1,
+    pubsubTopicToSingleShardInfo(autoshardingPubsubTopic1)
+  );
   const customEncoder2 = createEncoder({
     contentTopic: customContentTopic2,
-    pubsubTopicShardInfo: { clusterId: 3 }
+    pubsubTopicShardInfo: pubsubTopicToSingleShardInfo(autoshardingPubsubTopic2)
   });
-  const customDecoder2 = createDecoder(customContentTopic2, { clusterId: 3 });
+  const customDecoder2 = createDecoder(
+    customContentTopic2,
+    pubsubTopicToSingleShardInfo(autoshardingPubsubTopic2)
+  );
   const contentTopicInfoBothShards: ContentTopicInfo = {
     clusterId: 3,
     contentTopics: [customContentTopic1, customContentTopic2]
diff --git a/packages/tests/tests/sharding/running_nodes.spec.ts b/packages/tests/tests/sharding/running_nodes.spec.ts
diff --git a/packages/tests/tests/store/multiple_pubsub.spec.ts b/packages/tests/tests/store/multiple_pubsub.spec.ts
diff --git a/packages/utils/src/common/sharding.ts b/packages/utils/src/common/sharding.ts
diff --git a/packages/utils/src/libp2p/index.ts b/packages/utils/src/libp2p/index.ts

Original file line number	Diff line number	Diff line change
`@@ -132,7 +132,7 @@ class Metadata extends BaseProtocol implements IMetadata {`
`132`	`132`	`}`
`133`	`133`
`134`	`134`	`export function wakuMetadata(`
`135`		`- shardInfo: ShardingParams`
	`135`	`+ shardInfo: ShardInfo`
`136`	`136`	`): (components: Libp2pComponents) => IMetadata {`
`137`	`137`	`return (components: Libp2pComponents) => new Metadata(shardInfo, components);`
`138`	`138`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,7 @@ export interface SingleShardInfo {`
`5`	`5`	`/**`
`6`	`6`	`* Specifying this field indicates to the encoder/decoder that static sharding must be used.`
`7`	`7`	`*/`
`8`		`- shard?: number;`
	`8`	`+ shard: number;`
`9`	`9`	`}`
`10`	`10`
`11`	`11`	`export interface IRateLimitProof {`
Original file line number	Diff line number	Diff line change
`@@ -60,7 +60,7 @@ export type ProtocolCreateOptions = {`
`60`	`60`	`* See [Waku v2 Topic Usage Recommendations](https://rfc.vac.dev/spec/23/) for details.`
`61`	`61`	`*`
`62`	`62`	`*/`
`63`		`- shardInfo?: ShardingParams;`
	`63`	`+ shardInfo?: Partial<ShardingParams>;`
`64`	`64`	`/**`
`65`	`65`	* You can pass options to the `Libp2p` instance used by {@link @waku/core!WakuNode} using the `libp2p` property.
`66`	`66`	* This property is the same type as the one passed to [`Libp2p.create`](https://github.com/libp2p/js-libp2p/blob/master/doc/API.md#create)