feat: toSubscriptionIterator impl for IReceiver (#1307)

Author: weboko

package-lock.json

@@ -3,9 +3,9 @@
   "private": true,
   "type": "module",
   "workspaces": [
+    "packages/interfaces",
     "packages/utils",
     "packages/proto",
-    "packages/interfaces",
     "packages/enr",
     "packages/core",
     "packages/message-hash",

package.json

@@ -4,13 +4,15 @@ import type { IncomingStreamData } from "@libp2p/interface-registrar";
 import type {
   ActiveSubscriptions,
   Callback,
+  IAsyncIterator,
   IDecodedMessage,
   IDecoder,
   IFilter,
   ProtocolCreateOptions,
   ProtocolOptions,
 } from "@waku/interfaces";
 import { WakuMessage as WakuMessageProto } from "@waku/proto";
+import { toAsyncIterator } from "@waku/utils";
 import debug from "debug";
 import all from "it-all";
 import * as lp from "it-length-prefixed";
@@ -124,6 +126,13 @@ class Filter extends BaseProtocol implements IFilter {
     };
   }
 
+  public toSubscriptionIterator<T extends IDecodedMessage>(
+    decoders: IDecoder<T> | IDecoder<T>[],
+    opts?: ProtocolOptions | undefined
+  ): Promise<IAsyncIterator<T>> {
+    return toAsyncIterator(this, decoders, opts);
+  }
+
   public getActiveSubscriptions(): ActiveSubscriptions {
     const map: ActiveSubscriptions = new Map();
     const subscriptions = this.subscriptions as Map<

packages/core/src/lib/filter/index.ts

@@ -12,14 +12,17 @@ import { sha256 } from "@noble/hashes/sha256";
 import type {
   ActiveSubscriptions,
   Callback,
+  IAsyncIterator,
   IDecodedMessage,
   IDecoder,
   IEncoder,
   IMessage,
   IRelay,
   ProtocolCreateOptions,
+  ProtocolOptions,
   SendResult,
 } from "@waku/interfaces";
+import { toAsyncIterator } from "@waku/utils";
 import debug from "debug";
 
 import { DefaultPubSubTopic } from "../constants.js";
@@ -146,6 +149,13 @@ class Relay implements IRelay {
     };
   }
 
+  public toSubscriptionIterator<T extends IDecodedMessage>(
+    decoders: IDecoder<T> | IDecoder<T>[],
+    opts?: ProtocolOptions | undefined
+  ): Promise<IAsyncIterator<T>> {
+    return toAsyncIterator(this, decoders, opts);
+  }
+
   public getActiveSubscriptions(): ActiveSubscriptions {
     const map = new Map();
     map.set(this.pubSubTopic, this.observers.keys());

packages/core/src/lib/relay/index.ts

@@ -10,3 +10,4 @@ export * from "./waku.js";
 export * from "./connection_manager.js";
 export * from "./sender.js";
 export * from "./receiver.js";
+export * from "./misc.js";

packages/interfaces/src/index.ts

@@ -0,0 +1,11 @@
+import type { IDecodedMessage } from "./message.js";
+
+export interface IAsyncIterator<T extends IDecodedMessage> {
+  iterator: AsyncIterator<T>;
+  stop: Unsubscribe;
+}
+
+export type Unsubscribe = () => void | Promise<void>;
+
+export type PubSubTopic = string;
+export type ContentTopic = string;

packages/interfaces/src/misc.ts

@@ -1,13 +1,19 @@
 import type { IDecodedMessage, IDecoder } from "./message.js";
+import type {
+  ContentTopic,
+  IAsyncIterator,
+  PubSubTopic,
+  Unsubscribe,
+} from "./misc.js";
 import type { Callback, ProtocolOptions } from "./protocols.js";
 
-type Unsubscribe = () => void | Promise<void>;
-type PubSubTopic = string;
-type ContentTopic = string;
-
 export type ActiveSubscriptions = Map<PubSubTopic, ContentTopic[]>;
 
 export interface IReceiver {
+  toSubscriptionIterator: <T extends IDecodedMessage>(
+    decoders: IDecoder<T> | IDecoder<T>[],
+    opts?: ProtocolOptions
+  ) => Promise<IAsyncIterator<T>>;
   subscribe: <T extends IDecodedMessage>(
     decoders: IDecoder<T> | IDecoder<T>[],
     callback: Callback<T>,

packages/interfaces/src/receiver.ts

@@ -0,0 +1,98 @@
+import {
+  createDecoder,
+  createEncoder,
+  DefaultPubSubTopic,
+  waitForRemotePeer,
+} from "@waku/core";
+import { createLightNode } from "@waku/create";
+import type { LightNode } from "@waku/interfaces";
+import { Protocols } from "@waku/interfaces";
+import { toAsyncIterator } from "@waku/utils";
+import { bytesToUtf8, utf8ToBytes } from "@waku/utils/bytes";
+import { expect } from "chai";
+
+import { makeLogFileName, NOISE_KEY_1, Nwaku } from "../src/index.js";
+
+const TestContentTopic = "/test/1/waku-filter";
+const TestEncoder = createEncoder({ contentTopic: TestContentTopic });
+const TestDecoder = createDecoder(TestContentTopic);
+
+describe("Util: toAsyncIterator", () => {
+  let waku: LightNode;
+  let nwaku: Nwaku;
+
+  beforeEach(async function () {
+    this.timeout(15000);
+    nwaku = new Nwaku(makeLogFileName(this));
+    await nwaku.start({ filter: true, lightpush: true, relay: true });
+    waku = await createLightNode({
+      staticNoiseKey: NOISE_KEY_1,
+      libp2p: { addresses: { listen: ["/ip4/0.0.0.0/tcp/0/ws"] } },
+    });
+    await waku.start();
+    await waku.dial(await nwaku.getMultiaddrWithId());
+    await waitForRemotePeer(waku, [Protocols.Filter, Protocols.LightPush]);
+  });
+
+  afterEach(async () => {
+    try {
+      await nwaku.stop();
+      await waku.stop();
+    } catch (err) {
+      console.log("Failed to stop", err);
+    }
+  });
+
+  it("creates an iterator", async function () {
+    const messageText = "hey, what's up?";
+    const sent = { payload: utf8ToBytes(messageText) };
+
+    const { iterator } = await toAsyncIterator(waku.filter, TestDecoder);
+
+    await waku.lightPush.send(TestEncoder, sent);
+    const { value } = await iterator.next();
+
+    expect(value.contentTopic).to.eq(TestContentTopic);
+    expect(value.pubSubTopic).to.eq(DefaultPubSubTopic);
+    expect(bytesToUtf8(value.payload)).to.eq(messageText);
+  });
+
+  it("handles multiple messages", async function () {
+    const { iterator } = await toAsyncIterator(waku.filter, TestDecoder);
+
+    await waku.lightPush.send(TestEncoder, {
+      payload: utf8ToBytes("Filtering works!"),
+    });
+    await waku.lightPush.send(TestEncoder, {
+      payload: utf8ToBytes("Filtering still works!"),
+    });
+
+    let result = await iterator.next();
+    expect(bytesToUtf8(result.value.payload)).to.eq("Filtering works!");
+
+    result = await iterator.next();
+    expect(bytesToUtf8(result.value.payload)).to.eq("Filtering still works!");
+  });
+
+  it("unsubscribes", async function () {
+    const { iterator, stop } = await toAsyncIterator(waku.filter, TestDecoder);
+
+    await waku.lightPush.send(TestEncoder, {
+      payload: utf8ToBytes("This should be received"),
+    });
+
+    await stop();
+
+    await waku.lightPush.send(TestEncoder, {
+      payload: utf8ToBytes("This should not be received"),
+    });
+
+    let result = await iterator.next();
+    expect(result.done).to.eq(true);
+    expect(bytesToUtf8(result.value.payload)).to.eq("This should be received");
+
+    result = await iterator.next();
+    expect(result.value).to.eq(undefined);
+    expect(result.done).to.eq(true);
+  });
+});

packages/tests/tests/utils.spec.ts

@@ -78,6 +78,7 @@
     "@typescript-eslint/eslint-plugin": "^5.57.0",
     "@typescript-eslint/parser": "^5.51.0",
     "@waku/build-utils": "*",
+    "@waku/interfaces": "*",
     "cspell": "^6.31.1",
     "eslint": "^8.35.0",
     "eslint-config-prettier": "^8.6.0",

packages/utils/package.json

@@ -1,2 +1,3 @@
 export * from "./is_defined.js";
 export * from "./random_subset.js";
+export * from "./to_async_iterator.js";

packages/utils/src/common/index.ts

@@ -0,0 +1,51 @@
+import type {
+  IAsyncIterator,
+  IDecodedMessage,
+  IDecoder,
+  IReceiver,
+  ProtocolOptions,
+  Unsubscribe,
+} from "@waku/interfaces";
+
+export async function toAsyncIterator<T extends IDecodedMessage>(
+  receiver: IReceiver,
+  decoder: IDecoder<T> | IDecoder<T>[],
+  options?: ProtocolOptions
+): Promise<IAsyncIterator<T>> {
+  const messages: T[] = [];
+
+  let unsubscribe: undefined | Unsubscribe;
+  unsubscribe = await receiver.subscribe(
+    decoder,
+    (message: T) => {
+      messages.push(message);
+    },
+    options
+  );
+
+  async function* iterator(): AsyncIterator<T> {
+    while (true) {
+      const message = messages.shift() as T;
+
+      if (!unsubscribe && messages.length === 0) {
+        return message;
+      }
+
+      if (!message && unsubscribe) {
+        continue;
+      }
+
+      yield message;
+    }
+  }
+
+  return {
+    iterator: iterator(),
+    async stop() {
+      if (unsubscribe) {
+        await unsubscribe();
+        unsubscribe = undefined;
+      }
+    },
+  };
+}