From ec1ac91ea0ac54e851176a8b83dcc4a9c1515b8f Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Wed, 13 Mar 2024 17:09:13 +0530 Subject: [PATCH 01/15] refactor the Store protocol into Core and SDK, simplify `queryGenerator()` --- packages/core/src/lib/store/index.ts | 415 +++++---------------------- packages/interfaces/src/store.ts | 19 +- packages/sdk/src/protocols/store.ts | 318 ++++++++++++++++++++ 3 files changed, 407 insertions(+), 345 deletions(-) create mode 100644 packages/sdk/src/protocols/store.ts diff --git a/packages/core/src/lib/store/index.ts b/packages/core/src/lib/store/index.ts index 8b77adb605..53a7b3097e 100644 --- a/packages/core/src/lib/store/index.ts +++ b/packages/core/src/lib/store/index.ts @@ -1,17 +1,14 @@ -import type { Stream } from "@libp2p/interface"; -import { sha256 } from "@noble/hashes/sha256"; +import type { Peer } from "@libp2p/interface"; import { Cursor, IDecodedMessage, IDecoder, - IStore, + IStoreCore, Libp2p, ProtocolCreateOptions } from "@waku/interfaces"; import { proto_store as proto } from "@waku/proto"; -import { ensurePubsubTopicIsConfigured, isDefined } from "@waku/utils"; import { Logger } from "@waku/utils"; -import { concat, utf8ToBytes } from "@waku/utils/bytes"; import all from "it-all"; import * as lp from "it-length-prefixed"; import { pipe } from "it-pipe"; @@ -28,8 +25,6 @@ const log = new Logger("store"); export const StoreCodec = "/vac/waku/store/2.0.0-beta4"; -export const DefaultPageSize = 10; - export { PageDirection }; export interface TimeFilter { @@ -72,361 +67,105 @@ export interface QueryOptions { * * The Waku Store protocol can be used to retrieved historical messages. */ -class Store extends BaseProtocol implements IStore { - private readonly NUM_PEERS_PROTOCOL = 1; - +export default class StoreCore extends BaseProtocol implements IStoreCore { constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { super(StoreCodec, libp2p.components, log, options!.pubsubTopics!, options); } - /** - * Processes messages based on the provided callback and options. - * @private - */ - private async processMessages( - messages: Promise[], - callback: (message: T) => Promise | boolean | void, - options?: QueryOptions - ): Promise { - let abort = false; - const messagesOrUndef: Array = await Promise.all(messages); - let processedMessages: Array = messagesOrUndef.filter(isDefined); - - if (this.shouldReverseOrder(options)) { - processedMessages = processedMessages.reverse(); - } - - await Promise.all( - processedMessages.map(async (msg) => { - if (msg && !abort) { - abort = Boolean(await callback(msg)); - } - }) - ); - - return abort; - } - - /** - * Determines whether to reverse the order of messages based on the provided options. - * - * Messages in pages are ordered from oldest (first) to most recent (last). - * https://github.com/vacp2p/rfc/issues/533 - * - * @private - */ - private shouldReverseOrder(options?: QueryOptions): boolean { - return ( - typeof options?.pageDirection === "undefined" || - options?.pageDirection === PageDirection.BACKWARD - ); - } - - /** - * @deprecated Use `queryWithOrderedCallback` instead - **/ - queryOrderedCallback = this.queryWithOrderedCallback; - - /** - * Do a query to a Waku Store to retrieve historical/missed messages. - * - * The callback function takes a `WakuMessage` in input, - * messages are processed in order: - * - oldest to latest if `options.pageDirection` == { @link PageDirection.FORWARD } - * - latest to oldest if `options.pageDirection` == { @link PageDirection.BACKWARD } - * - * The ordering may affect performance. - * The ordering depends on the behavior of the remote store node. - * If strong ordering is needed, you may need to handle this at application level - * and set your own timestamps too (the WakuMessage timestamps are not certified). - * - * @throws If not able to reach a Waku Store peer to query, - * or if an error is encountered when processing the reply, - * or if two decoders with the same content topic are passed. - */ - async queryWithOrderedCallback( - decoders: IDecoder[], - callback: (message: T) => Promise | boolean | void, - options?: QueryOptions - ): Promise { - for await (const promises of this.queryGenerator(decoders, options)) { - if (await this.processMessages(promises, callback, options)) break; - } - } - - /** - * Do a query to a Waku Store to retrieve historical/missed messages. - * The callback function takes a `Promise` in input, - * useful if messages need to be decrypted and performance matters. - * - * The order of the messages passed to the callback is as follows: - * - within a page, messages are expected to be ordered from oldest to most recent - * - pages direction depends on { @link QueryOptions.pageDirection } - * - * Do note that the resolution of the `Promise( - decoders: IDecoder[], - callback: ( - message: Promise - ) => Promise | boolean | void, - options?: QueryOptions - ): Promise { - let abort = false; - for await (const page of this.queryGenerator(decoders, options)) { - const _promises = page.map(async (msgPromise) => { - if (abort) return; - abort = Boolean(await callback(msgPromise)); - }); - - await Promise.all(_promises); - if (abort) break; - } - } - - /** - * Do a query to a Waku Store to retrieve historical/missed messages. - * - * This is a generator, useful if you want most control on how messages - * are processed. - * - * The order of the messages returned by the remote Waku node SHOULD BE - * as follows: - * - within a page, messages SHOULD be ordered from oldest to most recent - * - pages direction depends on { @link QueryOptions.pageDirection } - * @throws If not able to reach a Waku Store peer to query, - * or if an error is encountered when processing the reply, - * or if two decoders with the same content topic are passed. - * - * This API only supports querying a single pubsub topic at a time. - * If multiple decoders are provided, they must all have the same pubsub topic. - * @throws If multiple decoders with different pubsub topics are provided. - * @throws If no decoders are provided. - * @throws If no decoders are found for the provided pubsub topic. - */ - async *queryGenerator( - decoders: IDecoder[], - options?: QueryOptions + async *queryPerPage( + queryOpts: Params, + decoders: Map>, + peer: Peer, + cursor?: Cursor ): AsyncGenerator[]> { - if (decoders.length === 0) { - throw new Error("No decoders provided"); - } - - let startTime, endTime; - - if (options?.timeFilter) { - startTime = options.timeFilter.startTime; - endTime = options.timeFilter.endTime; - } - - // convert array to set to remove duplicates - const uniquePubsubTopicsInQuery = Array.from( - new Set(decoders.map((decoder) => decoder.pubsubTopic)) - ); - - // If multiple pubsub topics are provided, throw an error - if (uniquePubsubTopicsInQuery.length > 1) { - throw new Error( - "API does not support querying multiple pubsub topics at once" - ); - } - - // we can be certain that there is only one pubsub topic in the query - const pubsubTopicForQuery = uniquePubsubTopicsInQuery[0]; - - ensurePubsubTopicIsConfigured(pubsubTopicForQuery, this.pubsubTopics); - - // check that the pubsubTopic from the Cursor and Decoder match if ( - options?.cursor?.pubsubTopic && - options.cursor.pubsubTopic !== pubsubTopicForQuery + queryOpts.contentTopics.toString() !== + Array.from(decoders.keys()).toString() ) { throw new Error( - `Cursor pubsub topic (${options?.cursor?.pubsubTopic}) does not match decoder pubsub topic (${pubsubTopicForQuery})` + "Internal error, the decoders should match the query's content topics" ); } - const decodersAsMap = new Map(); - decoders.forEach((dec) => { - if (decodersAsMap.has(dec.contentTopic)) { - throw new Error( - "API does not support different decoder per content topic" - ); - } - decodersAsMap.set(dec.contentTopic, dec); - }); + let currentCursor = cursor; + while (true) { + queryOpts.cursor = currentCursor; - const contentTopics = decoders - .filter((decoder) => decoder.pubsubTopic === pubsubTopicForQuery) - .map((dec) => dec.contentTopic); + const historyRpcQuery = HistoryRpc.createQuery(queryOpts); - if (contentTopics.length === 0) { - throw new Error("No decoders found for topic " + pubsubTopicForQuery); - } - - const queryOpts = Object.assign( - { - pubsubTopic: pubsubTopicForQuery, - pageDirection: PageDirection.BACKWARD, - pageSize: DefaultPageSize - }, - options, - { contentTopics, startTime, endTime } - ); - - const peer = ( - await this.getPeers({ - numPeers: this.NUM_PEERS_PROTOCOL, - maxBootstrapPeers: 1 - }) - )[0]; - - for await (const messages of paginate( - this.getStream.bind(this, peer), - queryOpts, - decodersAsMap, - options?.cursor - )) { - yield messages; - } - } -} + const stream = await this.getStream(peer); -async function* paginate( - streamFactory: () => Promise, - queryOpts: Params, - decoders: Map>, - cursor?: Cursor -): AsyncGenerator[]> { - if ( - queryOpts.contentTopics.toString() !== - Array.from(decoders.keys()).toString() - ) { - throw new Error( - "Internal error, the decoders should match the query's content topics" - ); - } - - let currentCursor = cursor; - while (true) { - queryOpts.cursor = currentCursor; - - const historyRpcQuery = HistoryRpc.createQuery(queryOpts); - - log.info( - "Querying store peer", - `for (${queryOpts.pubsubTopic})`, - queryOpts.contentTopics - ); - - const stream = await streamFactory(); - - const res = await pipe( - [historyRpcQuery.encode()], - lp.encode, - stream, - lp.decode, - async (source) => await all(source) - ); - - const bytes = new Uint8ArrayList(); - res.forEach((chunk) => { - bytes.append(chunk); - }); - - const reply = historyRpcQuery.decode(bytes); + const res = await pipe( + [historyRpcQuery.encode()], + lp.encode, + stream, + lp.decode, + async (source) => await all(source) + ); - if (!reply.response) { - log.warn("Stopping pagination due to store `response` field missing"); - break; - } + const bytes = new Uint8ArrayList(); + res.forEach((chunk) => { + bytes.append(chunk); + }); - const response = reply.response as proto.HistoryResponse; + const reply = historyRpcQuery.decode(bytes); - if (response.error && response.error !== HistoryError.NONE) { - throw "History response contains an Error: " + response.error; - } + if (!reply.response) { + log.warn("Stopping pagination due to store `response` field missing"); + break; + } - if (!response.messages || !response.messages.length) { - log.warn( - "Stopping pagination due to store `response.messages` field missing or empty" - ); - break; - } + const response = reply.response as proto.HistoryResponse; - log.error(`${response.messages.length} messages retrieved from store`); + if (response.error && response.error !== HistoryError.NONE) { + throw "History response contains an Error: " + response.error; + } - yield response.messages.map((protoMsg) => { - const contentTopic = protoMsg.contentTopic; - if (typeof contentTopic !== "undefined") { - const decoder = decoders.get(contentTopic); - if (decoder) { - return decoder.fromProtoObj( - queryOpts.pubsubTopic, - toProtoMessage(protoMsg) - ); - } + if (!response.messages || !response.messages.length) { + log.warn( + "Stopping pagination due to store `response.messages` field missing or empty" + ); + break; } - return Promise.resolve(undefined); - }); - const nextCursor = response.pagingInfo?.cursor; - if (typeof nextCursor === "undefined") { - // If the server does not return cursor then there is an issue, - // Need to abort, or we end up in an infinite loop - log.warn( - "Stopping pagination due to `response.pagingInfo.cursor` missing from store response" - ); - break; - } + log.error(`${response.messages.length} messages retrieved from store`); + + yield response.messages.map((protoMsg) => { + const contentTopic = protoMsg.contentTopic; + if (typeof contentTopic !== "undefined") { + const decoder = decoders.get(contentTopic); + if (decoder) { + return decoder.fromProtoObj( + queryOpts.pubsubTopic, + toProtoMessage(protoMsg) + ); + } + } + return Promise.resolve(undefined); + }); - currentCursor = nextCursor; + const nextCursor = response.pagingInfo?.cursor; + if (typeof nextCursor === "undefined") { + // If the server does not return cursor then there is an issue, + // Need to abort, or we end up in an infinite loop + log.warn( + "Stopping pagination due to `response.pagingInfo.cursor` missing from store response" + ); + break; + } - const responsePageSize = response.pagingInfo?.pageSize; - const queryPageSize = historyRpcQuery.query?.pagingInfo?.pageSize; - if ( - // Response page size smaller than query, meaning this is the last page - responsePageSize && - queryPageSize && - responsePageSize < queryPageSize - ) { - break; + currentCursor = nextCursor; + + const responsePageSize = response.pagingInfo?.pageSize; + const queryPageSize = historyRpcQuery.query?.pagingInfo?.pageSize; + if ( + // Response page size smaller than query, meaning this is the last page + responsePageSize && + queryPageSize && + responsePageSize < queryPageSize + ) { + break; + } } } } - -export async function createCursor(message: IDecodedMessage): Promise { - if ( - !message || - !message.timestamp || - !message.payload || - !message.contentTopic - ) { - throw new Error("Message is missing required fields"); - } - - const contentTopicBytes = utf8ToBytes(message.contentTopic); - - const digest = sha256(concat([contentTopicBytes, message.payload])); - - const messageTime = BigInt(message.timestamp.getTime()) * BigInt(1000000); - - return { - digest, - pubsubTopic: message.pubsubTopic, - senderTime: messageTime, - receiverTime: messageTime - }; -} - -export function wakuStore( - init: Partial = {} -): (libp2p: Libp2p) => IStore { - return (libp2p: Libp2p) => new Store(libp2p, init); -} diff --git a/packages/interfaces/src/store.ts b/packages/interfaces/src/store.ts index 7780801301..ca79f38db3 100644 --- a/packages/interfaces/src/store.ts +++ b/packages/interfaces/src/store.ts @@ -1,5 +1,5 @@ import type { IDecodedMessage, IDecoder } from "./message.js"; -import type { IBaseProtocolCore } from "./protocols.js"; +import type { IBaseProtocolCore, IBaseProtocolSDK } from "./protocols.js"; export enum PageDirection { BACKWARD = "backward", @@ -45,7 +45,16 @@ export type StoreQueryOptions = { cursor?: Cursor; }; -export interface IStore extends IBaseProtocolCore { +export interface IStoreCore extends IBaseProtocolCore {} + +export type IStoreSDK = IBaseProtocolSDK & { + protocol: IBaseProtocolCore; + createCursor(message: IDecodedMessage): Cursor; + queryGenerator: ( + decoders: IDecoder[], + options?: StoreQueryOptions + ) => AsyncGenerator[]>; + queryWithOrderedCallback: ( decoders: IDecoder[], callback: (message: T) => Promise | boolean | void, @@ -58,8 +67,4 @@ export interface IStore extends IBaseProtocolCore { ) => Promise | boolean | void, options?: StoreQueryOptions ) => Promise; - queryGenerator: ( - decoders: IDecoder[], - options?: StoreQueryOptions - ) => AsyncGenerator[]>; -} +}; diff --git a/packages/sdk/src/protocols/store.ts b/packages/sdk/src/protocols/store.ts new file mode 100644 index 0000000000..971fcaa7e2 --- /dev/null +++ b/packages/sdk/src/protocols/store.ts @@ -0,0 +1,318 @@ +import { sha256 } from "@noble/hashes/sha256"; +import { + Cursor, + IDecodedMessage, + IDecoder, + IStoreSDK, + type Libp2p, + PageDirection, + type ProtocolCreateOptions +} from "@waku/interfaces"; +import { ensurePubsubTopicIsConfigured, isDefined } from "@waku/utils"; +import { concat } from "@waku/utils/bytes"; + +import { Params } from "../../../core/dist/lib/store/history_rpc.js"; +import StoreCore, { QueryOptions } from "../../../core/dist/lib/store/index.js"; +import { utf8ToBytes } from "../index.js"; + +import { BaseProtocolSDK } from "./base_protocol.js"; + +export const DefaultPageSize = 10; + +// const log = new Logger("sdk:store"); + +const DEFAULT_NUM_PEERS = 1; + +export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { + public readonly protocol: StoreCore; + + constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { + // options.numPeersToUse is disregarded: https://github.com/waku-org/js-waku/issues/1685 + super({ numPeersToUse: DEFAULT_NUM_PEERS }); + + this.protocol = new StoreCore(libp2p, options); + } + + /** + * Do a query to a Waku Store to retrieve historical/missed messages. + * + * This is a generator, useful if you want most control on how messages + * are processed. + * + * The order of the messages returned by the remote Waku node SHOULD BE + * as follows: + * - within a page, messages SHOULD be ordered from oldest to most recent + * - pages direction depends on { @link QueryOptions.pageDirection } + * @throws If not able to reach a Waku Store peer to query, + * or if an error is encountered when processing the reply, + * or if two decoders with the same content topic are passed. + * + * This API only supports querying a single pubsub topic at a time. + * If multiple decoders are provided, they must all have the same pubsub topic. + * @throws If multiple decoders with different pubsub topics are provided. + * @throws If no decoders are provided. + * @throws If no decoders are found for the provided pubsub topic. + */ + async *queryGenerator( + decoders: IDecoder[], + options?: QueryOptions + ): AsyncGenerator[]> { + const { pubsubTopic, contentTopics, decodersAsMap } = + this.validateDecodersAndPubsubTopic(decoders, options); + + const queryOpts = this.constructOptions( + pubsubTopic, + contentTopics, + options + ); + + const peer = ( + await this.protocol.getPeers({ + numPeers: this.numPeers, + maxBootstrapPeers: 1 + }) + )[0]; + + const responseGenerator = this.protocol.queryPerPage( + queryOpts, + decodersAsMap, + peer + ); + + for await (const messages of responseGenerator) { + yield messages; + } + } + + /** + * @deprecated Use `queryWithOrderedCallback` instead + **/ + queryOrderedCallback = this.queryWithOrderedCallback; + + /** + * Do a query to a Waku Store to retrieve historical/missed messages. + * + * The callback function takes a `WakuMessage` in input, + * messages are processed in order: + * - oldest to latest if `options.pageDirection` == { @link PageDirection.FORWARD } + * - latest to oldest if `options.pageDirection` == { @link PageDirection.BACKWARD } + * + * The ordering may affect performance. + * The ordering depends on the behavior of the remote store node. + * If strong ordering is needed, you may need to handle this at application level + * and set your own timestamps too (the WakuMessage timestamps are not certified). + * + * @throws If not able to reach a Waku Store peer to query, + * or if an error is encountered when processing the reply, + * or if two decoders with the same content topic are passed. + */ + async queryWithOrderedCallback( + decoders: IDecoder[], + callback: (message: T) => Promise | boolean | void, + options?: QueryOptions + ): Promise { + for await (const promises of this.queryGenerator(decoders, options)) { + if (await this.processMessages(promises, callback, options)) break; + } + } + + /** + * Do a query to a Waku Store to retrieve historical/missed messages. + * The callback function takes a `Promise` in input, + * useful if messages need to be decrypted and performance matters. + * + * The order of the messages passed to the callback is as follows: + * - within a page, messages are expected to be ordered from oldest to most recent + * - pages direction depends on { @link QueryOptions.pageDirection } + * + * Do note that the resolution of the `Promise( + decoders: IDecoder[], + callback: ( + message: Promise + ) => Promise | boolean | void, + options?: QueryOptions + ): Promise { + let abort = false; + for await (const page of this.queryGenerator(decoders, options)) { + const _promises = page.map(async (msgPromise) => { + if (abort) return; + abort = Boolean(await callback(msgPromise)); + }); + + await Promise.all(_promises); + if (abort) break; + } + } + + createCursor(message: IDecodedMessage): Cursor { + if ( + !message || + !message.timestamp || + !message.payload || + !message.contentTopic + ) { + throw new Error("Message is missing required fields"); + } + + const contentTopicBytes = utf8ToBytes(message.contentTopic); + + const digest = sha256(concat([contentTopicBytes, message.payload])); + + const messageTime = BigInt(message.timestamp.getTime()) * BigInt(1000000); + + return { + digest, + pubsubTopic: message.pubsubTopic, + senderTime: messageTime, + receiverTime: messageTime + }; + } + + private validateDecodersAndPubsubTopic( + decoders: IDecoder[], + options?: QueryOptions + ): { + pubsubTopic: string; + contentTopics: string[]; + decodersAsMap: Map>; + } { + if (decoders.length === 0) { + throw new Error("No decoders provided"); + } + + // convert array to set to remove duplicates + const uniquePubsubTopicsInQuery = Array.from( + new Set(decoders.map((decoder) => decoder.pubsubTopic)) + ); + // If multiple pubsub topics are provided, throw an error + if (uniquePubsubTopicsInQuery.length > 1) { + throw new Error( + "API does not support querying multiple pubsub topics at once" + ); + } + + // we can be certain that there is only one pubsub topic in the query + const pubsubTopicForQuery = uniquePubsubTopicsInQuery[0]; + + ensurePubsubTopicIsConfigured( + pubsubTopicForQuery, + this.protocol.pubsubTopics + ); + + // check that the pubsubTopic from the Cursor and Decoder match + if ( + options?.cursor?.pubsubTopic && + options.cursor.pubsubTopic !== pubsubTopicForQuery + ) { + throw new Error( + `Cursor pubsub topic (${options?.cursor?.pubsubTopic}) does not match decoder pubsub topic (${pubsubTopicForQuery})` + ); + } + + const decodersAsMap = new Map(); + decoders.forEach((dec) => { + if (decodersAsMap.has(dec.contentTopic)) { + throw new Error( + "API does not support different decoder per content topic" + ); + } + decodersAsMap.set(dec.contentTopic, dec); + }); + + const contentTopics = decoders + .filter((decoder) => decoder.pubsubTopic === pubsubTopicForQuery) + .map((dec) => dec.contentTopic); + + if (contentTopics.length === 0) { + throw new Error("No decoders found for topic " + pubsubTopicForQuery); + } + + return { + pubsubTopic: pubsubTopicForQuery, + contentTopics, + decodersAsMap + }; + } + + private constructOptions( + pubsubTopic: string, + contentTopics: string[], + options: QueryOptions = {} + ): Params { + let startTime, endTime; + + if (options?.timeFilter) { + startTime = options.timeFilter.startTime; + endTime = options.timeFilter.endTime; + } + + const queryOpts = Object.assign( + { + pubsubTopic: pubsubTopic, + pageDirection: PageDirection.BACKWARD, + pageSize: DefaultPageSize + }, + options, + { contentTopics, startTime, endTime } + ); + + return queryOpts; + } + + /** + * Processes messages based on the provided callback and options. + * @private + */ + private async processMessages( + messages: Promise[], + callback: (message: T) => Promise | boolean | void, + options?: QueryOptions + ): Promise { + let abort = false; + const messagesOrUndef: Array = await Promise.all(messages); + let processedMessages: Array = messagesOrUndef.filter(isDefined); + + if (this.shouldReverseOrder(options)) { + processedMessages = processedMessages.reverse(); + } + + await Promise.all( + processedMessages.map(async (msg) => { + if (msg && !abort) { + abort = Boolean(await callback(msg)); + } + }) + ); + + return abort; + } + + /** + * Determines whether to reverse the order of messages based on the provided options. + * + * Messages in pages are ordered from oldest (first) to most recent (last). + * https://github.com/vacp2p/rfc/issues/533 + * + * @private + */ + private shouldReverseOrder(options?: QueryOptions): boolean { + return ( + typeof options?.pageDirection === "undefined" || + options?.pageDirection === PageDirection.BACKWARD + ); + } +} + +export function wakuStore( + init: Partial = {} +): (libp2p: Libp2p) => IStoreSDK { + return (libp2p: Libp2p) => new StoreSDK(libp2p, init); +} From 7aab5be83ebbb0cdbf7ee4ce32307ab5acbae355 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Wed, 13 Mar 2024 17:11:02 +0530 Subject: [PATCH 02/15] update imports & types --- packages/core/src/index.ts | 2 +- packages/core/src/lib/wait_for_remote_peer.ts | 2 +- packages/interfaces/src/waku.ts | 8 ++++---- packages/sdk/src/light-node/index.ts | 3 ++- packages/sdk/src/relay-node/index.ts | 3 ++- packages/sdk/src/waku.ts | 8 ++++---- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 716b06c100..b94ce1ec89 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -14,7 +14,7 @@ export { LightPushCodec, LightPushCore } from "./lib/light_push/index.js"; export * as waku_store from "./lib/store/index.js"; -export { PageDirection, wakuStore, createCursor } from "./lib/store/index.js"; +export { PageDirection } from "./lib/store/index.js"; export { waitForRemotePeer } from "./lib/wait_for_remote_peer.js"; diff --git a/packages/core/src/lib/wait_for_remote_peer.ts b/packages/core/src/lib/wait_for_remote_peer.ts index 487df7c243..ac812cb556 100644 --- a/packages/core/src/lib/wait_for_remote_peer.ts +++ b/packages/core/src/lib/wait_for_remote_peer.ts @@ -51,7 +51,7 @@ export async function waitForRemotePeer( if (!waku.store) throw new Error("Cannot wait for Store peer: protocol not mounted"); promises.push( - waitForConnectedPeer(waku.store, waku.libp2p.services.metadata) + waitForConnectedPeer(waku.store.protocol, waku.libp2p.services.metadata) ); } diff --git a/packages/interfaces/src/waku.ts b/packages/interfaces/src/waku.ts index 0948d05369..2b03db79b5 100644 --- a/packages/interfaces/src/waku.ts +++ b/packages/interfaces/src/waku.ts @@ -7,12 +7,12 @@ import type { Libp2p } from "./libp2p.js"; import type { ILightPushSDK } from "./light_push.js"; import { Protocols } from "./protocols.js"; import type { IRelay } from "./relay.js"; -import type { IStore } from "./store.js"; +import type { IStoreSDK } from "./store.js"; export interface Waku { libp2p: Libp2p; relay?: IRelay; - store?: IStore; + store?: IStoreSDK; filter?: IFilter; lightPush?: ILightPushSDK; @@ -31,7 +31,7 @@ export interface Waku { export interface LightNode extends Waku { relay: undefined; - store: IStore; + store: IStoreSDK; filter: IFilter; lightPush: ILightPushSDK; } @@ -45,7 +45,7 @@ export interface RelayNode extends Waku { export interface FullNode extends Waku { relay: IRelay; - store: IStore; + store: IStoreSDK; filter: IFilter; lightPush: ILightPushSDK; } diff --git a/packages/sdk/src/light-node/index.ts b/packages/sdk/src/light-node/index.ts index 073b485f62..754f1a0f46 100644 --- a/packages/sdk/src/light-node/index.ts +++ b/packages/sdk/src/light-node/index.ts @@ -1,7 +1,8 @@ -import { wakuFilter, wakuStore } from "@waku/core"; +import { wakuFilter } from "@waku/core"; import { type Libp2pComponents, type LightNode } from "@waku/interfaces"; import { wakuLightPush } from "../protocols/light_push.js"; +import { wakuStore } from "../protocols/store.js"; import { createLibp2pAndUpdateOptions } from "../utils/libp2p.js"; import { CreateWakuNodeOptions, WakuNode, WakuOptions } from "../waku.js"; diff --git a/packages/sdk/src/relay-node/index.ts b/packages/sdk/src/relay-node/index.ts index 136713919a..4d7d040cc6 100644 --- a/packages/sdk/src/relay-node/index.ts +++ b/packages/sdk/src/relay-node/index.ts @@ -1,8 +1,9 @@ -import { wakuFilter, wakuStore } from "@waku/core"; +import { wakuFilter } from "@waku/core"; import { type FullNode, type RelayNode } from "@waku/interfaces"; import { RelayCreateOptions, wakuRelay } from "@waku/relay"; import { wakuLightPush } from "../protocols/light_push.js"; +import { wakuStore } from "../protocols/store.js"; import { createLibp2pAndUpdateOptions } from "../utils/libp2p.js"; import { CreateWakuNodeOptions, WakuNode, WakuOptions } from "../waku.js"; diff --git a/packages/sdk/src/waku.ts b/packages/sdk/src/waku.ts index 8fa9aba555..dac5579998 100644 --- a/packages/sdk/src/waku.ts +++ b/packages/sdk/src/waku.ts @@ -8,7 +8,7 @@ import type { IFilterSubscription, ILightPushSDK, IRelay, - IStore, + IStoreSDK, Libp2p, LightNode, ProtocolCreateOptions, @@ -55,7 +55,7 @@ export type CreateWakuNodeOptions = ProtocolCreateOptions & export class WakuNode implements Waku { public libp2p: Libp2p; public relay?: IRelay; - public store?: IStore; + public store?: IStoreSDK; public filter?: IFilter; public lightPush?: ILightPushSDK; public connectionManager: ConnectionManager; @@ -64,7 +64,7 @@ export class WakuNode implements Waku { constructor( options: WakuOptions, libp2p: Libp2p, - store?: (libp2p: Libp2p) => IStore, + store?: (libp2p: Libp2p) => IStoreSDK, lightPush?: (libp2p: Libp2p) => ILightPushSDK, filter?: (libp2p: Libp2p) => IFilter, relay?: (libp2p: Libp2p) => IRelay @@ -148,7 +148,7 @@ export class WakuNode implements Waku { } if (_protocols.includes(Protocols.Store)) { if (this.store) { - codecs.push(this.store.multicodec); + codecs.push(this.store.protocol.multicodec); } else { log.error( "Store codec not included in dial codec: protocol not mounted locally" From 5a431e23b69e2bbe2deac4200ddb7ef54b97c8d0 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Wed, 13 Mar 2024 17:12:08 +0530 Subject: [PATCH 03/15] chore: `@noble/hashes` moves to `sdk` --- package-lock.json | 8 ++++---- packages/core/package.json | 1 - packages/sdk/package.json | 7 ++++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index f7ddb418f8..ba0b346d00 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27834,7 +27834,6 @@ "license": "MIT OR Apache-2.0", "dependencies": { "@libp2p/ping": "^1.0.12", - "@noble/hashes": "^1.3.2", "@waku/enr": "^0.0.21", "@waku/interfaces": "0.0.22", "@waku/message-hash": "^0.1.11", @@ -28345,11 +28344,12 @@ "license": "MIT OR Apache-2.0", "dependencies": { "@chainsafe/libp2p-noise": "^14.1.0", - "@libp2p/bootstrap": "^10.0.11", + "@libp2p/bootstrap": "^10", "@libp2p/identify": "^1.0.11", "@libp2p/mplex": "^10.0.12", "@libp2p/ping": "^1.0.12", "@libp2p/websockets": "^8.0.11", + "@noble/hashes": "^1.3.3", "@waku/core": "0.0.27", "@waku/discovery": "0.0.1", "@waku/dns-discovery": "0.0.21", @@ -32245,7 +32245,6 @@ "requires": { "@libp2p/ping": "^1.0.12", "@multiformats/multiaddr": "^12.0.0", - "@noble/hashes": "^1.3.2", "@rollup/plugin-commonjs": "^25.0.7", "@rollup/plugin-json": "^6.0.0", "@rollup/plugin-node-resolve": "^15.2.3", @@ -32515,11 +32514,12 @@ "requires": { "@chainsafe/libp2p-gossipsub": "^12.0.0", "@chainsafe/libp2p-noise": "^14.1.0", - "@libp2p/bootstrap": "^10.0.11", + "@libp2p/bootstrap": "^10", "@libp2p/identify": "^1.0.11", "@libp2p/mplex": "^10.0.12", "@libp2p/ping": "^1.0.12", "@libp2p/websockets": "^8.0.11", + "@noble/hashes": "^1.3.3", "@rollup/plugin-commonjs": "^25.0.7", "@rollup/plugin-json": "^6.0.0", "@rollup/plugin-node-resolve": "^15.2.3", diff --git a/packages/core/package.json b/packages/core/package.json index 87ccda4e56..cd9ebed704 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -73,7 +73,6 @@ }, "dependencies": { "@libp2p/ping": "^1.0.12", - "@noble/hashes": "^1.3.2", "@waku/enr": "^0.0.21", "@waku/interfaces": "0.0.22", "@waku/message-hash": "^0.1.11", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 63a49a6c94..cfaebbf52a 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -62,20 +62,21 @@ }, "dependencies": { "@chainsafe/libp2p-noise": "^14.1.0", + "@libp2p/bootstrap": "^10", "@libp2p/identify": "^1.0.11", "@libp2p/mplex": "^10.0.12", "@libp2p/ping": "^1.0.12", "@libp2p/websockets": "^8.0.11", - "@waku/discovery": "0.0.1", + "@noble/hashes": "^1.3.3", "@waku/core": "0.0.27", + "@waku/discovery": "0.0.1", "@waku/dns-discovery": "0.0.21", "@waku/interfaces": "0.0.22", "@waku/local-peer-cache-discovery": "^1.0.0", "@waku/peer-exchange": "^0.0.20", "@waku/relay": "0.0.10", "@waku/utils": "0.0.15", - "libp2p": "^1.1.2", - "@libp2p/bootstrap": "^10.0.11" + "libp2p": "^1.1.2" }, "devDependencies": { "@chainsafe/libp2p-gossipsub": "^12.0.0", From efc3ac3837f171c0762ba711d506a772e832aa1b Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Wed, 13 Mar 2024 17:12:17 +0530 Subject: [PATCH 04/15] chore: update tests --- packages/tests/tests/store/cursor.node.spec.ts | 10 +++++----- packages/tests/tests/wait_for_remote_peer.node.spec.ts | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/tests/tests/store/cursor.node.spec.ts b/packages/tests/tests/store/cursor.node.spec.ts index af82dcfd3c..d5abb91a7a 100644 --- a/packages/tests/tests/store/cursor.node.spec.ts +++ b/packages/tests/tests/store/cursor.node.spec.ts @@ -1,4 +1,4 @@ -import { createCursor, DecodedMessage } from "@waku/core"; +import { DecodedMessage } from "@waku/core"; import type { LightNode } from "@waku/interfaces"; import { DefaultPubsubTopic } from "@waku/interfaces"; import { bytesToUtf8 } from "@waku/utils/bytes"; @@ -63,7 +63,7 @@ describe("Waku Store, cursor", function () { } // create cursor to extract messages after the cursorIndex - const cursor = await createCursor(messages[cursorIndex]); + const cursor = waku.store.createCursor(messages[cursorIndex]); const messagesAfterCursor: DecodedMessage[] = []; for await (const page of waku.store.queryGenerator([TestDecoder], { @@ -108,7 +108,7 @@ describe("Waku Store, cursor", function () { } // create cursor to extract messages after the cursorIndex - const cursor = await createCursor(messages[5]); + const cursor = waku.store.createCursor(messages[5]); // query node2 with the cursor from node1 const messagesAfterCursor: DecodedMessage[] = []; @@ -142,7 +142,7 @@ describe("Waku Store, cursor", function () { messages.push(msg as DecodedMessage); } } - const cursor = await createCursor(messages[5]); + const cursor = waku.store.createCursor(messages[5]); // setting a wrong digest cursor.digest = new Uint8Array([]); @@ -185,7 +185,7 @@ describe("Waku Store, cursor", function () { } } messages[5].pubsubTopic = customShardedPubsubTopic1; - const cursor = await createCursor(messages[5]); + const cursor = waku.store.createCursor(messages[5]); try { for await (const page of waku.store.queryGenerator([TestDecoder], { diff --git a/packages/tests/tests/wait_for_remote_peer.node.spec.ts b/packages/tests/tests/wait_for_remote_peer.node.spec.ts index d50c5e29eb..d86406aee1 100644 --- a/packages/tests/tests/wait_for_remote_peer.node.spec.ts +++ b/packages/tests/tests/wait_for_remote_peer.node.spec.ts @@ -115,7 +115,7 @@ describe("Wait for remote peer", function () { await delay(1000); await waitForRemotePeer(waku2, [Protocols.Store]); - const peers = (await waku2.store.connectedPeers()).map((peer) => + const peers = (await waku2.store.protocol.connectedPeers()).map((peer) => peer.id.toString() ); const nimPeerId = multiAddrWithId.getPeerId(); @@ -144,7 +144,7 @@ describe("Wait for remote peer", function () { await waku2.dial(multiAddrWithId); await waitPromise; - const peers = (await waku2.store.connectedPeers()).map((peer) => + const peers = (await waku2.store.protocol.connectedPeers()).map((peer) => peer.id.toString() ); @@ -235,8 +235,8 @@ describe("Wait for remote peer", function () { const filterPeers = (await waku2.filter.connectedPeers()).map((peer) => peer.id.toString() ); - const storePeers = (await waku2.store.connectedPeers()).map((peer) => - peer.id.toString() + const storePeers = (await waku2.store.protocol.connectedPeers()).map( + (peer) => peer.id.toString() ); const lightPushPeers = ( await waku2.lightPush.protocol.connectedPeers() From 23306ea43f7ae75ca20fc3559dfe2b1cced7360a Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Wed, 13 Mar 2024 19:16:41 +0530 Subject: [PATCH 05/15] chore: update size-limit import path --- .size-limit.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.size-limit.cjs b/.size-limit.cjs index b77b56f9b5..dcba50b6ef 100644 --- a/.size-limit.cjs +++ b/.size-limit.cjs @@ -54,7 +54,7 @@ module.exports = [ }, { name: "History retrieval protocols", - path: "packages/core/bundle/index.js", + path: "packages/sdk/bundle/index.js", import: "{ wakuStore }", }, { From 8a5c3ccd3103069a972e08e65d0caf3b84ee2556 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Wed, 13 Mar 2024 19:32:55 +0530 Subject: [PATCH 06/15] fix: cursor tests, use `Cursor` type from `proto.Index` instead of redefining --- package-lock.json | 6 +++++- package.json | 2 +- packages/core/src/lib/store/index.ts | 5 ++--- packages/interfaces/package.json | 5 ++++- packages/interfaces/src/store.ts | 4 +++- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index ba0b346d00..68ba255f85 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,10 +6,10 @@ "": { "name": "@waku/root", "workspaces": [ + "packages/proto", "packages/interfaces", "packages/utils", "packages/message-hash", - "packages/proto", "packages/enr", "packages/core", "packages/relay", @@ -28037,6 +28037,9 @@ "name": "@waku/interfaces", "version": "0.0.22", "license": "MIT OR Apache-2.0", + "dependencies": { + "@waku/proto": "^0.0.6" + }, "devDependencies": { "@chainsafe/libp2p-gossipsub": "^12.0.0", "@multiformats/multiaddr": "^12.0.0", @@ -32393,6 +32396,7 @@ "requires": { "@chainsafe/libp2p-gossipsub": "^12.0.0", "@multiformats/multiaddr": "^12.0.0", + "@waku/proto": "^0.0.6", "cspell": "^8.6.0", "libp2p": "^1.1.2", "npm-run-all": "^4.1.5" diff --git a/package.json b/package.json index b153d88939..a4f38db53a 100644 --- a/package.json +++ b/package.json @@ -3,10 +3,10 @@ "private": true, "type": "module", "workspaces": [ + "packages/proto", "packages/interfaces", "packages/utils", "packages/message-hash", - "packages/proto", "packages/enr", "packages/core", "packages/relay", diff --git a/packages/core/src/lib/store/index.ts b/packages/core/src/lib/store/index.ts index 53a7b3097e..82d227145e 100644 --- a/packages/core/src/lib/store/index.ts +++ b/packages/core/src/lib/store/index.ts @@ -75,8 +75,7 @@ export default class StoreCore extends BaseProtocol implements IStoreCore { async *queryPerPage( queryOpts: Params, decoders: Map>, - peer: Peer, - cursor?: Cursor + peer: Peer ): AsyncGenerator[]> { if ( queryOpts.contentTopics.toString() !== @@ -87,7 +86,7 @@ export default class StoreCore extends BaseProtocol implements IStoreCore { ); } - let currentCursor = cursor; + let currentCursor = queryOpts.cursor; while (true) { queryOpts.cursor = currentCursor; diff --git a/packages/interfaces/package.json b/packages/interfaces/package.json index 1c95cb60f1..363a1d72fa 100644 --- a/packages/interfaces/package.json +++ b/packages/interfaces/package.json @@ -62,5 +62,8 @@ "CHANGELOG.md", "LICENSE", "README.md" - ] + ], + "dependencies": { + "@waku/proto": "^0.0.6" + } } diff --git a/packages/interfaces/src/store.ts b/packages/interfaces/src/store.ts index ca79f38db3..a1e58f58d9 100644 --- a/packages/interfaces/src/store.ts +++ b/packages/interfaces/src/store.ts @@ -1,3 +1,5 @@ +import { proto_store as proto } from "@waku/proto"; + import type { IDecodedMessage, IDecoder } from "./message.js"; import type { IBaseProtocolCore, IBaseProtocolSDK } from "./protocols.js"; @@ -42,7 +44,7 @@ export type StoreQueryOptions = { * Cursor as an index to start a query from. Must be generated from a Waku * Message. */ - cursor?: Cursor; + cursor?: proto.Index; }; export interface IStoreCore extends IBaseProtocolCore {} From d881173cd544d35a28638eb02ae494e17d431ff9 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Wed, 13 Mar 2024 20:07:55 +0530 Subject: [PATCH 07/15] export wakuStore from sdk --- packages/sdk/src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 4ef5d65353..c263ef731b 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -13,6 +13,7 @@ export * from "./waku.js"; export { createLightNode, createNode } from "./light-node/index.js"; export { wakuLightPush } from "./protocols/light_push.js"; +export { wakuStore } from "./protocols/store.js"; export * as waku from "@waku/core"; export * as utils from "@waku/utils"; From b865da93ec03ee5afbd143f689bef24dd2c549d1 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Fri, 22 Mar 2024 13:35:20 +0800 Subject: [PATCH 08/15] fix: imports --- packages/core/src/index.ts | 1 + packages/core/src/lib/store/index.ts | 4 ++-- packages/sdk/src/protocols/store.ts | 19 +++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index b94ce1ec89..f6a4ca2eab 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -13,6 +13,7 @@ export * as waku_light_push from "./lib/light_push/index.js"; export { LightPushCodec, LightPushCore } from "./lib/light_push/index.js"; export * as waku_store from "./lib/store/index.js"; +export { StoreCore } from "./lib/store/index.js"; export { PageDirection } from "./lib/store/index.js"; diff --git a/packages/core/src/lib/store/index.ts b/packages/core/src/lib/store/index.ts index 82d227145e..5d90882b98 100644 --- a/packages/core/src/lib/store/index.ts +++ b/packages/core/src/lib/store/index.ts @@ -25,7 +25,7 @@ const log = new Logger("store"); export const StoreCodec = "/vac/waku/store/2.0.0-beta4"; -export { PageDirection }; +export { PageDirection, Params }; export interface TimeFilter { startTime: Date; @@ -67,7 +67,7 @@ export interface QueryOptions { * * The Waku Store protocol can be used to retrieved historical messages. */ -export default class StoreCore extends BaseProtocol implements IStoreCore { +export class StoreCore extends BaseProtocol implements IStoreCore { constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { super(StoreCodec, libp2p.components, log, options!.pubsubTopics!, options); } diff --git a/packages/sdk/src/protocols/store.ts b/packages/sdk/src/protocols/store.ts index 971fcaa7e2..c9c0337a62 100644 --- a/packages/sdk/src/protocols/store.ts +++ b/packages/sdk/src/protocols/store.ts @@ -1,4 +1,5 @@ import { sha256 } from "@noble/hashes/sha256"; +import { StoreCore, waku_store } from "@waku/core"; import { Cursor, IDecodedMessage, @@ -11,8 +12,6 @@ import { import { ensurePubsubTopicIsConfigured, isDefined } from "@waku/utils"; import { concat } from "@waku/utils/bytes"; -import { Params } from "../../../core/dist/lib/store/history_rpc.js"; -import StoreCore, { QueryOptions } from "../../../core/dist/lib/store/index.js"; import { utf8ToBytes } from "../index.js"; import { BaseProtocolSDK } from "./base_protocol.js"; @@ -55,7 +54,7 @@ export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { */ async *queryGenerator( decoders: IDecoder[], - options?: QueryOptions + options?: waku_store.QueryOptions ): AsyncGenerator[]> { const { pubsubTopic, contentTopics, decodersAsMap } = this.validateDecodersAndPubsubTopic(decoders, options); @@ -109,7 +108,7 @@ export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { async queryWithOrderedCallback( decoders: IDecoder[], callback: (message: T) => Promise | boolean | void, - options?: QueryOptions + options?: waku_store.QueryOptions ): Promise { for await (const promises of this.queryGenerator(decoders, options)) { if (await this.processMessages(promises, callback, options)) break; @@ -138,7 +137,7 @@ export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { callback: ( message: Promise ) => Promise | boolean | void, - options?: QueryOptions + options?: waku_store.QueryOptions ): Promise { let abort = false; for await (const page of this.queryGenerator(decoders, options)) { @@ -178,7 +177,7 @@ export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { private validateDecodersAndPubsubTopic( decoders: IDecoder[], - options?: QueryOptions + options?: waku_store.QueryOptions ): { pubsubTopic: string; contentTopics: string[]; @@ -245,8 +244,8 @@ export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { private constructOptions( pubsubTopic: string, contentTopics: string[], - options: QueryOptions = {} - ): Params { + options: waku_store.QueryOptions = {} + ): waku_store.Params { let startTime, endTime; if (options?.timeFilter) { @@ -274,7 +273,7 @@ export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { private async processMessages( messages: Promise[], callback: (message: T) => Promise | boolean | void, - options?: QueryOptions + options?: waku_store.QueryOptions ): Promise { let abort = false; const messagesOrUndef: Array = await Promise.all(messages); @@ -303,7 +302,7 @@ export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { * * @private */ - private shouldReverseOrder(options?: QueryOptions): boolean { + private shouldReverseOrder(options?: waku_store.QueryOptions): boolean { return ( typeof options?.pageDirection === "undefined" || options?.pageDirection === PageDirection.BACKWARD From e7caa5316f1d95eb6bb8c4af79a17c10cb311358 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 28 Mar 2024 13:53:30 +0530 Subject: [PATCH 09/15] chore: use specific version for package --- package-lock.json | 28 ++++++++++++++-------------- packages/sdk/package.json | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/package-lock.json b/package-lock.json index 68ba255f85..5d6dc20690 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2709,14 +2709,14 @@ "license": "MIT" }, "node_modules/@libp2p/bootstrap": { - "version": "10.0.11", - "resolved": "https://registry.npmjs.org/@libp2p/bootstrap/-/bootstrap-10.0.11.tgz", - "integrity": "sha512-uFqfMFtCDLIFUNOOvBFUzcSSkJx9y428jYzxpyLoWv0XH4pd3gaHcPgEvK9ZddhNysg1BDslivsFw6ZyE3Tvsg==", + "version": "10.0.16", + "resolved": "https://registry.npmjs.org/@libp2p/bootstrap/-/bootstrap-10.0.16.tgz", + "integrity": "sha512-ZFuq5OtQfdeZVjfWrJpW/OuPVOuAflu1nzq9g6/UiVfSvBaZtwe8hcMCQDXv21V8fCVsd703sblzkBwBYi17rQ==", "dependencies": { - "@libp2p/interface": "^1.1.1", - "@libp2p/peer-id": "^4.0.4", + "@libp2p/interface": "^1.1.4", + "@libp2p/peer-id": "^4.0.7", "@multiformats/mafmt": "^12.1.6", - "@multiformats/multiaddr": "^12.1.10" + "@multiformats/multiaddr": "^12.1.14" } }, "node_modules/@libp2p/crypto": { @@ -28347,7 +28347,7 @@ "license": "MIT OR Apache-2.0", "dependencies": { "@chainsafe/libp2p-noise": "^14.1.0", - "@libp2p/bootstrap": "^10", + "@libp2p/bootstrap": "^10.0.16", "@libp2p/identify": "^1.0.11", "@libp2p/mplex": "^10.0.12", "@libp2p/ping": "^1.0.12", @@ -30309,14 +30309,14 @@ "version": "2.0.4" }, "@libp2p/bootstrap": { - "version": "10.0.11", - "resolved": "https://registry.npmjs.org/@libp2p/bootstrap/-/bootstrap-10.0.11.tgz", - "integrity": "sha512-uFqfMFtCDLIFUNOOvBFUzcSSkJx9y428jYzxpyLoWv0XH4pd3gaHcPgEvK9ZddhNysg1BDslivsFw6ZyE3Tvsg==", + "version": "10.0.16", + "resolved": "https://registry.npmjs.org/@libp2p/bootstrap/-/bootstrap-10.0.16.tgz", + "integrity": "sha512-ZFuq5OtQfdeZVjfWrJpW/OuPVOuAflu1nzq9g6/UiVfSvBaZtwe8hcMCQDXv21V8fCVsd703sblzkBwBYi17rQ==", "requires": { - "@libp2p/interface": "^1.1.1", - "@libp2p/peer-id": "^4.0.4", + "@libp2p/interface": "^1.1.4", + "@libp2p/peer-id": "^4.0.7", "@multiformats/mafmt": "^12.1.6", - "@multiformats/multiaddr": "^12.1.10" + "@multiformats/multiaddr": "^12.1.14" } }, "@libp2p/crypto": { @@ -32518,7 +32518,7 @@ "requires": { "@chainsafe/libp2p-gossipsub": "^12.0.0", "@chainsafe/libp2p-noise": "^14.1.0", - "@libp2p/bootstrap": "^10", + "@libp2p/bootstrap": "^10.0.16", "@libp2p/identify": "^1.0.11", "@libp2p/mplex": "^10.0.12", "@libp2p/ping": "^1.0.12", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index cfaebbf52a..e09386fb20 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -62,7 +62,7 @@ }, "dependencies": { "@chainsafe/libp2p-noise": "^14.1.0", - "@libp2p/bootstrap": "^10", + "@libp2p/bootstrap": "^10.0.16", "@libp2p/identify": "^1.0.11", "@libp2p/mplex": "^10.0.12", "@libp2p/ping": "^1.0.12", From 82c924ed97cc670aad31d921485a0a4b04f32b6b Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 28 Mar 2024 13:56:56 +0530 Subject: [PATCH 10/15] chore: handle error for peer access --- packages/sdk/src/protocols/store.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/sdk/src/protocols/store.ts b/packages/sdk/src/protocols/store.ts index c9c0337a62..60df7da5a0 100644 --- a/packages/sdk/src/protocols/store.ts +++ b/packages/sdk/src/protocols/store.ts @@ -72,6 +72,8 @@ export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { }) )[0]; + if (!peer) throw new Error("No peers available to query"); + const responseGenerator = this.protocol.queryPerPage( queryOpts, decodersAsMap, From fb32ede25559f1991782e72e05c7197405425172 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 28 Mar 2024 14:07:53 +0530 Subject: [PATCH 11/15] use type instead of interface --- packages/interfaces/src/store.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interfaces/src/store.ts b/packages/interfaces/src/store.ts index a1e58f58d9..ac6486995f 100644 --- a/packages/interfaces/src/store.ts +++ b/packages/interfaces/src/store.ts @@ -47,7 +47,7 @@ export type StoreQueryOptions = { cursor?: proto.Index; }; -export interface IStoreCore extends IBaseProtocolCore {} +export type IStoreCore = IBaseProtocolCore; export type IStoreSDK = IBaseProtocolSDK & { protocol: IBaseProtocolCore; From f60356aa4ae0cc0dbb99f527d323b576b22ce70a Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 28 Mar 2024 14:11:29 +0530 Subject: [PATCH 12/15] rm: comment --- packages/sdk/src/protocols/store.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/sdk/src/protocols/store.ts b/packages/sdk/src/protocols/store.ts index 60df7da5a0..f465a5dcd2 100644 --- a/packages/sdk/src/protocols/store.ts +++ b/packages/sdk/src/protocols/store.ts @@ -18,8 +18,6 @@ import { BaseProtocolSDK } from "./base_protocol.js"; export const DefaultPageSize = 10; -// const log = new Logger("sdk:store"); - const DEFAULT_NUM_PEERS = 1; export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { From b151e94f6416af48cd626d691b67fccf07b5641b Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 28 Mar 2024 14:12:02 +0530 Subject: [PATCH 13/15] add TODO --- packages/sdk/src/protocols/store.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/sdk/src/protocols/store.ts b/packages/sdk/src/protocols/store.ts index f465a5dcd2..d5c247edc3 100644 --- a/packages/sdk/src/protocols/store.ts +++ b/packages/sdk/src/protocols/store.ts @@ -24,7 +24,7 @@ export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { public readonly protocol: StoreCore; constructor(libp2p: Libp2p, options?: ProtocolCreateOptions) { - // options.numPeersToUse is disregarded: https://github.com/waku-org/js-waku/issues/1685 + // TODO: options.numPeersToUse is disregarded: https://github.com/waku-org/js-waku/issues/1685 super({ numPeersToUse: DEFAULT_NUM_PEERS }); this.protocol = new StoreCore(libp2p, options); From d9232c3891ade3544487c014c8d3b7a5be869669 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Thu, 28 Mar 2024 14:14:53 +0530 Subject: [PATCH 14/15] chore!: remove deprecated function definition --- packages/sdk/src/protocols/store.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/sdk/src/protocols/store.ts b/packages/sdk/src/protocols/store.ts index d5c247edc3..f8981f5842 100644 --- a/packages/sdk/src/protocols/store.ts +++ b/packages/sdk/src/protocols/store.ts @@ -83,11 +83,6 @@ export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { } } - /** - * @deprecated Use `queryWithOrderedCallback` instead - **/ - queryOrderedCallback = this.queryWithOrderedCallback; - /** * Do a query to a Waku Store to retrieve historical/missed messages. * From 1863e0d29220f605d72c8ac8ea4a3f7ec41e6e30 Mon Sep 17 00:00:00 2001 From: danisharora099 Date: Mon, 1 Apr 2024 16:32:18 +0530 Subject: [PATCH 15/15] chore: improve logging --- packages/sdk/src/protocols/store.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/protocols/store.ts b/packages/sdk/src/protocols/store.ts index f8981f5842..774fe6a168 100644 --- a/packages/sdk/src/protocols/store.ts +++ b/packages/sdk/src/protocols/store.ts @@ -9,7 +9,7 @@ import { PageDirection, type ProtocolCreateOptions } from "@waku/interfaces"; -import { ensurePubsubTopicIsConfigured, isDefined } from "@waku/utils"; +import { ensurePubsubTopicIsConfigured, isDefined, Logger } from "@waku/utils"; import { concat } from "@waku/utils/bytes"; import { utf8ToBytes } from "../index.js"; @@ -20,6 +20,8 @@ export const DefaultPageSize = 10; const DEFAULT_NUM_PEERS = 1; +const log = new Logger("waku:store:protocol"); + export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { public readonly protocol: StoreCore; @@ -248,6 +250,13 @@ export class StoreSDK extends BaseProtocolSDK implements IStoreSDK { endTime = options.timeFilter.endTime; } + if (!startTime) { + log.warn("No start time provided"); + } + if (!endTime) { + log.warn("No end time provided"); + } + const queryOpts = Object.assign( { pubsubTopic: pubsubTopic,