feat: create r2d2 fetch for web3 provider (#6443)

* feat: add r2d2 fetch function in background

* feat: serialize response object for r2d2 fetch

* refactor: review feedback

* feat: add request serializer

* feat: some api provider use r2d2 fetch

* fix: cspell

cspell.json

@@ -171,6 +171,7 @@
         "realise",
         "rebalance",
         "redpacket",
+        "regedit",
         "repayer",
         "replacestate",
         "repost",

packages/mask/background/services/helper/index.ts

@@ -4,3 +4,4 @@ export { openPopupWindow, removePopupWindow, openDashboard } from './popup-opene
 export { __deprecated__getStorage, __deprecated__setStorage } from './deprecated-storage'
 export { queryExtensionPermission, requestExtensionPermission } from './request-permission'
 export { saveFileFromBuffer, type SaveFileOptions } from '../../../shared/helpers/download'
+export { r2d2Fetch } from './r2d2Fetch'

packages/mask/background/services/helper/r2d2Fetch.ts

@@ -0,0 +1,39 @@
+export const r2d2URL = 'r2d2.to'
+
+export enum R2d2Workers {
+    opensea = 'opensea-proxy',
+    gitcoin = 'gitcoin-agent',
+    coinMarketCap = 'coinmarketcap-agent',
+    goPlusLabs = 'gopluslabs',
+}
+
+type R2d2WorkerMatchTuple = [string, R2d2Workers]
+
+const matchers: R2d2WorkerMatchTuple[] = [
+    ['https://api.opensea.io', R2d2Workers.opensea],
+    ['https://gitcoin.co', R2d2Workers.gitcoin],
+    ['https://web-api.coinmarketcap.com', R2d2Workers.coinMarketCap],
+    ['https://api.gopluslabs.io', R2d2Workers.goPlusLabs],
+]
+
+/**
+ * Why use r2d2 fetch: some third api provider will be block in Firefox and protect api key
+ * @returns fetch response
+ * @param input
+ * @param init
+ */
+export async function r2d2Fetch(input: RequestInfo, init?: RequestInit): Promise<Response> {
+    const url = init instanceof Request ? init.url : (input as string)
+    if (url.includes('r2d2.to')) return globalThis.fetch(input, init)
+
+    const r2deWorkerType = matchers.find((x) => url.startsWith(x[0]))?.[1]
+
+    if (!r2deWorkerType) {
+        return globalThis.fetch(input, init)
+    }
+
+    const origin = new URL(url).origin
+    const r2d2ProxyURL = url.replace(origin, `https://${r2deWorkerType}.${r2d2URL}`)
+
+    return globalThis.fetch(r2d2ProxyURL, init)
+}

packages/mask/src/plugins/Gitcoin/apis/index.ts

@@ -40,6 +40,8 @@ export interface GitcoinGrant {
 
 export async function fetchGrant(id: string) {
     if (!/\d+/.test(id)) return
+    const fetch = globalThis.r2d2Fetch ?? globalThis.fetch
+
     const response = await fetch(urlcat(GITCOIN_API_GRANTS_V1, { id }))
     const { grants } = (await response.json()) as {
         grants: GitcoinGrant

packages/mask/src/plugins/Gitcoin/constants.ts

@@ -4,6 +4,4 @@ export const PLUGIN_ID = PluginId.Gitcoin
 export const PLUGIN_META_KEY = `${PluginId.Gitcoin}:1`
 export const PLUGIN_NAME = 'Gitcoin'
 export const PLUGIN_DESCRIPTION = 'Gitcoin grants sustain web3 projects with quadratic funding.'
-
-// proxy for: https://gitcoin.co/grants/v1/api/grant/
-export const GITCOIN_API_GRANTS_V1 = 'https://gitcoin-agent.r2d2.to/grants/v1/api/grant/:id'
+export const GITCOIN_API_GRANTS_V1 = 'https://gitcoin.co/grants/v1/api/grant/:id'

packages/mask/src/plugins/Gitcoin/hooks/useGrant.ts

@@ -1,6 +1,6 @@
 import { useAsyncRetry } from 'react-use'
-import { PluginGitcoinRPC } from '../messages'
+import { fetchGrant } from '../apis'
 
 export function useGrant(id: string) {
-    return useAsyncRetry(() => PluginGitcoinRPC.fetchGrant(id))
+    return useAsyncRetry(() => fetchGrant(id))
 }

packages/mask/src/plugins/Wallet/constants.ts

@@ -1,3 +1 @@
 export { PLUGIN_ID, HD_PATH_WITHOUT_INDEX_ETHEREUM, UPDATE_CHAIN_STATE_DELAY } from '@masknet/plugin-wallet'
-
-export const OPENSEA_API_KEY = 'c38fe2446ee34f919436c32db480a2e3'

packages/mask/src/setup.ui.0.ts

@@ -21,3 +21,6 @@ const indexedDB: KVStorageBackend = {
     },
 }
 setupMaskKVStorageBackend(indexedDB, memory)
+
+// Temporary, will be removed when the Mask SDK is ready
+Reflect.set(globalThis, 'r2d2Fetch', Services.Helper.r2d2Fetch)

packages/polyfills/types/__all.d.ts

@@ -1,2 +1,3 @@
 /// <reference path="./env.d.ts" />
 /// <reference path="./intl.d.ts" />
+/// <reference path="./global.d.ts" />

packages/polyfills/types/global.d.ts

@@ -0,0 +1 @@
+declare function r2d2Fetch(url: RequestInfo, init?: RequestInit): Promise<Response>

packages/shared-base/src/serializer/index.ts

@@ -1,12 +1,15 @@
 /// <reference path="./typeson.d.ts" />
-import { Typeson } from 'typeson'
+import { Typeson, TypesonPromise } from 'typeson'
 import type { Serialization } from 'async-call-rpc'
-import { Ok, Err, Some, None } from 'ts-results'
+import { Err, None, Ok, Some } from 'ts-results'
 import * as BN from 'bignumber.js'
 
 // @ts-ignore
-import { builtin, blob, file, filelist, imagebitmap, specialNumbers, cryptokey } from 'typeson-registry'
+import { blob, builtin, cryptokey, file, filelist, imagebitmap, specialNumbers } from 'typeson-registry'
 import { Identifier } from '../Identifier'
+import { responseRegedit } from './response'
+import { readableStreamRegedit } from './readableStream'
+import { requestRegedit } from './request'
 
 const pendingRegister = new Set<() => void>()
 let typeson: Typeson | undefined
@@ -27,6 +30,9 @@ function setup() {
 
     typeson.register({
         Identifier: [(x) => x instanceof Identifier, (x: Identifier) => x.toText(), (x) => Identifier.from(x).unwrap()],
+        ReadableStream: [...readableStreamRegedit],
+        Response: [...responseRegedit],
+        Request: [...requestRegedit],
     })
 
     for (const a of pendingRegister) a()
@@ -52,7 +58,7 @@ export function registerSerializableClass(name: string, constructor: NewableFunc
 export function registerSerializableClass<T, Q>(
     name: string,
     isT: (x: unknown) => boolean,
-    ser: (x: T) => Q,
+    ser: (x: T) => Q | TypesonPromise<Q>,
     de_ser: (x: Q) => T,
 ): void
 export function registerSerializableClass(name: string, a: any, b?: any, c?: any): void {

packages/shared-base/src/serializer/readableStream.ts

@@ -0,0 +1,39 @@
+import { TypesonPromise } from 'typeson'
+
+export const is = (x: any) => x instanceof ReadableStream
+
+export const serializer = (x: ReadableStream) => {
+    return new TypesonPromise<Uint8Array[]>(async (resolve, reject) => {
+        const reader = x.getReader()
+        const output = []
+        let isDone = false
+        if (reader) {
+            try {
+                while (!isDone) {
+                    const { done, value } = await reader.read()
+                    if (!done) {
+                        output.push(value)
+                    } else {
+                        isDone = true
+                    }
+                }
+            } catch (error) {
+                reject(error)
+            }
+        }
+        resolve(output)
+    })
+}
+
+export const deserializer = (x: Uint8Array[]) => {
+    return new ReadableStream({
+        start(controller) {
+            for (const binary of x) {
+                controller.enqueue(binary)
+            }
+            controller.close()
+        },
+    })
+}
+
+export const readableStreamRegedit = [is, serializer, deserializer] as const

packages/shared-base/src/serializer/request.ts

@@ -0,0 +1,25 @@
+export const is = (x: any) => x instanceof Request
+export const serializer = (x: Request) => {
+    const { url, method, body, headers, mode, credentials, cache, redirect, referrer, integrity } = x
+    return {
+        input: url,
+        init: {
+            method,
+            // body maybe is a Blob, a BufferSource, a FormData, a URLSearchParams, a string, or a ReadableStream object, should handle different object type later
+            body,
+            headers,
+            mode,
+            credentials,
+            cache,
+            redirect,
+            referrer,
+            integrity,
+        },
+    }
+}
+
+export const deserializer = (x: { input: string; init: RequestInit }) => {
+    return new Request(x.input, x.init)
+}
+
+export const requestRegedit = [is, serializer, deserializer] as const

packages/shared-base/src/serializer/response.ts

@@ -0,0 +1,25 @@
+export const is = (x: any) => x instanceof Response
+export const serializer = (x: Response) => {
+    return {
+        body: x.body,
+        init: {
+            status: x.status,
+            statusText: x.statusText,
+            headers: x.headers,
+        },
+    }
+}
+
+export const deserializer = (x: { body: Uint8Array[]; init: ResponseInit }) => {
+    const body = new ReadableStream({
+        start(controller) {
+            for (const binary of x.body) {
+                controller.enqueue(binary)
+            }
+            controller.close()
+        },
+    })
+    return new Response(body, x.init)
+}
+
+export const responseRegedit = [is, serializer, deserializer] as const

packages/shared-base/src/serializer/typeson.d.ts

@@ -5,6 +5,9 @@ declare module 'typeson' {
         (x: InternalRepresentation) => Type,
     ]
     export class Undefined {}
+    export class TypesonPromise<T> {
+        constructor(executor: (resolve: (value: T | PromiseLike<T>) => void, reject: (reason?: any) => void) => void)
+    }
     export class Typeson {
         constructor(options?: {
             cyclic?: boolean

packages/web3-providers/src/NextID/helper.ts

@@ -17,16 +17,18 @@ export async function fetchJSON<T = unknown>(
 ): Promise<Result<T, string>> {
     type FetchCache = LRU<string, Promise<Response> | T>
 
+    const fetch = globalThis.r2d2Fetch ?? globalThis.fetch
     const cached = enableCache ? (fetchCache as FetchCache).get(url) : undefined
     const isPending = cached instanceof Promise
+
     if (cached && !isPending) {
         return Ok(cached)
     }
     let pendingResponse: Promise<Response>
     if (isPending) {
         pendingResponse = cached
     } else {
-        pendingResponse = globalThis.fetch(url, { mode: 'cors', ...requestInit })
+        pendingResponse = fetch(url, requestInit)
         if (enableCache) {
             fetchCache.set(url, pendingResponse)
         }

packages/web3-providers/src/gopluslabs/constants.ts

@@ -1 +1 @@
-export const GO_PLUS_LABS_ROOT_URL = 'https://gopluslabs.r2d2.to'
+export const GO_PLUS_LABS_ROOT_URL = 'https://api.gopluslabs.io'

packages/web3-providers/src/helpers.ts

@@ -8,8 +8,9 @@ export function isProxyENV() {
     }
 }
 
-export async function fetchJSON<T = unknown>(requestInfo: RequestInfo, requestInit?: RequestInit): Promise<T> {
-    const res = await globalThis.fetch(requestInfo, requestInit)
+export async function fetchJSON<T = unknown>(requestInfo: string, requestInit?: RequestInit): Promise<T> {
+    const fetch = globalThis.r2d2Fetch ?? globalThis.fetch
+    const res = await fetch(requestInfo, requestInit)
     return res.json()
 }
 

packages/web3-providers/src/opensea/constants.ts

@@ -1,3 +1,2 @@
 export const OPENSEA_ACCOUNT_URL = 'https://opensea.io/accounts/:address'
-export const OPENSEA_API_KEY = '1cf95be40f1e45449c0b63ccb4b64cef'
 export const OPENSEA_API_URL = 'https://api.opensea.io'

packages/web3-providers/src/opensea/index.ts

@@ -33,18 +33,15 @@ import type {
     OpenSeaResponse,
 } from './types'
 import { getOrderUSDPrice, toImage } from './utils'
-import { OPENSEA_ACCOUNT_URL, OPENSEA_API_KEY, OPENSEA_API_URL } from './constants'
-import { isProxyENV } from '../helpers'
+import { OPENSEA_ACCOUNT_URL, OPENSEA_API_URL } from './constants'
 
 async function fetchFromOpenSea<T>(url: string, chainId: ChainId, apiKey?: string) {
     if (![ChainId.Mainnet, ChainId.Rinkeby, ChainId.Matic].includes(chainId)) return
+    const fetch = globalThis.r2d2Fetch ?? globalThis.fetch
 
     try {
-        const response = await fetch(urlcat(OPENSEA_API_URL, url), {
-            method: 'GET',
-            headers: { 'x-api-key': apiKey ?? OPENSEA_API_KEY, Accept: 'application/json' },
-            ...(!isProxyENV() && { mode: 'cors' }),
-        })
+        // TODO: backend fix 500
+        const response = await fetch(urlcat(OPENSEA_API_URL, url), { method: 'GET' })
         if (response.ok) {
             return (await response.json()) as T
         }