feat(ipfs-manager): adapt RPC usage with POST methods

.circleci/config.yml

@@ -7,7 +7,7 @@ references:
   node_image: &node_image
     image: cimg/node:16.15
   ipfs_image: &ipfs_image
-    image: requestnetwork/request-ipfs:0.4.23-2
+    image: requestnetwork/request-ipfs:v0.13.0
   ganache_image: &ganache_image
     image: trufflesuite/ganache-cli:v6.8.2
     command:

packages/ethereum-storage/package.json

@@ -44,7 +44,6 @@
     "@requestnetwork/types": "0.35.0",
     "@requestnetwork/utils": "0.35.0",
     "axios": "0.27.2",
-    "axios-retry": "3.2.5",
     "bluebird": "3.7.2",
     "ethers": "5.5.1",
     "form-data": "3.0.0",

packages/ethereum-storage/src/config.ts

@@ -3,9 +3,9 @@ import { BigNumber } from 'ethers';
 
 // This contains default values used to use Ethereum Network and IPFS
 // if information are not provided by the user
-const config: any = {
+const config = {
   ethereum: {
-    default: 'private',
+    default: 'private' as const,
     gasPriceDefault: '100000000000',
     maxRetries: 5,
     nodeUrlDefault: {
@@ -22,22 +22,19 @@ const config: any = {
     defaultNode: {
       host: 'localhost',
       port: 5001,
-      protocol: 'http',
+      protocol: 'http' as StorageTypes.IpfsGatewayProtocol,
       timeout: 30000,
     },
     errorHandling: {
       delayBetweenRetries: 500,
       maxRetries: 3,
     },
-    expectedBootstrapNodes: [
-      '/dns4/ipfs-bootstrap.request.network/tcp/4001/ipfs/QmaSrBXFBaupfeGMTuigswtKtsthbVaSonurjTV967Fdxx',
-
-      '/dns4/ipfs-bootstrap-2.request.network/tcp/4001/ipfs/QmYdcSoVNU1axgSnkRAyHtwsKiSvFHXeVvRonGCAV9LVEj',
-
-      '/dns4/ipfs-2.request.network/tcp/4001/ipfs/QmPBPgTDVjveRu6KjGVMYixkCSgGtVyV8aUe6wGQeLZFVd',
-
-      '/dns4/ipfs-survival.request.network/tcp/4001/ipfs/Qmb6a5DH45k8JwLdLVZUhRhv1rnANpsbXjtsH41esGhNCh',
-    ],
+    expectedBootstrapNodes: {
+      'ipfs-bootstrap.request.network': 'QmaSrBXFBaupfeGMTuigswtKtsthbVaSonurjTV967Fdxx',
+      'ipfs-bootstrap-2.request.network': 'QmYdcSoVNU1axgSnkRAyHtwsKiSvFHXeVvRonGCAV9LVEj',
+      'ipfs-2.request.network': 'QmPBPgTDVjveRu6KjGVMYixkCSgGtVyV8aUe6wGQeLZFVd',
+      'ipfs-survival.request.network': 'Qmb6a5DH45k8JwLdLVZUhRhv1rnANpsbXjtsH41esGhNCh',
+    },
     maxIpfsReadRetry: 1,
     pinRequest: {
       delayBetweenCalls: 1000,
@@ -147,10 +144,12 @@ export function getIpfsErrorHandlingConfig(): StorageTypes.IIpfsErrorHandlingCon
 
 /**
  * Retrieve from config the ipfs bootstrap nodes of the ipfs node
- * @returns array of the swarm addresses
+ * @returns array of the swarm addresses regexes
  */
-export function getIpfsExpectedBootstrapNodes(): string[] {
-  return config.ipfs.expectedBootstrapNodes;
+export function getIpfsExpectedBootstrapNodes(): RegExp[] {
+  return Object.entries(config.ipfs.expectedBootstrapNodes).map(
+    ([host, id]) => new RegExp(`/dns4/${host}/tcp/4001/(ipfs|p2p)/${id}`),
+  );
 }
 
 /**

packages/ethereum-storage/src/ipfs-manager.ts

@@ -1,13 +1,24 @@
 import { UnixFS } from 'ipfs-unixfs';
 import * as qs from 'qs';
-import axios, { AxiosInstance, AxiosResponse } from 'axios';
+import axios, { AxiosError, AxiosInstance, AxiosRequestConfig } from 'axios';
 import Utils from '@requestnetwork/utils';
 import { LogTypes, StorageTypes } from '@requestnetwork/types';
 
 import { getDefaultIpfs, getIpfsErrorHandlingConfig } from './config';
-import axiosRetry from 'axios-retry';
 import * as FormData from 'form-data';
 
+/** A mapping between IPFS Paths and the response type */
+type IpfsPaths = {
+  id: string;
+  add: { Hash: string };
+  'object/get':
+    | { Type: 'error'; Message: string }
+    | { Type: undefined; Data: string; Links: string[] };
+  'object/stat': { DataSize: number };
+  'pin/add': { Pins: string[] };
+  'bootstrap/list': { Peers: string[] };
+};
+
 /**
  * Manages Ipfs communication used as storage
  */
@@ -17,14 +28,7 @@ export default class IpfsManager {
   private readonly ipfsGatewayConnection: StorageTypes.IIpfsGatewayConnection;
   private readonly ipfsErrorHandling: StorageTypes.IIpfsErrorHandlingConfiguration;
 
-  public readonly IPFS_API_ADD: string = '/api/v0/add';
-  public readonly IPFS_API_CAT: string = '/api/v0/object/get';
-  public readonly IPFS_API_STAT: string = '/api/v0/object/stat';
-  public readonly IPFS_API_CONNECT_SWARM: string = '/api/v0/swarm/connect';
-
-  public readonly IPFS_API_ID: string = '/api/v0/id';
-  public readonly IPFS_API_PIN: string = '/api/v0/pin/add';
-  public readonly IPFS_API_BOOTSTRAP_LIST: string = '/api/v0/bootstrap/list';
+  public readonly BASE_PATH: string = 'api/v0';
 
   /**
    * Constructor
@@ -42,17 +46,33 @@ export default class IpfsManager {
     this.ipfsGatewayConnection = options?.ipfsGatewayConnection || getDefaultIpfs();
     this.ipfsErrorHandling = options?.ipfsErrorHandling || getIpfsErrorHandlingConfig();
     this.logger = options?.logger || new Utils.SimpleLogger();
+
     this.axiosInstance = axios.create({
-      baseURL: `${this.ipfsGatewayConnection.protocol}://${this.ipfsGatewayConnection.host}:${this.ipfsGatewayConnection.port}`,
+      baseURL: `${this.ipfsGatewayConnection.protocol}://${this.ipfsGatewayConnection.host}:${this.ipfsGatewayConnection.port}/${this.BASE_PATH}/`,
       timeout: this.ipfsGatewayConnection.timeout,
       paramsSerializer: function (params) {
         return qs.stringify(params, { arrayFormat: 'repeat' });
       },
     });
-    axiosRetry(this.axiosInstance, {
-      retries: this.ipfsErrorHandling.maxRetries,
-      retryDelay: () => this.ipfsErrorHandling.delayBetweenRetries,
+  }
+
+  private async ipfs<T extends keyof IpfsPaths>(path: T, config?: AxiosRequestConfig) {
+    const _post = Utils.retry(this.axiosInstance.post, {
+      context: this.axiosInstance,
+      maxRetries: this.ipfsErrorHandling.maxRetries,
+      retryDelay: this.ipfsErrorHandling.delayBetweenRetries,
     });
+    try {
+      const { data, ...rest } = config || {};
+      const response = await _post<IpfsPaths[T]>(path, data, rest);
+      return response.data;
+    } catch (e) {
+      const axiosError = e as AxiosError<{ Message?: string }>;
+      if (axiosError.isAxiosError && axiosError.response?.data?.Message) {
+        throw new Error(axiosError.response.data.Message);
+      }
+      throw e;
+    }
   }
 
   /**
@@ -61,8 +81,7 @@ export default class IpfsManager {
    */
   public async getIpfsNodeId(): Promise<string> {
     try {
-      const response = await this.axiosInstance.get(this.IPFS_API_ID);
-      return response.data;
+      return await this.ipfs('id');
     } catch (e) {
       this.logger.error(`Failed to retrieve IPFS node ID: ${e.message}`, ['ipfs']);
       throw e;
@@ -78,11 +97,12 @@ export default class IpfsManager {
     try {
       const addForm = new FormData();
       addForm.append('file', Buffer.from(content));
-      const response = await this.axiosInstance.post(this.IPFS_API_ADD, addForm, {
+      const response = await this.ipfs('add', {
+        data: addForm,
         headers: addForm.getHeaders(),
       });
       // Return the hash of the response
-      const hash = response.data.Hash;
+      const hash = response.Hash;
       if (!hash) {
         throw new Error('response has no Hash field');
       }
@@ -115,17 +135,17 @@ export default class IpfsManager {
         const base64StringMaxLength = ((4 * maxSize) / 3 + 3) & ~3; // https://stackoverflow.com/a/32140193/16270345
         maxSize = base64StringMaxLength + jsonMetadataSize;
       }
-      const response: AxiosResponse = await this.axiosInstance.get(this.IPFS_API_CAT, {
+      const response = await this.ipfs('object/get', {
         params: { arg: hash, 'data-encoding': 'base64' },
         maxContentLength: maxSize,
       });
-      if (response.data.Type === 'error') {
-        throw new Error(response.data.Message);
+      if (response.Type === 'error') {
+        throw new Error(response.Message);
       }
-      const ipfsDataBuffer = Buffer.from(response.data.Data, 'base64');
+      const ipfsDataBuffer = Buffer.from(response.Data, 'base64');
       const content = IpfsManager.getContentFromMarshaledData(ipfsDataBuffer);
       const ipfsSize = ipfsDataBuffer.length;
-      const ipfsLinks = response.data.Links;
+      const ipfsLinks = response.Links;
       return { content, ipfsSize, ipfsLinks };
     } catch (e) {
       this.logger.error(`Failed to read IPFS file: ${e.message}`, ['ipfs']);
@@ -141,11 +161,11 @@ export default class IpfsManager {
    */
   public async pin(hashes: string[], timeout?: number): Promise<string[]> {
     try {
-      const response = await this.axiosInstance.get(this.IPFS_API_PIN, {
+      const response = await this.ipfs('pin/add', {
         params: { arg: hashes },
         timeout,
       });
-      const pins = response.data.Pins;
+      const pins = response.Pins;
       if (!pins) {
         throw new Error('Ipfs pin request response has no Pins field');
       }
@@ -163,12 +183,12 @@ export default class IpfsManager {
    */
   public async getContentLength(hash: string): Promise<number> {
     try {
-      const response = await this.axiosInstance.get(this.IPFS_API_STAT, { params: { arg: hash } });
-      const length = response.data.DataSize;
+      const response = await this.ipfs('object/stat', { params: { arg: hash } });
+      const length = response.DataSize;
       if (!length) {
         throw new Error('Ipfs stat request response has no DataSize field');
       }
-      return parseInt(length, 10);
+      return length;
     } catch (e) {
       this.logger.error(`Failed to retrieve IPFS file size: ${e.message}`, ['ipfs']);
       throw e;
@@ -181,8 +201,8 @@ export default class IpfsManager {
    */
   public async getBootstrapList(): Promise<string[]> {
     try {
-      const response = await this.axiosInstance.get(this.IPFS_API_BOOTSTRAP_LIST);
-      const peers = response.data.Peers;
+      const response = await this.ipfs('bootstrap/list');
+      const peers = response.Peers;
       if (!peers) {
         throw new Error('Ipfs bootstrap list request response has no Peers field');
       }

packages/ethereum-storage/src/ipfs-storage.ts

@@ -125,12 +125,7 @@ export class IpfsStorage implements StorageTypes.IIpfsStorage {
     this.logger.info('Checking ipfs network', ['ipfs', 'sanity']);
     try {
       const bootstrapList = await this.ipfsManager.getBootstrapList();
-
-      const bootstrapNodeFoundCount: number = getIpfsExpectedBootstrapNodes().filter(
-        (nodeExpected) => bootstrapList.includes(nodeExpected),
-      ).length;
-
-      if (bootstrapNodeFoundCount !== getIpfsExpectedBootstrapNodes().length) {
+      if (!IpfsStorage.hasRequiredBootstrapNodes(bootstrapList)) {
         throw Error(
           `The list of bootstrap node in the ipfs config don't match the expected bootstrap nodes`,
         );
@@ -139,4 +134,11 @@ export class IpfsStorage implements StorageTypes.IIpfsStorage {
       throw Error(`IPFS node bootstrap node check failed: ${error}`);
     }
   }
+
+  static hasRequiredBootstrapNodes(actualList: string[]): boolean {
+    const expectedList = getIpfsExpectedBootstrapNodes();
+    return expectedList.every((nodeExpected) =>
+      actualList.some((actual) => nodeExpected.test(actual)),
+    );
+  }
 }

packages/ethereum-storage/test/ipfs-manager.test.ts

@@ -14,7 +14,7 @@ const invalidHostIpfsGatewayConnection: StorageTypes.IIpfsGatewayConnection = {
   host: 'nonexistent',
   port: 5001,
   protocol: StorageTypes.IpfsGatewayProtocol.HTTP,
-  timeout: 1000,
+  timeout: 1500,
 };
 
 const testErrorHandling: StorageTypes.IIpfsErrorHandlingConfiguration = {
@@ -76,24 +76,24 @@ describe('Ipfs manager', () => {
   it('allows to pin one file ipfs', async () => {
     await ipfsManager.add(content);
     const pinnedHash = await ipfsManager.pin([hash]);
-    expect(hash).toBe(pinnedHash[0]);
+    expect(pinnedHash[0]).toBe(hash);
   });
 
   it('allows to pin multiple files to ipfs', async () => {
     await ipfsManager.add(content);
     await ipfsManager.add(content2);
     const pinnedHashes = await ipfsManager.pin([hash, hash2]);
-    expect([hash, hash2]).toMatchObject(pinnedHashes);
+    expect(pinnedHashes).toMatchObject([hash, hash2]);
   });
 
   it('allows to read files from ipfs', async () => {
     await ipfsManager.add(content);
     let contentReturned = await ipfsManager.read(hash, 36);
-    expect(content).toBe(contentReturned.content);
+    expect(contentReturned.content).toBe(content);
 
     await ipfsManager.add(content2);
     contentReturned = await ipfsManager.read(hash2);
-    expect(content2).toBe(contentReturned.content);
+    expect(contentReturned.content).toBe(content2);
   });
 
   it('must throw if max size reached', async () => {
@@ -108,11 +108,11 @@ describe('Ipfs manager', () => {
   it('allows to get file size from ipfs', async () => {
     await ipfsManager.add(content);
     let sizeReturned = await ipfsManager.getContentLength(hash);
-    expect(contentLengthOnIpfs).toEqual(sizeReturned);
+    expect(sizeReturned).toEqual(contentLengthOnIpfs);
 
     await ipfsManager.add(content2);
     sizeReturned = await ipfsManager.getContentLength(hash2);
-    expect(contentLengthOnIpfs2).toEqual(sizeReturned);
+    expect(sizeReturned).toEqual(contentLengthOnIpfs2);
   });
 
   it('operations with a invalid host network should throw ENOTFOUND errors', async () => {
@@ -146,10 +146,10 @@ describe('Ipfs manager', () => {
     const axiosInstanceMock = new MockAdapter(axiosInstance);
     axiosInstanceMock.onAny().networkError();
     await expect(ipfsManager.read(hash)).rejects.toThrowError('Network Error');
-    expect(axiosInstanceMock.history.get.length).toBe(retryTestErrorHandling.maxRetries + 1);
+    expect(axiosInstanceMock.history.post.length).toBe(retryTestErrorHandling.maxRetries + 1);
   });
 
-  it('timeout errors should not generate any retry', async () => {
+  it('timeout errors should generate retry', async () => {
     ipfsManager = new IpfsManager({
       ipfsGatewayConnection: { ...ipfsGatewayConnection, timeout: 1 },
       ipfsErrorHandling: retryTestErrorHandling,
@@ -158,8 +158,7 @@ describe('Ipfs manager', () => {
     const axiosInstanceMock = new MockAdapter(axiosInstance);
     axiosInstanceMock.onAny().timeout();
     await expect(ipfsManager.add('test')).rejects.toThrowError('timeout of 1ms exceeded');
-    // only one request should have been sent, no retry should happen on timeouts
-    expect(axiosInstanceMock.history.post.length).toBe(1);
+    expect(axiosInstanceMock.history.post.length).toBe(retryTestErrorHandling.maxRetries + 1);
   });
 
   it('added and read files should have the same size and content', async () => {

packages/ethereum-storage/test/ipfs-storage.test.ts

@@ -98,4 +98,41 @@ describe('IPFS Storage', () => {
       ['ipfs'],
     );
   });
+
+  describe('compareBootstrapNodes', () => {
+    describe.each(['ipfs', 'p2p'])('It supports the %s path', (path) => {
+      it('Returns true for same list', () => {
+        expect(
+          IpfsStorage.hasRequiredBootstrapNodes([
+            `/dns4/ipfs-survival.request.network/tcp/4001/${path}/Qmb6a5DH45k8JwLdLVZUhRhv1rnANpsbXjtsH41esGhNCh`,
+            `/dns4/ipfs-2.request.network/tcp/4001/${path}/QmPBPgTDVjveRu6KjGVMYixkCSgGtVyV8aUe6wGQeLZFVd`,
+            `/dns4/ipfs-bootstrap-2.request.network/tcp/4001/${path}/QmYdcSoVNU1axgSnkRAyHtwsKiSvFHXeVvRonGCAV9LVEj`,
+            `/dns4/ipfs-bootstrap.request.network/tcp/4001/${path}/QmaSrBXFBaupfeGMTuigswtKtsthbVaSonurjTV967Fdxx`,
+          ]),
+        ).toBeTruthy();
+      });
+
+      it('Returns false for additional items', () => {
+        expect(
+          IpfsStorage.hasRequiredBootstrapNodes([
+            `/dns4/ipfs-survival.request.network/tcp/4001/${path}/Qmb6a5DH45k8JwLdLVZUhRhv1rnANpsbXjtsH41esGhNCh`,
+            `/dns4/ipfs-2.request.network/tcp/4001/${path}/QmPBPgTDVjveRu6KjGVMYixkCSgGtVyV8aUe6wGQeLZFVd`,
+            `/dns4/ipfs-bootstrap-2.request.network/tcp/4001/${path}/QmYdcSoVNU1axgSnkRAyHtwsKiSvFHXeVvRonGCAV9LVEj`,
+            `/dns4/ipfs-bootstrap.request.network/tcp/4001/${path}/QmaSrBXFBaupfeGMTuigswtKtsthbVaSonurjTV967Fdxx`,
+            `/dns4/ipfs-bootstrap-NONEXISTANT.request.network/tcp/4001/${path}/QmaSrBXFBaupfeGMTuigswtKtsthbVaSonurjTV967FaKe`,
+          ]),
+        ).toBeTruthy();
+      });
+
+      it('Returns false for missing items', () => {
+        expect(
+          IpfsStorage.hasRequiredBootstrapNodes([
+            `/dns4/ipfs-survival.request.network/tcp/4001/${path}/Qmb6a5DH45k8JwLdLVZUhRhv1rnANpsbXjtsH41esGhNCh`,
+            `/dns4/ipfs-2.request.network/tcp/4001/${path}/QmPBPgTDVjveRu6KjGVMYixkCSgGtVyV8aUe6wGQeLZFVd`,
+            `/dns4/ipfs-bootstrap-2.request.network/tcp/4001/${path}/QmYdcSoVNU1axgSnkRAyHtwsKiSvFHXeVvRonGCAV9LVEj`,
+          ]),
+        ).toBeFalsy();
+      });
+    });
+  });
 });

packages/request-node/README.md

@@ -314,8 +314,6 @@ yarn build
 
 #### 3. On a new terminal, launch a local IPFS node
 
-Note: only IPFS v0.4.\* supported, from the [IPFS Installation docs](https://docs.ipfs.io/install/), replace the binary URL with the good one from the following list: https://github.com/ipfs/go-ipfs/releases/tag/v0.4.23
-
 ```bash
 ipfs daemon
 ```