Feature/refresh (#273)

* Adds initial support for guppy refresh

* Fix Formatting

* Adds create method checking, updates tests

* fix: test

* Addresses PR feedback

* chore(deps): bump braces from 3.0.2 to 3.0.3 (#271)

* chore(deps): bump braces from 3.0.2 to 3.0.3

Bumps [braces](https://github.com/micromatch/braces) from 3.0.2 to 3.0.3.
- [Changelog](https://github.com/micromatch/braces/blob/master/CHANGELOG.md)
- [Commits](micromatch/braces@3.0.2...3.0.3)

---
updated-dependencies:
- dependency-name: braces
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

* trigger gh action

* trigger travis

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Mingfei Shao <mshao1@uchicago.edu>

* chore(deps-dev): bump ws from 6.2.2 to 6.2.3 (#272)

Bumps [ws](https://github.com/websockets/ws) from 6.2.2 to 6.2.3.
- [Release notes](https://github.com/websockets/ws/releases)
- [Commits](websockets/ws@6.2.2...6.2.3)

---
updated-dependencies:
- dependency-name: ws
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Mingfei Shao <2475897+mfshao@users.noreply.github.com>

* address recent feedback

* fix apply hide number resolver to as text agg results (#274)

* fix apply hide number resolver to as text agg results

* update dockerfile

* Fix param ordering in server.js (#277)

Param order was wrong on changed line, put err first.
Refer to: https://expressjs.com/en/guide/error-handling.html

* fix: package.json & package-lock.json to reduce vulnerabilities (#280)

The following vulnerabilities are fixed with an upgrade:
- https://snyk.io/vuln/SNYK-JS-SEND-7926862

Co-authored-by: snyk-bot <snyk-bot@snyk.io>

* Update integration_tests.yaml (#284)

* (PPS-1564 PPS-1567 PPS-1585): update body-parser and package lock file to use newer dependencies (#285)

* PPS-1564 PPS-1567 PPS-1585: update body-parser and package lock file to use newer dependencies
* PPS-1564 PPS-1567 PPS-1585: add file so nvm can set node version

* Update quickstart_helm.md (#289)

Fix a dead link

* Adds initial support for guppy refresh

* run linter

* bug fixes

* fix es-lint errors

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Mingfei Shao <mshao1@uchicago.edu>
Co-authored-by: Mingfei Shao <2475897+mfshao@users.noreply.github.com>
Co-authored-by: Albert Snow <ajsnow2012@gmail.com>
Co-authored-by: Binam Bajracharya <44302895+BinamB@users.noreply.github.com>
Co-authored-by: snyk-bot <snyk-bot@snyk.io>
Co-authored-by: Krishna Agarwal <159047652+krishnaa05@users.noreply.github.com>
Co-authored-by: Shawn O'Connor <ocshawn@ocshawn.com>
Co-authored-by: smvgarcia <111767892+smvgarcia@users.noreply.github.com>

Author: 9 people

src/server/__mocks__/mockDataFromES.js

@@ -8,9 +8,19 @@ import mockNestedTermsAndMissingAggs from './mockESData/mockNestedTermsAndMissin
 import mockNestedAggs from './mockESData/mockNestedAggs';
 
 const mockPing = () => {
-  nock(config.esConfig.host)
-    .head('/')
-    .reply(200, 'hello');
+  nock(config.esConfig.host).head('/').reply(200, 'hello');
+};
+
+const mockRefresh = () => {
+  if (config.allowRefresh) {
+    nock(config.esConfig.host)
+      .post('/_refresh')
+      .reply(200, '[Server] guppy refreshed successfully');
+  } else {
+    nock(config.esConfig.host)
+      .post('/_refresh')
+      .reply(404, '[Server] guppy _refresh functionality is not enabled');
+  }
 };
 
 const mockResourcePath = () => {
@@ -35,12 +45,8 @@ const mockResourcePath = () => {
       },
     },
     highlight: {
-      pre_tags: [
-        '<em>',
-      ],
-      post_tags: [
-        '</em>',
-      ],
+      pre_tags: ['<em>'],
+      post_tags: ['</em>'],
       fields: {
         '*.analyzed': {},
       },
@@ -111,12 +117,8 @@ const mockResourcePath = () => {
       },
     },
     highlight: {
-      pre_tags: [
-        '<em>',
-      ],
-      post_tags: [
-        '</em>',
-      ],
+      pre_tags: ['<em>'],
+      post_tags: ['</em>'],
       fields: {
         '*.analyzed': {},
       },
@@ -172,12 +174,8 @@ const mockResourcePath = () => {
       },
     },
     highlight: {
-      pre_tags: [
-        '<em>',
-      ],
-      post_tags: [
-        '</em>',
-      ],
+      pre_tags: ['<em>'],
+      post_tags: ['</em>'],
       fields: {
         '*.analyzed': {},
       },
@@ -212,7 +210,8 @@ const mockArborist = () => {
     .persist()
     .post('/auth/mapping')
     .reply(200, {
-      'internal-project-1': [ // accessible
+      'internal-project-1': [
+        // accessible
         {
           service: '*',
           method: 'create',
@@ -234,13 +233,15 @@ const mockArborist = () => {
           method: 'update',
         },
       ],
-      'internal-project-2': [ // accessible
+      'internal-project-2': [
+        // accessible
         {
           service: '*',
           method: 'read',
         },
       ],
-      'internal-project-3': [ // not accessible since method does not match
+      'internal-project-3': [
+        // not accessible since method does not match
         {
           service: '*',
           method: 'create',
@@ -258,19 +259,22 @@ const mockArborist = () => {
           method: 'update',
         },
       ],
-      'internal-project-4': [ // accessible
+      'internal-project-4': [
+        // accessible
         {
           service: '*',
           method: '*',
         },
       ],
-      'internal-project-5': [ // accessible
+      'internal-project-5': [
+        // accessible
         {
           service: 'guppy',
           method: '*',
         },
       ],
-      'internal-project-6': [ // not accessible since service does not match
+      'internal-project-6': [
+        // not accessible since service does not match
         {
           service: 'indexd',
           method: '*',
@@ -376,7 +380,9 @@ const mockESMapping = () => {
 };
 
 const mockArrayConfig = () => {
-  const arrayConfigQuery = { query: { ids: { values: ['gen3-dev-subject', 'gen3-dev-file'] } } };
+  const arrayConfigQuery = {
+    query: { ids: { values: ['gen3-dev-subject', 'gen3-dev-file'] } },
+  };
   const fakeArrayConfig = {
     hits: {
       total: 1,
@@ -387,10 +393,7 @@ const mockArrayConfig = () => {
           _id: 'gen3-dev-subject',
           _score: 1.0,
           _source: {
-            array: [
-              'some_array_integer_field',
-              'some_array_string_field',
-            ],
+            array: ['some_array_integer_field', 'some_array_string_field'],
           },
         },
       ],
@@ -405,6 +408,7 @@ const mockArrayConfig = () => {
 const setup = () => {
   mockArborist();
   mockPing();
+  mockRefresh();
   mockResourcePath();
   mockESMapping();
   mockArrayConfig();

src/server/__tests__/config.test.js

@@ -34,15 +34,19 @@ describe('config', () => {
 
   test('should show error if invalid tier access level', async () => {
     process.env.TIER_ACCESS_LEVEL = 'invalid-level';
-    expect(() => (require('../config'))).toThrow(new Error(`Invalid TIER_ACCESS_LEVEL "${process.env.TIER_ACCESS_LEVEL}"`));
+    expect(() => require('../config')).toThrow(
+      new Error(`Invalid TIER_ACCESS_LEVEL "${process.env.TIER_ACCESS_LEVEL}"`),
+    );
   });
 
   test('should show error if invalid tier access level in guppy block', async () => {
     process.env.TIER_ACCESS_LEVEL = null;
     const fileName = './testConfigFiles/test-invalid-index-scoped-tier-access.json';
     process.env.GUPPY_CONFIG_FILEPATH = `${__dirname}/${fileName}`;
     const invalidItemType = 'subject_private';
-    expect(() => (require('../config'))).toThrow(new Error(`tier_access_level invalid for index ${invalidItemType}.`));
+    expect(() => require('../config')).toThrow(
+      new Error(`tier_access_level invalid for index ${invalidItemType}.`),
+    );
   });
 
   test('clears out site-wide default tiered-access setting if index-scoped levels set', async () => {
@@ -54,7 +58,9 @@ describe('config', () => {
     const { indices } = require(fileName);
     expect(config.tierAccessLevel).toBeUndefined();
     expect(config.tierAccessLimit).toEqual(50);
-    expect(JSON.stringify(config.esConfig.indices)).toEqual(JSON.stringify(indices));
+    expect(JSON.stringify(config.esConfig.indices)).toEqual(
+      JSON.stringify(indices),
+    );
   });
 
   /* --------------- For whitelist --------------- */
@@ -97,4 +103,11 @@ describe('config', () => {
     expect(config.esConfig.aggregationIncludeMissingData).toBe(true);
     expect(config.esConfig.missingDataAlias).toEqual(alias);
   });
+
+  /* --------------- For _refresh testing --------------- */
+  test('could not access _refresh method if not in config', async () => {
+    process.env.GUPPY_CONFIG_FILEPATH = `${__dirname}/testConfigFiles/test-no-refresh-option-provided.json`;
+    const config = require('../config').default;
+    expect(config.allowRefresh).toBe(false);
+  });
 });

src/server/__tests__/testConfigFiles/test-no-refresh-option-provided.json

@@ -0,0 +1 @@
+{}

src/server/auth/__tests__/authHelper.test.js

@@ -1,5 +1,5 @@
 // eslint-disable-next-line
-import nock from 'nock'; // must import this to enable mock data by nock 
+import nock from 'nock'; // must import this to enable mock data by nock
 import getAuthHelperInstance from '../authHelper';
 import esInstance from '../../es/index';
 import setupMockDataEndpoint from '../../__mocks__/mockDataFromES';
@@ -12,6 +12,7 @@ setupMockDataEndpoint();
 describe('AuthHelper', () => {
   test('could create auth helper instance', async () => {
     const authHelper = await getAuthHelperInstance('fake-jwt');
+    expect(authHelper.getCanRefresh()).toEqual(false);
     expect(authHelper.getAccessibleResources()).toEqual(['internal-project-1', 'internal-project-2', 'internal-project-4', 'internal-project-5']);
     expect(authHelper.getAccessibleResources()).not.toContain(['internal-project-3', 'internal-project-6']);
     expect(authHelper.getUnaccessibleResources()).toEqual(['external-project-1', 'external-project-2']);

src/server/auth/arboristClient.js

@@ -8,10 +8,10 @@ class ArboristClient {
     this.baseEndpoint = arboristEndpoint;
   }
 
-  listAuthorizedResources(jwt) {
+  listAuthMapping(jwt) {
     // Make request to arborist for list of resources with access
     const resourcesEndpoint = `${this.baseEndpoint}/auth/mapping`;
-    log.debug('[ArboristClient] listAuthorizedResources jwt: ', jwt);
+    log.debug('[ArboristClient] listAuthMapping jwt: ', jwt);
 
     const headers = (jwt) ? { Authorization: `bearer ${jwt}` } : {};
     return fetch(
@@ -40,29 +40,6 @@ class ArboristClient {
         log.error(err);
         throw new CodedError(500, err);
       },
-    ).then(
-      (result) => {
-        const data = {
-          resources: [],
-        };
-        Object.keys(result).forEach((key) => {
-        // logic: you have access to a project if you have the following access:
-        // method 'read' (or '*' - all methods) to service 'guppy' (or '*' - all services)
-        // on the project resource.
-          if (result[key] && result[key].some((x) => (
-            (x.method === 'read' || x.method === '*')
-          && (x.service === 'guppy' || x.service === '*')
-          ))) {
-            data.resources.push(key);
-          }
-        });
-        log.debug('[ArboristClient] data: ', data);
-        return data;
-      },
-      (err) => {
-        log.error(err);
-        throw new CodedError(500, err);
-      },
     );
   }
 }

src/server/auth/authHelper.js

@@ -5,6 +5,7 @@ import {
   getRequestResourceListFromFilter,
   buildFilterWithResourceList,
   getAccessibleResourcesFromArboristasync,
+  checkIfUserCanRefreshServer,
 } from './utils';
 import config from '../config';
 
@@ -15,8 +16,12 @@ export class AuthHelper {
 
   async initialize() {
     try {
-      this._accessibleResourceList = await getAccessibleResourcesFromArboristasync(this._jwt);
+      const [accessibleResourceList, arboristResources] = await getAccessibleResourcesFromArboristasync(this._jwt);
+      this._accessibleResourceList = accessibleResourceList;
+      this._arboristResources = arboristResources;
       log.debug('[AuthHelper] accessible resources:', this._accessibleResourceList);
+      this._canRefresh = await checkIfUserCanRefreshServer(this._arboristResources);
+      log.debug('[AuthHelper] can user refresh:', this._canRefresh);
 
       const promiseList = [];
       config.esConfig.indices.forEach(({ index, type }) => {
@@ -39,6 +44,10 @@ export class AuthHelper {
     return this._accessibleResourceList;
   }
 
+  getCanRefresh() {
+    return this._canRefresh;
+  }
+
   getUnaccessibleResources() {
     return this._unaccessibleResourceList;
   }

src/server/auth/utils.js

@@ -6,6 +6,23 @@ import arboristClient from './arboristClient';
 import CodedError from '../utils/error';
 import config from '../config';
 
+export const resourcePathsWithServiceMethodCombination = (userAuthMapping, services, methods = {}) => {
+  const data = {
+    resources: [],
+  };
+  Object.keys(userAuthMapping).forEach((key) => {
+    // logic: you have access to a project if you have
+    // access to any of the combinations made by the method and service lists
+    if (userAuthMapping[key] && userAuthMapping[key].some((x) => (
+      methods.includes(x.method)
+      && services.includes(x.service)
+    ))) {
+      data.resources.push(key);
+    }
+  });
+  return data;
+};
+
 export const getAccessibleResourcesFromArboristasync = async (jwt) => {
   let data;
   if (config.internalLocalTest) {
@@ -16,7 +33,7 @@ export const getAccessibleResourcesFromArboristasync = async (jwt) => {
       ],
     };
   } else {
-    data = await arboristClient.listAuthorizedResources(jwt);
+    data = await arboristClient.listAuthMapping(jwt);
   }
 
   log.debug('[authMiddleware] list resources: ', JSON.stringify(data, null, 4));
@@ -27,8 +44,35 @@ export const getAccessibleResourcesFromArboristasync = async (jwt) => {
     }
     throw new CodedError(data.error.code, data.error.message);
   }
-  const resources = data.resources ? _.uniq(data.resources) : [];
-  return resources;
+
+  const read = resourcePathsWithServiceMethodCombination(data, ['guppy', '*'], ['read', '*']);
+  const readResources = read.resources ? _.uniq(read.resources) : [];
+  return [readResources, data];
+};
+
+export const checkIfUserCanRefreshServer = async (passedData) => {
+  let data = passedData;
+  if (config.internalLocalTest) {
+    data = {
+      resources: [ // these are just for testing
+        '/programs/DEV/projects/test',
+        '/programs/jnkns/projects/jenkins',
+      ],
+    };
+  }
+
+  log.debug('[authMiddleware] list resources: ', JSON.stringify(data, null, 4));
+  if (data && data.error) {
+    // if user is not in arborist db, assume has no access to any
+    if (data.error.code === 404) {
+      return false;
+    }
+    throw new CodedError(data.error.code, data.error.message);
+  }
+  const adminAccess = resourcePathsWithServiceMethodCombination(data, ['guppy'], ['admin_access', '*']);
+
+  // Only guppy_admin resource path can control guppy admin access
+  return adminAccess.resources ? adminAccess.resources.includes('/guppy_admin') : false;
 };
 
 export const getRequestResourceListFromFilter = async (