diff --git a/docs/limited_flie_pfb_export.md b/docs/limited_flie_pfb_export.md new file mode 100644 index 0000000000..3fdf185acb --- /dev/null +++ b/docs/limited_flie_pfb_export.md @@ -0,0 +1,116 @@ +Limited File PFB export adds a limited ability to export PFBs of data files from the Files tab. The limitation is that users cannot export data files of different data types (e.g. `Aligned Reads`, `Imaging Files`, etc.). (More specifically, users cannot export data files that are on different nodes in the graph.) We're accepting this limitation in order to avoid a significant rewrite of the pelican-export job, which assumes all entities to export are on the same node. + +- Link to Jira ticket: [PXP-6544](https://ctds-planx.atlassian.net/browse/PXP-6544) +- Link to design doc: https://docs.google.com/document/d/12FkAYOpDuSdQScEgYBxXsPUdm8GUuUZgD6xU7EQga5A/edit#heading=h.53vab1pwrz1y +- Link to relevant Pelican PR: https://github.com/uc-cdis/pelican/pull/33 +- Link to manual test plan: https://github.com/uc-cdis/gen3-qa/pull/454 + +### How to deploy +- Limited File PFB Export requires Pelican >= 0.5.0, Tube >= 0.4.1 +- Limited File PFB Export is enabled by setting `fileExplorerConfig.enableLimitedFilePFBExport: { sourceNodeField: "source_node" }` and by adding buttons of buttonType `export-files` (Export to Terra), `export-files-to-pfb` (Download PFB), or `export-files-to-seven-bridges` (Export to Seven Bridges) to `fileExplorerConfig.buttons`. Example: +``` + "fileExplorerConfig": { + ... + "buttons": [ + .... + { + "enabled": true, + "type": "export-files-to-pfb", + "title": "Export All to PFB", + "rightIcon": "external-link", + "tooltipText": "You have not selected any cases to export. Please use the checkboxes on the left to select specific cases you would like to export." + }, + { + "enabled": true, + "type": "export-files", + "title": "Export All to Terra", + "rightIcon": "external-link", + "tooltipText": "You have not selected any cases to export. Please use the checkboxes on the left to select specific cases you would like to export." + } + ], + "enableLimitedFilePFBExport": {"sourceNodeField": "source_node"}, +``` +- Limited File PFB Export requires the `source_node` property to be ETL'd -- `source_node` should be added to `props` in the `file` section of `etlMapping.yaml`. Example: +``` + - name: mpingram_file + doc_type: file + type: collector + root: None + category: data_file + props: + - name: object_id + - name: md5sum + ... + - name: source_node +``` +- Limited File PFB Export requires a new `export-files` job block to be added to the Sower config in manifest.json. The $ROOT_NODE environment variable must be set to `"file"`, or the name of the Guppy file index if it isn't "file". *For BioDataCatalyst, the EXTRA_NODES environment variable must be set to `""`*. (This is for backwards compatibility: pelican-export includes `reference_file` by default on all BDCat PFB exports unless $EXTRA_NODES is set to an empty string). Example sower config: +``` + { + "name": "pelican-export-files", + "action": "export-files", + "container": { + "name": "job-task", + "image": "quay.io/cdis/pelican-export:0.5.0", + "pull_policy": "Always", + "env": [ + { + "name": "DICTIONARY_URL", + "valueFrom": { + "configMapKeyRef": { + "name": "manifest-global", + "key": "dictionary_url" + } + } + }, + { + "name": "GEN3_HOSTNAME", + "valueFrom": { + "configMapKeyRef": { + "name": "manifest-global", + "key": "hostname" + } + } + }, + { + "name": "ROOT_NODE", + "value": "file" + }, + { + "name": "EXTRA_NODES", + "value": "" + } + ], + "volumeMounts": [ + { + "name": "pelican-creds-volume", + "readOnly": true, + "mountPath": "/pelican-creds.json", + "subPath": "config.json" + }, + { + "name": "peregrine-creds-volume", + "readOnly": true, + "mountPath": "/peregrine-creds.json", + "subPath": "creds.json" + } + ], + "cpu-limit": "1", + "memory-limit": "4Gi" + }, + "volumes": [ + { + "name": "pelican-creds-volume", + "secret": { + "secretName": "pelicanservice-g3auto" + } + }, + { + "name": "peregrine-creds-volume", + "secret": { + "secretName": "peregrine-creds" + } + } + ], + "restart_policy": "Never" + }, +``` diff --git a/src/GuppyDataExplorer/ExplorerButtonGroup/ExplorerButtonGroup.css b/src/GuppyDataExplorer/ExplorerButtonGroup/ExplorerButtonGroup.css index ec4b2de92e..1cac0576d3 100644 --- a/src/GuppyDataExplorer/ExplorerButtonGroup/ExplorerButtonGroup.css +++ b/src/GuppyDataExplorer/ExplorerButtonGroup/ExplorerButtonGroup.css @@ -22,3 +22,8 @@ width: unset; font-size: medium; } + +.explorer-button-group__toaster-dl-link { + color: var(--g3-color__white); + text-decoration: underline; +} diff --git a/src/GuppyDataExplorer/ExplorerButtonGroup/index.jsx b/src/GuppyDataExplorer/ExplorerButtonGroup/index.jsx index c249c80713..5847118218 100644 --- a/src/GuppyDataExplorer/ExplorerButtonGroup/index.jsx +++ b/src/GuppyDataExplorer/ExplorerButtonGroup/index.jsx @@ -32,16 +32,39 @@ class ExplorerButtonGroup extends React.Component { exportPFBURL: '', pfbStartText: 'Your export is in progress.', pfbWarning: 'Please do not navigate away from this page until your export is finished.', - pfbSuccessText: 'Your cohort has been exported to PFB! The URL is displayed below.', + pfbSuccessText: 'Your cohort has been exported to PFB.', + // for export to PFB in Files tab + sourceNodesInCohort: [], // for export to workspace exportingToWorkspace: false, exportWorkspaceFileName: null, exportWorkspaceStatus: null, workspaceSuccessText: 'Your cohort has been saved! In order to view and run analysis on this cohort, please go to the workspace.', }; + + // Display misconfiguration warnings if Export PFB to Terra/SBG buttons are present + // but no URL was configured to send the PFBs to. + const exportToTerraButtonPresent = + props.buttonConfig && props.buttonConfig.buttons && + props.buttonConfig.buttons.some(btn => btn.type === 'export' || btn.type === 'export-files'); + if (exportToTerraButtonPresent && !this.props.buttonConfig.terraExportURL) { + console.error('Misconfiguration error: Export to Terra button is present, but there is no `terraExportURL` specified in the portal config.'); // eslint-disable-line no-console + } + const exportToSevenBridgesButtonPresent = + props.buttonConfig && props.buttonConfig.buttons && + props.buttonConfig.buttons.some(btn => btn.type === 'export-to-seven-bridges' || btn.type === 'export-files-to-seven-bridges'); + if (exportToSevenBridgesButtonPresent && !this.props.buttonConfig.sevenBridgesExportURL) { + console.error('Misconfiguration error: Export to Seven Bridges button is present, but there is no `sevenBridgesExportURL` specified in the portal config.'); // eslint-disable-line no-console + } } componentWillReceiveProps(nextProps) { + if (nextProps.job && nextProps.job.status === 'Failed' && this.props.job.status !== 'Failed') { + this.setState(prevState => ({ + toasterOpen: true, + toasterHeadline: prevState.toasterErrorText, + })); + } if (nextProps.job && nextProps.job.status === 'Completed' && this.props.job.status !== 'Completed') { this.fetchJobResult() .then((res) => { @@ -74,6 +97,14 @@ class ExplorerButtonGroup extends React.Component { && nextProps.totalCount) { this.refreshManifestEntryCount(); } + if (this.props.buttonConfig.enableLimitedFilePFBExport + && nextProps.filter !== this.props.filter) { + const sourceNodeField = this.props.buttonConfig.enableLimitedFilePFBExport.sourceNodeField; + if (!sourceNodeField) { + throw new Error('Limited File PFB Export is enabled, but \'sourceNodeField\' has not been specified. Check the portal config.'); + } + this.refreshSourceNodes(nextProps.filter, sourceNodeField); + } } componentWillUnmount() { @@ -101,12 +132,28 @@ class ExplorerButtonGroup extends React.Component { clickFunc = this.exportToTerra; } } + if (buttonConfig.type === 'export-files') { + // REMOVE THIS CODE WHEN TERRA EXPORT WORKS + // ======================================= + if (terraExportWarning) { + clickFunc = this.exportFilesToTerraWithWarning; + } else { + // ======================================= + clickFunc = this.exportFilesToTerra; + } + } if (buttonConfig.type === 'export-to-seven-bridges') { clickFunc = this.exportToSevenBridges; } + if (buttonConfig.type === 'export-files-to-seven-bridges') { + clickFunc = this.exportFilesToSevenBridges; + } if (buttonConfig.type === 'export-to-pfb') { clickFunc = this.exportToPFB; } + if (buttonConfig.type === 'export-files-to-pfb') { + clickFunc = this.exportFilesToPFB; + } if (buttonConfig.type === 'export-to-workspace') { clickFunc = this.exportToWorkspace; } @@ -115,7 +162,6 @@ class ExplorerButtonGroup extends React.Component { } return clickFunc; }; - getManifest = async (indexType) => { if (!this.props.guppyConfig.manifestMapping || !this.props.guppyConfig.manifestMapping.referenceIdFieldInDataIndex) { @@ -222,7 +268,7 @@ class ExplorerButtonGroup extends React.Component { : null } { (this.state.exportPFBURL) ? -
Most recent PFB URL: { this.state.exportPFBURL }
+ Click here to download your PFB. : null } { (this.state.toasterError) ? @@ -320,6 +366,16 @@ class ExplorerButtonGroup extends React.Component { this.exportToTerra(); } } + exportFilesToTerraWithWarning = () => { + // If the number of subjects is over the threshold, warn the user that their + // export to Terra job might fail. + if (this.props.totalCount >= terraExportWarning.subjectThreshold) { + this.setState({ enableTerraWarningPopup: true }); + } else { + // If the number is below the threshold, proceed as normal + this.exportFilesToTerra(); + } + } // ========================================== exportToTerra = () => { @@ -328,12 +384,24 @@ class ExplorerButtonGroup extends React.Component { }); }; + exportFilesToTerra = () => { + this.setState({ exportingToTerra: true }, () => { + this.exportFilesToPFB(); + }); + }; + exportToSevenBridges = () => { this.setState({ exportingToSevenBridges: true }, () => { this.exportToPFB(); }); } + exportFilesToSevenBridges = () => { + this.setState({ exportingToSevenBridges: true }, () => { + this.exportFilesToPFB(); + }); + } + sendPFBToTerra = () => { const url = encodeURIComponent(this.state.exportPFBURL); let templateParam = ''; @@ -354,10 +422,35 @@ class ExplorerButtonGroup extends React.Component { exportToPFB = () => { this.props.submitJob({ action: 'export', input: { filter: getGQLFilter(this.props.filter) } }); this.props.checkJobStatus(); - this.setState({ + this.setState(prevState => ({ toasterOpen: true, - toasterHeadline: this.state.pfbStartText, - }); + toasterHeadline: prevState.pfbStartText, + })); + }; + + exportFilesToPFB = () => { + if (this.props.buttonConfig.enableLimitedFilePFBExport) { + if (!this.state.sourceNodesInCohort || this.state.sourceNodesInCohort.length !== 1) { + return; + } + const rootNode = this.state.sourceNodesInCohort[0]; + this.props.submitJob({ + action: 'export-files', + input: { + filter: getGQLFilter(this.props.filter), + root_node: rootNode, + }, + }); + this.props.checkJobStatus(); + this.setState({ + toasterOpen: true, + toasterHeadline: this.state.pfbStartText, + }); + } else { + /* eslint-disable no-console */ + console.error(`Error: Missing \`enableLimitedFilePFBExport\` in the portal config. +Currently, in order to export a File PFB, \`enableLimitedFilePFBExport\` must be set in the portal config.`); + } }; exportToWorkspace = async (indexType) => { @@ -396,12 +489,12 @@ class ExplorerButtonGroup extends React.Component { }; exportToWorkspaceErrorHandler = (status) => { - this.setState({ + this.setState(prevState => ({ toasterOpen: true, - toasterHeadline: this.state.toasterErrorText, + toasterHeadline: prevState.toasterErrorText, exportWorkspaceStatus: status, exportingToWorkspace: false, - }); + })); }; exportToWorkspaceMessageHandler = (status, message) => { @@ -462,6 +555,43 @@ class ExplorerButtonGroup extends React.Component { } }; + refreshSourceNodes = async (filter, sourceNodeField) => { + try { + const indexType = this.props.guppyConfig.dataType; + const query = `query ($filter: JSON) { + _aggregation { + ${indexType} (filter: $filter) { + ${sourceNodeField} { + histogram { + key + count + } + } + } + } + }`; + const body = { query, variables: { filter: getGQLFilter(filter) } }; + const res = await fetchWithCreds({ + path: guppyGraphQLUrl, + method: 'POST', + body: JSON.stringify(body), + }); + // eslint-disable-next-line no-underscore-dangle + const sourceNodesHistogram = res.data.data._aggregation[indexType][sourceNodeField].histogram; + const sourceNodes = []; + sourceNodesHistogram.forEach(({ key, count }) => { + if (count > 0) { + sourceNodes.push(key); + } + }); + this.setState({ + sourceNodesInCohort: sourceNodes, + }); + } catch (err) { + throw Error(`Error when getting data types: ${err}`); + } + }; + // check if the user has access to this resource isButtonDisplayed = (buttonConfig) => { if (buttonConfig.type === 'export-to-workspace' || buttonConfig.type === 'export-files-to-workspace') { @@ -479,31 +609,57 @@ class ExplorerButtonGroup extends React.Component { if (buttonConfig.type === 'manifest') { return this.state.manifestEntryCount > 0; } + const pfbJobIsRunning = this.state.exportingToTerra + || this.state.exportingToSevenBridges + || this.isPFBRunning(); if (buttonConfig.type === 'export-to-pfb') { // disable the pfb export button if any other pfb export jobs are running - return !(this.state.exportingToTerra || this.state.exportingToSevenBridges); + return !pfbJobIsRunning; } - if (buttonConfig.type === 'export') { - if (!this.props.buttonConfig.terraExportURL) { - console.error('Export to Terra button is present, but there is no `terraExportURL` specified in the portal config. Disabling the export to Terra button.'); // eslint-disable-line no-console + if (buttonConfig.type === 'export-files-to-pfb') { + // disable the pfb export button if any other pfb export jobs are running + if (pfbJobIsRunning) { return false; } + // If limited file PFB export is enabled, disable the button if the selected + // data files are on more than one source node. (See https://github.com/uc-cdis/data-portal/pull/729) + if (this.props.buttonConfig.enableLimitedFilePFBExport) { + return this.state.sourceNodesInCohort.length === 1; + } + } + if (buttonConfig.type === 'export') { // disable the terra export button if any of the // pfb export operations are running. - return !(this.state.exportingToTerra - || this.state.exportingToSevenBridges - || this.isPFBRunning()); + return !pfbJobIsRunning; } - if (buttonConfig.type === 'export-to-seven-bridges') { - if (!this.props.buttonConfig.sevenBridgesExportURL) { - console.error('Export to Terra button is present, but there is no `terraExportURL` specified in the portal config. Disabling the export to Terra button.'); // eslint-disable-line no-console + if (buttonConfig.type === 'export-files') { + // disable the terra export button if any of the + // pfb export operations are running. + if (pfbJobIsRunning) { return false; } + // If limited file PFB export is enabled, disable the button if the selected + // data files are on more than one source node. (See https://github.com/uc-cdis/data-portal/pull/729) + if (this.props.buttonConfig.enableLimitedFilePFBExport) { + return this.state.sourceNodesInCohort.length === 1; + } + } + if (buttonConfig.type === 'export-to-seven-bridges') { + // disable the seven bridges export buttons if any of the + // pfb export operations are running. + return !pfbJobIsRunning; + } + if (buttonConfig.type === 'export-files-to-seven-bridges') { // disable the seven bridges export buttons if any of the // pfb export operations are running. - return !(this.state.exportingToTerra - || this.state.exportingToSevenBridges - || this.isPFBRunning()); + if (pfbJobIsRunning) { + return false; + } + // If limited file PFB export is enabled, disable the button if the selected + // data files are on more than one source node. (See https://github.com/uc-cdis/data-portal/pull/729) + if (this.props.buttonConfig.enableLimitedFilePFBExport) { + return this.state.sourceNodesInCohort.length === 1; + } } if (buttonConfig.type === 'export-to-workspace') { return this.state.manifestEntryCount > 0; @@ -519,19 +675,19 @@ class ExplorerButtonGroup extends React.Component { if (buttonConfig.type === 'export-to-workspace' || buttonConfig.type === 'export-files-to-workspace') { return this.state.exportingToWorkspace; } - if (buttonConfig.type === 'export-to-pfb') { + if (buttonConfig.type === 'export-to-pfb' || buttonConfig.type === 'export-files-to-pfb') { // export to pfb button is pending if a pfb export job is running and it's // neither an export to terra job or an export to seven bridges job. return this.isPFBRunning() && !(this.state.exportingToTerra || this.state.exportingToSevenBridges); } - if (buttonConfig.type === 'export') { + if (buttonConfig.type === 'export' || buttonConfig.type === 'export-files') { // export to terra button is pending if a pfb export job is running and // it's an exporting to terra job. return this.isPFBRunning() && this.state.exportingToTerra; } - if (buttonConfig.type === 'export-to-seven-bridges') { + if (buttonConfig.type === 'export-to-seven-bridges' || buttonConfig.type === 'export-files-to-seven-bridges') { // export to seven bridges button is pending if a pfb export job is running // and it's an export to seven bridges job. return this.isPFBRunning() @@ -553,7 +709,22 @@ class ExplorerButtonGroup extends React.Component { } else if (buttonConfig.type === 'manifest' && this.state.manifestEntryCount > 0) { buttonTitle = `${buttonConfig.title} (${humanizeNumber(this.state.manifestEntryCount)})`; } - const btnTooltipText = (this.props.isLocked) ? 'You only have access to summary data' : buttonConfig.tooltipText; + + let tooltipEnabled = buttonConfig.tooltipText ? !this.isButtonEnabled(buttonConfig) : false; + let btnTooltipText = (this.props.isLocked) ? 'You only have access to summary data' : buttonConfig.tooltipText; + + // If limited file PFB export is enabled, PFB export buttons will be disabled + // if the user selects multiple files that are on different nodes in the graph. + // (See https://github.com/uc-cdis/data-portal/pull/729). + // If the user has selected multiple files on different nodes, display a + // tooltip explaining that the user can only export files of the same type. + const isFilePFBButton = buttonConfig.type === 'export-files' || buttonConfig.type === 'export-files-to-pfb' || buttonConfig.type === 'export-files-to-seven-bridges'; + if (this.props.buttonConfig.enableLimitedFilePFBExport + && isFilePFBButton + && this.state.sourceNodesInCohort.length > 1) { + tooltipEnabled = true; + btnTooltipText = 'Currently you cannot export files with different Data Types. Please choose a single Data Type from the Data Type filter on the left.'; + } return (