Feat/marxan 1452 set up volume for cloning files storage

api/api.Dockerfile

@@ -19,6 +19,9 @@ COPY --chown=$USER:$USER nodemon.json tsconfig.json tsconfig.build.json nest-cli
 COPY --chown=$USER:$USER apps ./apps
 COPY --chown=$USER:$USER libs ./libs
 
+RUN mkdir -p /opt/marxan-project-cloning
+RUN chown -R $USER:$USER /opt/marxan-project-cloning
+
 RUN yarn prestart:prod
 
 EXPOSE 3000

api/apps/api/config/custom-environment-variables.json

@@ -53,6 +53,9 @@
     "sharedFileStorage": {
       "localPath": "API_SHARED_FILE_STORAGE_LOCAL_PATH",
       "cleanupTemporaryFolders": "BACKEND_CLEANUP_TEMPORARY_FOLDERS"
+    },
+    "cloningFileStorage": {
+      "localPath": "CLONING_FILE_STORAGE_LOCAL_PATH"
     }
   },
   "marxan": {

api/apps/api/config/default.json

@@ -4,17 +4,15 @@
   },
   "network": {
     "cors": {
-      "origins": [
-        "http://localhost:3010"
-      ]
+      "origins": ["http://localhost:3010"]
     }
   },
   "redis": {
     "host": "marxan-redis",
     "port": 6379,
     "password": null,
     "useTLS": false,
-    "concurrency" : 50
+    "concurrency": 50
   },
   "postgresApi": {
     "url": null,
@@ -60,6 +58,9 @@
   "storage": {
     "sharedFileStorage": {
       "localPath": "/tmp/storage"
+    },
+    "cloningFileStorage": {
+      "localPath": "/opt/marxan-project-cloning"
     }
   },
   "marxan": {
@@ -77,8 +78,7 @@
     }
   },
   "fileUploads": {
-    "limits": {
-    }
+    "limits": {}
   },
   "sparkpost": {
     "apikey": "invalidSparkpostApikey",
@@ -95,4 +95,3 @@
     "tokenPrefix": "/auth/sign-up-confirmation?token="
   }
 }
-

api/apps/api/src/app.module.ts

@@ -34,7 +34,7 @@ import { ScenarioSpecificationModule } from './modules/scenario-specification';
 import { PublishedProjectModule } from '@marxan-api/modules/published-project/published-project.module';
 import { BlmValuesModule } from '@marxan-api/modules/blm';
 import { AccessControlModule } from '@marxan-api/modules/access-control';
-import { FileRepositoryModule } from '@marxan/files-repository';
+import { CloningFileSRepositoryModule } from '@marxan/cloning-files-repository';
 import { CloneModule } from './modules/clone';
 
 @Module({
@@ -69,7 +69,7 @@ import { CloneModule } from './modules/clone';
     PublishedProjectModule,
     BlmValuesModule,
     AccessControlModule,
-    FileRepositoryModule,
+    CloningFileSRepositoryModule,
     CloneModule,
   ],
   controllers: [AppController, PingController],

api/apps/api/src/modules/clone/export/adapters/export-adapters.module.ts

@@ -1,4 +1,4 @@
-import { FileRepositoryModule } from '@marxan/files-repository';
+import { CloningFileSRepositoryModule } from '@marxan/cloning-files-repository';
 import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { Project } from '../../../projects/project.api.entity';
@@ -14,7 +14,7 @@ import { TypeormExportRepository } from './typeorm-export.repository';
 
 @Module({
   imports: [
-    FileRepositoryModule,
+    CloningFileSRepositoryModule,
     TypeOrmModule.forFeature([
       Project,
       ExportEntity,

api/apps/api/src/modules/clone/export/adapters/node-archive-creator.ts

@@ -4,7 +4,7 @@ import * as archiver from 'archiver';
 import { PassThrough } from 'stream';
 
 import { ArchiveLocation } from '@marxan/cloning/domain';
-import { FileRepository } from '@marxan/files-repository';
+import { CloningFilesRepository } from '@marxan/cloning-files-repository';
 
 import {
   ArchiveCreationError,
@@ -19,14 +19,14 @@ import {
 export class NodeArchiveCreator extends ArchiveCreator {
   private readonly logger = new Logger(this.constructor.name);
 
-  constructor(private readonly fileRepository: FileRepository) {
+  constructor(private readonly fileRepository: CloningFilesRepository) {
     super();
   }
 
   async zip(
     files: { uri: string; relativeDestination: string }[],
   ): Promise<Either<ArchiveCreationError, ArchiveLocation>> {
-    let archivePersistencePromise: ReturnType<FileRepository['save']>;
+    let archivePersistencePromise: ReturnType<CloningFilesRepository['save']>;
     const passThrough = new PassThrough();
     const onPersistenceFinished = async (
       resolvePromise: (

api/apps/api/src/modules/clone/export/application/get-archive.handler.ts

@@ -1,4 +1,4 @@
-import { FileRepository } from '@marxan/files-repository';
+import { CloningFilesRepository } from '@marxan/cloning-files-repository';
 import { IInferredQueryHandler, QueryHandler } from '@nestjs/cqrs';
 import { Either, left } from 'fp-ts/Either';
 import { Readable } from 'stream';
@@ -14,7 +14,7 @@ export class GetArchiveHandler
   implements IInferredQueryHandler<GetExportArchive> {
   constructor(
     private readonly exportRepo: ExportRepository,
-    private readonly fileRepo: FileRepository,
+    private readonly fileRepo: CloningFilesRepository,
   ) {}
 
   async execute({

api/apps/api/src/modules/clone/export/application/get-archive.query.ts

@@ -1,4 +1,4 @@
-import { GetFileError } from '@marxan/files-repository';
+import { GetFileError } from '@marxan/cloning-files-repository';
 import { Query } from '@nestjs-architects/typed-cqrs';
 import { Either } from 'fp-ts/Either';
 import { Readable } from 'stream';

api/apps/api/src/modules/clone/export/export.module.ts

@@ -1,11 +1,11 @@
-import { FileRepositoryModule } from '@marxan/files-repository';
+import { CloningFileSRepositoryModule } from '@marxan/cloning-files-repository';
 import { Module } from '@nestjs/common';
 import { ExportAdaptersModule } from './adapters/export-adapters.module';
 import { ExportApplicationModule } from './application/export-application.module';
 
 const ExportApplication = ExportApplicationModule.for([
   ExportAdaptersModule,
-  FileRepositoryModule,
+  CloningFileSRepositoryModule,
 ]);
 
 @Module({

api/apps/api/src/modules/clone/import/adapters/import-adapters.module.ts

@@ -1,4 +1,4 @@
-import { FileRepositoryModule } from '@marxan/files-repository';
+import { CloningFileSRepositoryModule } from '@marxan/cloning-files-repository';
 import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { ImportResourcePieces } from '../application/import-resource-pieces.port';
@@ -16,7 +16,7 @@ import { TypeormImportRepository } from './typeorm-import.repository.adapter';
       ImportComponentEntity,
       ImportComponentLocationEntity,
     ]),
-    FileRepositoryModule,
+    CloningFileSRepositoryModule,
   ],
   providers: [
     {

api/apps/api/src/modules/clone/import/application/export-config-reader.spec.ts

@@ -14,7 +14,7 @@ import {
   ExportConfigContent,
   exportVersion,
 } from '@marxan/cloning/infrastructure/clone-piece-data/export-config';
-import { fileNotFound } from '@marxan/files-repository/file.repository';
+import { fileNotFound } from '@marxan/cloning-files-repository';
 import { FixtureType } from '@marxan/utils/tests/fixture-type';
 import { Test } from '@nestjs/testing';
 import * as archiver from 'archiver';

api/apps/api/src/modules/clone/infra/import/import.infra.module.ts

@@ -1,4 +1,4 @@
-import { FileRepositoryModule } from '@marxan/files-repository';
+import { CloningFileSRepositoryModule } from '@marxan/cloning-files-repository';
 import { Logger, Module, Scope } from '@nestjs/common';
 import { CqrsModule } from '@nestjs/cqrs';
 import { ApiEventsModule } from '../../../api-events';
@@ -30,7 +30,7 @@ import { UploadExportFileHandler } from './upload-export-file.handler';
     QueueApiEventsModule,
     CqrsModule,
     ImportAdaptersModule,
-    FileRepositoryModule,
+    CloningFileSRepositoryModule,
   ],
   providers: [
     PieceImportRequestedSaga,

api/apps/api/src/modules/clone/infra/import/upload-export-file.command.ts

@@ -1,5 +1,5 @@
 import { ArchiveLocation } from '@marxan/cloning/domain';
-import { unknownError } from '@marxan/files-repository';
+import { unknownError } from '@marxan/cloning-files-repository';
 import { Command } from '@nestjs-architects/typed-cqrs';
 import { Either } from 'fp-ts/lib/Either';
 

api/apps/api/src/modules/clone/infra/import/upload-export-file.handler.ts

@@ -1,5 +1,8 @@
 import { ArchiveLocation } from '@marxan/cloning/domain';
-import { FileRepository, unknownError } from '@marxan/files-repository';
+import {
+  CloningFilesRepository,
+  unknownError,
+} from '@marxan/cloning-files-repository';
 import { Logger } from '@nestjs/common';
 import { CommandHandler, IInferredCommandHandler } from '@nestjs/cqrs';
 import { Either, isLeft, left, right } from 'fp-ts/lib/Either';
@@ -10,7 +13,7 @@ import { UploadExportFile } from './upload-export-file.command';
 export class UploadExportFileHandler
   implements IInferredCommandHandler<UploadExportFile> {
   constructor(
-    private readonly fileRepository: FileRepository,
+    private readonly fileRepository: CloningFilesRepository,
     private readonly logger: Logger,
   ) {
     this.logger.setContext(UploadExportFileHandler.name);

api/apps/api/src/modules/projects/projects.controller.ts

@@ -104,12 +104,14 @@ import {
 } from './dto/export.project.dto';
 import { ScenarioLockResultPlural } from '@marxan-api/modules/access-control/scenarios-acl/locks/dto/scenario.lock.dto';
 import { RequestProjectImportResponseDto } from './dto/import.project.response.dto';
-import { unknownError as fileRepositoryUnknownError } from '@marxan/files-repository';
+import {
+  unknownError as fileRepositoryUnknownError,
+  fileNotFound,
+} from '@marxan/cloning-files-repository';
 import {
   archiveCorrupted,
   invalidFiles,
 } from '@marxan/cloning/infrastructure/archive-reader.port';
-import { fileNotFound } from '@marxan/files-repository/file.repository';
 import { ProxyService } from '@marxan-api/modules/proxy/proxy.service';
 import { TilesOpenApi } from '@marxan/tiles';
 import { mapAclDomainToHttpError } from '@marxan-api/utils/acl.utils';

api/apps/api/src/modules/projects/projects.service.ts

@@ -52,7 +52,7 @@ import {
   ChangeProjectRangeErrors,
 } from '@marxan-api/modules/projects/blm';
 import { UploadExportFile } from '../clone/infra/import/upload-export-file.command';
-import { unknownError } from '@marxan/files-repository';
+import { unknownError } from '@marxan/cloning-files-repository';
 import {
   ImportProject,
   ImportProjectCommandResult,

api/apps/api/test/files-repository/files-repository.e2e-spec.ts

@@ -5,10 +5,10 @@ import { Test } from '@nestjs/testing';
 
 import { FixtureType } from '@marxan/utils/tests/fixture-type';
 import {
-  FileRepository,
-  FileRepositoryModule,
-  TempStorageRepository,
-} from '@marxan/files-repository';
+  CloningFilesRepository,
+  CloningFileSRepositoryModule,
+  VolumeCloningFilesStorage,
+} from '@marxan/cloning-files-repository';
 
 let fixtures: FixtureType<typeof getFixtures>;
 
@@ -36,12 +36,12 @@ const getFixtures = async () => {
    * to avoid communicating with potential storage provider
    */
   const app = await Test.createTestingModule({
-    imports: [FileRepositoryModule],
+    imports: [CloningFileSRepositoryModule],
   })
-    .overrideProvider(FileRepository)
-    .useClass(TempStorageRepository)
+    .overrideProvider(CloningFilesRepository)
+    .useClass(VolumeCloningFilesStorage)
     .compile();
-  const sut = app.get(FileRepository);
+  const sut = app.get(CloningFilesRepository);
 
   return {
     cleanup: async () => app.close(),

api/apps/api/test/jest-e2e.json

@@ -63,8 +63,8 @@
     "@marxan/projects": "<rootDir>/../../../libs/projects/src",
     "@marxan/blm-calibration/(.*)": "<rootDir>/../../../libs/blm-calibration/src/$1",
     "@marxan/blm-calibration": "<rootDir>/../../../libs/blm-calibration/src",
-    "@marxan/files-repository/(.*)": "<rootDir>/../../../libs/files-repository/src/$1",
-    "@marxan/files-repository": "<rootDir>/../../../libs/files-repository/src",
+    "@marxan/cloning-files-repository/(.*)": "<rootDir>/../../../libs/cloning-files-repository/src/$1",
+    "@marxan/cloning-files-repository": "<rootDir>/../../../libs/cloning-files-repository/src",
     "@marxan/cloning/(.*)": "<rootDir>/../../../libs/cloning/src/$1",
     "@marxan/cloning": "<rootDir>/../../../libs/cloning/src",
     "@marxan/webshot/(.*)": "<rootDir>../../../libs/webshot/src/$1",

api/apps/api/test/project/get-export-file.e2e-spec.ts

@@ -10,7 +10,7 @@ import {
 } from '@marxan-api/modules/clone/export/domain';
 import { PublishedProject } from '@marxan-api/modules/published-project/entities/published-project.api.entity';
 import { ComponentId, ComponentLocation } from '@marxan/cloning/domain';
-import { FileRepository } from '@marxan/files-repository';
+import { CloningFilesRepository } from '@marxan/cloning-files-repository';
 import { FixtureType } from '@marxan/utils/tests/fixture-type';
 import { CommandBus, CqrsModule } from '@nestjs/cqrs';
 import { getRepositoryToken } from '@nestjs/typeorm';
@@ -95,7 +95,7 @@ export const getFixtures = async () => {
   eventBusTestUtils.startInspectingEvents();
   const commandBus = app.get(CommandBus);
   const exportRepo = app.get(ExportRepository);
-  const fileRepo = app.get(FileRepository);
+  const fileRepo = app.get(CloningFilesRepository);
   const userProjectsRepo = app.get<Repository<UsersProjectsApiEntity>>(
     getRepositoryToken(UsersProjectsApiEntity),
   );

api/apps/api/test/project/get-latest-export.e2e-spec.ts

@@ -10,7 +10,7 @@ import {
 } from '@marxan-api/modules/clone/export/domain';
 import { PublishedProject } from '@marxan-api/modules/published-project/entities/published-project.api.entity';
 import { ComponentId, ComponentLocation } from '@marxan/cloning/domain';
-import { FileRepository } from '@marxan/files-repository';
+import { CloningFilesRepository } from '@marxan/cloning-files-repository';
 import { FixtureType } from '@marxan/utils/tests/fixture-type';
 import { CommandBus, CqrsModule } from '@nestjs/cqrs';
 import { getRepositoryToken } from '@nestjs/typeorm';
@@ -100,7 +100,7 @@ export const getFixtures = async () => {
   const eventBusTestUtils = app.get(EventBusTestUtils);
   const commandBus = app.get(CommandBus);
   const exportRepo = app.get(ExportRepository);
-  const fileRepo = app.get(FileRepository);
+  const fileRepo = app.get(CloningFilesRepository);
   const userProjectsRepo = app.get<Repository<UsersProjectsApiEntity>>(
     getRepositoryToken(UsersProjectsApiEntity),
   );

api/apps/api/test/utils/api-application.ts

@@ -2,9 +2,9 @@ import { AppModule } from '@marxan-api/app.module';
 import { ProjectChecker } from '@marxan-api/modules/projects/project-checker/project-checker.service';
 import { QueueBuilder } from '@marxan-api/modules/queue/queue.builder';
 import {
-  FileRepository,
-  TempStorageRepository,
-} from '@marxan/files-repository';
+  CloningFilesRepository,
+  VolumeCloningFilesStorage,
+} from '@marxan/cloning-files-repository';
 import { INestApplication, ValidationPipe } from '@nestjs/common';
 import { ModuleMetadata } from '@nestjs/common/interfaces/modules/module-metadata.interface';
 import { Test, TestingModuleBuilder } from '@nestjs/testing';
@@ -29,7 +29,10 @@ const defaultOverrides: Overrides = {
     { provider: QueueBuilder, implementation: FakeQueueBuilder },
     { provider: ProjectChecker, implementation: ProjectCheckerFake },
     { provider: ScenarioChecker, implementation: ScenarioCheckerFake },
-    { provider: FileRepository, implementation: TempStorageRepository },
+    {
+      provider: CloningFilesRepository,
+      implementation: VolumeCloningFilesStorage,
+    },
     {
       provider: ScenarioCalibrationRepo,
       implementation: FakeScenarioCalibrationRepo,

api/apps/geoprocessing/config/custom-environment-variables.json

@@ -45,6 +45,9 @@
     "sharedFileStorage": {
       "localPath": "API_SHARED_FILE_STORAGE_LOCAL_PATH",
       "cleanupTemporaryFolders": "BACKEND_CLEANUP_TEMPORARY_FOLDERS"
+    },
+    "cloningFileStorage": {
+      "localPath": "CLONING_FILE_STORAGE_LOCAL_PATH"
     }
   },
   "fileUploads": {

api/apps/geoprocessing/config/default.json

@@ -4,7 +4,7 @@
     "host": "marxan-redis",
     "port": 6379,
     "password": null,
-    "concurrency" : 50
+    "concurrency": 50
   },
   "postgresApi": {
     "url": null,
@@ -38,6 +38,9 @@
     "sharedFileStorage": {
       "localPath": "/tmp/storage",
       "cleanupTemporaryFolders": true
+    },
+    "cloningFileStorage": {
+      "localPath": "/opt/marxan-project-cloning"
     }
   },
   "fileUploads": {