feat(cogify): Update cogify to support the topo raster processes. BM-…

…1116 (#3388)

### Motivation

As a Basemaps user, I want to consume the NZTopo 50 & 250 Maps [as a
tile service].

### Modifications

This work comprises a series of additions and modifications to the
`basemaps/cogify` package.

These changes allow us to generate [STAC] files for an NZTopo Map Series
imagery collection. They also allow us to generate standardised and
cloud-optimised GeoTIFFs from the map sheet files of such collections.
We have adjusted the `basemaps/cogify` package to handle the following
collections stored in the AWS S3 [TopoReleaseArchive] directory:

```
s3://topographic-upload/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/
s3://topographic-upload/TopoReleaseArchive/NZTopo250_GeoTif_Gridless/
``` 

There is also a [new Argo workflow in development][pr-argo-workflow]
that depends on this work as part of its process flow. The workflow's
purpose is to automate the standardisation of an NZTopo Map Series
imagery collection.

### Generating STAC files

This works defines a new process for generating a nested directory
structure of STAC files for an NZTopo Map Series imagery collection. The
process operates as follows:

1. Loads the collection of GeoTiff images into memory
6. Extracts the map code, version, and metadata (e.g. EPSG) from each
image
7. Identifies the latest version of each map sheet by code
8. Groups the images by EPSG
9. Generates a StacItem file for each image, and a StacCollection file
for each EPSG grouping
10. Structures the groups of StacItem and StacCollection files into a
directory tree
11. Saves the files as they are structured to a target location

#### Outputs

The command groups the images by EPSG and then structures the generated
StacItem and StacCollection files, as illustrated:

| Schema | Example |
| - | - |
| ![][schema] | ![][example] |

The command then saves the generated tree of folders and files into the
target location directory.

### Processing Map Sheet files

This work extends the `create` CLI command of the `basemaps/cogify`
package. This extension allows the command to recognise and process the
map sheet files of an NZTopo Map Series imagery collection by way of
particular properties added to the accompanying STAC files.

[as a tile service]:
#3365 (comment)
[STAC]: https://stacspec.org/en
[TopoReleaseArchive]:
https://linz-topographic-upload.s3.ap-southeast-2.amazonaws.com/topographic/TopoReleaseArchive/

[pr-argo-workflow]: linz/topo-workflows#932

[schema]:
https://github.com/user-attachments/assets/2f4eddd7-eb2c-4796-8ab7-b3d9eca9140f
[example]:
https://github.com/user-attachments/assets/8f111ba6-befd-44b5-9566-db07a07089ed

---------

Co-authored-by: Blayne Chard <bchard@linz.govt.nz>
Co-authored-by: Tawera Manaena <TManaena@linz.govt.nz>

packages/cogify/src/cogify/__test__/covering.test.ts

@@ -4,7 +4,7 @@ import { describe, it } from 'node:test';
 import { GoogleTms, QuadKey } from '@basemaps/geo';
 
 import { gsdToMeter } from '../cli/cli.cover.js';
-import { addChildren, addSurrounding } from '../covering.js';
+import { addChildren, addSurrounding } from '../covering/covering.js';
 
 describe('getChildren', () => {
   it('should get children', () => {

packages/cogify/src/cogify/__test__/extract.test.ts

@@ -0,0 +1,37 @@
+import { strictEqual, throws } from 'node:assert';
+import { describe, it } from 'node:test';
+
+import { extractMapCodeAndVersion } from '../topo/extract.js';
+
+describe('extractMapCodeAndVersion', () => {
+  const FakeDomain = 's3://topographic/fake-domain';
+  const validFiles = [
+    { input: `${FakeDomain}/MB07_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_GRIDLESS_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_TIFFv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_TIFF_600v1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    {
+      input: `${FakeDomain}/AX32ptsAX31AY31AY32_GeoTifv1-00.tif`,
+      expected: { mapCode: 'AX32ptsAX31AY31AY32', version: 'v1-00' },
+    },
+    {
+      input: `${FakeDomain}/AZ36ptsAZ35BA35BA36_GeoTifv1-00.tif`,
+      expected: { mapCode: 'AZ36ptsAZ35BA35BA36', version: 'v1-00' },
+    },
+  ];
+  const invalidFiles = [`${FakeDomain}/MB07_GeoTif1-00.tif`, `${FakeDomain}/MB07_TIFF_600v1.tif`];
+
+  it('should parse the correct MapSheet Names', () => {
+    for (const file of validFiles) {
+      const output = extractMapCodeAndVersion(new URL(file.input));
+      strictEqual(output.mapCode, file.expected.mapCode, 'Map code does not match');
+      strictEqual(output.version, file.expected.version, 'Version does not match');
+    }
+  });
+
+  it('should not able to parse a version from file', () => {
+    for (const file of invalidFiles) {
+      throws(() => extractMapCodeAndVersion(new URL(file)), new Error(`Version not found in the file name: "${file}"`));
+    }
+  });
+});

packages/cogify/src/cogify/cli.ts

@@ -3,6 +3,7 @@ import { subcommands } from 'cmd-ts';
 import { BasemapsCogifyCreateCommand } from './cli/cli.cog.js';
 import { BasemapsCogifyConfigCommand } from './cli/cli.config.js';
 import { BasemapsCogifyCoverCommand } from './cli/cli.cover.js';
+import { TopoStacCreationCommand } from './cli/cli.topo.js';
 import { BasemapsCogifyValidateCommand } from './cli/cli.validate.js';
 
 export const CogifyCli = subcommands({
@@ -12,5 +13,6 @@ export const CogifyCli = subcommands({
     create: BasemapsCogifyCreateCommand,
     config: BasemapsCogifyConfigCommand,
     validate: BasemapsCogifyValidateCommand,
+    topo: TopoStacCreationCommand,
   },
 });

packages/cogify/src/cogify/cli/__test__/cli.topo.test.ts

@@ -0,0 +1,69 @@
+import assert from 'node:assert';
+import { beforeEach, describe, it } from 'node:test';
+
+import { fsa, FsMemory, LogConfig } from '@basemaps/shared';
+import { TestTiff } from '@basemaps/test';
+import { StacCollection } from 'stac-ts';
+
+import { TopoStacCreationCommand } from '../cli.topo.js';
+
+describe('cli.topo', () => {
+  const fsMemory = new FsMemory();
+
+  beforeEach(async () => {
+    LogConfig.get().level = 'silent';
+    fsa.register('memory://', fsMemory);
+    fsMemory.files.clear();
+
+    await fsa.write(new URL('memory://source/CJ10_GRIDLESS_GeoTifv1-00.tif'), fsa.readStream(TestTiff.Nztm2000));
+    await fsa.write(new URL('memory://source/CJ10_GRIDLESS_GeoTifv1-01.tif'), fsa.readStream(TestTiff.Nztm2000));
+  });
+
+  const baseArgs = {
+    paths: [new URL('memory://source/')],
+    target: new URL('memory://target/'),
+    mapSeries: 'topo50',
+    latestOnly: false,
+    title: undefined,
+    output: undefined,
+
+    // extra logging arguments
+    verbose: false,
+    extraVerbose: false,
+  };
+
+  it('should generate a covering', async () => {
+    const ret = await TopoStacCreationCommand.handler({ ...baseArgs }).catch((e) => String(e));
+    assert.equal(ret, undefined); // no errors returned
+
+    const files = [...fsMemory.files.keys()];
+    files.sort();
+
+    assert.deepEqual(files, [
+      'memory://source/CJ10_GRIDLESS_GeoTifv1-00.tif',
+      'memory://source/CJ10_GRIDLESS_GeoTifv1-01.tif',
+      'memory://target/topo50/gridless_600dpi/2193/CJ10_v1-00.json',
+      'memory://target/topo50/gridless_600dpi/2193/CJ10_v1-01.json',
+      'memory://target/topo50/gridless_600dpi/2193/collection.json',
+      'memory://target/topo50_latest/gridless_600dpi/2193/CJ10.json',
+      'memory://target/topo50_latest/gridless_600dpi/2193/collection.json',
+    ]);
+
+    const collectionJson = await fsa.readJson<StacCollection>(
+      new URL('memory://target/topo50/gridless_600dpi/2193/collection.json'),
+    );
+    assert.equal(collectionJson['description'], 'Topographic maps of New Zealand');
+    assert.equal(collectionJson['linz:slug'], 'topo50-new-zealand-mainland');
+    assert.equal(collectionJson['linz:region'], 'new-zealand');
+
+    const latestItemUrl = new URL('memory://target/topo50_latest/gridless_600dpi/2193/CJ10.json');
+    const latestVersion = await fsa.readJson<StacCollection>(latestItemUrl);
+
+    // Latest file should be derived_from the source file
+    const derived = latestVersion.links.filter((f) => f.rel === 'derived_from');
+    assert.equal(derived.length, 1);
+
+    const derivedFile = new URL(derived[0].href, latestItemUrl);
+    assert.equal(derivedFile.href, 'memory://target/topo50/gridless_600dpi/2193/CJ10_v1-01.json');
+  });
+});

packages/cogify/src/cogify/cli/cli.cog.ts

@@ -11,18 +11,20 @@ import path from 'path';
 import { StacAsset, StacCollection } from 'stac-ts';
 import { pathToFileURL } from 'url';
 
-import { CutlineOptimizer } from '../../cutline.js';
 import { SourceDownloader } from '../../download.js';
 import { HashTransform } from '../../hash.stream.js';
 import { getLogger, logArguments } from '../../log.js';
-import { gdalBuildCog, gdalBuildVrt, gdalBuildVrtWarp, gdalCreate } from '../gdal.command.js';
-import { GdalRunner } from '../gdal.runner.js';
+import { CutlineOptimizer } from '../covering/cutline.js';
+import {
+  gdalBuildCog,
+  gdalBuildTopoRasterCommands,
+  gdalBuildVrt,
+  gdalBuildVrtWarp,
+  gdalCreate,
+} from '../gdal/gdal.command.js';
+import { GdalRunner } from '../gdal/gdal.runner.js';
 import { Url, UrlArrayJsonFile } from '../parsers.js';
-import { CogifyCreationOptions, CogifyStacItem, getCutline, getSources } from '../stac.js';
-
-function extractSourceFiles(item: CogifyStacItem, baseUrl: URL): URL[] {
-  return item.links.filter((link) => link.rel === 'linz_basemaps:source').map((link) => new URL(link.href, baseUrl));
-}
+import { CogifyCreationOptions, CogifyStacItem, getCutline, getSources, isTopoStacItem } from '../stac.js';
 
 const Collections = new Map<string, Promise<StacCollection>>();
 
@@ -153,16 +155,17 @@ export const BasemapsCogifyCreateCommand = command({
 
       const promises = filtered.map(async (f) => {
         const { item, url } = f;
+
         const cutlineLink = getCutline(item.links);
         const options = item.properties['linz_basemaps:options'];
-        const tileId = TileId.fromTile(options.tile);
+        const tileId = isTopoStacItem(item) ? item.id : TileId.fromTile(options.tile);
 
         // Location to where the tiff should be stored
         const tiffPath = new URL(tileId + '.tiff', url);
         const itemStacPath = new URL(tileId + '.json', url);
         const tileMatrix = TileMatrixSets.find(options.tileMatrix);
         if (tileMatrix == null) throw new Error('Failed to find tileMatrix: ' + options.tileMatrix);
-        const sourceFiles = extractSourceFiles(item, url);
+        const sourceFiles = getSources(item.links);
 
         // Skip creating the COG if the item STAC contains no source tiffs
         if (sourceFiles.length === 0) {
@@ -174,10 +177,31 @@ export const BasemapsCogifyCreateCommand = command({
         const outputTiffPath = await Q(async () => {
           metrics.start(tileId); // Only start the timer when the cog is actually being processed
 
+          // Download all tiff files needed for the processing
+          const sourceLocations = await Promise.all(
+            sourceFiles.map((link) => sources.get(new URL(link.href, url), logger)),
+          );
+
           const cutline = await CutlineOptimizer.loadFromLink(cutlineLink, tileMatrix);
-          const sourceLocations = await Promise.all(sourceFiles.map((f) => sources.get(f, logger)));
+          if (isTopoStacItem(item)) {
+            if (sourceFiles.length !== 1) {
+              throw new Error('Topo MapSheet procesing is limited to one input file, found: ' + sourceLocations.length);
+            }
+            const width = sourceFiles[0]['linz_basemaps:source_width'];
+            const height = sourceFiles[0]['linz_basemaps:source_height'];
+            return createTopoCog({
+              tileId,
+              options,
+              tempFolder: tmpFolder,
+              sourceFiles: sourceLocations,
+              cutline,
+              size: { width, height },
+              logger,
+            });
+          }
 
           return createCog({
+            tileId,
             options,
             tempFolder: tmpFolder,
             sourceFiles: sourceLocations,
@@ -190,8 +214,9 @@ export const BasemapsCogifyCreateCommand = command({
         logger.debug({ files: sourceFiles.length }, 'Cog:Cleanup');
         const deleted = await Promise.all(
           sourceFiles.map(async (f) => {
-            const asset = sources.items.get(f.href);
-            await sources.done(f, item.id, logger);
+            const sourceLocation = new URL(f.href, url);
+            const asset = sources.items.get(sourceLocation.href);
+            await sources.done(sourceLocation, item.id, logger);
             // Update the STAC Document with the checksum and file size of the files used to create this asset
             if (asset == null || asset.size == null || asset.hash == null) return;
             const link = item.links.find((link) => new URL(link.href, url).href === asset.url.href);
@@ -268,14 +293,18 @@ export const BasemapsCogifyCreateCommand = command({
       {
         count: toCreate.length,
         created: filtered.length,
-        files: filtered.map((f) => TileId.fromTile(f.item.properties['linz_basemaps:options'].tile)),
+        files: filtered.map((f) => {
+          return isTopoStacItem(f.item) ? f.item.id : TileId.fromTile(f.item.properties['linz_basemaps:options'].tile);
+        }),
       },
       'Cog:Done',
     );
   },
 });
 
 export interface CogCreationContext {
+  /** TileId for the file name */
+  tileId: string;
   /** COG Creation options */
   options: CogifyCreationOptions;
   /** Location to store all the temporary files */
@@ -284,15 +313,17 @@ export interface CogCreationContext {
   sourceFiles: URL[];
   /** Optional cutline to cut the imagery too */
   cutline: CutlineOptimizer;
+  /** Optional Source imagery size for topo raster trim pixel */
+  size?: { width: number; height: number };
   /** Optional logger */
   logger?: LogType;
 }
 
-/** Create a cog from the creation options */
+/** Create a generic COG from the creation options */
 async function createCog(ctx: CogCreationContext): Promise<URL> {
   const options = ctx.options;
   await ProjectionLoader.load(options.sourceEpsg);
-  const tileId = TileId.fromTile(options.tile);
+  const tileId = ctx.tileId;
 
   const logger = ctx.logger?.child({ tileId });
 
@@ -309,7 +340,7 @@ async function createCog(ctx: CogCreationContext): Promise<URL> {
   logger?.debug({ tileId }, 'Cog:Create:VrtWarp');
 
   const cutlineProperties: { url: URL | null; blend: number } = { url: null, blend: ctx.cutline.blend };
-  if (ctx.cutline.path) {
+  if (ctx.cutline) {
     logger?.debug('Cog:Cutline');
     const optimizedCutline = ctx.cutline.optimize(options.tile);
     if (optimizedCutline) {
@@ -355,6 +386,32 @@ async function createCog(ctx: CogCreationContext): Promise<URL> {
   return cogCreateCommand.output;
 }
 
+/** Create a COG specific to LINZ's Topographic 50k and 250k map series from the creation options */
+async function createTopoCog(ctx: CogCreationContext): Promise<URL> {
+  const options = ctx.options;
+  await ProjectionLoader.load(options.sourceEpsg);
+  const tileId = ctx.tileId;
+
+  const logger = ctx.logger?.child({ tileId });
+
+  logger?.debug({ tileId }, 'TopoCog:Create:VrtSource');
+  // Create the vrt of all the source files
+  const vrtSourceCommand = gdalBuildVrt(new URL(`${tileId}-source.vrt`, ctx.tempFolder), ctx.sourceFiles, true);
+  await new GdalRunner(vrtSourceCommand).run(logger);
+
+  // Create the COG from the vrt file
+  if (ctx.size == null) throw new Error('TopoCog: Source image size is required for pixel trim');
+  const cogCreateCommand = gdalBuildTopoRasterCommands(
+    new URL(`${tileId}.tiff`, ctx.tempFolder),
+    vrtSourceCommand.output,
+    options,
+    ctx.size?.width,
+    ctx.size?.height,
+  );
+  await new GdalRunner(cogCreateCommand).run(logger);
+  return cogCreateCommand.output;
+}
+
 /**
  * Very basic checking for the output tiff to ensure it was uploaded ok
  * Just open it as a COG and ensure the metadata looks about  right

packages/cogify/src/cogify/cli/cli.cover.ts

@@ -7,10 +7,10 @@ import { Metrics } from '@linzjs/metrics';
 import { command, flag, number, oneOf, option, optional, restPositionals, string } from 'cmd-ts';
 
 import { isArgo } from '../../argo.js';
-import { CutlineOptimizer } from '../../cutline.js';
 import { getLogger, logArguments } from '../../log.js';
 import { Presets } from '../../preset.js';
-import { createTileCover, TileCoverContext } from '../../tile.cover.js';
+import { CutlineOptimizer } from '../covering/cutline.js';
+import { createTileCover, TileCoverContext } from '../covering/tile.cover.js';
 import { RgbaType, Url, UrlFolder } from '../parsers.js';
 import { createFileStats } from '../stac.js';
 
@@ -84,6 +84,9 @@ export const BasemapsCogifyCoverCommand = command({
       throw new Error(`No collection.json found with imagery: ${im.url.href}`);
     }
 
+    const slug = im.collection?.['linz:slug'];
+    if (slug != null) im.name = slug as string;
+
     const tms = SupportedTileMatrix.find((f) => f.identifier.toLowerCase() === args.tileMatrix.toLowerCase());
     if (tms == null) throw new Error('--tile-matrix: ' + args.tileMatrix + ' not found');
 
@@ -144,11 +147,13 @@ export const BasemapsCogifyCoverCommand = command({
     const items = [];
     const tilesByZoom: number[] = [];
     for (const item of res.items) {
-      const tileId = TileId.fromTile(item.properties['linz_basemaps:options'].tile);
+      const tile = item.properties['linz_basemaps:options'].tile;
+      if (tile == null) throw new Error('Tile not found in item');
+      const tileId = TileId.fromTile(tile);
       const itemPath = new URL(`${tileId}.json`, targetPath);
       items.push({ path: itemPath });
       await fsa.write(itemPath, JSON.stringify(item, null, 2));
-      const z = item.properties['linz_basemaps:options'].tile.z;
+      const z = tile.z;
       tilesByZoom[z] = (tilesByZoom[z] ?? 0) + 1;
       ctx.logger?.trace({ path: itemPath }, 'Imagery:Stac:Item:Write');
     }