GTiff: internal overview building: do not set PHOTOMETRIC=YCBCR when …

…COMPRESS_OVERVIEW != JPEG

Fixes comment #11555 (comment)
but not the core issue itself

autotest/gcore/tiff_ovr.py

@@ -1676,6 +1676,33 @@ def test_tiff_ovr_43(tmp_path, both_endian):
     assert cs == 642, "did not get expected checksum"
 
 
+###############################################################################
+# Test that we do not propagate PHOTOMETRIC=YCBCR on overviews when
+# COMPRESS_OVERVIEW != JPEG
+
+
+@pytest.mark.require_creation_option("GTiff", "JPEG")
+@gdaltest.enable_exceptions()
+def test_tiff_ovr_do_not_propagate_photometric_ycbcr_if_ovr_if_not_jpeg(tmp_path):
+
+    tif_fname = str(tmp_path / "test.tif")
+
+    ds = gdal.GetDriverByName("GTiff").Create(
+        tif_fname, 8, 8, 3, options=["COMPRESS=JPEG", "PHOTOMETRIC=YCBCR"]
+    )
+    ds.GetRasterBand(1).Fill(255)
+    ds.GetRasterBand(2).Fill(255)
+    ds.GetRasterBand(3).Fill(255)
+    ds = None
+
+    with gdal.Open(tif_fname, gdal.GA_Update) as ds:
+        with gdal.config_option("COMPRESS_OVERVIEW", "DEFLATE"):
+            ds.BuildOverviews("NEAR", [2])
+
+    with gdal.Open(tif_fname) as ds:
+        assert ds.GetRasterBand(1).GetOverview(0).ComputeRasterMinMax() == (255, 255)
+
+
 ###############################################################################
 # Test that we can change overview block size through GDAL_TIFF_OVR_BLOCKSIZE configuration
 # option

frmts/gtiff/gtiffdataset_write.cpp

@@ -2709,7 +2709,10 @@ bool GTiffDataset::GetOverviewParameters(
     /* -------------------------------------------------------------------- */
     /*      Determine photometric tag                                       */
     /* -------------------------------------------------------------------- */
-    nPhotometric = m_nPhotometric;
+    if (m_nPhotometric == PHOTOMETRIC_YCBCR && nCompression != COMPRESSION_JPEG)
+        nPhotometric = PHOTOMETRIC_RGB;
+    else
+        nPhotometric = m_nPhotometric;
     const char *pszPhotometric =
         GetOptionValue("PHOTOMETRIC", "PHOTOMETRIC_OVERVIEW", &pszOptionKey);
     if (!GTIFFUpdatePhotometric(pszPhotometric, pszOptionKey, nCompression,