Merge pull request #5930 from rouault/fix_751

/vsistdin/: make size of buffered area configurable (fixes #751)

autotest/gcore/vsistdin.py

@@ -32,6 +32,7 @@
 
 
 from osgeo import gdal
+import gdaltest
 import test_cli_utilities
 import pytest
 
@@ -128,5 +129,76 @@ def test_vsistdin_4():
     gdal.Unlink("tmp/vsistdin_4_out.tif")
 
 
+###############################################################################
+
 
+def test_vsistdin_5():
+
+    f = open('tmp/test_vsistdin_5.bin', 'wb')
+    f.write(b'0123456789' * (1024 * 1024))
+    f.close()
 
+    with gdaltest.config_options({'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin',
+                                  'CPL_VSISTDIN_RESET_POSITION': 'YES',
+                                  'CPL_VSISTDIN_FILE_CLOSE': 'YES'}):
+        f = gdal.VSIFOpenL('/vsistdin?buffer_limit=10MB', 'rb')
+        assert f is not None
+        assert gdal.VSIFEofL(f) == 0
+        assert gdal.VSIFReadL(10, 1, f) == b'0123456789'
+        assert gdal.VSIFTellL(f) == 10
+        assert gdal.VSIFSeekL(f, 5, 0) == 0
+        assert gdal.VSIFTellL(f) == 5
+        assert gdal.VSIFReadL(3, 1, f) == b'567'
+        assert gdal.VSIFTellL(f) == 8
+        assert gdal.VSIFReadL(4, 1, f) == b'8901'
+        assert gdal.VSIFTellL(f) == 12
+        assert gdal.VSIFSeekL(f, 0, 2) == 0
+        assert gdal.VSIFTellL(f) == 10*1024 * 1024
+        assert gdal.VSIFReadL(1, 1, f) == b''
+        assert gdal.VSIFEofL(f) == 1
+        assert gdal.VSIFTellL(f) == 10*1024 * 1024
+        assert gdal.VSIFSeekL(f, 5, 0) == 0
+        assert gdal.VSIFTellL(f) == 5
+        assert gdal.VSIFReadL(3, 1, f) == b'567'
+        assert gdal.VSIFTellL(f) == 8
+        assert gdal.VSIFSeekL(f, 10*1024 * 1024 - 10 + 1, 0) == 0
+        assert gdal.VSIFReadL(3, 1, f) == b'123'
+        gdal.VSIFCloseL(f)
+
+    with gdaltest.config_options({'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin',
+                                  'CPL_VSISTDIN_RESET_POSITION': 'YES',
+                                  'CPL_VSISTDIN_FILE_CLOSE': 'YES'}):
+        f = gdal.VSIFOpenL('/vsistdin?buffer_limit=-1', 'rb')
+        assert f is not None
+        assert gdal.VSIFSeekL(f, 0, 2) == 0
+        assert gdal.VSIFTellL(f) == 10*1024 * 1024
+        gdal.VSIFCloseL(f)
+
+        assert gdal.VSIStatL('/vsistdin?buffer_limit=-1').size == 10*1024 * 1024
+
+    with gdaltest.config_options({'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin',
+                                  'CPL_VSISTDIN_RESET_POSITION': 'YES',
+                                  'CPL_VSISTDIN_FILE_CLOSE': 'YES'}):
+        f = gdal.VSIFOpenL('/vsistdin?buffer_limit=1GB', 'rb')
+        assert f is not None
+        assert gdal.VSIFSeekL(f, 0, 2) == 0
+        assert gdal.VSIFTellL(f) == 10*1024 * 1024
+        gdal.VSIFCloseL(f)
+
+        assert gdal.VSIStatL('/vsistdin?buffer_limit=-1').size == 10*1024 * 1024
+
+    with gdaltest.config_options({'CPL_VSISTDIN_FILE': 'tmp/test_vsistdin_5.bin',
+                                  'CPL_VSISTDIN_RESET_POSITION': 'YES',
+                                  'CPL_VSISTDIN_FILE_CLOSE': 'YES'}):
+        f = gdal.VSIFOpenL('/vsistdin?buffer_limit=10', 'rb')
+        assert f is not None
+        assert gdal.VSIFSeekL(f, 0, 2) == 0
+        assert gdal.VSIFTellL(f) == 10*1024 * 1024
+        assert gdal.VSIFSeekL(f, 0, 0) == 0
+        assert gdal.VSIFReadL(5, 1, f) == b'01234'
+        assert gdal.VSIFReadL(3, 1, f) == b'567'
+        with gdaltest.error_handler():
+            assert gdal.VSIFReadL(3, 1, f) == b''
+        gdal.VSIFCloseL(f)
+
+    os.unlink('tmp/test_vsistdin_5.bin')

doc/source/user/virtual_file_systems.rst

@@ -581,7 +581,23 @@ This file system handler also allows sequential writing of files (no seeks or re
 
 The filename syntax must be only :file:`/vsistdin/`.
 
-The file operations available are of course limited to Read() and forward Seek(). Full seek in the first MB of a file is possible, and it is cached so that closing, re-opening :file:`/vsistdin/` and reading within this first megabyte is possible multiple times in the same process.
+The file operations available are of course limited to Read() and forward Seek().
+Full seek in the first MB of a file is possible, and it is cached so that closing,
+re-opening :file:`/vsistdin/` and reading within this first megabyte is possible
+multiple times in the same process.
+
+Starting with GDAL 3.6, the size of the in-memory cache can be controlled with
+the :decl_configoption:`CPL_VSISTDIN_BUFFER_LIMIT` configuration option, by
+specifying a size in bytes (or using a MB or GB suffix, e.g. "1GB"), or -1 for unlimited.
+The "/vsistdin?buffer_limit=value" syntax can also be used.
+
+/vsistdin filenames can be combined with other file system. For example, to
+read a file within a potentially big ZIP file streamed to gdal_translate:
+
+::
+
+    cat file.tif.zip | gdal_translate /vsizip/{/vsistdin?buffer_limit=-1}/path/to/some.tif out.tif
+
 
 .. _vsistdout:
 

port/cpl_conv.cpp

@@ -3289,7 +3289,7 @@ int CPLCheckForFile( char *pszFilename, char **papszSiblingFiles )
     {
         VSIStatBufL sStatBuf;
 
-        return VSIStatL(pszFilename, &sStatBuf) == 0;
+        return VSIStatExL(pszFilename, &sStatBuf, VSI_STAT_EXISTS_FLAG) == 0;
     }
 
 /* -------------------------------------------------------------------- */