interpreter: materialize Files from generated sources

Instead of keeping them as strings, lets be smarter. This code attempts
to do the following:

 - detect that a string is a path into the build directory
 - match that string to the output of a target or configure_file
 - convert the string to a file
 - give a helpful warning and/or deprecation message
 - error if the string cannot be matched

This allows us to get rid of the strict check to
`source_strings_to_files`, since we only allowed that in one case
(inside BuildTarget), and moves to fully deprecating that now that the
only known case has been removed (vala gir)

Author: dcbaker

mesonbuild/interpreter/interpreter.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2012-2021 The Meson development team
-# Copyright © 2023 Intel Corporation
+# Copyright © 2023-2024 Intel Corporation
 
 from __future__ import annotations
 
@@ -2038,7 +2038,7 @@ def func_custom_target(self, node: mparser.FunctionNode, args: T.Tuple[str],
             # string in the meantime.
             FeatureNew.single_use('custom_target() with no name argument', '0.60.0', self.subproject, location=node)
             name = ''
-        inputs = self.source_strings_to_files(kwargs['input'], strict=False)
+        inputs = self.source_strings_to_files(self.materialize_generated_sources(kwargs['input']))
         command = kwargs['command']
         if command and isinstance(command[0], str):
             command[0] = self.find_program_impl([command[0]])
@@ -3078,21 +3078,18 @@ def validate_installable_file(fpath: Path) -> bool:
             raise InterpreterException(f'Sandbox violation: Tried to grab {inputtype} {norm.name} from a nested subproject.')
 
     @T.overload
-    def source_strings_to_files(self, sources: T.List['mesonlib.FileOrString'], strict: bool = True) -> T.List['mesonlib.File']: ...
-
-    @T.overload
-    def source_strings_to_files(self, sources: T.List['mesonlib.FileOrString'], strict: bool = False) -> T.List['mesonlib.FileOrString']: ... # noqa: F811
+    def source_strings_to_files(self, sources: T.List['mesonlib.FileOrString']) -> T.List['mesonlib.File']: ...
 
     @T.overload
     def source_strings_to_files(self, sources: T.List[T.Union[mesonlib.FileOrString, build.GeneratedTypes]]) -> T.List[T.Union[mesonlib.File, build.GeneratedTypes]]: ... # noqa: F811
 
     @T.overload
-    def source_strings_to_files(self, sources: T.List['SourceInputs'], strict: bool = True) -> T.List['SourceOutputs']: ... # noqa: F811
+    def source_strings_to_files(self, sources: T.List['SourceInputs']) -> T.List['SourceOutputs']: ... # noqa: F811
 
     @T.overload
-    def source_strings_to_files(self, sources: T.List[SourcesVarargsType], strict: bool = True) -> T.List['SourceOutputs']: ... # noqa: F811
+    def source_strings_to_files(self, sources: T.List[SourcesVarargsType]) -> T.List['SourceOutputs']: ... # noqa: F811
 
-    def source_strings_to_files(self, sources: T.List['SourceInputs'], strict: bool = True) -> T.List['SourceOutputs']: # noqa: F811
+    def source_strings_to_files(self, sources: T.List['SourceInputs']) -> T.List['SourceOutputs']: # noqa: F811
         """Lower inputs to a list of Targets and Files, replacing any strings.
 
         :param sources: A raw (Meson DSL) list of inputs (targets, files, and
@@ -3106,13 +3103,8 @@ def source_strings_to_files(self, sources: T.List['SourceInputs'], strict: bool
         results: T.List['SourceOutputs'] = []
         for s in sources:
             if isinstance(s, str):
-                if not strict and s.startswith(self.environment.get_build_dir()):
-                    results.append(s)
-                    mlog.warning(f'Source item {s!r} cannot be converted to File object, because it is a generated file. '
-                                 'This will become a hard error in the future.', location=self.current_node)
-                else:
-                    self.validate_within_subproject(self.subdir, s)
-                    results.append(mesonlib.File.from_source_file(self.environment.source_dir, self.subdir, s))
+                self.validate_within_subproject(self.subdir, s)
+                results.append(mesonlib.File.from_source_file(self.environment.source_dir, self.subdir, s))
             elif isinstance(s, mesonlib.File):
                 results.append(s)
             elif isinstance(s, (build.GeneratedList, build.BuildTarget,
@@ -3124,6 +3116,60 @@ def source_strings_to_files(self, sources: T.List['SourceInputs'], strict: bool
                                            'string or File-type object')
         return results
 
+    def materialize_generated_sources(self, sources: T.Sequence[SourceInputs]) -> T.List[SourceInputs]:
+        """Attempt to materialize Files for generated sources passed as strings.
+
+        This attempts to match a file passed as a string to a Target to the
+        output of a Target or configure_file call, and then create a File, while
+        providing warnings and deprecations.
+
+        We have to do this for backwards compatbility for Vala .gir files, and
+        while we're here we should just do it in general.
+        """
+        if isinstance(sources, str):
+            sources = [sources]
+        res: T.List[SourceInputs] = []
+        for s in sources:
+            if not isinstance(s, str):
+                res.append(s)
+                continue
+            if not s.startswith(self.environment.build_dir):
+                res.append(s)
+                continue
+            # Convert the absolute path into a relative path
+            source = os.path.relpath(s, self.environment.build_dir)
+
+            if source in self.configure_file_outputs:
+                res.append(mesonlib.File.from_built_relative(source))
+                FeatureDeprecated.single_use('building paths to the output of a configure_file', '1.4.0', self.subproject,
+                                             'Use the returned value from configure_file() instead', self.current_node)
+                continue
+
+            subdir, source = os.path.split(source)
+
+            self.validate_within_subproject(self.subdir, source)
+            t = mesonlib.first(self.build.targets.values(), lambda b: subdir == b.subdir and source in b.outputs)
+            if t:
+                res.append(mesonlib.File.from_built_file(subdir, source))
+                if s.endswith('.gir'):
+                    FeatureDeprecated.single_use('building paths to Vala generated gir files', '1.4.0', self.subproject,
+                                                 'Use target.get_gir() instead', self.current_node)
+                else:
+                    mlog.deprecation('Building a string path to a generated file that is part of', t.name,
+                                     'Use the target directly as as source, or, for custom_targets, index into the output.',
+                                     'If you cannot get the desired output for some reason, please file an issue.',
+                                     location=self.current_node)
+                continue
+
+            res.append(mesonlib.File.from_absolute_file(s, built=True))
+            mlog.warning(f'Attempted to build a path to a file, {s!r}, in the build directory which is not',
+                         'a declared output of a target or configure_file. This is likely to lead to',
+                         'race conditions and other intermittant build failures. You should use the outputs',
+                         f'of the target creating {s!r}. If you cannot get a reference to the desired output',
+                         'for some reason, please file an issue.', location=self.current_node)
+
+        return res
+
     @staticmethod
     def validate_forbidden_targets(name: str) -> None:
         if name.startswith('meson-internal__'):

mesonbuild/utils/universal.py

@@ -403,8 +403,8 @@ def from_built_relative(relative: str) -> 'File':
         return File(True, dirpart, fnamepart)
 
     @staticmethod
-    def from_absolute_file(fname: str) -> 'File':
-        return File(False, '', fname)
+    def from_absolute_file(fname: str, *, built: bool = False) -> 'File':
+        return File(built, '', fname)
 
     @lru_cache(maxsize=None)
     def rel_to_builddir(self, build_to_src: str) -> str:

test cases/common/49 custom target/meson.build

@@ -1,4 +1,4 @@
-project('custom target')
+project('custom target', meson_version : '>= 1.4')
 
 python = find_program('python3', required : false)
 if not python.found()
@@ -20,6 +20,13 @@ install : true,
 install_dir : 'subdir'
 )
 
+custom_target(
+  'warn',
+  input : meson.current_build_dir() / 'data.dat',
+  output : 'data.copy',
+  command : [python, '-c', 'import shutil, sys; shutil.copyfile(sys.argv[1], sys.argv[2])', '@INPUT@', '@OUTPUT@'],
+)
+
 has_not_changed = false
 if is_disabler(mytarget)
     has_not_changed = true
@@ -60,6 +67,14 @@ install : true,
 install_dir : 'subdir'
 )
 
+custom_target(
+  'uses-non-existant-input',
+  input : meson.current_build_dir() / 'non-existant-file',
+  output : 'non-existant-output',
+  command : [python, '-c', 'raise RuntimeError("Dont actually try to run this, mkay?")', '@INPUT@', '@OUTPUT@'],
+  build_by_default : false,
+)
+
 assert(is_disabler(mytarget_disabler), 'Disabled custom target is not a disabler.')
 
 if mytarget_disabler.found()

test cases/common/49 custom target/test.json

@@ -1,4 +1,13 @@
 {
+  "stdout": [
+    {
+      "line": "test cases/common/49 custom target/meson.build:23: DEPRECATION: Building a string path to a generated file that is part of bindat Use the target directly as as source, or, for custom_targets, index into the output. If you cannot get the desired output for some reason, please file an issue."
+    },
+    {
+      "line": "test cases/common/49 custom target/meson.build:70: WARNING: Attempted to build a path to a file, '.*/non-existant-file', in the build directory which is not a declared output of a target or configure_file. This is likely to lead to race conditions and other intermittant build failures. You should use the outputs of the target creating '.*/non-existant-file'. If you cannot get a reference to the desired output for some reason, please file an issue.",
+      "match": "re"
+    }
+  ],
   "installed": [
     {"type": "file", "file": "usr/subdir/data.dat"}
   ]

test cases/common/69 configure file in custom target/meson.build

@@ -1,4 +1,4 @@
-project('conf file in custom target')
+project('conf file in custom target', meson_version : '>= 1.4')
 
 subdir('inc')
 subdir('src')

test cases/common/69 configure file in custom target/src/meson.build

@@ -15,6 +15,13 @@ compiler = configure_file(input : 'mycompiler.py',
   copy: true)
 
 custom_target('thing2',
-output : 'final2.dat',
-input : cfile,
-command : [py3, compiler, '@INPUT@', '@OUTPUT@'])
+  output : 'final2.dat',
+  input : cfile,
+  command : [py3, compiler, '@INPUT@', '@OUTPUT@'])
+
+custom_target(
+  'thing2-warn',
+  output : 'final2-warn.dat',
+  input : meson.current_build_dir() / 'mycompiler2.py',
+  command : [py3, compiler, '@INPUT@', '@OUTPUT@']
+)

test cases/vala/9 gir/meson.build

@@ -1,4 +1,4 @@
-project('foo', 'c', 'vala')
+project('foo', 'c', 'vala', meson_version : '>= 1.4.0')
 
 glib = dependency('glib-2.0')
 gobject = dependency('gobject-2.0')
@@ -10,6 +10,12 @@ foo = shared_library('foo', 'foo.vala',
                      vala_gir: 'Foo-1.0.gir',
                      dependencies: [glib, gobject])
 
+custom_target('warn-typelib',
+              command: [g_ir_compiler, '--output', '@OUTPUT@', '@INPUT@'],
+              input: meson.current_build_dir() / 'Foo-1.0.gir',
+              output: 'Warn-1.0.typelib',
+              depends: foo)
+
 custom_target('foo-typelib',
               command: [g_ir_compiler, '--output', '@OUTPUT@', '@INPUT@'],
               input: foo.get_gir(),

test cases/vala/9 gir/test.json

@@ -1,7 +1,13 @@
 {
+  "stdout": [
+    {
+      "line": "test cases/vala/9 gir/meson.build:13: WARNING: Project targets '>= 1.4.0' but uses feature deprecated since '1.4.0': building paths to Vala generated gir files. Use target.get_gir() instead"
+    }
+  ],
   "installed": [
-    {"type": "expr", "platform": "gcc", "file": "usr/lib/?libfoo.so"},
-    {"type": "file", "platform": "cygwin", "file": "usr/lib/libfoo.dll.a"},
-    {"type": "file", "file": "usr/share/gir-1.0/Foo-1.0.gir"}
+    { "type": "shared_lib", "platform": "gcc", "file": "usr/lib/foo" },
+    { "type": "shared_lib", "platform": "msvc", "file": "usr/bin/foo" },
+    { "type": "implib", "file": "usr/lib/foo" },
+    { "type": "file", "file": "usr/share/gir-1.0/Foo-1.0.gir" }
   ]
 }