JSON Compilation Database: support case sensitive paths

Depending on whether the file system is case sensitive or case insentitive, paths/comparisons of paths must be handled differently.
- case sensitive: (default Linux): Paths must not be changed.
- case insentitive (default Windows):  Here, paths in reports from tools are often stored with lower case letters. The original spelling in the file system must be restored before a path comparison.
- add additional integration test samples for Windows/Linux using upper case filenames
- close #2691

cxx-squid/src/main/java/org/sonar/cxx/config/CxxSquidConfiguration.java

@@ -25,12 +25,14 @@
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Locale;
 import java.util.Optional;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
@@ -130,6 +132,9 @@ public class CxxSquidConfiguration extends SquidConfiguration {
 
   private static final Logger LOG = Loggers.get(CxxSquidConfiguration.class);
 
+  // case-sensitive filesystem or not
+  private static boolean isCaseSensitive = true;
+
   private final XPathFactory xFactory = XPathFactory.instance();
   private Document document;
 
@@ -159,6 +164,12 @@ public CxxSquidConfiguration(String baseDir, Charset encoding) {
     super(encoding);
     this.baseDir = baseDir;
 
+    try {
+      isCaseSensitive = fileSystemIsCaseSensitive();
+    } catch (IOException e) {
+      isCaseSensitive = true;
+    }
+
     var root = new Element(ROOT);
     root.setAttribute(new Attribute("version", "1.0"));
     document = new Document(root);
@@ -236,8 +247,8 @@ public boolean isUnitsEmpty() {
    * @param level The level parameter defines the level on which the data should be inserted. For level a predefined
    * name can be used or a new one can be defined. <br>
    * - If level is an identifier, the information is created in an element with the level-name directly under root.<br>
-   * - If level is a path, the information is stored on Units level. In that case the level-string is normalized and
-   * converted to lower case letters to simplify the following search.
+   * - If level is a path, the information is stored on Units level. In that case the level-string is normalized to
+   * simplify the following search.
    * @param key the key to be placed into the database.
    * @param value the value corresponding to key. Several values can be assigned to one key. Internally a value-list for
    * key is created. The method can be called several times for this, but more effective is the method
@@ -550,10 +561,34 @@ public void readJsonCompilationDb() {
     }
   }
 
+  /**
+   * Check if file system is case sensitive
+   *
+   * @return true if running on a case sensitive filesystem
+   * @throws IOException
+   */
+  public static boolean fileSystemIsCaseSensitive() throws IOException {
+    Path a = null, b = null;
+    try {
+      Path tempDir = Files.createTempDirectory("test");
+      a = Files.createFile(tempDir.resolve(Paths.get("test.test")));
+      b = Files.createFile(tempDir.resolve(Paths.get("TEST.TEST")));
+    } catch (FileAlreadyExistsException e) {
+      return false;
+    } finally {
+      Files.deleteIfExists(a);
+      if (b != null) {
+        Files.deleteIfExists(b);
+      }
+    }
+    return true;
+  }
+
   /**
    * Create uniform notation of path names.
    *
-   * Normalize path and replace file separators by forward slash. Resulting string is converted to lower case.
+   * Normalize path and replace file separators by forward slash.
+   * Use lowercase path on case insensitive file systems.
    *
    * @param path to unify
    * @return unified path
@@ -563,7 +598,11 @@ private static String unifyPath(String path) {
     if (result == null) {
       result = "unknown";
     }
-    return result.toLowerCase(Locale.getDefault());
+    if (isCaseSensitive) {
+      return result;
+    } else {
+      return result.toLowerCase();
+    }
   }
 
   /**

cxx-squid/src/main/java/org/sonar/cxx/config/JsonCompilationDatabase.java

@@ -98,7 +98,6 @@ private static String[] tokenizeCommandLine(String cmdLine) {
   /**
    * Set up the given CxxSquidConfiguration from the JSON compilation database
    *
-   * @param squidConfig
    * @param compileCommandsFile
    * @throws IOException
    */

cxx-squid/src/test/java/org/sonar/cxx/config/CxxSquidConfigurationTest.java

@@ -288,8 +288,8 @@ void testPathNames() {
     squidConfig.add("C:\\X\\Y\\Z.cpp", "key4", "value4");
     Optional<String> value1 = squidConfig.get("/a/b/c.cpp", "key1");
     Optional<String> value2 = squidConfig.get("c:\\a\\b\\c.cpp", "key2");
-    Optional<String> value3 = squidConfig.get("/x/y/z.cpp", "key3");
-    Optional<String> value4 = squidConfig.get("c:/x/y/z.cpp", "key4");
+    Optional<String> value3 = squidConfig.get("/X/Y/Z.cpp", "key3");
+    Optional<String> value4 = squidConfig.get("C:/X/Y/Z.cpp", "key4");
 
     var softly = new SoftAssertions();
     softly.assertThat(value1).isNotEmpty();

cxx-squid/src/test/java/org/sonar/cxx/config/JsonCompilationDatabaseTest.java

@@ -46,6 +46,10 @@ void testGlobalSettings() throws Exception {
                                                   CxxSquidConfiguration.INCLUDE_DIRECTORIES);
     List<Path> files = squidConfig.getFiles();
 
+    var cwd = Path.of(".");
+    var absPath = cwd.resolve("TEST-argument-parser.cpp");
+    var filename = absPath.toAbsolutePath().normalize();
+
     assertThat(defines)
       .isNotEmpty()
       .doesNotContain("UNIT_DEFINE 1")
@@ -56,6 +60,8 @@ void testGlobalSettings() throws Exception {
       .contains(unifyPath("/usr/include"));
     assertThat(files)
       .hasSize(7);
+    assertThat(files)
+      .contains(filename);
   }
 
   @Test
@@ -148,7 +154,7 @@ void testArgumentParser() throws Exception {
     jsonDb.parse(file);
 
     var cwd = Path.of(".");
-    var absPath = cwd.resolve("test-argument-parser.cpp");
+    var absPath = cwd.resolve("TEST-argument-parser.cpp");
     var filename = absPath.toAbsolutePath().normalize().toString();
 
     List<String> defines = squidConfig.getValues(filename, CxxSquidConfiguration.DEFINES);

cxx-squid/src/test/resources/jsondb/compile_commands.json

@@ -12,7 +12,7 @@
   {
     "_comment_": "test argument parser",
     "directory": ".",
-    "file": "test-argument-parser.cpp",
+    "file": "TEST-argument-parser.cpp",
     "arguments": [
       "-DMACRO1", "-DMACRO2=2",
       "-D", "MACRO3", "-D", "MACRO4=4",

integration-tests/features/json-db.feature

@@ -3,8 +3,9 @@ Feature: JSON Compilation Database support
 
   As a CXX Plugin user, I want to use the JSON Compilation Database to analyze SonarQube projects
 
-  Scenario: Analyze only the source files contained in the JSON Compilation Database.
-    Given the project "json_db_project"
+  Scenario: Analyze only the source files contained in the JSON Compilation Database (case insensitive filesystem).
+    Given the project "json_db_project_case_insensitive"
+    And platform is "Windows"
     When I run sonar-scanner with "-X"
     Then the analysis finishes successfully
     And the analysis in server has completed
@@ -13,7 +14,24 @@ Feature: JSON Compilation Database support
       .*WARN.*Unable to get a valid mac address, will use a dummy address
       """
     And the following metrics have following values:
-      | metric     | value |          
+      | metric     | value |
+      | ncloc      | 9     |
+      | lines      | 24    |
+      | statements | 3     |
+      | functions  | 3     |
+
+  Scenario: Analyze only the source files contained in the JSON Compilation Database (case sensitive filesystem).
+    Given the project "json_db_project_case_sensitive"
+    And platform is not "Windows"
+    When I run sonar-scanner with "-X"
+    Then the analysis finishes successfully
+    And the analysis in server has completed
+    And the analysis log contains no error/warning messages except those matching:
+      """
+      .*WARN.*Unable to get a valid mac address, will use a dummy address
+      """
+    And the following metrics have following values:
+      | metric     | value |
       | ncloc      | 9     |
       | lines      | 24    |
       | statements | 3     |

integration-tests/testdata/json_db_project_case_insensitive/sonar-project.properties

@@ -0,0 +1,18 @@
+# metadata
+sonar.projectKey=json_db_project_case_insensitive
+
+# disable SCM support
+sonar.scm.disabled=true
+
+# disable XML sensor
+sonar.xml.file.suffixes=.disable-xml
+
+# file extensions assigned to the cxx programming language
+sonar.cxx.file.suffixes=.cxx,.cpp,.cc,.c,.hxx,.hpp,.hh,.h
+
+# comma-separated paths to directories containing source files
+sonar.sources=src
+
+# read a JSON Compilation Database file
+sonar.cxx.jsonCompilationDatabase=compile_commands.json
+sonar.cxx.jsonCompilationDatabase.analyzeOnlyContainedFiles=true

integration-tests/testdata/json_db_project_case_sensitive/compile_commands.json

@@ -0,0 +1,33 @@
+[
+  {
+    "_comment_": "example extension to define global defines and includes for headers and for files which are not compiled",
+    "file": "__global__",
+    "defines": {
+      "GLOBAL_DEFINE": "1"
+    },
+    "includes": [
+      "/usr/include"
+    ]
+  },
+  {
+    "_comment_": "source file to use",
+    "directory": "./src",
+    "file": "file1.cc",
+    "command": "gcc -o output1 -I/usr/local/include -I /another/include/dir -DSIMPLE1 -DCOMMAND_DEFINE=1 file1.cpp",
+    "output": "output1"
+  },
+  {
+    "_comment_": "source file to use",
+    "directory": "./src",
+    "file": "FILE2.cc",
+    "command": "gcc -o output2 -I/usr/local/include -I /another/include/dir -DSIMPLE2 -DCOMMAND_DEFINE=2 FILE2.cpp",
+    "output": "output2"
+  },
+  {
+    "_comment_": "source file to use",
+    "directory": "./src",
+    "file": "File3.cc",
+    "command": "gcc -o output3 -I/usr/local/include -I /another/include/dir -DSIMPLE3 -DCOMMAND_DEFINE=3 File3.cpp",
+    "output": "output3"
+  }
+]

integration-tests/testdata/json_db_project/sonar-project.properties → integration-tests/testdata/json_db_project_case_sensitive/sonar-project.properties

@@ -1,5 +1,5 @@
 # metadata
-sonar.projectKey=json_db_project
+sonar.projectKey=json_db_project_case_sensitive
 
 # disable SCM support
 sonar.scm.disabled=true

integration-tests/testdata/json_db_project_case_sensitive/src/FILE2.cc

@@ -0,0 +1,3 @@
+int func2(){
+    return 2;
+}

integration-tests/testdata/json_db_project_case_sensitive/src/File3.cc

@@ -0,0 +1,3 @@
+int func3(){
+    return 3;
+}

integration-tests/testdata/json_db_project_case_sensitive/src/file1.cc

@@ -0,0 +1,3 @@
+int func1(){
+    return 1;
+}

integration-tests/testdata/json_db_project_case_sensitive/src/file4.cc

@@ -0,0 +1,3 @@
+int func4(){
+    return 4;
+}

integration-tests/testdata/json_db_project_case_sensitive/src/file5.cc

@@ -0,0 +1,3 @@
+int func5(){
+    return 5;
+}

integration-tests/testdata/json_db_project_case_sensitive/src/file6.cc

@@ -0,0 +1,3 @@
+int func6(){
+    return 6;
+}

sonar-cxx-plugin/src/main/java/org/sonar/plugins/cxx/CxxSquidSensor.java

@@ -21,7 +21,9 @@
 
 import com.sonar.cxx.sslr.api.Grammar;
 import java.io.File;
+import java.io.IOException;
 import java.io.Serializable;
+import java.nio.file.LinkOption;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -396,9 +398,18 @@ private Iterable<InputFile> getInputFiles(SensorContext context, CxxSquidConfigu
 
     if (context.config().hasKey(JSON_COMPILATION_DATABASE_KEY)
           && context.config().getBoolean(JSON_COMPILATION_DATABASE_ONLY_CONTAINED_FILES_KEY).orElse(Boolean.FALSE)) {
+
       // if the source of the configuration is JSON Compilation Database and analyzeOnlyContainedFiles=True,
       // then analyze only the files contained in the db.
-      var inputFilesInConfig = squidConfig.getFiles();
+      var inputFilesInConfig = new ArrayList<Path>();
+      for (var inputfile : squidConfig.getFiles()) {
+        try {
+          // resolution of symbolic links and case-sensitive paths: In Json DB the path/filenames are often lowercase
+          inputFilesInConfig.add(inputfile.toRealPath(LinkOption.NOFOLLOW_LINKS));
+        } catch (IOException | RuntimeException e) {
+          // ...
+        }
+      }
       var result = StreamSupport.stream(inputFiles.spliterator(), false)
         .filter(f -> inputFilesInConfig.contains(Path.of(f.uri())))
         .collect(Collectors.toList());