fix #2118

- BOM support - support UTF8, UTF16, ... - improve check, regex start searching from the beginning of a line now (add `^` to the regex) - better unit test samples - add integration tests
SonarOpenCommunity · Apr 30, 2021 · fe90235 · fe90235
1 parent 5b241ad
commit fe90235
Show file tree

Hide file tree

Showing 34 changed files with 344 additions and 172 deletions.
diff --git a/cxx-checks/pom.xml b/cxx-checks/pom.xml
@@ -27,6 +27,14 @@
       <artifactId>cxx-squid</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-lang</groupId>
+      <artifactId>commons-lang</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.sonarsource.sslr</groupId>
       <artifactId>sslr-testing-harness</artifactId>

diff --git a/cxx-checks/src/main/java/org/sonar/cxx/checks/file/TabCharacterCheck.java b/cxx-checks/src/main/java/org/sonar/cxx/checks/file/TabCharacterCheck.java
@@ -22,14 +22,13 @@
 import com.sonar.sslr.api.AstNode;
 import com.sonar.sslr.api.Grammar;
 import java.io.BufferedReader;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import org.sonar.check.Priority;
 import org.sonar.check.Rule;
 import org.sonar.check.RuleProperty;
+import org.sonar.cxx.checks.utils.CheckUtils;
 import org.sonar.cxx.tag.Tag;
 import org.sonar.cxx.visitors.CxxCharsetAwareVisitor;
 import org.sonar.squidbridge.annotations.ActivatedByDefault;
@@ -59,19 +58,16 @@ public class TabCharacterCheck extends SquidCheck<Grammar> implements CxxCharset
     description = "Create violations per line (default is one per file)",
     defaultValue = "" + DEFAULT_CREATE_LINE_VIOLATION)
   public boolean createLineViolation = DEFAULT_CREATE_LINE_VIOLATION;
-  private Charset charset = StandardCharsets.UTF_8;
+  private Charset defaultCharset = StandardCharsets.UTF_8;
 
   @Override
   public void setCharset(Charset charset) {
-    this.charset = charset;
+    this.defaultCharset = charset;
   }
 
   @Override
   public void visitFile(AstNode astNode) {
-
-    // use onMalformedInput(CodingErrorAction.REPLACE) / onUnmappableCharacter(CodingErrorAction.REPLACE)
-    try ( var br = new BufferedReader(
-      new InputStreamReader(new FileInputStream(getContext().getFile()), charset))) {
+    try ( var br = new BufferedReader(CheckUtils.getInputSteam(getContext().getFile(), defaultCharset))) {
       String line;
       int nr = 0;
 

diff --git a/cxx-checks/src/main/java/org/sonar/cxx/checks/metrics/TooLongLineCheck.java b/cxx-checks/src/main/java/org/sonar/cxx/checks/metrics/TooLongLineCheck.java
@@ -22,14 +22,13 @@
 import com.sonar.sslr.api.AstNode;
 import com.sonar.sslr.api.Grammar;
 import java.io.BufferedReader;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import org.sonar.check.Priority;
 import org.sonar.check.Rule;
 import org.sonar.check.RuleProperty;
+import org.sonar.cxx.checks.utils.CheckUtils;
 import org.sonar.cxx.tag.Tag;
 import org.sonar.cxx.visitors.CxxCharsetAwareVisitor;
 import org.sonar.squidbridge.annotations.ActivatedByDefault;
@@ -70,19 +69,16 @@ public class TooLongLineCheck extends SquidCheck<Grammar> implements CxxCharsetA
     defaultValue = "" + DEFAULT_TAB_WIDTH)
   public int tabWidth = DEFAULT_TAB_WIDTH;
 
-  private Charset charset = StandardCharsets.UTF_8;
+  private Charset defaultCharset = StandardCharsets.UTF_8;
 
   @Override
   public void setCharset(Charset charset) {
-    this.charset = charset;
+    this.defaultCharset = charset;
   }
 
   @Override
   public void visitFile(AstNode astNode) {
-
-    // use onMalformedInput(CodingErrorAction.REPLACE) / onUnmappableCharacter(CodingErrorAction.REPLACE)
-    try ( var br = new BufferedReader(
-      new InputStreamReader(new FileInputStream(getContext().getFile()), charset))) {
+    try ( var br = new BufferedReader(CheckUtils.getInputSteam(getContext().getFile(), defaultCharset))) {
       String line;
       int nr = 0;
 

diff --git a/cxx-checks/src/main/java/org/sonar/cxx/checks/regex/FileHeaderCheck.java b/cxx-checks/src/main/java/org/sonar/cxx/checks/regex/FileHeaderCheck.java
@@ -22,14 +22,11 @@
 import com.sonar.sslr.api.AstNode;
 import com.sonar.sslr.api.Grammar;
 import java.io.BufferedReader;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 import org.sonar.check.Priority;
 import org.sonar.check.Rule;
 import org.sonar.check.RuleProperty;
@@ -74,7 +71,7 @@ public class FileHeaderCheck extends SquidCheck<Grammar> implements CxxCharsetAw
     defaultValue = "false")
   public boolean isRegularExpression = false;
 
-  private Charset charset = StandardCharsets.UTF_8;
+  private Charset defaultCharset = StandardCharsets.UTF_8;
   private String[] expectedLines = null;
   private Pattern searchPattern = null;
 
@@ -90,37 +87,46 @@ private static boolean matches(String[] expectedLines, BufferedReader br) throws
 
   @Override
   public void setCharset(Charset charset) {
-    this.charset = charset;
+    this.defaultCharset = charset;
   }
 
   @Override
   public void init() {
     if (isRegularExpression) {
-      searchPattern = CheckUtils.compileUserRegexp(headerFormat, Pattern.DOTALL);
+      if (searchPattern == null) {
+        searchPattern = CheckUtils.compileUserRegexp(getHeaderFormat(), Pattern.DOTALL);
+      }
     } else {
       expectedLines = headerFormat.split("\\R");
     }
   }
 
   @Override
   public void visitFile(AstNode astNode) {
-
-    // use onMalformedInput(CodingErrorAction.REPLACE) / onUnmappableCharacter(CodingErrorAction.REPLACE)
-    try (var br = new BufferedReader(new InputStreamReader(new FileInputStream(getContext().getFile()), charset))) {
-
+    try {
       if (isRegularExpression) {
-        String fileContent = br.lines().collect(Collectors.joining(System.lineSeparator()));
+        String fileContent = CheckUtils.getFileContent(getContext().getFile(), defaultCharset);
         checkRegularExpression(fileContent);
       } else {
-        if (!matches(expectedLines, br)) {
-          getContext().createFileViolation(this, MESSAGE);
+        try ( var br = new BufferedReader(CheckUtils.getInputSteam(getContext().getFile(), defaultCharset))) {
+          if (!matches(expectedLines, br)) {
+            getContext().createFileViolation(this, MESSAGE);
+          }
         }
       }
     } catch (IOException e) {
       throw new IllegalStateException(e);
     }
   }
 
+  private String getHeaderFormat() {
+    String format = headerFormat;
+    if (format.charAt(0) != '^') {
+      format = "^" + format;
+    }
+    return format;
+  }
+
   private void checkRegularExpression(String fileContent) {
     Matcher matcher = searchPattern.matcher(fileContent);
     if (!matcher.find() || matcher.start() != 0) {

diff --git a/cxx-checks/src/main/java/org/sonar/cxx/checks/regex/FileRegularExpressionCheck.java b/cxx-checks/src/main/java/org/sonar/cxx/checks/regex/FileRegularExpressionCheck.java
@@ -21,15 +21,11 @@
 
 import com.sonar.sslr.api.AstNode;
 import com.sonar.sslr.api.Grammar;
-import java.io.BufferedReader;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 import org.sonar.api.utils.PathUtils;
 import org.sonar.api.utils.WildcardPattern;
 import org.sonar.check.Priority;
@@ -103,7 +99,7 @@ public class FileRegularExpressionCheck extends SquidCheck<Grammar> implements C
     description = "The violation message",
     defaultValue = DEFAULT_MESSAGE)
   public String message = DEFAULT_MESSAGE;
-  private Charset charset = StandardCharsets.UTF_8;
+  private Charset defaultCharset = StandardCharsets.UTF_8;
   private Pattern pattern = null;
 
   private static boolean compare(boolean invert, boolean condition) {
@@ -117,18 +113,19 @@ public void init() {
 
   @Override
   public void setCharset(Charset charset) {
-    this.charset = charset;
+    this.defaultCharset = charset;
   }
 
   @Override
   public void visitFile(AstNode fileNode) {
     if (!compare(invertFilePattern, matchFile())) {
       return;
     }
-    // use onMalformedInput(CodingErrorAction.REPLACE) / onUnmappableCharacter(CodingErrorAction.REPLACE)
-    try ( var br = new BufferedReader(new InputStreamReader(new FileInputStream(getContext().getFile()), charset))) {
-      final String fileContent = br.lines().collect(Collectors.joining(System.lineSeparator()));
+
+    try {
+      String fileContent = CheckUtils.getFileContent(getContext().getFile(), defaultCharset);
       Matcher matcher = pattern.matcher(fileContent);
+
       if (compare(invertRegularExpression, matcher.find())) {
         getContext().createFileViolation(this, message);
       }

diff --git a/cxx-checks/src/main/java/org/sonar/cxx/checks/regex/LineRegularExpressionCheck.java b/cxx-checks/src/main/java/org/sonar/cxx/checks/regex/LineRegularExpressionCheck.java
@@ -22,9 +22,7 @@
 import com.sonar.sslr.api.AstNode;
 import com.sonar.sslr.api.Grammar;
 import java.io.BufferedReader;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.regex.Matcher;
@@ -102,7 +100,7 @@ public class LineRegularExpressionCheck extends SquidCheck<Grammar> implements C
     description = "The violation message",
     defaultValue = DEFAULT_MESSAGE)
   public String message = DEFAULT_MESSAGE;
-  private Charset charset = StandardCharsets.UTF_8;
+  private Charset defaultCharset = StandardCharsets.UTF_8;
   private Pattern pattern = null;
 
   private static boolean compare(boolean invert, boolean condition) {
@@ -116,14 +114,15 @@ public void init() {
 
   @Override
   public void setCharset(Charset charset) {
-    this.charset = charset;
+    this.defaultCharset = charset;
   }
 
   @Override
   public void visitFile(AstNode fileNode) {
     if (compare(invertFilePattern, matchFile())) {
+
       // use onMalformedInput(CodingErrorAction.REPLACE) / onUnmappableCharacter(CodingErrorAction.REPLACE)
-      try ( var br = new BufferedReader(new InputStreamReader(new FileInputStream(getContext().getFile()), charset))) {
+      try ( var br = new BufferedReader(CheckUtils.getInputSteam(getContext().getFile(), defaultCharset))) {
         String line;
         int nr = 0;
 

diff --git a/cxx-checks/src/main/java/org/sonar/cxx/checks/utils/CheckUtils.java b/cxx-checks/src/main/java/org/sonar/cxx/checks/utils/CheckUtils.java
@@ -21,9 +21,20 @@
 
 import com.sonar.sslr.api.AstNode;
 import com.sonar.sslr.api.GenericTokenType;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
+import java.util.Scanner;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
+import java.util.stream.Collectors;
+import org.apache.commons.io.ByteOrderMark;
+import org.apache.commons.io.input.BOMInputStream;
 import org.sonar.cxx.parser.CxxGrammarImpl;
 import org.sonar.cxx.parser.CxxKeyword;
 import org.sonar.cxx.parser.CxxPunctuator;
@@ -86,4 +97,32 @@ public static boolean isFunctionDefinition(AstNode node) {
     return false;
   }
 
+  public static List<String> getFileLines(File source, Charset defaultCharset) throws IOException {
+    List<String> lines = new ArrayList<>();
+    try ( Scanner scanner = new Scanner(getInputSteam(source, defaultCharset))) {
+      while (scanner.hasNextLine()) {
+        lines.add(scanner.nextLine());
+      }
+    }
+    return lines;
+  }
+
+  public static String getFileContent(File source, Charset defaultCharset) throws IOException {
+    return getFileLines(source, defaultCharset).stream().collect(Collectors.joining(System.lineSeparator()));
+  }
+
+  public static InputStreamReader getInputSteam(File source, Charset defaultCharset) throws IOException {
+    BOMInputStream bomInputStream = new BOMInputStream(new FileInputStream(source),
+                                                       ByteOrderMark.UTF_8,
+                                                       ByteOrderMark.UTF_16LE,
+                                                       ByteOrderMark.UTF_16BE,
+                                                       ByteOrderMark.UTF_32LE,
+                                                       ByteOrderMark.UTF_32BE);
+
+    ByteOrderMark bom = bomInputStream.getBOM();
+    Charset charset = bom != null ? Charset.forName(bom.getCharsetName()) : defaultCharset;
+
+    return new InputStreamReader(bomInputStream, charset);
+  }
+
 }