From 47072dee061ba43985b073abb035af1a28b5d44a Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Mon, 3 Mar 2025 14:17:09 -0800 Subject: [PATCH 01/20] Various bug fixes. --- docs/ReleaseHistory.md | 6 ++ ...002_Unclassified16ByteHexadecimalString.cs | 2 +- ...EC000_003_Unclassified32CharacterString.cs | 102 ------------------ .../SEC101_102_AdoPat.cs | 8 +- .../WellKnownRegexPatterns.cs | 5 +- .../AdoLegacyPatTests.cs | 22 ++++ ...nclassified16ByteHexadecimalStringTests.cs | 36 +++++++ .../WellKnownRegexPatternsTests.cs | 82 ++++++++++---- 8 files changed, 135 insertions(+), 128 deletions(-) delete mode 100644 src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_003_Unclassified32CharacterString.cs create mode 100644 src/Tests.Microsoft.Security.Utilities.Core/AdoLegacyPatTests.cs create mode 100644 src/Tests.Microsoft.Security.Utilities.Core/Unclassified16ByteHexadecimalStringTests.cs diff --git a/docs/ReleaseHistory.md b/docs/ReleaseHistory.md index 4f3f9495..b520b929 100644 --- a/docs/ReleaseHistory.md +++ b/docs/ReleaseHistory.md @@ -11,6 +11,12 @@ - FPS => False positive reduction in static analysis. - FNS => False negative reduction in static analysis. +# 1.16.0 - 03/03/2025 +- BRK: Eliminate `SEC000/101.Unclassified32CharacterString` as noisy and not useful. +- BRK: Rename `SEC101/102.AdoPat` friendly name to `AdoLegacyPat`. +- BUG: Correct `SEC000/002.Unclassified16ByteHexadecimalString` id and rule name on calling `GetMatchIdAndName` (where `SEC000/001.Unclassified64ByteBase64String` was returned incorrectly before). +- BUG: Resolve `System.FormatException: The input is not a valid Base-46 string` errors calling `SEC101/102.AdoPat.GetMatchIdAndName` by swallowing correct exception kind `ArgumentException` in `IsChecksumValid` helper. + # 1.15.0 - 03/03/2025 - BRK: Regular expression syntax has been standardized in JSON to conform to how the overwhelming majority of patterns were already defined. - `refine` is used now used throughout as the name of the capture group used to isolate an actual find from the full expression that also matches delimiting characters. `secret` was previously used in some instances. diff --git a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_002_Unclassified16ByteHexadecimalString.cs b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_002_Unclassified16ByteHexadecimalString.cs index 8606a984..37c36331 100644 --- a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_002_Unclassified16ByteHexadecimalString.cs +++ b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_002_Unclassified16ByteHexadecimalString.cs @@ -22,7 +22,7 @@ public Unclassified16ByteHexadecimalString() DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.Unclassified | DetectionMetadata.LowConfidence; } - public override Tuple? GetMatchIdAndName(string match) => new Tuple("SEC000/001", "Unclassified64ByteBase64String"); + public override Tuple? GetMatchIdAndName(string match) => new Tuple("SEC000/002", nameof(Unclassified16ByteHexadecimalString)); public override IEnumerable GenerateTruePositiveExamples() { diff --git a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_003_Unclassified32CharacterString.cs b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_003_Unclassified32CharacterString.cs deleted file mode 100644 index f2453ef3..00000000 --- a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_003_Unclassified32CharacterString.cs +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections.Generic; - -namespace Microsoft.Security.Utilities -{ - public class Unclassified32CharacterString : RegexPattern - { - public const string AzureContainerRegistryLegacyKey = nameof(AzureContainerRegistryLegacyKey); - public const string AadClientAppLegacyCredentials = nameof(AadClientAppLegacyCredentials); - - /// - /// Detect 32-character Azure Active Directory client application legacy credentials. - /// The generated key is a 32-character string that contains alphanumeric characters - /// as well as symbols from the set: .=\-:[_@\*]+? - /// - public Unclassified32CharacterString() - { - Id = "SEC000/003"; - Name = nameof(Unclassified32CharacterString); - DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.Unclassified | DetectionMetadata.LowConfidence; - Pattern = $"(?i)[a-z0-9.=\\-:[_@\\/*\\]+?]{{32}}$"; - } - - public override Tuple GetMatchIdAndName(string match) - { - if (DateTime.TryParse(match, out DateTime result)) - { - return null; - } - - if (HasAtLeastOneNonBase64EncodingSymbol(match)) - { - return new Tuple("SEC101/101", AadClientAppLegacyCredentials); - } - - return new Tuple("SEC101/109", AzureContainerRegistryLegacyKey); - } - - private const string symbols = ".=-:[_@/*]+?"; - private static readonly HashSet symbolChars = new HashSet(symbols.ToCharArray()); - - public override IEnumerable GenerateTruePositiveExamples() - { - int sampleSize = 5; - - string alphabet = $"={WellKnownRegexPatterns.Base64}"; - yield return $"{WellKnownRegexPatterns.GenerateString(alphabet, 32)}"; - - while (sampleSize > 0) - { - string key = WellKnownRegexPatterns.GenerateString($"{WellKnownRegexPatterns.Base62}{symbols}", 32); - if (AadClientAppLegacyCredentials34.HasAtLeastOneSymbol(key)) - { - sampleSize--; - yield return key; - } - } - } - - internal static bool HasAtLeastOneSymbol(string text) - { - foreach (char c in text) - { - - if (symbolChars.Contains(c)) - { - return true; - } - } - - return false; - } - - public override IEnumerable GenerateFalsePositiveExamples() - { - yield return DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ"); - yield return "2024-03-07T02:50:56.464790+00:00"; - - yield return WellKnownRegexPatterns.GenerateString($"{WellKnownRegexPatterns.Base62}.=-:[_@/*]+?", 31); - yield return WellKnownRegexPatterns.GenerateString($"{WellKnownRegexPatterns.Base62}.=-:[_@/*]+?", 31); - } - - private static bool HasAtLeastOneNonBase64EncodingSymbol(string text) - { - foreach (char c in text) - { - - if (c == '.' || c == '-' || c == ':' || - c == '[' || c == '_' || c == '@' || - c == '*' || c == ']' || c == '?') - { - return true; - } - } - - return false; - } - } -} \ No newline at end of file diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_102_AdoPat.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_102_AdoPat.cs index 15346f86..4558b273 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_102_AdoPat.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_102_AdoPat.cs @@ -5,14 +5,14 @@ namespace Microsoft.Security.Utilities { - public class AdoPat : RegexPattern + public class AdoLegacyPat : RegexPattern { private static readonly byte[] EmptyByteArray = new byte[0]; - public AdoPat() + public AdoLegacyPat() { Id = "SEC101/102"; - Name = nameof(AdoPat); + Name = nameof(AdoLegacyPat); DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.EmbeddedChecksum; Pattern = "(?:[^2-7a-z]|^)(?[2-7a-z]{52})(?:[^2-7a-z]|$)"; } @@ -42,7 +42,7 @@ private static bool IsChecksumValid(string input, uint magicNumber) { inputBytes = ConvertFromBase32(input); } - catch (FormatException) + catch (ArgumentException) { return false; } diff --git a/src/Microsoft.Security.Utilities.Core/WellKnownRegexPatterns.cs b/src/Microsoft.Security.Utilities.Core/WellKnownRegexPatterns.cs index a1759c52..4b310e07 100644 --- a/src/Microsoft.Security.Utilities.Core/WellKnownRegexPatterns.cs +++ b/src/Microsoft.Security.Utilities.Core/WellKnownRegexPatterns.cs @@ -53,10 +53,9 @@ public static IEnumerable HighConfidenceSecurityModelsIterator() new UrlCredentials(), new LooseSasSecret(), new OAuth2BearerToken(), - new Unclassified32CharacterString(), new Unclassified32ByteBase64String(), new Unclassified64ByteBase64String(), - new AadClientAppLegacyCredentials34(), // SEC101/101 + new AadClientAppLegacyCredentials34(), // SEC101/101 legacy generated passwords. new Pkcs12CertificatePrivateKeyBundle(), new Unclassified16ByteHexadecimalString(), }; @@ -85,7 +84,7 @@ public static IEnumerable HighConfidenceSecurityModelsIterator() new AzureCacheForRedisIdentifiableKey(), new AzureContainerRegistryIdentifiableKey(), new NuGetApiKey(), - new AdoPat(), // SEC101/102 + new AdoLegacyPat(), // SEC101/102 new AzureCosmosDBLegacyCredentials(), // SEC101/104 new AzureStorageAccountLegacyCredentials(), // SEC101/106 new AzureMessageLegacyCredentials(), diff --git a/src/Tests.Microsoft.Security.Utilities.Core/AdoLegacyPatTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/AdoLegacyPatTests.cs new file mode 100644 index 00000000..1a49e557 --- /dev/null +++ b/src/Tests.Microsoft.Security.Utilities.Core/AdoLegacyPatTests.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Diagnostics.CodeAnalysis; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.Security.Utilities.Core +{ + [TestClass] + public class AdoLegacyPatTests + { + [TestMethod] + public void AdoLegacyPat_InvalidBase32Input() + { + var classifier = new AdoLegacyPat(); + string invalidInput = "=22222222222222222222222222"; + var result = classifier.GetMatchIdAndName(invalidInput); + Assert.IsNull(result); + } + } +} diff --git a/src/Tests.Microsoft.Security.Utilities.Core/Unclassified16ByteHexadecimalStringTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/Unclassified16ByteHexadecimalStringTests.cs new file mode 100644 index 00000000..4fbea19b --- /dev/null +++ b/src/Tests.Microsoft.Security.Utilities.Core/Unclassified16ByteHexadecimalStringTests.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#nullable enable + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using System; +using System.Linq; + +namespace Microsoft.Security.Utilities.Core +{ + [TestClass] + public class Unclassified16ByteHexadecimalStringTests + { + + [TestMethod] + public void Unclassified16ByteHexadecimalString_ValidInput() + { + var classifier = new Unclassified16ByteHexadecimalString(); + string validInput = "0123456789abcdef0123456789abcdef"; + + var result = classifier.GetMatchIdAndName(validInput); + + Assert.IsNotNull(result); + Assert.AreEqual("SEC000/002", result.Item1); + Assert.AreEqual("Unclassified16ByteHexadecimalString", result.Item2); + + var detection = + classifier.GetDetections(validInput, generateCrossCompanyCorrelatingIds: false).FirstOrDefault(); + + Assert.IsNotNull(detection); + Assert.AreEqual("SEC000/002", detection.Id); + } + } +} diff --git a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs index ae4fce1b..bfce0f0d 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs @@ -26,6 +26,54 @@ public class WellKnownRegexPatternsTests "SEC101/109.AzureContainerRegistryLegacyKey" }; + [TestMethod] + public void WellKnownRegexPatterns_AllRuleIdsAndNamesAreUnique() + { + using var assertionScope = new AssertionScope(); + + var patterns = GetAllPatterns(); + + HashSet ruleIds = new HashSet(StringComparer.OrdinalIgnoreCase); + HashSet ruleNames = new HashSet(StringComparer.OrdinalIgnoreCase); + + bool result; + foreach (RegexPattern pattern in patterns) + { + result = ruleIds.Contains(pattern.Id); + result.Should().BeFalse(because: $"Pattern '{pattern.GetType().Name}' should not share its Id with another rule: '{pattern.Id}'"); + + result = ruleNames.Contains(pattern.Name); + result.Should().BeFalse(because: $"Pattern '{pattern.GetType().Name}' should not share its Name with another rule: '{pattern.Name}'"); + } + } + + [TestMethod] + public void WellKnownRegexPatterns_MonikerRuleIdsAndNamesMatchDeclared() + { + using var assertionScope = new AssertionScope(); + + var patterns = GetAllPatterns(); + + foreach (RegexPattern pattern in patterns) + { + foreach (string example in pattern.GenerateTruePositiveExamples()) + { + var detection = pattern.GetDetections(example, generateCrossCompanyCorrelatingIds: false).FirstOrDefault(); + Assert.AreNotEqual(default, detection); + + string preciseMatch = example.Substring(detection.Start, detection.Length); + + var idAndName = pattern.GetMatchIdAndName(preciseMatch); + + Assert.AreEqual(pattern.Id, idAndName.Item1, + $"Pattern '{pattern.GetType().Name}' id did not match 'GetMatchIdAndName' result"); + + Assert.AreEqual(pattern.Name, idAndName.Item2, + $"Pattern '{pattern.GetType().Name}' name did not match 'GetMatchIdAndName' result"); + } + } + } + [TestMethod] public void WellKnownRegexPatterns_GetMatchMonikerHardenedForOutOfOrderExecution() { @@ -41,7 +89,7 @@ public void WellKnownRegexPatterns_GetMatchMonikerHardenedForOutOfOrderExecution generateCorrelatingIds: true, RE2RegexEngine.Instance); - foreach (var pattern in patterns) + foreach (RegexPattern pattern in patterns) { foreach (string example in pattern.GenerateTruePositiveExamples()) { @@ -57,26 +105,13 @@ public void WellKnownRegexPatterns_GetMatchMonikerHardenedForOutOfOrderExecution string moniker = pattern.GetMatchMoniker(example); var detection = masker.DetectSecrets(example).FirstOrDefault((d) => d.Id == pattern.Id); - - // Currently, some rules are tuned not to double-fire, i.e., they determine - // whether a separate rule might identify a pattern and, if so, the rule - // drops the result. This is a problematic design. For one thing, we don't - // to see if the more precise rule is enabled. This gives the appearance of - // false negatives in these low-level checks. This is a subtle topic - // potentially arguing for redesign of the engine or our test expectations. - bool result = - detection != default || - pattern.Name == nameof(Unclassified32CharacterString) || - pattern.Name == nameof(Unclassified16ByteHexadecimalString); + bool result = detection != default; result.Should().BeTrue(because: $"pattern '{pattern.GetType().Name}' should match '{example}'"); - if (moniker == null) - { - string matched = example.Substring(detection.Start, detection.End - detection.Start); - moniker = pattern.GetMatchMoniker(matched); - moniker.Should().NotBeNull(because: $"'{matched}' should produce a non-null moniker for {pattern.GetType().Name}' test data"); - } + string matched = example.Substring(detection.Start, detection.End - detection.Start); + moniker = pattern.GetMatchMoniker(matched); + moniker.Should().NotBeNull(because: $"'{matched}' should produce a non-null moniker for {pattern.GetType().Name}' test data"); } } } @@ -271,5 +306,16 @@ public void WellKnownRegexPatterns_EnsureAllPatternsHaveCorrectCaptureGroups() } } } + + private static List GetAllPatterns() + { + var patterns = new List(); + + patterns.AddRange(WellKnownRegexPatterns.DataClassification); + patterns.AddRange(WellKnownRegexPatterns.PreciselyClassifiedSecurityKeys); + patterns.AddRange(WellKnownRegexPatterns.UnclassifiedPotentialSecurityKeys); + + return patterns; + } } } From b9446016f1f2f79256ecac087cfb97587d8d72b2 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Mon, 3 Mar 2025 14:30:50 -0800 Subject: [PATCH 02/20] Update version.json to v1.16.0 --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index cd3edf04..60ae3040 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", - "version": "1.15.0", + "version": "1.16.0", "publicReleaseRefSpec": [ "^refs/heads/main$", "^refs/heads/release/v\\d+\\.\\d+\\.\\d+$" From 5cea3092c007d4d2dbac4665eed50a58c462976d Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Mon, 3 Mar 2025 14:36:11 -0800 Subject: [PATCH 03/20] Remove unnecessary double fire check --- GeneratedRegexPatterns/LowConfidenceSecurityModels.json | 7 ------- .../PreciselyClassifiedSecurityKeys.json | 2 +- .../UnclassifiedPotentialSecurityKeys.json | 7 ------- .../WellKnownRegexPatternsTests.cs | 2 +- 4 files changed, 2 insertions(+), 16 deletions(-) diff --git a/GeneratedRegexPatterns/LowConfidenceSecurityModels.json b/GeneratedRegexPatterns/LowConfidenceSecurityModels.json index 97d9e3c2..ba3b7041 100644 --- a/GeneratedRegexPatterns/LowConfidenceSecurityModels.json +++ b/GeneratedRegexPatterns/LowConfidenceSecurityModels.json @@ -6,13 +6,6 @@ "Signatures": null, "DetectionMetadata": "LowConfidence" }, - { - "Pattern": "(?i)[a-z0-9.=\\-:[_@\\/*\\]+?]{32}$", - "Id": "SEC000/003", - "Name": "Unclassified32CharacterString", - "Signatures": null, - "DetectionMetadata": "HighEntropy, Unclassified, LowConfidence" - }, { "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{43}=", "Id": "SEC000/000", diff --git a/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json b/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json index f73d4d11..4b4a87aa 100644 --- a/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json +++ b/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json @@ -330,7 +330,7 @@ { "Pattern": "(?:[^2-7a-z]|^)(?[2-7a-z]{52})(?:[^2-7a-z]|$)", "Id": "SEC101/102", - "Name": "AdoPat", + "Name": "AdoLegacyPat", "Signatures": null, "DetectionMetadata": "HighEntropy, EmbeddedChecksum" }, diff --git a/GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json b/GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json index 562ed237..444577ed 100644 --- a/GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json +++ b/GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json @@ -37,13 +37,6 @@ "Signatures": null, "DetectionMetadata": "LowConfidence" }, - { - "Pattern": "(?i)[a-z0-9.=\\-:[_@\\/*\\]+?]{32}$", - "Id": "SEC000/003", - "Name": "Unclassified32CharacterString", - "Signatures": null, - "DetectionMetadata": "HighEntropy, Unclassified, LowConfidence" - }, { "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{43}=", "Id": "SEC000/000", diff --git a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs index bfce0f0d..9c2b54a4 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs @@ -104,7 +104,7 @@ public void WellKnownRegexPatterns_GetMatchMonikerHardenedForOutOfOrderExecution // is no longer null post-detection. string moniker = pattern.GetMatchMoniker(example); - var detection = masker.DetectSecrets(example).FirstOrDefault((d) => d.Id == pattern.Id); + var detection = masker.DetectSecrets(example).FirstOrDefault(); bool result = detection != default; result.Should().BeTrue(because: $"pattern '{pattern.GetType().Name}' should match '{example}'"); From 33f8a72ecbe333353d5599acbe85abb39ab9eb81 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Mon, 3 Mar 2025 14:37:51 -0800 Subject: [PATCH 04/20] PR feedback --- src/Microsoft.Security.Utilities.Core/WellKnownRegexPatterns.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Security.Utilities.Core/WellKnownRegexPatterns.cs b/src/Microsoft.Security.Utilities.Core/WellKnownRegexPatterns.cs index 4b310e07..8b6f70dd 100644 --- a/src/Microsoft.Security.Utilities.Core/WellKnownRegexPatterns.cs +++ b/src/Microsoft.Security.Utilities.Core/WellKnownRegexPatterns.cs @@ -84,7 +84,7 @@ public static IEnumerable HighConfidenceSecurityModelsIterator() new AzureCacheForRedisIdentifiableKey(), new AzureContainerRegistryIdentifiableKey(), new NuGetApiKey(), - new AdoLegacyPat(), // SEC101/102 + new AdoLegacyPat(), // SEC101/102 new AzureCosmosDBLegacyCredentials(), // SEC101/104 new AzureStorageAccountLegacyCredentials(), // SEC101/106 new AzureMessageLegacyCredentials(), From 4193f31e1cda66d2cc1baec88b92d4179c103241 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Tue, 4 Mar 2025 11:44:36 -0800 Subject: [PATCH 05/20] Restore RE2 '?P' named group pattern. --- .../HighConfidenceSecurityModels.json | 52 ++++++++-------- .../LowConfidenceSecurityModels.json | 2 +- .../MediumConfidenceSecurityModels.json | 6 +- .../PreciselyClassifiedSecurityKeys.json | 60 +++++++++---------- .../UnclassifiedPotentialSecurityKeys.json | 6 +- .../RE2RegexEngine.cs | 30 ++++++++-- .../CachedDotNetRegex.cs | 12 ++++ .../SEC101_060_LooseSasSecret.cs | 2 +- .../SEC101_061_LooseOAuth2BearerToken.cs | 2 +- .../Azure32ByteIdentifiableKey.cs | 2 +- .../Azure32ByteIdentifiableKeys.cs | 2 +- .../Azure64ByteIdentifiableKey.cs | 2 +- .../Azure64ByteIdentifiableKeys.cs | 2 +- .../SEC101_031_NuGetApiKey.cs | 2 +- .../SEC101_050_NpmAuthorKey.cs | 2 +- .../SEC101_102_AdoPat.cs | 2 +- ...C101_104_AzureCosmosDBLegacyCredentials.cs | 2 +- ...101_105_AzureMessagingLegacyCredentials.cs | 2 +- ...06_AzureStorageAccountLegacyCredentials.cs | 2 +- .../SEC101_110.AzureDatabricksPat.cs | 2 +- ...6_AzureClientAppIdentifiableCredentials.cs | 2 +- ...SEC101_158_AzureFunctionIdentifiableKey.cs | 2 +- ...101_166_AzureSearchIdentifiableQueryKey.cs | 2 +- ...6_AzureContainerRegistryIdentifiableKey.cs | 2 +- .../SEC101_200_CommonAnnotatedSecurityKey.cs | 2 +- .../SEC101_565_SecretScanningSampleToken.cs | 2 +- .../UrlCredentials.cs | 2 +- .../AdoLegacyPatTests.cs | 2 - .../RE2RegexEngine.cs | 43 +++++++++---- .../RegexPatternTests.cs | 2 +- .../SecretMaskerTests.cs | 3 +- .../WellKnownRegexPatternsTests.cs | 8 +++ 32 files changed, 161 insertions(+), 105 deletions(-) diff --git a/GeneratedRegexPatterns/HighConfidenceSecurityModels.json b/GeneratedRegexPatterns/HighConfidenceSecurityModels.json index 1906bd4f..92852edf 100644 --- a/GeneratedRegexPatterns/HighConfidenceSecurityModels.json +++ b/GeneratedRegexPatterns/HighConfidenceSecurityModels.json @@ -1,6 +1,6 @@ [ { - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{52}JQQJ9(?:9|D|H)[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890][A-L][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{16}[A-Za-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{7}(?:[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{2}==)?)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{52}JQQJ9(?:9|D|H)[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890][A-L][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{16}[A-Za-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{7}(?:[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{2}==)?)", "Id": "SEC101/200", "Name": "CommonAnnotatedSecurityKey", "Signatures": [ @@ -9,7 +9,7 @@ "DetectionMetadata": "Identifiable" }, { - "Pattern": "(^|[^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/])(?[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{3}(7|8)Q~[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{31,34})([^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/]|$)", + "Pattern": "(^|[^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/])(?P[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{3}(7|8)Q~[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{31,34})([^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/]|$)", "Id": "SEC101/156", "Name": "AadClientAppIdentifiableCredentials", "Signatures": [ @@ -28,7 +28,7 @@ 5575864757416767536, 6014965721085063216 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{44}AzFu[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{44}AzFu[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]|$)", "KeyLength": 40, "RegexNormalizedSignature": "AzFu", "Id": "SEC101/158", @@ -43,7 +43,7 @@ "ChecksumSeeds": [ 5869709231681187888 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "KeyLength": 39, "RegexNormalizedSignature": "AzSe", "Id": "SEC101/166", @@ -58,7 +58,7 @@ "Signatures": [ "AzSe" ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "KeyLength": 39, "RegexNormalizedSignature": "AzSe", "Id": "SEC101/167", @@ -74,7 +74,7 @@ 5506058963192262704, 5575859178286952496 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ARm[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ARm[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ARm", "KeyLength": 32, "EncodeForUrl": false, @@ -91,7 +91,7 @@ 5506058963192262704, 5575859178286952496 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+AEh[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+AEh[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+AEh", "KeyLength": 32, "EncodeForUrl": false, @@ -108,7 +108,7 @@ 5506058963192262704, 5575859178286952496 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ASb[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ASb[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ASb", "KeyLength": 32, "EncodeForUrl": false, @@ -123,7 +123,7 @@ "ChecksumSeeds": [ 5291540757367369776 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AIoT", "KeyLength": 32, "EncodeForUrl": false, @@ -138,7 +138,7 @@ "ChecksumSeeds": [ 4928475562238095408 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AIoT", "KeyLength": 32, "EncodeForUrl": false, @@ -153,7 +153,7 @@ "ChecksumSeeds": [ 4931568359632875568 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AIoT", "KeyLength": 32, "EncodeForUrl": false, @@ -169,7 +169,7 @@ 4928457935994778672 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ASt[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ASt[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ASt", "EncodeForUrl": false, "Id": "SEC101/152", @@ -190,7 +190,7 @@ 6014965720764854320 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}ACDb[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}ACDb[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "ACDb", "EncodeForUrl": false, "Id": "SEC101/160", @@ -205,7 +205,7 @@ 4711400055309086768 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ABa[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ABa[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ABa", "EncodeForUrl": false, "Id": "SEC101/163", @@ -220,7 +220,7 @@ 4858365246511342384 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+AMC[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+AMC[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+AMC", "EncodeForUrl": false, "Id": "SEC101/170", @@ -235,7 +235,7 @@ 6081388236577714224 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/181", @@ -250,7 +250,7 @@ 5291540757367369776 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/182", @@ -265,7 +265,7 @@ 5143520228578766896 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/183", @@ -280,7 +280,7 @@ 5145771916421312560 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/184", @@ -294,7 +294,7 @@ "ChecksumSeeds": [ 4711718922539446320 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AzCa[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AzCa[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AzCa", "KeyLength": 32, "EncodeForUrl": false, @@ -309,7 +309,7 @@ "ChecksumSeeds": [ 4702692889634567216 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{42}\\+ACR[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{42}\\+ACR[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "KeyLength": 39, "RegexNormalizedSignature": "\\+ACR", "EncodeForUrl": false, @@ -318,7 +318,7 @@ "DetectionMetadata": "Identifiable" }, { - "Pattern": "(?i)(^|[^a-z0-9])(?oy2[a-z2-7]{43})([^a-z0-9]|$)", + "Pattern": "(?i)(^|[^a-z0-9])(?Poy2[a-z2-7]{43})([^a-z0-9]|$)", "Id": "SEC101/031", "Name": "NuGetApiKey", "Signatures": [ @@ -328,7 +328,7 @@ "DetectionMetadata": "HighEntropy, FixedSignature, HighConfidence" }, { - "Pattern": "(?:^|[^0-9a-f\\-])(?dapi[0-9a-f\\-]{32,34})(?:[^0-9a-f\\-]|$)", + "Pattern": "(?:^|[^0-9a-f\\-])(?Pdapi[0-9a-f\\-]{32,34})(?:[^0-9a-f\\-]|$)", "Id": "SEC101/110", "Name": "AzureDatabricksPat", "Signatures": [ @@ -343,7 +343,7 @@ "ChecksumSeeds": [ 4928457935994778672 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AZEG[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AZEG[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AZEG", "KeyLength": 32, "EncodeForUrl": false, @@ -352,7 +352,7 @@ "DetectionMetadata": "Identifiable" }, { - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?npm_[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{36})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?Pnpm_[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{36})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", "Id": "SEC101/050", "Name": "NpmAuthorKey", "Signatures": [ @@ -361,7 +361,7 @@ "DetectionMetadata": "HighEntropy, FixedSignature, HighConfidence" }, { - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?secret_scanning_ab85fc6f8d7638cf1c11da812da308d43_[0-9A-Za-z]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?Psecret_scanning_ab85fc6f8d7638cf1c11da812da308d43_[0-9A-Za-z]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", "Id": "SEC101/565", "Name": "SecretScanningSampleToken", "Signatures": [ diff --git a/GeneratedRegexPatterns/LowConfidenceSecurityModels.json b/GeneratedRegexPatterns/LowConfidenceSecurityModels.json index ba3b7041..2c327ee0 100644 --- a/GeneratedRegexPatterns/LowConfidenceSecurityModels.json +++ b/GeneratedRegexPatterns/LowConfidenceSecurityModels.json @@ -1,6 +1,6 @@ [ { - "Pattern": "(?i)authorization:(\\s|%20)bearer(\\s|%20)(?[0-9a-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+\\/=]*)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+/=]|$)", + "Pattern": "(?i)authorization:(\\s|%20)bearer(\\s|%20)(?P[0-9a-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+\\/=]*)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+/=]|$)", "Id": "SEC101/061", "Name": "OAuth2BearerToken", "Signatures": null, diff --git a/GeneratedRegexPatterns/MediumConfidenceSecurityModels.json b/GeneratedRegexPatterns/MediumConfidenceSecurityModels.json index b3809922..c6210440 100644 --- a/GeneratedRegexPatterns/MediumConfidenceSecurityModels.json +++ b/GeneratedRegexPatterns/MediumConfidenceSecurityModels.json @@ -1,6 +1,6 @@ [ { - "Pattern": "(?i)\\.servicebus\\.windows.+[^0-9a-z\\/+](?[0-9a-z\\/+]{43}=)(?:[^=]|$)", + "Pattern": "(?i)\\.servicebus\\.windows.+[^0-9a-z\\/+](?P[0-9a-z\\/+]{43}=)(?:[^=]|$)", "Id": "SEC101/105", "Name": "AzureMessageLegacyCredentials", "Signatures": [ @@ -20,7 +20,7 @@ "DetectionMetadata": "HighEntropy, MediumConfidence" }, { - "Pattern": "($|\\b)(ftps?|https?):\\/\\/(?[^:@\\/]+:[^:@?\\/]+)@", + "Pattern": "($|\\b)(ftps?|https?):\\/\\/(?P[^:@\\/]+:[^:@?\\/]+)@", "Id": "SEC101/127", "Name": "UrlCredentials", "Signatures": [ @@ -30,7 +30,7 @@ "DetectionMetadata": "MediumConfidence" }, { - "Pattern": "(?i)(?:^|[?;&])(?:dsas_secret|sig)=(?[0-9a-z\\/+%]{43,129}(?:=|%3d))", + "Pattern": "(?i)(?:^|[?;&])(?:dsas_secret|sig)=(?P[0-9a-z\\/+%]{43,129}(?:=|%3d))", "Id": "SEC101/060", "Name": "LooseSasSecret", "Signatures": [ diff --git a/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json b/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json index 4b4a87aa..ad0c66ad 100644 --- a/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json +++ b/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json @@ -1,6 +1,6 @@ [ { - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{52}JQQJ9(?:9|D|H)[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890][A-L][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{16}[A-Za-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{7}(?:[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{2}==)?)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{52}JQQJ9(?:9|D|H)[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890][A-L][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{16}[A-Za-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{7}(?:[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{2}==)?)", "Id": "SEC101/200", "Name": "CommonAnnotatedSecurityKey", "Signatures": [ @@ -9,7 +9,7 @@ "DetectionMetadata": "Identifiable" }, { - "Pattern": "(^|[^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/])(?[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{3}(7|8)Q~[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{31,34})([^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/]|$)", + "Pattern": "(^|[^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/])(?P[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{3}(7|8)Q~[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{31,34})([^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/]|$)", "Id": "SEC101/156", "Name": "AadClientAppIdentifiableCredentials", "Signatures": [ @@ -28,7 +28,7 @@ 5575864757416767536, 6014965721085063216 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{44}AzFu[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{44}AzFu[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]|$)", "KeyLength": 40, "RegexNormalizedSignature": "AzFu", "Id": "SEC101/158", @@ -43,7 +43,7 @@ "ChecksumSeeds": [ 5869709231681187888 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "KeyLength": 39, "RegexNormalizedSignature": "AzSe", "Id": "SEC101/166", @@ -58,7 +58,7 @@ "Signatures": [ "AzSe" ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "KeyLength": 39, "RegexNormalizedSignature": "AzSe", "Id": "SEC101/167", @@ -74,7 +74,7 @@ 5506058963192262704, 5575859178286952496 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ARm[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ARm[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ARm", "KeyLength": 32, "EncodeForUrl": false, @@ -91,7 +91,7 @@ 5506058963192262704, 5575859178286952496 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+AEh[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+AEh[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+AEh", "KeyLength": 32, "EncodeForUrl": false, @@ -108,7 +108,7 @@ 5506058963192262704, 5575859178286952496 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ASb[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ASb[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ASb", "KeyLength": 32, "EncodeForUrl": false, @@ -123,7 +123,7 @@ "ChecksumSeeds": [ 5291540757367369776 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AIoT", "KeyLength": 32, "EncodeForUrl": false, @@ -138,7 +138,7 @@ "ChecksumSeeds": [ 4928475562238095408 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AIoT", "KeyLength": 32, "EncodeForUrl": false, @@ -153,7 +153,7 @@ "ChecksumSeeds": [ 4931568359632875568 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AIoT", "KeyLength": 32, "EncodeForUrl": false, @@ -169,7 +169,7 @@ 4928457935994778672 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ASt[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ASt[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ASt", "EncodeForUrl": false, "Id": "SEC101/152", @@ -190,7 +190,7 @@ 6014965720764854320 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}ACDb[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}ACDb[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "ACDb", "EncodeForUrl": false, "Id": "SEC101/160", @@ -205,7 +205,7 @@ 4711400055309086768 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ABa[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ABa[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ABa", "EncodeForUrl": false, "Id": "SEC101/163", @@ -220,7 +220,7 @@ 4858365246511342384 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+AMC[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+AMC[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+AMC", "EncodeForUrl": false, "Id": "SEC101/170", @@ -235,7 +235,7 @@ 6081388236577714224 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/181", @@ -250,7 +250,7 @@ 5291540757367369776 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/182", @@ -265,7 +265,7 @@ 5143520228578766896 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/183", @@ -280,7 +280,7 @@ 5145771916421312560 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/184", @@ -294,7 +294,7 @@ "ChecksumSeeds": [ 4711718922539446320 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AzCa[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AzCa[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AzCa", "KeyLength": 32, "EncodeForUrl": false, @@ -309,7 +309,7 @@ "ChecksumSeeds": [ 4702692889634567216 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{42}\\+ACR[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{42}\\+ACR[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "KeyLength": 39, "RegexNormalizedSignature": "\\+ACR", "EncodeForUrl": false, @@ -318,7 +318,7 @@ "DetectionMetadata": "Identifiable" }, { - "Pattern": "(?i)(^|[^a-z0-9])(?oy2[a-z2-7]{43})([^a-z0-9]|$)", + "Pattern": "(?i)(^|[^a-z0-9])(?Poy2[a-z2-7]{43})([^a-z0-9]|$)", "Id": "SEC101/031", "Name": "NuGetApiKey", "Signatures": [ @@ -328,28 +328,28 @@ "DetectionMetadata": "HighEntropy, FixedSignature, HighConfidence" }, { - "Pattern": "(?:[^2-7a-z]|^)(?[2-7a-z]{52})(?:[^2-7a-z]|$)", + "Pattern": "(?:[^2-7a-z]|^)(?P[2-7a-z]{52})(?:[^2-7a-z]|$)", "Id": "SEC101/102", "Name": "AdoLegacyPat", "Signatures": null, "DetectionMetadata": "HighEntropy, EmbeddedChecksum" }, { - "Pattern": "(?i)\\.documents\\.azure\\.com.+(?:^|[^0-9a-z\\/+])(?[0-9a-z\\/+]{86}==)(?:[^=]|$)", + "Pattern": "(?i)\\.documents\\.azure\\.com.+(?:^|[^0-9a-z\\/+])(?P[0-9a-z\\/+]{86}==)(?:[^=]|$)", "Id": "SEC101/104", "Name": "AzureCosmosDBLegacyCredentials", "Signatures": null, "DetectionMetadata": "ObsoleteFormat, HighEntropy" }, { - "Pattern": "(?i)(?:AccountName|StorageName|StorageAccount)\\s*=.+(?:Account|Storage)Key\\s*=\\s*(?[0-9a-z\\\\\\/+]{86}==)(?:[^=]|$)", + "Pattern": "(?i)(?:AccountName|StorageName|StorageAccount)\\s*=.+(?:Account|Storage)Key\\s*=\\s*(?P[0-9a-z\\\\\\/+]{86}==)(?:[^=]|$)", "Id": "SEC101/106", "Name": "AzureStorageAccountLegacyCredentials", "Signatures": null, "DetectionMetadata": "HighEntropy" }, { - "Pattern": "(?i)\\.servicebus\\.windows.+[^0-9a-z\\/+](?[0-9a-z\\/+]{43}=)(?:[^=]|$)", + "Pattern": "(?i)\\.servicebus\\.windows.+[^0-9a-z\\/+](?P[0-9a-z\\/+]{43}=)(?:[^=]|$)", "Id": "SEC101/105", "Name": "AzureMessageLegacyCredentials", "Signatures": [ @@ -358,7 +358,7 @@ "DetectionMetadata": "ObsoleteFormat, HighEntropy, MediumConfidence" }, { - "Pattern": "(?:^|[^0-9a-f\\-])(?dapi[0-9a-f\\-]{32,34})(?:[^0-9a-f\\-]|$)", + "Pattern": "(?:^|[^0-9a-f\\-])(?Pdapi[0-9a-f\\-]{32,34})(?:[^0-9a-f\\-]|$)", "Id": "SEC101/110", "Name": "AzureDatabricksPat", "Signatures": [ @@ -373,7 +373,7 @@ "ChecksumSeeds": [ 4928457935994778672 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AZEG[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AZEG[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AZEG", "KeyLength": 32, "EncodeForUrl": false, @@ -382,7 +382,7 @@ "DetectionMetadata": "Identifiable" }, { - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?npm_[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{36})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?Pnpm_[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{36})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", "Id": "SEC101/050", "Name": "NpmAuthorKey", "Signatures": [ @@ -391,7 +391,7 @@ "DetectionMetadata": "HighEntropy, FixedSignature, HighConfidence" }, { - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?secret_scanning_ab85fc6f8d7638cf1c11da812da308d43_[0-9A-Za-z]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?Psecret_scanning_ab85fc6f8d7638cf1c11da812da308d43_[0-9A-Za-z]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", "Id": "SEC101/565", "Name": "SecretScanningSampleToken", "Signatures": [ diff --git a/GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json b/GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json index 444577ed..ae61c0c0 100644 --- a/GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json +++ b/GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json @@ -11,7 +11,7 @@ "DetectionMetadata": "HighEntropy, MediumConfidence" }, { - "Pattern": "($|\\b)(ftps?|https?):\\/\\/(?[^:@\\/]+:[^:@?\\/]+)@", + "Pattern": "($|\\b)(ftps?|https?):\\/\\/(?P[^:@\\/]+:[^:@?\\/]+)@", "Id": "SEC101/127", "Name": "UrlCredentials", "Signatures": [ @@ -21,7 +21,7 @@ "DetectionMetadata": "MediumConfidence" }, { - "Pattern": "(?i)(?:^|[?;&])(?:dsas_secret|sig)=(?[0-9a-z\\/+%]{43,129}(?:=|%3d))", + "Pattern": "(?i)(?:^|[?;&])(?:dsas_secret|sig)=(?P[0-9a-z\\/+%]{43,129}(?:=|%3d))", "Id": "SEC101/060", "Name": "LooseSasSecret", "Signatures": [ @@ -31,7 +31,7 @@ "DetectionMetadata": "HighEntropy, MediumConfidence" }, { - "Pattern": "(?i)authorization:(\\s|%20)bearer(\\s|%20)(?[0-9a-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+\\/=]*)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+/=]|$)", + "Pattern": "(?i)authorization:(\\s|%20)bearer(\\s|%20)(?P[0-9a-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+\\/=]*)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+/=]|$)", "Id": "SEC101/061", "Name": "OAuth2BearerToken", "Signatures": null, diff --git a/src/Microsoft.Security.Utilities.Benchmarks/RE2RegexEngine.cs b/src/Microsoft.Security.Utilities.Benchmarks/RE2RegexEngine.cs index e9135933..22105e83 100644 --- a/src/Microsoft.Security.Utilities.Benchmarks/RE2RegexEngine.cs +++ b/src/Microsoft.Security.Utilities.Benchmarks/RE2RegexEngine.cs @@ -18,13 +18,9 @@ public class RE2RegexEngine : IRegexEngine #endif public IEnumerable Matches(string input, string pattern, RegexOptions options = RegexOptionsDefaults, TimeSpan timeout = default, string? captureGroup = null) { - foreach (FlexMatch flexMatch in RE2Regex.Instance.Matches(input, pattern, options, timeout, captureGroup)) + if (captureGroup == null) { - if (captureGroup != null) - { - yield return CachedDotNetRegex.Instance.Matches(input, pattern, options, timeout, captureGroup).First(); - } - else + foreach (FlexMatch flexMatch in RE2Regex.Instance.Matches(input, pattern, options, timeout, captureGroup)) { yield return new UniversalMatch { @@ -35,6 +31,28 @@ public IEnumerable Matches(string input, string pattern, RegexOp }; } } + else + { + if (Regex2.Matches(pattern, input, out List> matches, 256L * 1024L * 1024L)) + { + foreach (Dictionary match in matches) + { + FlexMatch flexMatch = match["0"]; + if (match.TryGetValue("refine", out FlexMatch refineMatch)) + { + flexMatch = refineMatch; + } + + yield return new UniversalMatch + { + Index = flexMatch.Index, + Length = flexMatch.Length, + Value = flexMatch.Value, + Success = flexMatch.Success + }; + } + } + } } } } diff --git a/src/Microsoft.Security.Utilities.Core/CachedDotNetRegex.cs b/src/Microsoft.Security.Utilities.Core/CachedDotNetRegex.cs index 704d4383..f1e8d754 100644 --- a/src/Microsoft.Security.Utilities.Core/CachedDotNetRegex.cs +++ b/src/Microsoft.Security.Utilities.Core/CachedDotNetRegex.cs @@ -27,6 +27,8 @@ private CachedDotNetRegex() public static Regex GetOrCreateRegex(string pattern, RegexOptions options) { + pattern = NormalizeGroupsPattern(pattern); + var key = (pattern, options); #if NET7_0_OR_GREATER return RegexCache.GetOrAdd(key, key => new Regex(key.Pattern, key.Options | RegexOptions.Compiled | RegexOptions.NonBacktracking)); @@ -35,6 +37,16 @@ public static Regex GetOrCreateRegex(string pattern, RegexOptions options) #endif } + internal static string NormalizeGroupsPattern(string pattern) + { + if (pattern.IndexOf("?P<") != -1) + { + return pattern.Replace("?P<", "?<"); + } + + return pattern; + } + public bool IsMatch(string input, string pattern, RegexOptions options = RegexDefaults.DefaultOptionsCaseSensitive, TimeSpan timeout = default, string captureGroup = null) { // Note: Instance Regex.IsMatch has no timeout overload. diff --git a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_060_LooseSasSecret.cs b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_060_LooseSasSecret.cs index 234ec9f0..3a4f54ce 100644 --- a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_060_LooseSasSecret.cs +++ b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_060_LooseSasSecret.cs @@ -13,7 +13,7 @@ public LooseSasSecret() Id = "SEC101/060"; Name = nameof(LooseSasSecret); DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.MediumConfidence; - Pattern = @$"(?i)(?:^|[?;&])(?:dsas_secret|sig)=(?[0-9a-z\/+%]{{43,129}}(?:=|%3d))"; + Pattern = @$"(?i)(?:^|[?;&])(?:dsas_secret|sig)=(?P[0-9a-z\/+%]{{43,129}}(?:=|%3d))"; Signatures = new HashSet(new[] { "sig=", "ret=" }); } diff --git a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_061_LooseOAuth2BearerToken.cs b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_061_LooseOAuth2BearerToken.cs index 9244edeb..c2d2dffe 100644 --- a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_061_LooseOAuth2BearerToken.cs +++ b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_061_LooseOAuth2BearerToken.cs @@ -14,7 +14,7 @@ public OAuth2BearerToken() DetectionMetadata = DetectionMetadata.LowConfidence; // https://datatracker.ietf.org/doc/html/rfc6750#section-2.1 - Pattern = @$"(?i)authorization:(\s|%20)bearer(\s|%20)(?[0-9a-z][{WellKnownRegexPatterns.UrlUnreserved}+\/=]*)([^{WellKnownRegexPatterns.UrlUnreserved}+/=]|$)"; + Pattern = @$"(?i)authorization:(\s|%20)bearer(\s|%20)(?P[0-9a-z][{WellKnownRegexPatterns.UrlUnreserved}+\/=]*)([^{WellKnownRegexPatterns.UrlUnreserved}+/=]|$)"; } public override IEnumerable GenerateTruePositiveExamples() diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKey.cs index 8010659f..cdcb5381 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKey.cs @@ -13,7 +13,7 @@ public abstract class Azure32ByteIdentifiableKey : IdentifiableKey { public override string Pattern { - get => @$"{WellKnownRegexPatterns.PrefixAllBase64}(?[{WellKnownRegexPatterns.Base64}]{{33}}{RegexNormalizedSignature}[A-P][{WellKnownRegexPatterns.Base64}]{{5}}=){WellKnownRegexPatterns.SuffixAllBase64}"; + get => @$"{WellKnownRegexPatterns.PrefixAllBase64}(?P[{WellKnownRegexPatterns.Base64}]{{33}}{RegexNormalizedSignature}[A-P][{WellKnownRegexPatterns.Base64}]{{5}}=){WellKnownRegexPatterns.SuffixAllBase64}"; protected set => base.Pattern = value; } } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKeys.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKeys.cs index 814038ed..240dee7b 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKeys.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKeys.cs @@ -17,7 +17,7 @@ internal sealed class Azure32ByteIdentifiableKeys : RegexPattern public Azure32ByteIdentifiableKeys() { Pattern = $@"{WellKnownRegexPatterns.PrefixAllBase64}" + - $@"(?[{WellKnownRegexPatterns.Base64}]{{33}}(AIoT|\+(ASb|AEh|ARm))[A-P][{WellKnownRegexPatterns.Base64}]{{5}}=)" + + $@"(?P[{WellKnownRegexPatterns.Base64}]{{33}}(AIoT|\+(ASb|AEh|ARm))[A-P][{WellKnownRegexPatterns.Base64}]{{5}}=)" + $@"{WellKnownRegexPatterns.SuffixAllBase64}"; RotationPeriod = TimeSpan.FromDays(365 * 2); diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKey.cs index a0319572..bb63a0da 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKey.cs @@ -15,7 +15,7 @@ public abstract class Azure64ByteIdentifiableKey : IdentifiableKey public override string Pattern { - get => $@"{WellKnownRegexPatterns.PrefixAllBase64}(?[{WellKnownRegexPatterns.Base64}]{{76}}{RegexNormalizedSignature}[{WellKnownRegexPatterns.Base64}]{{5}}[AQgw]==){WellKnownRegexPatterns.SuffixAllBase64}"; + get => $@"{WellKnownRegexPatterns.PrefixAllBase64}(?P[{WellKnownRegexPatterns.Base64}]{{76}}{RegexNormalizedSignature}[{WellKnownRegexPatterns.Base64}]{{5}}[AQgw]==){WellKnownRegexPatterns.SuffixAllBase64}"; protected set => base.Pattern = value; } } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKeys.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKeys.cs index c3d8bb6f..d7efe7eb 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKeys.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKeys.cs @@ -15,7 +15,7 @@ internal sealed class Azure64ByteIdentifiableKeys : RegexPattern public Azure64ByteIdentifiableKeys() { Pattern = $@"{WellKnownRegexPatterns.PrefixAllBase64}" + - $@"(?[{WellKnownRegexPatterns.Base64}]{{76}}(APIM|ACDb|\+(ABa|AMC|ASt))[{WellKnownRegexPatterns.Base64}]{{5}}[AQgw]==)" + + $@"(?P[{WellKnownRegexPatterns.Base64}]{{76}}(APIM|ACDb|\+(ABa|AMC|ASt))[{WellKnownRegexPatterns.Base64}]{{5}}[AQgw]==)" + $@"{WellKnownRegexPatterns.SuffixAllBase64}"; RotationPeriod = TimeSpan.FromDays(365 * 2); diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_031_NuGetApiKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_031_NuGetApiKey.cs index 46899916..5b93fbe7 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_031_NuGetApiKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_031_NuGetApiKey.cs @@ -19,7 +19,7 @@ public NuGetApiKey() // This is the ApiKeyV4 format implemented here: // https://github.com/NuGet/NuGetGallery/blob/main/src/NuGetGallery.Services/Authentication/ApiKeyV4.cs - Pattern = "(?i)(^|[^a-z0-9])(?oy2[a-z2-7]{43})([^a-z0-9]|$)"; + Pattern = "(?i)(^|[^a-z0-9])(?Poy2[a-z2-7]{43})([^a-z0-9]|$)"; Signatures = new HashSet(new[] { "oy2", "OY2" }); } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_050_NpmAuthorKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_050_NpmAuthorKey.cs index ded6cfcb..b7af439f 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_050_NpmAuthorKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_050_NpmAuthorKey.cs @@ -12,7 +12,7 @@ public NpmAuthorKey() Id = "SEC101/050"; Name = nameof(NpmAuthorKey); DetectionMetadata = DetectionMetadata.FixedSignature | DetectionMetadata.HighEntropy | DetectionMetadata.HighConfidence; - Pattern = @$"{WellKnownRegexPatterns.PrefixBase62}(?npm_[{WellKnownRegexPatterns.Base62}]{{36}}){WellKnownRegexPatterns.SuffixBase62}"; + Pattern = @$"{WellKnownRegexPatterns.PrefixBase62}(?Pnpm_[{WellKnownRegexPatterns.Base62}]{{36}}){WellKnownRegexPatterns.SuffixBase62}"; Signatures = new HashSet(new[] { "npm_" }); } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_102_AdoPat.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_102_AdoPat.cs index 4558b273..dd05cfee 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_102_AdoPat.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_102_AdoPat.cs @@ -14,7 +14,7 @@ public AdoLegacyPat() Id = "SEC101/102"; Name = nameof(AdoLegacyPat); DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.EmbeddedChecksum; - Pattern = "(?:[^2-7a-z]|^)(?[2-7a-z]{52})(?:[^2-7a-z]|$)"; + Pattern = "(?:[^2-7a-z]|^)(?P[2-7a-z]{52})(?:[^2-7a-z]|$)"; } public override Tuple GetMatchIdAndName(string match) diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_104_AzureCosmosDBLegacyCredentials.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_104_AzureCosmosDBLegacyCredentials.cs index 2fda7d20..ab9aa707 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_104_AzureCosmosDBLegacyCredentials.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_104_AzureCosmosDBLegacyCredentials.cs @@ -12,7 +12,7 @@ public AzureCosmosDBLegacyCredentials() Id = "SEC101/104"; Name = nameof(AzureCosmosDBLegacyCredentials); DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.ObsoleteFormat; - Pattern = "(?i)\\.documents\\.azure\\.com.+(?:^|[^0-9a-z\\/+])(?[0-9a-z\\/+]{86}==)(?:[^=]|$)"; + Pattern = "(?i)\\.documents\\.azure\\.com.+(?:^|[^0-9a-z\\/+])(?P[0-9a-z\\/+]{86}==)(?:[^=]|$)"; } public override Tuple GetMatchIdAndName(string match) diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_105_AzureMessagingLegacyCredentials.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_105_AzureMessagingLegacyCredentials.cs index d9ae2ad8..4d00d463 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_105_AzureMessagingLegacyCredentials.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_105_AzureMessagingLegacyCredentials.cs @@ -13,7 +13,7 @@ public AzureMessageLegacyCredentials() Id = "SEC101/105"; Name = nameof(AzureMessageLegacyCredentials); DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.ObsoleteFormat | DetectionMetadata.MediumConfidence; - Pattern = "(?i)\\.servicebus\\.windows.+[^0-9a-z\\/+](?[0-9a-z\\/+]{43}=)(?:[^=]|$)"; + Pattern = "(?i)\\.servicebus\\.windows.+[^0-9a-z\\/+](?P[0-9a-z\\/+]{43}=)(?:[^=]|$)"; Signatures = new HashSet(new[] { ".servicebus" }); } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_106_AzureStorageAccountLegacyCredentials.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_106_AzureStorageAccountLegacyCredentials.cs index 829932af..103a3b0f 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_106_AzureStorageAccountLegacyCredentials.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_106_AzureStorageAccountLegacyCredentials.cs @@ -14,7 +14,7 @@ public AzureStorageAccountLegacyCredentials() Id = "SEC101/106"; Name = nameof(AzureStorageAccountLegacyCredentials); DetectionMetadata = DetectionMetadata.HighEntropy; - Pattern = "(?i)(?:AccountName|StorageName|StorageAccount)\\s*=.+(?:Account|Storage)Key\\s*=\\s*(?[0-9a-z\\\\\\/+]{86}==)(?:[^=]|$)"; + Pattern = "(?i)(?:AccountName|StorageName|StorageAccount)\\s*=.+(?:Account|Storage)Key\\s*=\\s*(?P[0-9a-z\\\\\\/+]{86}==)(?:[^=]|$)"; } public override Tuple GetMatchIdAndName(string match) diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_110.AzureDatabricksPat.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_110.AzureDatabricksPat.cs index 9f79c6c3..e1441fbb 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_110.AzureDatabricksPat.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_110.AzureDatabricksPat.cs @@ -12,7 +12,7 @@ public AzureDatabricksPat() Id = "SEC101/110"; Name = nameof(AzureDatabricksPat); DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.HighConfidence; - Pattern = $"(?:^|[^0-9a-f\\-])(?dapi[0-9a-f\\-]{{32,34}})(?:[^0-9a-f\\-]|$)"; + Pattern = $"(?:^|[^0-9a-f\\-])(?Pdapi[0-9a-f\\-]{{32,34}})(?:[^0-9a-f\\-]|$)"; Signatures = new HashSet(new[] { "dapi" }); } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_156_AzureClientAppIdentifiableCredentials.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_156_AzureClientAppIdentifiableCredentials.cs index 0f28183e..3fbe3ce9 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_156_AzureClientAppIdentifiableCredentials.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_156_AzureClientAppIdentifiableCredentials.cs @@ -12,7 +12,7 @@ public AadClientAppIdentifiableCredentials() Id = "SEC101/156"; Name = "AadClientAppIdentifiableCredentials"; DetectionMetadata = DetectionMetadata.Identifiable; - Pattern = $"{WellKnownRegexPatterns.PrefixUrlUnreserved}(?[{WellKnownRegexPatterns.RegexEncodedUrlUnreserved}]{{3}}(7|8)Q~[{WellKnownRegexPatterns.RegexEncodedUrlUnreserved}]{{31,34}}){WellKnownRegexPatterns.SuffixUrlUnreserved}"; + Pattern = $"{WellKnownRegexPatterns.PrefixUrlUnreserved}(?P[{WellKnownRegexPatterns.RegexEncodedUrlUnreserved}]{{3}}(7|8)Q~[{WellKnownRegexPatterns.RegexEncodedUrlUnreserved}]{{31,34}}){WellKnownRegexPatterns.SuffixUrlUnreserved}"; Signatures = new HashSet(new[] { "8Q~", "7Q~" }); } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_158_AzureFunctionIdentifiableKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_158_AzureFunctionIdentifiableKey.cs index 6354e9b8..f21fb732 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_158_AzureFunctionIdentifiableKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_158_AzureFunctionIdentifiableKey.cs @@ -27,7 +27,7 @@ public AzureFunctionIdentifiableKey() public override string Pattern { - get => @$"{WellKnownRegexPatterns.PrefixUrlSafeBase64}(?[{WellKnownRegexPatterns.RegexEncodedUrlSafeBase64}]{{44}}{Signatures!.First()}[{WellKnownRegexPatterns.RegexEncodedUrlSafeBase64}]{{5}}[AQgw]==){WellKnownRegexPatterns.SuffixUrlSafeBase64}"; + get => @$"{WellKnownRegexPatterns.PrefixUrlSafeBase64}(?P[{WellKnownRegexPatterns.RegexEncodedUrlSafeBase64}]{{44}}{Signatures!.First()}[{WellKnownRegexPatterns.RegexEncodedUrlSafeBase64}]{{5}}[AQgw]==){WellKnownRegexPatterns.SuffixUrlSafeBase64}"; protected set => base.Pattern = value; } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_166_AzureSearchIdentifiableQueryKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_166_AzureSearchIdentifiableQueryKey.cs index c562f176..bf012138 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_166_AzureSearchIdentifiableQueryKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_166_AzureSearchIdentifiableQueryKey.cs @@ -22,7 +22,7 @@ public AzureSearchIdentifiableQueryKey() public override string Pattern { - get => @$"{WellKnownRegexPatterns.PrefixAllBase64}(?[{WellKnownRegexPatterns.Base62}]{{42}}{Signatures!.First()}[A-D][{WellKnownRegexPatterns.Base62}]{{5}}){WellKnownRegexPatterns.SuffixAllBase64}"; + get => @$"{WellKnownRegexPatterns.PrefixAllBase64}(?P[{WellKnownRegexPatterns.Base62}]{{42}}{Signatures!.First()}[A-D][{WellKnownRegexPatterns.Base62}]{{5}}){WellKnownRegexPatterns.SuffixAllBase64}"; protected set => base.Pattern = value; } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_176_AzureContainerRegistryIdentifiableKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_176_AzureContainerRegistryIdentifiableKey.cs index 0f390c05..0712aee3 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_176_AzureContainerRegistryIdentifiableKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_176_AzureContainerRegistryIdentifiableKey.cs @@ -19,7 +19,7 @@ public AzureContainerRegistryIdentifiableKey() public override string Pattern { - get => @$"{WellKnownRegexPatterns.PrefixAllBase64}(?[{WellKnownRegexPatterns.Base64}]{{42}}{RegexNormalizedSignature}[A-D][{WellKnownRegexPatterns.Base64}]{{5}}){WellKnownRegexPatterns.SuffixAllBase64}"; + get => @$"{WellKnownRegexPatterns.PrefixAllBase64}(?P[{WellKnownRegexPatterns.Base64}]{{42}}{RegexNormalizedSignature}[A-D][{WellKnownRegexPatterns.Base64}]{{5}}){WellKnownRegexPatterns.SuffixAllBase64}"; protected set => base.Pattern = value; } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_200_CommonAnnotatedSecurityKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_200_CommonAnnotatedSecurityKey.cs index c2890cf5..9550f5b5 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_200_CommonAnnotatedSecurityKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_200_CommonAnnotatedSecurityKey.cs @@ -13,7 +13,7 @@ public CommonAnnotatedSecurityKey() Id = "SEC101/200"; Name = nameof(CommonAnnotatedSecurityKey); DetectionMetadata = DetectionMetadata.Identifiable; - Pattern = $"{WellKnownRegexPatterns.PrefixBase62}(?[{WellKnownRegexPatterns.Base62}]{{52}}JQQJ9(?:9|D|H)[{WellKnownRegexPatterns.Base62}][A-L][{WellKnownRegexPatterns.Base62}]{{16}}[A-Za-z][{WellKnownRegexPatterns.Base62}]{{7}}(?:[{WellKnownRegexPatterns.Base62}]{{2}}==)?)"; + Pattern = $"{WellKnownRegexPatterns.PrefixBase62}(?P[{WellKnownRegexPatterns.Base62}]{{52}}JQQJ9(?:9|D|H)[{WellKnownRegexPatterns.Base62}][A-L][{WellKnownRegexPatterns.Base62}]{{16}}[A-Za-z][{WellKnownRegexPatterns.Base62}]{{7}}(?:[{WellKnownRegexPatterns.Base62}]{{2}}==)?)"; Signatures = "JQQJ9".ToSet(); } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_565_SecretScanningSampleToken.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_565_SecretScanningSampleToken.cs index c8d90f8c..e2d4e839 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_565_SecretScanningSampleToken.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_565_SecretScanningSampleToken.cs @@ -18,7 +18,7 @@ public SecretScanningSampleToken() Id = "SEC101/565"; Name = nameof(SecretScanningSampleToken); DetectionMetadata = DetectionMetadata.FixedSignature | DetectionMetadata.HighEntropy | DetectionMetadata.HighConfidence; - Pattern = @$"{WellKnownRegexPatterns.PrefixBase62}(?secret_scanning_ab85fc6f8d7638cf1c11da812da308d43_[0-9A-Za-z]{{5}}){WellKnownRegexPatterns.SuffixBase62}"; + Pattern = @$"{WellKnownRegexPatterns.PrefixBase62}(?Psecret_scanning_ab85fc6f8d7638cf1c11da812da308d43_[0-9A-Za-z]{{5}}){WellKnownRegexPatterns.SuffixBase62}"; Signatures = "ab85".ToSet(); } diff --git a/src/Microsoft.Security.Utilities.Core/UrlCredentials.cs b/src/Microsoft.Security.Utilities.Core/UrlCredentials.cs index c346faf0..9c958f5d 100644 --- a/src/Microsoft.Security.Utilities.Core/UrlCredentials.cs +++ b/src/Microsoft.Security.Utilities.Core/UrlCredentials.cs @@ -14,7 +14,7 @@ public UrlCredentials() Name = nameof(UrlCredentials); - Pattern = @"($|\b)(ftps?|https?):\/\/(?[^:@\/]+:[^:@?\/]+)@"; + Pattern = @"($|\b)(ftps?|https?):\/\/(?P[^:@\/]+:[^:@?\/]+)@"; DetectionMetadata = DetectionMetadata.MediumConfidence; diff --git a/src/Tests.Microsoft.Security.Utilities.Core/AdoLegacyPatTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/AdoLegacyPatTests.cs index 1a49e557..1f9288f4 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/AdoLegacyPatTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/AdoLegacyPatTests.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Diagnostics.CodeAnalysis; - using Microsoft.VisualStudio.TestTools.UnitTesting; namespace Microsoft.Security.Utilities.Core diff --git a/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs index 68b3faa7..7b578bb9 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs @@ -2,30 +2,27 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.RE2.Managed; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using System; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; +using System; using System.Text.RegularExpressions; namespace Microsoft.Security.Utilities { - [ExcludeFromCodeCoverage] public class RE2RegexEngine : IRegexEngine { public static IRegexEngine Instance = new RE2RegexEngine(); - public IEnumerable Matches(string input, string pattern, RegexOptions options = RegexOptions.ExplicitCapture | RegexOptions.Compiled | RegexOptions.CultureInvariant, TimeSpan timeout = default, string captureGroup = null) +#if NET7_0_OR_GREATER + public const RegexOptions RegexOptionsDefaults = RegexOptions.ExplicitCapture | RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.NonBacktracking; +#else + public const RegexOptions RegexOptionsDefaults = RegexOptions.ExplicitCapture | RegexOptions.Compiled | RegexOptions.CultureInvariant; +#endif + public IEnumerable Matches(string input, string pattern, RegexOptions options = RegexOptionsDefaults, TimeSpan timeout = default, string? captureGroup = null) { - foreach (FlexMatch flexMatch in RE2Regex.Instance.Matches(input, pattern, options, timeout, captureGroup)) + if (captureGroup == null) { - if (captureGroup != null) - { - yield return CachedDotNetRegex.Instance.Matches(input, pattern, options, timeout, captureGroup).First(); - } - else + foreach (FlexMatch flexMatch in RE2Regex.Instance.Matches(input, pattern, options, timeout, captureGroup)) { yield return new UniversalMatch { @@ -36,6 +33,28 @@ public IEnumerable Matches(string input, string pattern, RegexOp }; } } + else + { + if (Regex2.Matches(pattern, input, out List> matches, 256L * 1024L * 1024L)) + { + foreach (Dictionary match in matches) + { + FlexMatch flexMatch = match["0"]; + if (match.TryGetValue("refine", out FlexMatch refineMatch)) + { + flexMatch = refineMatch; + } + + yield return new UniversalMatch + { + Index = flexMatch.Index, + Length = flexMatch.Length, + Value = flexMatch.Value, + Success = flexMatch.Success + }; + } + } + } } } } diff --git a/src/Tests.Microsoft.Security.Utilities.Core/RegexPatternTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/RegexPatternTests.cs index d6a4c3ed..81786a74 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/RegexPatternTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/RegexPatternTests.cs @@ -383,7 +383,7 @@ public void RegexPatterns_GetDetections_ReturnsEmpty_WhenNoMatchesExist() public void RegexPatterns_GetDetections_Returns_RefinedDetection() { // Arrange - var secret = new RegexPattern(Id, Name, DetectionMetadata.Identifiable, "a(?b)c"); + var secret = new RegexPattern(Id, Name, DetectionMetadata.Identifiable, "a(?Pb)c"); var input = "abc"; var match = "b"; diff --git a/src/Tests.Microsoft.Security.Utilities.Core/SecretMaskerTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/SecretMaskerTests.cs index cda84513..4e9c0d6b 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/SecretMaskerTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/SecretMaskerTests.cs @@ -157,13 +157,14 @@ public void SecretMasker_LowConfidenceSecurityModels_Masking() private void ValidateSecurityModelsMasking(IEnumerable patterns, IRegexEngine engine, bool lowEntropyModels) { + using var assertionScope = new AssertionScope(); + // These tests generate randomized values. It may be useful to // bump up the # of iterations on an ad hoc basis to flush // out non-deterministic failures (typically based on the // characters chosen from the secret alphabet for the pattern). for (int i = 0; i < 1; i++) { - using var assertionScope = new AssertionScope(); foreach (IRegexEngine regexEngine in new[] { RE2RegexEngine.Instance, CachedDotNetRegex.Instance }) { foreach (bool generateCrossCompanyCorrelatingIds in new[] { true, false }) diff --git a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs index 9c2b54a4..45bfed73 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs @@ -81,9 +81,13 @@ public void WellKnownRegexPatterns_GetMatchMonikerHardenedForOutOfOrderExecution var patterns = new List(); + /* patterns.AddRange(WellKnownRegexPatterns.DataClassification); patterns.AddRange(WellKnownRegexPatterns.PreciselyClassifiedSecurityKeys); patterns.AddRange(WellKnownRegexPatterns.UnclassifiedPotentialSecurityKeys); + */ + + patterns.Add(new AzureFunctionIdentifiableKey()); var masker = new SecretMasker(patterns, generateCorrelatingIds: true, @@ -303,6 +307,10 @@ public void WellKnownRegexPatterns_EnsureAllPatternsHaveCorrectCaptureGroups() groupNames.Length.Should().Be(1, because: $"Pattern '{pattern.GetType().Name}' should not have more than one capture group"); groupNames[0].Should().Be("refine", because: $"Pattern '{pattern.GetType().Name}' capture group should be named 'refine'"); + + pattern.Pattern.Should() + .Contain("(?P", + because: $"Pattern '{pattern.GetType().Name}' should contain an RE2-compatible '(?P...)' named capture group"); } } } From 635fc0dcf4ea070e376507d7abbf799f4106a468 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Tue, 4 Mar 2025 11:46:42 -0800 Subject: [PATCH 06/20] Update release notes --- docs/ReleaseHistory.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/ReleaseHistory.md b/docs/ReleaseHistory.md index b520b929..39dfda06 100644 --- a/docs/ReleaseHistory.md +++ b/docs/ReleaseHistory.md @@ -11,11 +11,12 @@ - FPS => False positive reduction in static analysis. - FNS => False negative reduction in static analysis. -# 1.16.0 - 03/03/2025 +# 1.16.0 - 03/04/2025 - BRK: Eliminate `SEC000/101.Unclassified32CharacterString` as noisy and not useful. - BRK: Rename `SEC101/102.AdoPat` friendly name to `AdoLegacyPat`. - BUG: Correct `SEC000/002.Unclassified16ByteHexadecimalString` id and rule name on calling `GetMatchIdAndName` (where `SEC000/001.Unclassified64ByteBase64String` was returned incorrectly before). - BUG: Resolve `System.FormatException: The input is not a valid Base-46 string` errors calling `SEC101/102.AdoPat.GetMatchIdAndName` by swallowing correct exception kind `ArgumentException` in `IsChecksumValid` helper. +- BUG: `?P` is now used throughout for named captures as this is required currently for RE2 compatibility. This reverts a clean-up in this area (that broke RE2) in 1.15.0. # 1.15.0 - 03/03/2025 - BRK: Regular expression syntax has been standardized in JSON to conform to how the overwhelming majority of patterns were already defined. From 5d2244eb419ef915e76c4f1aace94e634501d591 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Tue, 4 Mar 2025 12:15:02 -0800 Subject: [PATCH 07/20] Add test file. --- .../AzureFunctionIdentifiableKeyTests.cs | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 src/Tests.Microsoft.Security.Utilities.Core/AzureFunctionIdentifiableKeyTests.cs diff --git a/src/Tests.Microsoft.Security.Utilities.Core/AzureFunctionIdentifiableKeyTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/AzureFunctionIdentifiableKeyTests.cs new file mode 100644 index 00000000..512d5604 --- /dev/null +++ b/src/Tests.Microsoft.Security.Utilities.Core/AzureFunctionIdentifiableKeyTests.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using FluentAssertions; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using System.Linq; + + +namespace Microsoft.Security.Utilities.Core +{ + [TestClass] + public class AzureFunctionIdentifiableKeyTests + { + [TestMethod] + public void AzureFunctionIdentifiableKey_GenerateTruePositiveExamples() + { + var classifier = new AzureFunctionIdentifiableKey(); + + foreach (string example in classifier.GenerateTruePositiveExamples()) + { + var masker = new SecretMasker([classifier]); + var detection = masker.DetectSecrets(example).FirstOrDefault(); + detection.Should().NotBe(default); + + string refined = example.Substring(detection.Start, detection.Length); + var result = classifier.GetMatchIdAndName(refined); + result.Should().NotBeNull(); + } + } + } +} + + From 05cb6da671c1ff079272d946589afaffa574b025 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Tue, 4 Mar 2025 13:26:21 -0800 Subject: [PATCH 08/20] Add named group test. --- .../RE2RegexEngineTests.cs | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngineTests.cs diff --git a/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngineTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngineTests.cs new file mode 100644 index 00000000..94b818f7 --- /dev/null +++ b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngineTests.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using FluentAssertions; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using System.Linq; + +namespace Microsoft.Security.Utilities.Core +{ + [TestClass] + public class RE2RegexEngineTests + { + [TestMethod] + public void RE2RegexEngine_RefineGroup() + { + // Scan data has a discrete component with + // a leading and trailing 'x' character. + string scanData = $"x{nameof(scanData)}x"; + + // The regex pattern defines a named group to extract + // the scan data from between its encapsulating chars. + string regex = $"x(?P{nameof(scanData)})x"; + + var regexPattern = new RegexPattern(id: "1001", name: "MyRule", DetectionMetadata.None, regex); + var masker = new SecretMasker([regexPattern], regexEngine: RE2RegexEngine.Instance); + + var detection = masker.DetectSecrets(scanData).FirstOrDefault(); + detection.Should().NotBe(default); + + string refined = scanData.Substring(detection.Start, detection.Length); + refined.Should().Be(nameof(scanData)); + + // Run the same test directly against the engine. + var re2RegexEngine = new RE2RegexEngine(); + UniversalMatch flexMatch = re2RegexEngine.Matches(scanData, regex, captureGroup: "refine").FirstOrDefault(); + flexMatch.Should().NotBeNull(); + flexMatch.Value.Should().Be(nameof(scanData)); + } + } +} From 12eb88c712cb00d75e6de00241eaff8655877d59 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Tue, 4 Mar 2025 13:47:08 -0800 Subject: [PATCH 09/20] PR feedback --- .../Microsoft.Security.Utilities.Benchmarks.csproj | 1 + .../CachedDotNetRegex.cs | 13 +++---------- .../RE2RegexEngine.cs | 4 ++-- .../Tests.Microsoft.Security.Utilities.Core.csproj | 4 ++-- .../WellKnownRegexPatternsTests.cs | 12 +++++++----- 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/Microsoft.Security.Utilities.Benchmarks/Microsoft.Security.Utilities.Benchmarks.csproj b/src/Microsoft.Security.Utilities.Benchmarks/Microsoft.Security.Utilities.Benchmarks.csproj index 32d8b01a..17f0b048 100644 --- a/src/Microsoft.Security.Utilities.Benchmarks/Microsoft.Security.Utilities.Benchmarks.csproj +++ b/src/Microsoft.Security.Utilities.Benchmarks/Microsoft.Security.Utilities.Benchmarks.csproj @@ -14,6 +14,7 @@ + diff --git a/src/Microsoft.Security.Utilities.Core/CachedDotNetRegex.cs b/src/Microsoft.Security.Utilities.Core/CachedDotNetRegex.cs index f1e8d754..07940fae 100644 --- a/src/Microsoft.Security.Utilities.Core/CachedDotNetRegex.cs +++ b/src/Microsoft.Security.Utilities.Core/CachedDotNetRegex.cs @@ -27,24 +27,17 @@ private CachedDotNetRegex() public static Regex GetOrCreateRegex(string pattern, RegexOptions options) { - pattern = NormalizeGroupsPattern(pattern); - var key = (pattern, options); #if NET7_0_OR_GREATER - return RegexCache.GetOrAdd(key, key => new Regex(key.Pattern, key.Options | RegexOptions.Compiled | RegexOptions.NonBacktracking)); + return RegexCache.GetOrAdd(key, key => new Regex(NormalizeGroupsPattern(key.Pattern), key.Options | RegexOptions.Compiled | RegexOptions.NonBacktracking)); #else - return RegexCache.GetOrAdd(key, key => new Regex(key.Pattern, key.Options | RegexOptions.Compiled)); + return RegexCache.GetOrAdd(key, key => new Regex(NormalizeGroupsPattern(key.Pattern), key.Options | RegexOptions.Compiled)); #endif } internal static string NormalizeGroupsPattern(string pattern) { - if (pattern.IndexOf("?P<") != -1) - { - return pattern.Replace("?P<", "?<"); - } - - return pattern; + return pattern.Replace("?P<", "?<"); } public bool IsMatch(string input, string pattern, RegexOptions options = RegexDefaults.DefaultOptionsCaseSensitive, TimeSpan timeout = default, string captureGroup = null) diff --git a/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs index 7b578bb9..1675efc9 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs @@ -3,8 +3,8 @@ using Microsoft.RE2.Managed; -using System.Collections.Generic; using System; +using System.Collections.Generic; using System.Text.RegularExpressions; namespace Microsoft.Security.Utilities @@ -40,7 +40,7 @@ public IEnumerable Matches(string input, string pattern, RegexOp foreach (Dictionary match in matches) { FlexMatch flexMatch = match["0"]; - if (match.TryGetValue("refine", out FlexMatch refineMatch)) + if (match.TryGetValue(captureGroup, out FlexMatch refineMatch)) { flexMatch = refineMatch; } diff --git a/src/Tests.Microsoft.Security.Utilities.Core/Tests.Microsoft.Security.Utilities.Core.csproj b/src/Tests.Microsoft.Security.Utilities.Core/Tests.Microsoft.Security.Utilities.Core.csproj index 7619c0bc..443c08f1 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/Tests.Microsoft.Security.Utilities.Core.csproj +++ b/src/Tests.Microsoft.Security.Utilities.Core/Tests.Microsoft.Security.Utilities.Core.csproj @@ -14,8 +14,8 @@ - - + + diff --git a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs index 45bfed73..333d6013 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs @@ -33,17 +33,19 @@ public void WellKnownRegexPatterns_AllRuleIdsAndNamesAreUnique() var patterns = GetAllPatterns(); - HashSet ruleIds = new HashSet(StringComparer.OrdinalIgnoreCase); - HashSet ruleNames = new HashSet(StringComparer.OrdinalIgnoreCase); + HashSet ruleIdsObserved = new HashSet(StringComparer.OrdinalIgnoreCase); + HashSet ruleNamesObserved = new HashSet(StringComparer.OrdinalIgnoreCase); - bool result; foreach (RegexPattern pattern in patterns) { - result = ruleIds.Contains(pattern.Id); + bool result = ruleIdsObserved.Contains(pattern.Id); result.Should().BeFalse(because: $"Pattern '{pattern.GetType().Name}' should not share its Id with another rule: '{pattern.Id}'"); - result = ruleNames.Contains(pattern.Name); + result = ruleNamesObserved.Contains(pattern.Name); result.Should().BeFalse(because: $"Pattern '{pattern.GetType().Name}' should not share its Name with another rule: '{pattern.Name}'"); + + ruleIdsObserved.Add(pattern.Id); + ruleNamesObserved.Add(pattern.Name); } } From 135f2f0c2f16d3480ce7cc25ccb887e77827fe30 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Tue, 4 Mar 2025 14:14:43 -0800 Subject: [PATCH 10/20] Remove duplicated class source --- .../RE2RegexEngine.cs | 58 ------------------- 1 file changed, 58 deletions(-) delete mode 100644 src/Microsoft.Security.Utilities.Benchmarks/RE2RegexEngine.cs diff --git a/src/Microsoft.Security.Utilities.Benchmarks/RE2RegexEngine.cs b/src/Microsoft.Security.Utilities.Benchmarks/RE2RegexEngine.cs deleted file mode 100644 index 22105e83..00000000 --- a/src/Microsoft.Security.Utilities.Benchmarks/RE2RegexEngine.cs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using Microsoft.RE2.Managed; - -using System.Text.RegularExpressions; - -namespace Microsoft.Security.Utilities -{ - public class RE2RegexEngine : IRegexEngine - { - public static IRegexEngine Instance = new RE2RegexEngine(); - -#if NET7_0_OR_GREATER - public const RegexOptions RegexOptionsDefaults = RegexOptions.ExplicitCapture | RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.NonBacktracking; -#else - public const RegexOptions RegexOptionsDefaults = RegexOptions.ExplicitCapture | RegexOptions.Compiled | RegexOptions.CultureInvariant; -#endif - public IEnumerable Matches(string input, string pattern, RegexOptions options = RegexOptionsDefaults, TimeSpan timeout = default, string? captureGroup = null) - { - if (captureGroup == null) - { - foreach (FlexMatch flexMatch in RE2Regex.Instance.Matches(input, pattern, options, timeout, captureGroup)) - { - yield return new UniversalMatch - { - Index = flexMatch.Index, - Length = flexMatch.Length, - Value = flexMatch.Value, - Success = flexMatch.Success - }; - } - } - else - { - if (Regex2.Matches(pattern, input, out List> matches, 256L * 1024L * 1024L)) - { - foreach (Dictionary match in matches) - { - FlexMatch flexMatch = match["0"]; - if (match.TryGetValue("refine", out FlexMatch refineMatch)) - { - flexMatch = refineMatch; - } - - yield return new UniversalMatch - { - Index = flexMatch.Index, - Length = flexMatch.Length, - Value = flexMatch.Value, - Success = flexMatch.Success - }; - } - } - } - } - } -} From 32ac22e2c9aba172b8ca904837e41b471e00abcd Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Tue, 4 Mar 2025 15:36:02 -0800 Subject: [PATCH 11/20] Restore RE2 compatible named group for CASK rule --- .../SEC101_200_CommonAnnotatedSecurityKey.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_200_CommonAnnotatedSecurityKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_200_CommonAnnotatedSecurityKey.cs index e9f289bd..3f59834f 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_200_CommonAnnotatedSecurityKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_200_CommonAnnotatedSecurityKey.cs @@ -17,7 +17,7 @@ public CommonAnnotatedSecurityKey() Id = "SEC101/200"; Name = nameof(CommonAnnotatedSecurityKey); DetectionMetadata = DetectionMetadata.Identifiable; - Pattern = $"{WellKnownRegexPatterns.PrefixBase62}(?[{WellKnownRegexPatterns.Base62}]{{52}}JQQJ9(?:9|D|H)[{WellKnownRegexPatterns.Base62}][A-L][{WellKnownRegexPatterns.Base62}]{{16}}[A-Za-z][{WellKnownRegexPatterns.Base62}]{{7}}(?:[{WellKnownRegexPatterns.Base62}]{{2}}==)?)"; + Pattern = $"{WellKnownRegexPatterns.PrefixBase62}(?P[{WellKnownRegexPatterns.Base62}]{{52}}JQQJ9(?:9|D|H)[{WellKnownRegexPatterns.Base62}][A-L][{WellKnownRegexPatterns.Base62}]{{16}}[A-Za-z][{WellKnownRegexPatterns.Base62}]{{7}}(?:[{WellKnownRegexPatterns.Base62}]{{2}}==)?)"; Signatures = Signature.ToSet(); } From 1ce797078aace6a8899451614d7380bc20560489 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Tue, 4 Mar 2025 15:38:36 -0800 Subject: [PATCH 12/20] Fix release notes --- docs/ReleaseHistory.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/ReleaseHistory.md b/docs/ReleaseHistory.md index ad8d430d..5d2051ee 100644 --- a/docs/ReleaseHistory.md +++ b/docs/ReleaseHistory.md @@ -11,8 +11,9 @@ - FPS => False positive reduction in static analysis. - FNS => False negative reduction in static analysis. -# UNRELEASED - +# 1.16.0 - 03/05/2025 +- BRK: Eliminate `SEC000/101.Unclassified32CharacterString` as noisy and not useful. +- BRK: Rename `SEC101/102.AdoPat` friendly name to `AdoLegacyPat`. - BRK: `IdentifiableScan` no longer supports stream input. The following API are removed. Use `IdentifiableScan.DetectSecrets(string)`. - `IdentifiableScan.DetectSecrets(Stream)` - `IdentifiableScan.Start` @@ -22,10 +23,6 @@ - `IdentifiableScan.CheckPossibleMatchRange` - PRF: `IdentifiableScan` did not use high-performance scanning techniques for `SEC101/178.AzureIotHubIdentifiableKey` and `SEC101/200.CommonAnnotatedSecurityKey`. A bug triggered fallback to slower scanning due to incorrect signatures being used. - PRF: `IdentifiableScan` now implements high-performance scanning techniques in managed code. The performance has been found to be significantly better than the prior implementation via rust interop. This also reduces the size of the NuGet package size by a factor of 34 from 6.8 MB to 200 KB and adds support for non x86/x64 CPUs and non-Windows OSes. - -# 1.16.0 - 03/04/2025 -- BRK: Eliminate `SEC000/101.Unclassified32CharacterString` as noisy and not useful. -- BRK: Rename `SEC101/102.AdoPat` friendly name to `AdoLegacyPat`. - BUG: Correct `SEC000/002.Unclassified16ByteHexadecimalString` id and rule name on calling `GetMatchIdAndName` (where `SEC000/001.Unclassified64ByteBase64String` was returned incorrectly before). - BUG: Resolve `System.FormatException: The input is not a valid Base-46 string` errors calling `SEC101/102.AdoPat.GetMatchIdAndName` by swallowing correct exception kind `ArgumentException` in `IsChecksumValid` helper. - BUG: `?P` is now used throughout for named captures as this is required currently for RE2 compatibility. This reverts a clean-up in this area (that broke RE2) in 1.15.0. From 7b1e144e4967946db7f23131a9146cf086711bde Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Tue, 4 Mar 2025 15:45:56 -0800 Subject: [PATCH 13/20] Restore commented test --- .../WellKnownRegexPatternsTests.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs index 333d6013..8d13bcb7 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs @@ -82,12 +82,10 @@ public void WellKnownRegexPatterns_GetMatchMonikerHardenedForOutOfOrderExecution using var assertionScope = new AssertionScope(); var patterns = new List(); - - /* + patterns.AddRange(WellKnownRegexPatterns.DataClassification); patterns.AddRange(WellKnownRegexPatterns.PreciselyClassifiedSecurityKeys); - patterns.AddRange(WellKnownRegexPatterns.UnclassifiedPotentialSecurityKeys); - */ + patterns.AddRange(WellKnownRegexPatterns.UnclassifiedPotentialSecurityKeys); patterns.Add(new AzureFunctionIdentifiableKey()); From 350192912ad641b70f535f2f7a402f90219ebb22 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Tue, 4 Mar 2025 16:12:49 -0800 Subject: [PATCH 14/20] Eliminate nullable warning --- src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs index 1675efc9..1d40376c 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs @@ -18,7 +18,7 @@ public class RE2RegexEngine : IRegexEngine #else public const RegexOptions RegexOptionsDefaults = RegexOptions.ExplicitCapture | RegexOptions.Compiled | RegexOptions.CultureInvariant; #endif - public IEnumerable Matches(string input, string pattern, RegexOptions options = RegexOptionsDefaults, TimeSpan timeout = default, string? captureGroup = null) + public IEnumerable Matches(string input, string pattern, RegexOptions options = RegexOptionsDefaults, TimeSpan timeout = default, string captureGroup = null) { if (captureGroup == null) { From 176b6bdd3735499b15399eac6cd24f1f5de1f343 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Tue, 4 Mar 2025 16:32:45 -0800 Subject: [PATCH 15/20] Fix prefix to be RE2 compatible in high performance code path --- src/Microsoft.Security.Utilities.Core/RegexPattern.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Security.Utilities.Core/RegexPattern.cs b/src/Microsoft.Security.Utilities.Core/RegexPattern.cs index 42d84fcd..8254b9ec 100644 --- a/src/Microsoft.Security.Utilities.Core/RegexPattern.cs +++ b/src/Microsoft.Security.Utilities.Core/RegexPattern.cs @@ -389,7 +389,7 @@ private protected static string MakeHighPerformancePattern(string pattern, strin } } - const string refineStart = "(?"; + const string refineStart = "(?P"; if (pattern.StartsWith(refineStart)) { pattern = pattern.Substring(refineStart.Length, pattern.Length - refineStart.Length - 1); From bcd3be7e6de8304ae06ddbc78db6ffabc530792f Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Tue, 4 Mar 2025 17:13:28 -0800 Subject: [PATCH 16/20] Test resilience --- .../DetectionTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Tests.Microsoft.Security.Utilities.Core/DetectionTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/DetectionTests.cs index a414e588..000d36f0 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/DetectionTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/DetectionTests.cs @@ -208,13 +208,13 @@ public void Detection_EqualComparisonUpdateWhenPropertiesChange() previousDetection = new Detection(currentDetection); Assert.AreEqual(currentDetection, previousDetection); - currentDetection.Start = Math.Max(1, (int)DateTime.UtcNow.Ticks % 99); + currentDetection.Start++; Assert.AreNotEqual(currentDetection, previousDetection); previousDetection = new Detection(currentDetection); Assert.AreEqual(currentDetection, previousDetection); - currentDetection.Length = Math.Max(1, (int)DateTime.UtcNow.Ticks % 99); + currentDetection.Length++; Assert.AreNotEqual(currentDetection, previousDetection); previousDetection = new Detection(currentDetection); @@ -226,7 +226,7 @@ public void Detection_EqualComparisonUpdateWhenPropertiesChange() previousDetection = new Detection(currentDetection); Assert.AreEqual(currentDetection, previousDetection); - currentDetection.RotationPeriod = TimeSpan.FromSeconds(DateTime.UtcNow.Second); + currentDetection.RotationPeriod = TimeSpan.FromSeconds(currentDetection.RotationPeriod.TotalSeconds + 1); Assert.AreNotEqual(currentDetection, previousDetection); previousDetection = new Detection(currentDetection); From c326d78fedff5117a8075ddaff96e2e4c5b5937f Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Wed, 5 Mar 2025 08:29:03 -0800 Subject: [PATCH 17/20] Release history clean-up --- docs/ReleaseHistory.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ReleaseHistory.md b/docs/ReleaseHistory.md index 5d2051ee..76c5d035 100644 --- a/docs/ReleaseHistory.md +++ b/docs/ReleaseHistory.md @@ -25,7 +25,7 @@ - PRF: `IdentifiableScan` now implements high-performance scanning techniques in managed code. The performance has been found to be significantly better than the prior implementation via rust interop. This also reduces the size of the NuGet package size by a factor of 34 from 6.8 MB to 200 KB and adds support for non x86/x64 CPUs and non-Windows OSes. - BUG: Correct `SEC000/002.Unclassified16ByteHexadecimalString` id and rule name on calling `GetMatchIdAndName` (where `SEC000/001.Unclassified64ByteBase64String` was returned incorrectly before). - BUG: Resolve `System.FormatException: The input is not a valid Base-46 string` errors calling `SEC101/102.AdoPat.GetMatchIdAndName` by swallowing correct exception kind `ArgumentException` in `IsChecksumValid` helper. -- BUG: `?P` is now used throughout for named captures as this is required currently for RE2 compatibility. This reverts a clean-up in this area (that broke RE2) in 1.15.0. +- BUG: `?P` is now used throughout for named captures as this is required currently for RE2 compatibility. # 1.15.0 - 03/03/2025 - BRK: Regular expression syntax has been standardized in JSON to conform to how the overwhelming majority of patterns were already defined. From 3812c563d35c8a4edc6341832b7583633fa73518 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Wed, 5 Mar 2025 08:36:22 -0800 Subject: [PATCH 18/20] Add comment for memory cache value. --- .../RE2RegexEngine.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs index 1d40376c..f3fc3653 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs @@ -35,7 +35,11 @@ public IEnumerable Matches(string input, string pattern, RegexOp } else { - if (Regex2.Matches(pattern, input, out List> matches, 256L * 1024L * 1024L)) + // This value is to set a limit on memory used for its (deterministic) DFA caching. + // When a specific regex evaluation exceeds this limit, the regex engine will + // fall back to a non-deterministic approach with some unmeasured perf impact. + int maxMemoryInBytes = 256 * 1024 * 1024; // 256MB + if (Regex2.Matches(pattern, input, out List> matches, maxMemoryInBytes)) { foreach (Dictionary match in matches) { From 50b5f573ec22dda848d39e83280cb30d3071b50c Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Wed, 5 Mar 2025 09:32:52 -0800 Subject: [PATCH 19/20] PR feedback on test patterns. --- .../WellKnownRegexPatternsTests.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs index 8d13bcb7..3cdce044 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs @@ -81,13 +81,7 @@ public void WellKnownRegexPatterns_GetMatchMonikerHardenedForOutOfOrderExecution { using var assertionScope = new AssertionScope(); - var patterns = new List(); - - patterns.AddRange(WellKnownRegexPatterns.DataClassification); - patterns.AddRange(WellKnownRegexPatterns.PreciselyClassifiedSecurityKeys); - patterns.AddRange(WellKnownRegexPatterns.UnclassifiedPotentialSecurityKeys); - - patterns.Add(new AzureFunctionIdentifiableKey()); + var patterns = GetAllPatterns() var masker = new SecretMasker(patterns, generateCorrelatingIds: true, From 15255508c469ca202d60997498b643d47c3db2bf Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Wed, 5 Mar 2025 09:34:03 -0800 Subject: [PATCH 20/20] Test syntax error --- .../WellKnownRegexPatternsTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs index 3cdce044..1960653f 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs @@ -81,7 +81,7 @@ public void WellKnownRegexPatterns_GetMatchMonikerHardenedForOutOfOrderExecution { using var assertionScope = new AssertionScope(); - var patterns = GetAllPatterns() + var patterns = GetAllPatterns(); var masker = new SecretMasker(patterns, generateCorrelatingIds: true,