diff --git a/GeneratedRegexPatterns/HighConfidenceSecurityModels.json b/GeneratedRegexPatterns/HighConfidenceSecurityModels.json index 701980b8..36538a10 100644 --- a/GeneratedRegexPatterns/HighConfidenceSecurityModels.json +++ b/GeneratedRegexPatterns/HighConfidenceSecurityModels.json @@ -1,6 +1,6 @@ [ { - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{52}JQQJ9(?:9|D|H)[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890][A-L][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{16}[A-Za-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{7}(?:[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{2}==)?)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{52}JQQJ9(?:9|D|H)[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890][A-L][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{16}[A-Za-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{7}(?:[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{2}==)?)", "Id": "SEC101/200", "Name": "CommonAnnotatedSecurityKey", "Signatures": [ @@ -9,7 +9,7 @@ "DetectionMetadata": "Identifiable" }, { - "Pattern": "(^|[^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/])(?[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{3}(7|8)Q~[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{31,34})([^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/]|$)", + "Pattern": "(^|[^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/])(?P[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{3}(7|8)Q~[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{31,34})([^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/]|$)", "Id": "SEC101/156", "Name": "AadClientAppIdentifiableCredentials", "Signatures": [ @@ -28,7 +28,7 @@ 5575864757416767536, 6014965721085063216 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{44}AzFu[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{44}AzFu[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]|$)", "KeyLength": 40, "RegexNormalizedSignature": "AzFu", "Id": "SEC101/158", @@ -43,7 +43,7 @@ "ChecksumSeeds": [ 5869709231681187888 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "KeyLength": 39, "RegexNormalizedSignature": "AzSe", "Id": "SEC101/166", @@ -58,7 +58,7 @@ "Signatures": [ "AzSe" ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "KeyLength": 39, "RegexNormalizedSignature": "AzSe", "Id": "SEC101/167", @@ -74,7 +74,7 @@ 5506058963192262704, 5575859178286952496 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ARm[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ARm[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ARm", "KeyLength": 32, "EncodeForUrl": false, @@ -91,7 +91,7 @@ 5506058963192262704, 5575859178286952496 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+AEh[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+AEh[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+AEh", "KeyLength": 32, "EncodeForUrl": false, @@ -108,7 +108,7 @@ 5506058963192262704, 5575859178286952496 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ASb[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ASb[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ASb", "KeyLength": 32, "EncodeForUrl": false, @@ -123,7 +123,7 @@ "ChecksumSeeds": [ 5291540757367369776 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AIoT", "KeyLength": 32, "EncodeForUrl": false, @@ -138,7 +138,7 @@ "ChecksumSeeds": [ 4928475562238095408 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AIoT", "KeyLength": 32, "EncodeForUrl": false, @@ -153,7 +153,7 @@ "ChecksumSeeds": [ 4931568359632875568 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AIoT", "KeyLength": 32, "EncodeForUrl": false, @@ -169,7 +169,7 @@ 4928457935994778672 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ASt[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ASt[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ASt", "EncodeForUrl": false, "Id": "SEC101/152", @@ -190,7 +190,7 @@ 6014965720764854320 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}ACDb[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}ACDb[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "ACDb", "EncodeForUrl": false, "Id": "SEC101/160", @@ -205,7 +205,7 @@ 4711400055309086768 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ABa[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ABa[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ABa", "EncodeForUrl": false, "Id": "SEC101/163", @@ -220,7 +220,7 @@ 4858365246511342384 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+AMC[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+AMC[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+AMC", "EncodeForUrl": false, "Id": "SEC101/170", @@ -235,7 +235,7 @@ 6081388236577714224 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/181", @@ -250,7 +250,7 @@ 5291540757367369776 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/182", @@ -265,7 +265,7 @@ 5143520228578766896 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/183", @@ -280,7 +280,7 @@ 5145771916421312560 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/184", @@ -294,7 +294,7 @@ "ChecksumSeeds": [ 4711718922539446320 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AzCa[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AzCa[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AzCa", "KeyLength": 32, "EncodeForUrl": false, @@ -309,7 +309,7 @@ "ChecksumSeeds": [ 4702692889634567216 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{42}\\+ACR[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{42}\\+ACR[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "KeyLength": 39, "RegexNormalizedSignature": "\\+ACR", "EncodeForUrl": false, @@ -318,7 +318,7 @@ "DetectionMetadata": "Identifiable" }, { - "Pattern": "(?i)(^|[^a-z0-9])(?oy2[a-z2-7]{43})([^a-z0-9]|$)", + "Pattern": "(?i)(^|[^a-z0-9])(?Poy2[a-z2-7]{43})([^a-z0-9]|$)", "Id": "SEC101/031", "Name": "NuGetApiKey", "Signatures": [ @@ -328,7 +328,7 @@ "DetectionMetadata": "HighEntropy, FixedSignature, HighConfidence" }, { - "Pattern": "(?:^|[^0-9a-f\\-])(?dapi[0-9a-f\\-]{32,34})(?:[^0-9a-f\\-]|$)", + "Pattern": "(?:^|[^0-9a-f\\-])(?Pdapi[0-9a-f\\-]{32,34})(?:[^0-9a-f\\-]|$)", "Id": "SEC101/110", "Name": "AzureDatabricksPat", "Signatures": [ @@ -343,7 +343,7 @@ "ChecksumSeeds": [ 4928457935994778672 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AZEG[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AZEG[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AZEG", "KeyLength": 32, "EncodeForUrl": false, @@ -352,7 +352,7 @@ "DetectionMetadata": "Identifiable" }, { - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?npm_[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{36})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?Pnpm_[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{36})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", "Id": "SEC101/050", "Name": "NpmAuthorKey", "Signatures": [ @@ -361,7 +361,7 @@ "DetectionMetadata": "HighEntropy, FixedSignature, HighConfidence" }, { - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?secret_scanning_ab85fc6f8d7638cf1c11da812da308d43_[0-9A-Za-z]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?Psecret_scanning_ab85fc6f8d7638cf1c11da812da308d43_[0-9A-Za-z]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", "Id": "SEC101/565", "Name": "SecretScanningSampleToken", "Signatures": [ diff --git a/GeneratedRegexPatterns/LowConfidenceSecurityModels.json b/GeneratedRegexPatterns/LowConfidenceSecurityModels.json index 97d9e3c2..2c327ee0 100644 --- a/GeneratedRegexPatterns/LowConfidenceSecurityModels.json +++ b/GeneratedRegexPatterns/LowConfidenceSecurityModels.json @@ -1,18 +1,11 @@ [ { - "Pattern": "(?i)authorization:(\\s|%20)bearer(\\s|%20)(?[0-9a-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+\\/=]*)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+/=]|$)", + "Pattern": "(?i)authorization:(\\s|%20)bearer(\\s|%20)(?P[0-9a-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+\\/=]*)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+/=]|$)", "Id": "SEC101/061", "Name": "OAuth2BearerToken", "Signatures": null, "DetectionMetadata": "LowConfidence" }, - { - "Pattern": "(?i)[a-z0-9.=\\-:[_@\\/*\\]+?]{32}$", - "Id": "SEC000/003", - "Name": "Unclassified32CharacterString", - "Signatures": null, - "DetectionMetadata": "HighEntropy, Unclassified, LowConfidence" - }, { "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{43}=", "Id": "SEC000/000", diff --git a/GeneratedRegexPatterns/MediumConfidenceSecurityModels.json b/GeneratedRegexPatterns/MediumConfidenceSecurityModels.json index b3809922..c6210440 100644 --- a/GeneratedRegexPatterns/MediumConfidenceSecurityModels.json +++ b/GeneratedRegexPatterns/MediumConfidenceSecurityModels.json @@ -1,6 +1,6 @@ [ { - "Pattern": "(?i)\\.servicebus\\.windows.+[^0-9a-z\\/+](?[0-9a-z\\/+]{43}=)(?:[^=]|$)", + "Pattern": "(?i)\\.servicebus\\.windows.+[^0-9a-z\\/+](?P[0-9a-z\\/+]{43}=)(?:[^=]|$)", "Id": "SEC101/105", "Name": "AzureMessageLegacyCredentials", "Signatures": [ @@ -20,7 +20,7 @@ "DetectionMetadata": "HighEntropy, MediumConfidence" }, { - "Pattern": "($|\\b)(ftps?|https?):\\/\\/(?[^:@\\/]+:[^:@?\\/]+)@", + "Pattern": "($|\\b)(ftps?|https?):\\/\\/(?P[^:@\\/]+:[^:@?\\/]+)@", "Id": "SEC101/127", "Name": "UrlCredentials", "Signatures": [ @@ -30,7 +30,7 @@ "DetectionMetadata": "MediumConfidence" }, { - "Pattern": "(?i)(?:^|[?;&])(?:dsas_secret|sig)=(?[0-9a-z\\/+%]{43,129}(?:=|%3d))", + "Pattern": "(?i)(?:^|[?;&])(?:dsas_secret|sig)=(?P[0-9a-z\\/+%]{43,129}(?:=|%3d))", "Id": "SEC101/060", "Name": "LooseSasSecret", "Signatures": [ diff --git a/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json b/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json index 888747d1..0381599e 100644 --- a/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json +++ b/GeneratedRegexPatterns/PreciselyClassifiedSecurityKeys.json @@ -1,6 +1,6 @@ [ { - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{52}JQQJ9(?:9|D|H)[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890][A-L][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{16}[A-Za-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{7}(?:[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{2}==)?)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{52}JQQJ9(?:9|D|H)[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890][A-L][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{16}[A-Za-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{7}(?:[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{2}==)?)", "Id": "SEC101/200", "Name": "CommonAnnotatedSecurityKey", "Signatures": [ @@ -9,7 +9,7 @@ "DetectionMetadata": "Identifiable" }, { - "Pattern": "(^|[^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/])(?[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{3}(7|8)Q~[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{31,34})([^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/]|$)", + "Pattern": "(^|[^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/])(?P[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{3}(7|8)Q~[~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{31,34})([^~.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-+/]|$)", "Id": "SEC101/156", "Name": "AadClientAppIdentifiableCredentials", "Signatures": [ @@ -28,7 +28,7 @@ 5575864757416767536, 6014965721085063216 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{44}AzFu[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{44}AzFu[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_\\-]|$)", "KeyLength": 40, "RegexNormalizedSignature": "AzFu", "Id": "SEC101/158", @@ -43,7 +43,7 @@ "ChecksumSeeds": [ 5869709231681187888 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "KeyLength": 39, "RegexNormalizedSignature": "AzSe", "Id": "SEC101/166", @@ -58,7 +58,7 @@ "Signatures": [ "AzSe" ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{42}AzSe[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "KeyLength": 39, "RegexNormalizedSignature": "AzSe", "Id": "SEC101/167", @@ -74,7 +74,7 @@ 5506058963192262704, 5575859178286952496 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ARm[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ARm[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ARm", "KeyLength": 32, "EncodeForUrl": false, @@ -91,7 +91,7 @@ 5506058963192262704, 5575859178286952496 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+AEh[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+AEh[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+AEh", "KeyLength": 32, "EncodeForUrl": false, @@ -108,7 +108,7 @@ 5506058963192262704, 5575859178286952496 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ASb[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}\\+ASb[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ASb", "KeyLength": 32, "EncodeForUrl": false, @@ -123,7 +123,7 @@ "ChecksumSeeds": [ 5291540757367369776 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AIoT", "KeyLength": 32, "EncodeForUrl": false, @@ -138,7 +138,7 @@ "ChecksumSeeds": [ 4928475562238095408 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AIoT", "KeyLength": 32, "EncodeForUrl": false, @@ -153,7 +153,7 @@ "ChecksumSeeds": [ 4931568359632875568 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AIoT[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AIoT", "KeyLength": 32, "EncodeForUrl": false, @@ -169,7 +169,7 @@ 4928457935994778672 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ASt[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ASt[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ASt", "EncodeForUrl": false, "Id": "SEC101/152", @@ -190,7 +190,7 @@ 6014965720764854320 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}ACDb[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}ACDb[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "ACDb", "EncodeForUrl": false, "Id": "SEC101/160", @@ -205,7 +205,7 @@ 4711400055309086768 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ABa[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+ABa[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+ABa", "EncodeForUrl": false, "Id": "SEC101/163", @@ -220,7 +220,7 @@ 4858365246511342384 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+AMC[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}\\+AMC[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "\\+AMC", "EncodeForUrl": false, "Id": "SEC101/170", @@ -235,7 +235,7 @@ 6081388236577714224 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/181", @@ -250,7 +250,7 @@ 5291540757367369776 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/182", @@ -265,7 +265,7 @@ 5143520228578766896 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/183", @@ -280,7 +280,7 @@ 5145771916421312560 ], "KeyLength": 64, - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{76}APIM[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}[AQgw]==)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "APIM", "EncodeForUrl": false, "Id": "SEC101/184", @@ -294,7 +294,7 @@ "ChecksumSeeds": [ 4711718922539446320 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AzCa[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AzCa[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AzCa", "KeyLength": 32, "EncodeForUrl": false, @@ -309,7 +309,7 @@ "ChecksumSeeds": [ 4702692889634567216 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{42}\\+ACR[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{42}\\+ACR[A-D][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "KeyLength": 39, "RegexNormalizedSignature": "\\+ACR", "EncodeForUrl": false, @@ -318,7 +318,7 @@ "DetectionMetadata": "Identifiable" }, { - "Pattern": "(?i)(^|[^a-z0-9])(?oy2[a-z2-7]{43})([^a-z0-9]|$)", + "Pattern": "(?i)(^|[^a-z0-9])(?Poy2[a-z2-7]{43})([^a-z0-9]|$)", "Id": "SEC101/031", "Name": "NuGetApiKey", "Signatures": [ @@ -328,28 +328,28 @@ "DetectionMetadata": "HighEntropy, FixedSignature, HighConfidence" }, { - "Pattern": "(?:[^2-7a-z]|^)(?[2-7a-z]{52})(?:[^2-7a-z]|$)", + "Pattern": "(?:[^2-7a-z]|^)(?P[2-7a-z]{52})(?:[^2-7a-z]|$)", "Id": "SEC101/102", - "Name": "AdoPat", + "Name": "AdoLegacyPat", "Signatures": null, "DetectionMetadata": "HighEntropy, EmbeddedChecksum" }, { - "Pattern": "(?i)\\.documents\\.azure\\.com.+(?:^|[^0-9a-z\\/+])(?[0-9a-z\\/+]{86}==)(?:[^=]|$)", + "Pattern": "(?i)\\.documents\\.azure\\.com.+(?:^|[^0-9a-z\\/+])(?P[0-9a-z\\/+]{86}==)(?:[^=]|$)", "Id": "SEC101/104", "Name": "AzureCosmosDBLegacyCredentials", "Signatures": null, "DetectionMetadata": "ObsoleteFormat, HighEntropy" }, { - "Pattern": "(?i)(?:AccountName|StorageName|StorageAccount)\\s*=.+(?:Account|Storage)Key\\s*=\\s*(?[0-9a-z\\\\\\/+]{86}==)(?:[^=]|$)", + "Pattern": "(?i)(?:AccountName|StorageName|StorageAccount)\\s*=.+(?:Account|Storage)Key\\s*=\\s*(?P[0-9a-z\\\\\\/+]{86}==)(?:[^=]|$)", "Id": "SEC101/106", "Name": "AzureStorageAccountLegacyCredentials", "Signatures": null, "DetectionMetadata": "HighEntropy" }, { - "Pattern": "(?i)\\.servicebus\\.windows.+[^0-9a-z\\/+](?[0-9a-z\\/+]{43}=)(?:[^=]|$)", + "Pattern": "(?i)\\.servicebus\\.windows.+[^0-9a-z\\/+](?P[0-9a-z\\/+]{43}=)(?:[^=]|$)", "Id": "SEC101/105", "Name": "AzureMessageLegacyCredentials", "Signatures": [ @@ -358,7 +358,7 @@ "DetectionMetadata": "ObsoleteFormat, HighEntropy, MediumConfidence" }, { - "Pattern": "(?:^|[^0-9a-f\\-])(?dapi[0-9a-f\\-]{32,34})(?:[^0-9a-f\\-]|$)", + "Pattern": "(?:^|[^0-9a-f\\-])(?Pdapi[0-9a-f\\-]{32,34})(?:[^0-9a-f\\-]|$)", "Id": "SEC101/110", "Name": "AzureDatabricksPat", "Signatures": [ @@ -373,7 +373,7 @@ "ChecksumSeeds": [ 4928457935994778672 ], - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AZEG[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])(?P[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{33}AZEG[A-P][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{5}=)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_=\\-]|$)", "RegexNormalizedSignature": "AZEG", "KeyLength": 32, "EncodeForUrl": false, @@ -382,7 +382,7 @@ "DetectionMetadata": "Identifiable" }, { - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?npm_[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{36})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?Pnpm_[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]{36})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", "Id": "SEC101/050", "Name": "NpmAuthorKey", "Signatures": [ @@ -391,7 +391,7 @@ "DetectionMetadata": "HighEntropy, FixedSignature, HighConfidence" }, { - "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?secret_scanning_ab85fc6f8d7638cf1c11da812da308d43_[0-9A-Za-z]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", + "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890])(?Psecret_scanning_ab85fc6f8d7638cf1c11da812da308d43_[0-9A-Za-z]{5})([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890]|$)", "Id": "SEC101/565", "Name": "SecretScanningSampleToken", "Signatures": [ diff --git a/GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json b/GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json index 562ed237..ae61c0c0 100644 --- a/GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json +++ b/GeneratedRegexPatterns/UnclassifiedPotentialSecurityKeys.json @@ -11,7 +11,7 @@ "DetectionMetadata": "HighEntropy, MediumConfidence" }, { - "Pattern": "($|\\b)(ftps?|https?):\\/\\/(?[^:@\\/]+:[^:@?\\/]+)@", + "Pattern": "($|\\b)(ftps?|https?):\\/\\/(?P[^:@\\/]+:[^:@?\\/]+)@", "Id": "SEC101/127", "Name": "UrlCredentials", "Signatures": [ @@ -21,7 +21,7 @@ "DetectionMetadata": "MediumConfidence" }, { - "Pattern": "(?i)(?:^|[?;&])(?:dsas_secret|sig)=(?[0-9a-z\\/+%]{43,129}(?:=|%3d))", + "Pattern": "(?i)(?:^|[?;&])(?:dsas_secret|sig)=(?P[0-9a-z\\/+%]{43,129}(?:=|%3d))", "Id": "SEC101/060", "Name": "LooseSasSecret", "Signatures": [ @@ -31,19 +31,12 @@ "DetectionMetadata": "HighEntropy, MediumConfidence" }, { - "Pattern": "(?i)authorization:(\\s|%20)bearer(\\s|%20)(?[0-9a-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+\\/=]*)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+/=]|$)", + "Pattern": "(?i)authorization:(\\s|%20)bearer(\\s|%20)(?P[0-9a-z][abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+\\/=]*)([^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_~.\\-+/=]|$)", "Id": "SEC101/061", "Name": "OAuth2BearerToken", "Signatures": null, "DetectionMetadata": "LowConfidence" }, - { - "Pattern": "(?i)[a-z0-9.=\\-:[_@\\/*\\]+?]{32}$", - "Id": "SEC000/003", - "Name": "Unclassified32CharacterString", - "Signatures": null, - "DetectionMetadata": "HighEntropy, Unclassified, LowConfidence" - }, { "Pattern": "(^|[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/_\\-])[abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+/]{43}=", "Id": "SEC000/000", diff --git a/docs/ReleaseHistory.md b/docs/ReleaseHistory.md index 91c0c729..76c5d035 100644 --- a/docs/ReleaseHistory.md +++ b/docs/ReleaseHistory.md @@ -11,8 +11,9 @@ - FPS => False positive reduction in static analysis. - FNS => False negative reduction in static analysis. -# UNRELEASED - +# 1.16.0 - 03/05/2025 +- BRK: Eliminate `SEC000/101.Unclassified32CharacterString` as noisy and not useful. +- BRK: Rename `SEC101/102.AdoPat` friendly name to `AdoLegacyPat`. - BRK: `IdentifiableScan` no longer supports stream input. The following API are removed. Use `IdentifiableScan.DetectSecrets(string)`. - `IdentifiableScan.DetectSecrets(Stream)` - `IdentifiableScan.Start` @@ -22,6 +23,9 @@ - `IdentifiableScan.CheckPossibleMatchRange` - PRF: `IdentifiableScan` did not use high-performance scanning techniques for `SEC101/178.AzureIotHubIdentifiableKey` and `SEC101/200.CommonAnnotatedSecurityKey`. A bug triggered fallback to slower scanning due to incorrect signatures being used. - PRF: `IdentifiableScan` now implements high-performance scanning techniques in managed code. The performance has been found to be significantly better than the prior implementation via rust interop. This also reduces the size of the NuGet package size by a factor of 34 from 6.8 MB to 200 KB and adds support for non x86/x64 CPUs and non-Windows OSes. +- BUG: Correct `SEC000/002.Unclassified16ByteHexadecimalString` id and rule name on calling `GetMatchIdAndName` (where `SEC000/001.Unclassified64ByteBase64String` was returned incorrectly before). +- BUG: Resolve `System.FormatException: The input is not a valid Base-46 string` errors calling `SEC101/102.AdoPat.GetMatchIdAndName` by swallowing correct exception kind `ArgumentException` in `IsChecksumValid` helper. +- BUG: `?P` is now used throughout for named captures as this is required currently for RE2 compatibility. # 1.15.0 - 03/03/2025 - BRK: Regular expression syntax has been standardized in JSON to conform to how the overwhelming majority of patterns were already defined. diff --git a/src/Microsoft.Security.Utilities.Benchmarks/Microsoft.Security.Utilities.Benchmarks.csproj b/src/Microsoft.Security.Utilities.Benchmarks/Microsoft.Security.Utilities.Benchmarks.csproj index 8860241c..a69e362c 100644 --- a/src/Microsoft.Security.Utilities.Benchmarks/Microsoft.Security.Utilities.Benchmarks.csproj +++ b/src/Microsoft.Security.Utilities.Benchmarks/Microsoft.Security.Utilities.Benchmarks.csproj @@ -16,6 +16,7 @@ + diff --git a/src/Microsoft.Security.Utilities.Benchmarks/RE2RegexEngine.cs b/src/Microsoft.Security.Utilities.Benchmarks/RE2RegexEngine.cs deleted file mode 100644 index e9135933..00000000 --- a/src/Microsoft.Security.Utilities.Benchmarks/RE2RegexEngine.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using Microsoft.RE2.Managed; - -using System.Text.RegularExpressions; - -namespace Microsoft.Security.Utilities -{ - public class RE2RegexEngine : IRegexEngine - { - public static IRegexEngine Instance = new RE2RegexEngine(); - -#if NET7_0_OR_GREATER - public const RegexOptions RegexOptionsDefaults = RegexOptions.ExplicitCapture | RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.NonBacktracking; -#else - public const RegexOptions RegexOptionsDefaults = RegexOptions.ExplicitCapture | RegexOptions.Compiled | RegexOptions.CultureInvariant; -#endif - public IEnumerable Matches(string input, string pattern, RegexOptions options = RegexOptionsDefaults, TimeSpan timeout = default, string? captureGroup = null) - { - foreach (FlexMatch flexMatch in RE2Regex.Instance.Matches(input, pattern, options, timeout, captureGroup)) - { - if (captureGroup != null) - { - yield return CachedDotNetRegex.Instance.Matches(input, pattern, options, timeout, captureGroup).First(); - } - else - { - yield return new UniversalMatch - { - Index = flexMatch.Index, - Length = flexMatch.Length, - Value = flexMatch.Value, - Success = flexMatch.Success - }; - } - } - } - } -} diff --git a/src/Microsoft.Security.Utilities.Core/CachedDotNetRegex.cs b/src/Microsoft.Security.Utilities.Core/CachedDotNetRegex.cs index 704d4383..07940fae 100644 --- a/src/Microsoft.Security.Utilities.Core/CachedDotNetRegex.cs +++ b/src/Microsoft.Security.Utilities.Core/CachedDotNetRegex.cs @@ -29,12 +29,17 @@ public static Regex GetOrCreateRegex(string pattern, RegexOptions options) { var key = (pattern, options); #if NET7_0_OR_GREATER - return RegexCache.GetOrAdd(key, key => new Regex(key.Pattern, key.Options | RegexOptions.Compiled | RegexOptions.NonBacktracking)); + return RegexCache.GetOrAdd(key, key => new Regex(NormalizeGroupsPattern(key.Pattern), key.Options | RegexOptions.Compiled | RegexOptions.NonBacktracking)); #else - return RegexCache.GetOrAdd(key, key => new Regex(key.Pattern, key.Options | RegexOptions.Compiled)); + return RegexCache.GetOrAdd(key, key => new Regex(NormalizeGroupsPattern(key.Pattern), key.Options | RegexOptions.Compiled)); #endif } + internal static string NormalizeGroupsPattern(string pattern) + { + return pattern.Replace("?P<", "?<"); + } + public bool IsMatch(string input, string pattern, RegexOptions options = RegexDefaults.DefaultOptionsCaseSensitive, TimeSpan timeout = default, string captureGroup = null) { // Note: Instance Regex.IsMatch has no timeout overload. diff --git a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_002_Unclassified16ByteHexadecimalString.cs b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_002_Unclassified16ByteHexadecimalString.cs index 8606a984..37c36331 100644 --- a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_002_Unclassified16ByteHexadecimalString.cs +++ b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_002_Unclassified16ByteHexadecimalString.cs @@ -22,7 +22,7 @@ public Unclassified16ByteHexadecimalString() DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.Unclassified | DetectionMetadata.LowConfidence; } - public override Tuple? GetMatchIdAndName(string match) => new Tuple("SEC000/001", "Unclassified64ByteBase64String"); + public override Tuple? GetMatchIdAndName(string match) => new Tuple("SEC000/002", nameof(Unclassified16ByteHexadecimalString)); public override IEnumerable GenerateTruePositiveExamples() { diff --git a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_003_Unclassified32CharacterString.cs b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_003_Unclassified32CharacterString.cs deleted file mode 100644 index f2453ef3..00000000 --- a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC000_003_Unclassified32CharacterString.cs +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections.Generic; - -namespace Microsoft.Security.Utilities -{ - public class Unclassified32CharacterString : RegexPattern - { - public const string AzureContainerRegistryLegacyKey = nameof(AzureContainerRegistryLegacyKey); - public const string AadClientAppLegacyCredentials = nameof(AadClientAppLegacyCredentials); - - /// - /// Detect 32-character Azure Active Directory client application legacy credentials. - /// The generated key is a 32-character string that contains alphanumeric characters - /// as well as symbols from the set: .=\-:[_@\*]+? - /// - public Unclassified32CharacterString() - { - Id = "SEC000/003"; - Name = nameof(Unclassified32CharacterString); - DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.Unclassified | DetectionMetadata.LowConfidence; - Pattern = $"(?i)[a-z0-9.=\\-:[_@\\/*\\]+?]{{32}}$"; - } - - public override Tuple GetMatchIdAndName(string match) - { - if (DateTime.TryParse(match, out DateTime result)) - { - return null; - } - - if (HasAtLeastOneNonBase64EncodingSymbol(match)) - { - return new Tuple("SEC101/101", AadClientAppLegacyCredentials); - } - - return new Tuple("SEC101/109", AzureContainerRegistryLegacyKey); - } - - private const string symbols = ".=-:[_@/*]+?"; - private static readonly HashSet symbolChars = new HashSet(symbols.ToCharArray()); - - public override IEnumerable GenerateTruePositiveExamples() - { - int sampleSize = 5; - - string alphabet = $"={WellKnownRegexPatterns.Base64}"; - yield return $"{WellKnownRegexPatterns.GenerateString(alphabet, 32)}"; - - while (sampleSize > 0) - { - string key = WellKnownRegexPatterns.GenerateString($"{WellKnownRegexPatterns.Base62}{symbols}", 32); - if (AadClientAppLegacyCredentials34.HasAtLeastOneSymbol(key)) - { - sampleSize--; - yield return key; - } - } - } - - internal static bool HasAtLeastOneSymbol(string text) - { - foreach (char c in text) - { - - if (symbolChars.Contains(c)) - { - return true; - } - } - - return false; - } - - public override IEnumerable GenerateFalsePositiveExamples() - { - yield return DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ"); - yield return "2024-03-07T02:50:56.464790+00:00"; - - yield return WellKnownRegexPatterns.GenerateString($"{WellKnownRegexPatterns.Base62}.=-:[_@/*]+?", 31); - yield return WellKnownRegexPatterns.GenerateString($"{WellKnownRegexPatterns.Base62}.=-:[_@/*]+?", 31); - } - - private static bool HasAtLeastOneNonBase64EncodingSymbol(string text) - { - foreach (char c in text) - { - - if (c == '.' || c == '-' || c == ':' || - c == '[' || c == '_' || c == '@' || - c == '*' || c == ']' || c == '?') - { - return true; - } - } - - return false; - } - } -} \ No newline at end of file diff --git a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_060_LooseSasSecret.cs b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_060_LooseSasSecret.cs index 234ec9f0..3a4f54ce 100644 --- a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_060_LooseSasSecret.cs +++ b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_060_LooseSasSecret.cs @@ -13,7 +13,7 @@ public LooseSasSecret() Id = "SEC101/060"; Name = nameof(LooseSasSecret); DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.MediumConfidence; - Pattern = @$"(?i)(?:^|[?;&])(?:dsas_secret|sig)=(?[0-9a-z\/+%]{{43,129}}(?:=|%3d))"; + Pattern = @$"(?i)(?:^|[?;&])(?:dsas_secret|sig)=(?P[0-9a-z\/+%]{{43,129}}(?:=|%3d))"; Signatures = new HashSet(new[] { "sig=", "ret=" }); } diff --git a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_061_LooseOAuth2BearerToken.cs b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_061_LooseOAuth2BearerToken.cs index 9244edeb..c2d2dffe 100644 --- a/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_061_LooseOAuth2BearerToken.cs +++ b/src/Microsoft.Security.Utilities.Core/PotentialSecurityKeys/SEC101_061_LooseOAuth2BearerToken.cs @@ -14,7 +14,7 @@ public OAuth2BearerToken() DetectionMetadata = DetectionMetadata.LowConfidence; // https://datatracker.ietf.org/doc/html/rfc6750#section-2.1 - Pattern = @$"(?i)authorization:(\s|%20)bearer(\s|%20)(?[0-9a-z][{WellKnownRegexPatterns.UrlUnreserved}+\/=]*)([^{WellKnownRegexPatterns.UrlUnreserved}+/=]|$)"; + Pattern = @$"(?i)authorization:(\s|%20)bearer(\s|%20)(?P[0-9a-z][{WellKnownRegexPatterns.UrlUnreserved}+\/=]*)([^{WellKnownRegexPatterns.UrlUnreserved}+/=]|$)"; } public override IEnumerable GenerateTruePositiveExamples() diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKey.cs index 63ed45b4..47933e25 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKey.cs @@ -13,7 +13,7 @@ public abstract class Azure32ByteIdentifiableKey : IdentifiableKey { public override string Pattern { - get => @$"{WellKnownRegexPatterns.PrefixAllBase64}(?[{WellKnownRegexPatterns.Base64}]{{33}}{RegexNormalizedSignature}[A-P][{WellKnownRegexPatterns.Base64}]{{5}}=){WellKnownRegexPatterns.SuffixAllBase64}"; + get => @$"{WellKnownRegexPatterns.PrefixAllBase64}(?P[{WellKnownRegexPatterns.Base64}]{{33}}{RegexNormalizedSignature}[A-P][{WellKnownRegexPatterns.Base64}]{{5}}=){WellKnownRegexPatterns.SuffixAllBase64}"; protected set => base.Pattern = value; } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKeys.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKeys.cs index 814038ed..240dee7b 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKeys.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure32ByteIdentifiableKeys.cs @@ -17,7 +17,7 @@ internal sealed class Azure32ByteIdentifiableKeys : RegexPattern public Azure32ByteIdentifiableKeys() { Pattern = $@"{WellKnownRegexPatterns.PrefixAllBase64}" + - $@"(?[{WellKnownRegexPatterns.Base64}]{{33}}(AIoT|\+(ASb|AEh|ARm))[A-P][{WellKnownRegexPatterns.Base64}]{{5}}=)" + + $@"(?P[{WellKnownRegexPatterns.Base64}]{{33}}(AIoT|\+(ASb|AEh|ARm))[A-P][{WellKnownRegexPatterns.Base64}]{{5}}=)" + $@"{WellKnownRegexPatterns.SuffixAllBase64}"; RotationPeriod = TimeSpan.FromDays(365 * 2); diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKey.cs index 7ed17cff..bb5028db 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKey.cs @@ -14,7 +14,7 @@ public abstract class Azure64ByteIdentifiableKey : IdentifiableKey public override string Pattern { - get => $@"{WellKnownRegexPatterns.PrefixAllBase64}(?[{WellKnownRegexPatterns.Base64}]{{76}}{RegexNormalizedSignature}[{WellKnownRegexPatterns.Base64}]{{5}}[AQgw]==){WellKnownRegexPatterns.SuffixAllBase64}"; + get => $@"{WellKnownRegexPatterns.PrefixAllBase64}(?P[{WellKnownRegexPatterns.Base64}]{{76}}{RegexNormalizedSignature}[{WellKnownRegexPatterns.Base64}]{{5}}[AQgw]==){WellKnownRegexPatterns.SuffixAllBase64}"; protected set => base.Pattern = value; } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKeys.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKeys.cs index c3d8bb6f..d7efe7eb 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKeys.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/Azure64ByteIdentifiableKeys.cs @@ -15,7 +15,7 @@ internal sealed class Azure64ByteIdentifiableKeys : RegexPattern public Azure64ByteIdentifiableKeys() { Pattern = $@"{WellKnownRegexPatterns.PrefixAllBase64}" + - $@"(?[{WellKnownRegexPatterns.Base64}]{{76}}(APIM|ACDb|\+(ABa|AMC|ASt))[{WellKnownRegexPatterns.Base64}]{{5}}[AQgw]==)" + + $@"(?P[{WellKnownRegexPatterns.Base64}]{{76}}(APIM|ACDb|\+(ABa|AMC|ASt))[{WellKnownRegexPatterns.Base64}]{{5}}[AQgw]==)" + $@"{WellKnownRegexPatterns.SuffixAllBase64}"; RotationPeriod = TimeSpan.FromDays(365 * 2); diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_031_NuGetApiKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_031_NuGetApiKey.cs index 46899916..5b93fbe7 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_031_NuGetApiKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_031_NuGetApiKey.cs @@ -19,7 +19,7 @@ public NuGetApiKey() // This is the ApiKeyV4 format implemented here: // https://github.com/NuGet/NuGetGallery/blob/main/src/NuGetGallery.Services/Authentication/ApiKeyV4.cs - Pattern = "(?i)(^|[^a-z0-9])(?oy2[a-z2-7]{43})([^a-z0-9]|$)"; + Pattern = "(?i)(^|[^a-z0-9])(?Poy2[a-z2-7]{43})([^a-z0-9]|$)"; Signatures = new HashSet(new[] { "oy2", "OY2" }); } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_050_NpmAuthorKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_050_NpmAuthorKey.cs index ded6cfcb..b7af439f 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_050_NpmAuthorKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_050_NpmAuthorKey.cs @@ -12,7 +12,7 @@ public NpmAuthorKey() Id = "SEC101/050"; Name = nameof(NpmAuthorKey); DetectionMetadata = DetectionMetadata.FixedSignature | DetectionMetadata.HighEntropy | DetectionMetadata.HighConfidence; - Pattern = @$"{WellKnownRegexPatterns.PrefixBase62}(?npm_[{WellKnownRegexPatterns.Base62}]{{36}}){WellKnownRegexPatterns.SuffixBase62}"; + Pattern = @$"{WellKnownRegexPatterns.PrefixBase62}(?Pnpm_[{WellKnownRegexPatterns.Base62}]{{36}}){WellKnownRegexPatterns.SuffixBase62}"; Signatures = new HashSet(new[] { "npm_" }); } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_102_AdoPat.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_102_AdoPat.cs index 15346f86..dd05cfee 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_102_AdoPat.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_102_AdoPat.cs @@ -5,16 +5,16 @@ namespace Microsoft.Security.Utilities { - public class AdoPat : RegexPattern + public class AdoLegacyPat : RegexPattern { private static readonly byte[] EmptyByteArray = new byte[0]; - public AdoPat() + public AdoLegacyPat() { Id = "SEC101/102"; - Name = nameof(AdoPat); + Name = nameof(AdoLegacyPat); DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.EmbeddedChecksum; - Pattern = "(?:[^2-7a-z]|^)(?[2-7a-z]{52})(?:[^2-7a-z]|$)"; + Pattern = "(?:[^2-7a-z]|^)(?P[2-7a-z]{52})(?:[^2-7a-z]|$)"; } public override Tuple GetMatchIdAndName(string match) @@ -42,7 +42,7 @@ private static bool IsChecksumValid(string input, uint magicNumber) { inputBytes = ConvertFromBase32(input); } - catch (FormatException) + catch (ArgumentException) { return false; } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_104_AzureCosmosDBLegacyCredentials.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_104_AzureCosmosDBLegacyCredentials.cs index 2fda7d20..ab9aa707 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_104_AzureCosmosDBLegacyCredentials.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_104_AzureCosmosDBLegacyCredentials.cs @@ -12,7 +12,7 @@ public AzureCosmosDBLegacyCredentials() Id = "SEC101/104"; Name = nameof(AzureCosmosDBLegacyCredentials); DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.ObsoleteFormat; - Pattern = "(?i)\\.documents\\.azure\\.com.+(?:^|[^0-9a-z\\/+])(?[0-9a-z\\/+]{86}==)(?:[^=]|$)"; + Pattern = "(?i)\\.documents\\.azure\\.com.+(?:^|[^0-9a-z\\/+])(?P[0-9a-z\\/+]{86}==)(?:[^=]|$)"; } public override Tuple GetMatchIdAndName(string match) diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_105_AzureMessagingLegacyCredentials.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_105_AzureMessagingLegacyCredentials.cs index d9ae2ad8..4d00d463 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_105_AzureMessagingLegacyCredentials.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_105_AzureMessagingLegacyCredentials.cs @@ -13,7 +13,7 @@ public AzureMessageLegacyCredentials() Id = "SEC101/105"; Name = nameof(AzureMessageLegacyCredentials); DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.ObsoleteFormat | DetectionMetadata.MediumConfidence; - Pattern = "(?i)\\.servicebus\\.windows.+[^0-9a-z\\/+](?[0-9a-z\\/+]{43}=)(?:[^=]|$)"; + Pattern = "(?i)\\.servicebus\\.windows.+[^0-9a-z\\/+](?P[0-9a-z\\/+]{43}=)(?:[^=]|$)"; Signatures = new HashSet(new[] { ".servicebus" }); } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_106_AzureStorageAccountLegacyCredentials.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_106_AzureStorageAccountLegacyCredentials.cs index 829932af..103a3b0f 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_106_AzureStorageAccountLegacyCredentials.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_106_AzureStorageAccountLegacyCredentials.cs @@ -14,7 +14,7 @@ public AzureStorageAccountLegacyCredentials() Id = "SEC101/106"; Name = nameof(AzureStorageAccountLegacyCredentials); DetectionMetadata = DetectionMetadata.HighEntropy; - Pattern = "(?i)(?:AccountName|StorageName|StorageAccount)\\s*=.+(?:Account|Storage)Key\\s*=\\s*(?[0-9a-z\\\\\\/+]{86}==)(?:[^=]|$)"; + Pattern = "(?i)(?:AccountName|StorageName|StorageAccount)\\s*=.+(?:Account|Storage)Key\\s*=\\s*(?P[0-9a-z\\\\\\/+]{86}==)(?:[^=]|$)"; } public override Tuple GetMatchIdAndName(string match) diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_110.AzureDatabricksPat.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_110.AzureDatabricksPat.cs index 9f79c6c3..e1441fbb 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_110.AzureDatabricksPat.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_110.AzureDatabricksPat.cs @@ -12,7 +12,7 @@ public AzureDatabricksPat() Id = "SEC101/110"; Name = nameof(AzureDatabricksPat); DetectionMetadata = DetectionMetadata.HighEntropy | DetectionMetadata.HighConfidence; - Pattern = $"(?:^|[^0-9a-f\\-])(?dapi[0-9a-f\\-]{{32,34}})(?:[^0-9a-f\\-]|$)"; + Pattern = $"(?:^|[^0-9a-f\\-])(?Pdapi[0-9a-f\\-]{{32,34}})(?:[^0-9a-f\\-]|$)"; Signatures = new HashSet(new[] { "dapi" }); } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_156_AzureClientAppIdentifiableCredentials.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_156_AzureClientAppIdentifiableCredentials.cs index 035036e8..02669e4f 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_156_AzureClientAppIdentifiableCredentials.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_156_AzureClientAppIdentifiableCredentials.cs @@ -12,7 +12,7 @@ public AadClientAppIdentifiableCredentials() Id = "SEC101/156"; Name = "AadClientAppIdentifiableCredentials"; DetectionMetadata = DetectionMetadata.Identifiable; - Pattern = $"{WellKnownRegexPatterns.PrefixUrlUnreserved}(?[{WellKnownRegexPatterns.RegexEncodedUrlUnreserved}]{{3}}(7|8)Q~[{WellKnownRegexPatterns.RegexEncodedUrlUnreserved}]{{31,34}}){WellKnownRegexPatterns.SuffixUrlUnreserved}"; + Pattern = $"{WellKnownRegexPatterns.PrefixUrlUnreserved}(?P[{WellKnownRegexPatterns.RegexEncodedUrlUnreserved}]{{3}}(7|8)Q~[{WellKnownRegexPatterns.RegexEncodedUrlUnreserved}]{{31,34}}){WellKnownRegexPatterns.SuffixUrlUnreserved}"; Signatures = new HashSet(new[] { "8Q~", "7Q~" }); } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_158_AzureFunctionIdentifiableKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_158_AzureFunctionIdentifiableKey.cs index e8bebb5f..7db6b3b6 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_158_AzureFunctionIdentifiableKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_158_AzureFunctionIdentifiableKey.cs @@ -28,7 +28,7 @@ public AzureFunctionIdentifiableKey() public override string Pattern { - get => @$"{WellKnownRegexPatterns.PrefixUrlSafeBase64}(?[{WellKnownRegexPatterns.RegexEncodedUrlSafeBase64}]{{44}}{Signatures!.First()}[{WellKnownRegexPatterns.RegexEncodedUrlSafeBase64}]{{5}}[AQgw]==){WellKnownRegexPatterns.SuffixUrlSafeBase64}"; + get => @$"{WellKnownRegexPatterns.PrefixUrlSafeBase64}(?P[{WellKnownRegexPatterns.RegexEncodedUrlSafeBase64}]{{44}}{Signatures!.First()}[{WellKnownRegexPatterns.RegexEncodedUrlSafeBase64}]{{5}}[AQgw]==){WellKnownRegexPatterns.SuffixUrlSafeBase64}"; protected set => base.Pattern = value; } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_166_AzureSearchIdentifiableQueryKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_166_AzureSearchIdentifiableQueryKey.cs index 4cb50ff4..ef9e447e 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_166_AzureSearchIdentifiableQueryKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_166_AzureSearchIdentifiableQueryKey.cs @@ -22,7 +22,7 @@ public AzureSearchIdentifiableQueryKey() public override string Pattern { - get => @$"{WellKnownRegexPatterns.PrefixAllBase64}(?[{WellKnownRegexPatterns.Base62}]{{42}}{Signatures!.First()}[A-D][{WellKnownRegexPatterns.Base62}]{{5}}){WellKnownRegexPatterns.SuffixAllBase64}"; + get => @$"{WellKnownRegexPatterns.PrefixAllBase64}(?P[{WellKnownRegexPatterns.Base62}]{{42}}{Signatures!.First()}[A-D][{WellKnownRegexPatterns.Base62}]{{5}}){WellKnownRegexPatterns.SuffixAllBase64}"; protected set => base.Pattern = value; } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_176_AzureContainerRegistryIdentifiableKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_176_AzureContainerRegistryIdentifiableKey.cs index a9148f5e..4b13c964 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_176_AzureContainerRegistryIdentifiableKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_176_AzureContainerRegistryIdentifiableKey.cs @@ -20,7 +20,7 @@ public AzureContainerRegistryIdentifiableKey() public override string Pattern { - get => @$"{WellKnownRegexPatterns.PrefixAllBase64}(?[{WellKnownRegexPatterns.Base64}]{{42}}{RegexNormalizedSignature}[A-D][{WellKnownRegexPatterns.Base64}]{{5}}){WellKnownRegexPatterns.SuffixAllBase64}"; + get => @$"{WellKnownRegexPatterns.PrefixAllBase64}(?P[{WellKnownRegexPatterns.Base64}]{{42}}{RegexNormalizedSignature}[A-D][{WellKnownRegexPatterns.Base64}]{{5}}){WellKnownRegexPatterns.SuffixAllBase64}"; protected set => base.Pattern = value; } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_200_CommonAnnotatedSecurityKey.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_200_CommonAnnotatedSecurityKey.cs index e9f289bd..3f59834f 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_200_CommonAnnotatedSecurityKey.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_200_CommonAnnotatedSecurityKey.cs @@ -17,7 +17,7 @@ public CommonAnnotatedSecurityKey() Id = "SEC101/200"; Name = nameof(CommonAnnotatedSecurityKey); DetectionMetadata = DetectionMetadata.Identifiable; - Pattern = $"{WellKnownRegexPatterns.PrefixBase62}(?[{WellKnownRegexPatterns.Base62}]{{52}}JQQJ9(?:9|D|H)[{WellKnownRegexPatterns.Base62}][A-L][{WellKnownRegexPatterns.Base62}]{{16}}[A-Za-z][{WellKnownRegexPatterns.Base62}]{{7}}(?:[{WellKnownRegexPatterns.Base62}]{{2}}==)?)"; + Pattern = $"{WellKnownRegexPatterns.PrefixBase62}(?P[{WellKnownRegexPatterns.Base62}]{{52}}JQQJ9(?:9|D|H)[{WellKnownRegexPatterns.Base62}][A-L][{WellKnownRegexPatterns.Base62}]{{16}}[A-Za-z][{WellKnownRegexPatterns.Base62}]{{7}}(?:[{WellKnownRegexPatterns.Base62}]{{2}}==)?)"; Signatures = Signature.ToSet(); } diff --git a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_565_SecretScanningSampleToken.cs b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_565_SecretScanningSampleToken.cs index c8d90f8c..e2d4e839 100644 --- a/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_565_SecretScanningSampleToken.cs +++ b/src/Microsoft.Security.Utilities.Core/PreciselyClassifiedSecurityKeys/SEC101_565_SecretScanningSampleToken.cs @@ -18,7 +18,7 @@ public SecretScanningSampleToken() Id = "SEC101/565"; Name = nameof(SecretScanningSampleToken); DetectionMetadata = DetectionMetadata.FixedSignature | DetectionMetadata.HighEntropy | DetectionMetadata.HighConfidence; - Pattern = @$"{WellKnownRegexPatterns.PrefixBase62}(?secret_scanning_ab85fc6f8d7638cf1c11da812da308d43_[0-9A-Za-z]{{5}}){WellKnownRegexPatterns.SuffixBase62}"; + Pattern = @$"{WellKnownRegexPatterns.PrefixBase62}(?Psecret_scanning_ab85fc6f8d7638cf1c11da812da308d43_[0-9A-Za-z]{{5}}){WellKnownRegexPatterns.SuffixBase62}"; Signatures = "ab85".ToSet(); } diff --git a/src/Microsoft.Security.Utilities.Core/RegexPattern.cs b/src/Microsoft.Security.Utilities.Core/RegexPattern.cs index 42d84fcd..8254b9ec 100644 --- a/src/Microsoft.Security.Utilities.Core/RegexPattern.cs +++ b/src/Microsoft.Security.Utilities.Core/RegexPattern.cs @@ -389,7 +389,7 @@ private protected static string MakeHighPerformancePattern(string pattern, strin } } - const string refineStart = "(?"; + const string refineStart = "(?P"; if (pattern.StartsWith(refineStart)) { pattern = pattern.Substring(refineStart.Length, pattern.Length - refineStart.Length - 1); diff --git a/src/Microsoft.Security.Utilities.Core/UrlCredentials.cs b/src/Microsoft.Security.Utilities.Core/UrlCredentials.cs index c346faf0..9c958f5d 100644 --- a/src/Microsoft.Security.Utilities.Core/UrlCredentials.cs +++ b/src/Microsoft.Security.Utilities.Core/UrlCredentials.cs @@ -14,7 +14,7 @@ public UrlCredentials() Name = nameof(UrlCredentials); - Pattern = @"($|\b)(ftps?|https?):\/\/(?[^:@\/]+:[^:@?\/]+)@"; + Pattern = @"($|\b)(ftps?|https?):\/\/(?P[^:@\/]+:[^:@?\/]+)@"; DetectionMetadata = DetectionMetadata.MediumConfidence; diff --git a/src/Microsoft.Security.Utilities.Core/WellKnownRegexPatterns.cs b/src/Microsoft.Security.Utilities.Core/WellKnownRegexPatterns.cs index a2933f7e..2a806ec3 100644 --- a/src/Microsoft.Security.Utilities.Core/WellKnownRegexPatterns.cs +++ b/src/Microsoft.Security.Utilities.Core/WellKnownRegexPatterns.cs @@ -53,10 +53,9 @@ public static IEnumerable HighConfidenceSecurityModelsIterator() new UrlCredentials(), new LooseSasSecret(), new OAuth2BearerToken(), - new Unclassified32CharacterString(), new Unclassified32ByteBase64String(), new Unclassified64ByteBase64String(), - new AadClientAppLegacyCredentials34(), // SEC101/101 + new AadClientAppLegacyCredentials34(), // SEC101/101 legacy generated passwords. new Pkcs12CertificatePrivateKeyBundle(), new Unclassified16ByteHexadecimalString(), }; @@ -85,7 +84,7 @@ public static IEnumerable HighConfidenceSecurityModelsIterator() new AzureCacheForRedisIdentifiableKey(), new AzureContainerRegistryIdentifiableKey(), new NuGetApiKey(), - new AdoPat(), // SEC101/102 + new AdoLegacyPat(), // SEC101/102 new AzureCosmosDBLegacyCredentials(), // SEC101/104 new AzureStorageAccountLegacyCredentials(), // SEC101/106 new AzureMessageLegacyCredentials(), diff --git a/src/Tests.Microsoft.Security.Utilities.Core/AdoLegacyPatTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/AdoLegacyPatTests.cs new file mode 100644 index 00000000..1f9288f4 --- /dev/null +++ b/src/Tests.Microsoft.Security.Utilities.Core/AdoLegacyPatTests.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.Security.Utilities.Core +{ + [TestClass] + public class AdoLegacyPatTests + { + [TestMethod] + public void AdoLegacyPat_InvalidBase32Input() + { + var classifier = new AdoLegacyPat(); + string invalidInput = "=22222222222222222222222222"; + var result = classifier.GetMatchIdAndName(invalidInput); + Assert.IsNull(result); + } + } +} diff --git a/src/Tests.Microsoft.Security.Utilities.Core/AzureFunctionIdentifiableKeyTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/AzureFunctionIdentifiableKeyTests.cs new file mode 100644 index 00000000..512d5604 --- /dev/null +++ b/src/Tests.Microsoft.Security.Utilities.Core/AzureFunctionIdentifiableKeyTests.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using FluentAssertions; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using System.Linq; + + +namespace Microsoft.Security.Utilities.Core +{ + [TestClass] + public class AzureFunctionIdentifiableKeyTests + { + [TestMethod] + public void AzureFunctionIdentifiableKey_GenerateTruePositiveExamples() + { + var classifier = new AzureFunctionIdentifiableKey(); + + foreach (string example in classifier.GenerateTruePositiveExamples()) + { + var masker = new SecretMasker([classifier]); + var detection = masker.DetectSecrets(example).FirstOrDefault(); + detection.Should().NotBe(default); + + string refined = example.Substring(detection.Start, detection.Length); + var result = classifier.GetMatchIdAndName(refined); + result.Should().NotBeNull(); + } + } + } +} + + diff --git a/src/Tests.Microsoft.Security.Utilities.Core/DetectionTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/DetectionTests.cs index a414e588..000d36f0 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/DetectionTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/DetectionTests.cs @@ -208,13 +208,13 @@ public void Detection_EqualComparisonUpdateWhenPropertiesChange() previousDetection = new Detection(currentDetection); Assert.AreEqual(currentDetection, previousDetection); - currentDetection.Start = Math.Max(1, (int)DateTime.UtcNow.Ticks % 99); + currentDetection.Start++; Assert.AreNotEqual(currentDetection, previousDetection); previousDetection = new Detection(currentDetection); Assert.AreEqual(currentDetection, previousDetection); - currentDetection.Length = Math.Max(1, (int)DateTime.UtcNow.Ticks % 99); + currentDetection.Length++; Assert.AreNotEqual(currentDetection, previousDetection); previousDetection = new Detection(currentDetection); @@ -226,7 +226,7 @@ public void Detection_EqualComparisonUpdateWhenPropertiesChange() previousDetection = new Detection(currentDetection); Assert.AreEqual(currentDetection, previousDetection); - currentDetection.RotationPeriod = TimeSpan.FromSeconds(DateTime.UtcNow.Second); + currentDetection.RotationPeriod = TimeSpan.FromSeconds(currentDetection.RotationPeriod.TotalSeconds + 1); Assert.AreNotEqual(currentDetection, previousDetection); previousDetection = new Detection(currentDetection); diff --git a/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs index 68b3faa7..f3fc3653 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngine.cs @@ -2,30 +2,27 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.RE2.Managed; -using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; using System.Text.RegularExpressions; namespace Microsoft.Security.Utilities { - [ExcludeFromCodeCoverage] public class RE2RegexEngine : IRegexEngine { public static IRegexEngine Instance = new RE2RegexEngine(); - public IEnumerable Matches(string input, string pattern, RegexOptions options = RegexOptions.ExplicitCapture | RegexOptions.Compiled | RegexOptions.CultureInvariant, TimeSpan timeout = default, string captureGroup = null) +#if NET7_0_OR_GREATER + public const RegexOptions RegexOptionsDefaults = RegexOptions.ExplicitCapture | RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.NonBacktracking; +#else + public const RegexOptions RegexOptionsDefaults = RegexOptions.ExplicitCapture | RegexOptions.Compiled | RegexOptions.CultureInvariant; +#endif + public IEnumerable Matches(string input, string pattern, RegexOptions options = RegexOptionsDefaults, TimeSpan timeout = default, string captureGroup = null) { - foreach (FlexMatch flexMatch in RE2Regex.Instance.Matches(input, pattern, options, timeout, captureGroup)) + if (captureGroup == null) { - if (captureGroup != null) - { - yield return CachedDotNetRegex.Instance.Matches(input, pattern, options, timeout, captureGroup).First(); - } - else + foreach (FlexMatch flexMatch in RE2Regex.Instance.Matches(input, pattern, options, timeout, captureGroup)) { yield return new UniversalMatch { @@ -36,6 +33,32 @@ public IEnumerable Matches(string input, string pattern, RegexOp }; } } + else + { + // This value is to set a limit on memory used for its (deterministic) DFA caching. + // When a specific regex evaluation exceeds this limit, the regex engine will + // fall back to a non-deterministic approach with some unmeasured perf impact. + int maxMemoryInBytes = 256 * 1024 * 1024; // 256MB + if (Regex2.Matches(pattern, input, out List> matches, maxMemoryInBytes)) + { + foreach (Dictionary match in matches) + { + FlexMatch flexMatch = match["0"]; + if (match.TryGetValue(captureGroup, out FlexMatch refineMatch)) + { + flexMatch = refineMatch; + } + + yield return new UniversalMatch + { + Index = flexMatch.Index, + Length = flexMatch.Length, + Value = flexMatch.Value, + Success = flexMatch.Success + }; + } + } + } } } } diff --git a/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngineTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngineTests.cs new file mode 100644 index 00000000..94b818f7 --- /dev/null +++ b/src/Tests.Microsoft.Security.Utilities.Core/RE2RegexEngineTests.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using FluentAssertions; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using System.Linq; + +namespace Microsoft.Security.Utilities.Core +{ + [TestClass] + public class RE2RegexEngineTests + { + [TestMethod] + public void RE2RegexEngine_RefineGroup() + { + // Scan data has a discrete component with + // a leading and trailing 'x' character. + string scanData = $"x{nameof(scanData)}x"; + + // The regex pattern defines a named group to extract + // the scan data from between its encapsulating chars. + string regex = $"x(?P{nameof(scanData)})x"; + + var regexPattern = new RegexPattern(id: "1001", name: "MyRule", DetectionMetadata.None, regex); + var masker = new SecretMasker([regexPattern], regexEngine: RE2RegexEngine.Instance); + + var detection = masker.DetectSecrets(scanData).FirstOrDefault(); + detection.Should().NotBe(default); + + string refined = scanData.Substring(detection.Start, detection.Length); + refined.Should().Be(nameof(scanData)); + + // Run the same test directly against the engine. + var re2RegexEngine = new RE2RegexEngine(); + UniversalMatch flexMatch = re2RegexEngine.Matches(scanData, regex, captureGroup: "refine").FirstOrDefault(); + flexMatch.Should().NotBeNull(); + flexMatch.Value.Should().Be(nameof(scanData)); + } + } +} diff --git a/src/Tests.Microsoft.Security.Utilities.Core/RegexPatternTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/RegexPatternTests.cs index d6a4c3ed..81786a74 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/RegexPatternTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/RegexPatternTests.cs @@ -383,7 +383,7 @@ public void RegexPatterns_GetDetections_ReturnsEmpty_WhenNoMatchesExist() public void RegexPatterns_GetDetections_Returns_RefinedDetection() { // Arrange - var secret = new RegexPattern(Id, Name, DetectionMetadata.Identifiable, "a(?b)c"); + var secret = new RegexPattern(Id, Name, DetectionMetadata.Identifiable, "a(?Pb)c"); var input = "abc"; var match = "b"; diff --git a/src/Tests.Microsoft.Security.Utilities.Core/SecretMaskerTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/SecretMaskerTests.cs index cda84513..4e9c0d6b 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/SecretMaskerTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/SecretMaskerTests.cs @@ -157,13 +157,14 @@ public void SecretMasker_LowConfidenceSecurityModels_Masking() private void ValidateSecurityModelsMasking(IEnumerable patterns, IRegexEngine engine, bool lowEntropyModels) { + using var assertionScope = new AssertionScope(); + // These tests generate randomized values. It may be useful to // bump up the # of iterations on an ad hoc basis to flush // out non-deterministic failures (typically based on the // characters chosen from the secret alphabet for the pattern). for (int i = 0; i < 1; i++) { - using var assertionScope = new AssertionScope(); foreach (IRegexEngine regexEngine in new[] { RE2RegexEngine.Instance, CachedDotNetRegex.Instance }) { foreach (bool generateCrossCompanyCorrelatingIds in new[] { true, false }) diff --git a/src/Tests.Microsoft.Security.Utilities.Core/Tests.Microsoft.Security.Utilities.Core.csproj b/src/Tests.Microsoft.Security.Utilities.Core/Tests.Microsoft.Security.Utilities.Core.csproj index e01db3e4..53803b84 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/Tests.Microsoft.Security.Utilities.Core.csproj +++ b/src/Tests.Microsoft.Security.Utilities.Core/Tests.Microsoft.Security.Utilities.Core.csproj @@ -11,8 +11,8 @@ - - + + diff --git a/src/Tests.Microsoft.Security.Utilities.Core/Unclassified16ByteHexadecimalStringTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/Unclassified16ByteHexadecimalStringTests.cs new file mode 100644 index 00000000..4fbea19b --- /dev/null +++ b/src/Tests.Microsoft.Security.Utilities.Core/Unclassified16ByteHexadecimalStringTests.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#nullable enable + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using System; +using System.Linq; + +namespace Microsoft.Security.Utilities.Core +{ + [TestClass] + public class Unclassified16ByteHexadecimalStringTests + { + + [TestMethod] + public void Unclassified16ByteHexadecimalString_ValidInput() + { + var classifier = new Unclassified16ByteHexadecimalString(); + string validInput = "0123456789abcdef0123456789abcdef"; + + var result = classifier.GetMatchIdAndName(validInput); + + Assert.IsNotNull(result); + Assert.AreEqual("SEC000/002", result.Item1); + Assert.AreEqual("Unclassified16ByteHexadecimalString", result.Item2); + + var detection = + classifier.GetDetections(validInput, generateCrossCompanyCorrelatingIds: false).FirstOrDefault(); + + Assert.IsNotNull(detection); + Assert.AreEqual("SEC000/002", detection.Id); + } + } +} diff --git a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs index ae4fce1b..1960653f 100644 --- a/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs +++ b/src/Tests.Microsoft.Security.Utilities.Core/WellKnownRegexPatternsTests.cs @@ -27,21 +27,67 @@ public class WellKnownRegexPatternsTests }; [TestMethod] - public void WellKnownRegexPatterns_GetMatchMonikerHardenedForOutOfOrderExecution() + public void WellKnownRegexPatterns_AllRuleIdsAndNamesAreUnique() + { + using var assertionScope = new AssertionScope(); + + var patterns = GetAllPatterns(); + + HashSet ruleIdsObserved = new HashSet(StringComparer.OrdinalIgnoreCase); + HashSet ruleNamesObserved = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (RegexPattern pattern in patterns) + { + bool result = ruleIdsObserved.Contains(pattern.Id); + result.Should().BeFalse(because: $"Pattern '{pattern.GetType().Name}' should not share its Id with another rule: '{pattern.Id}'"); + + result = ruleNamesObserved.Contains(pattern.Name); + result.Should().BeFalse(because: $"Pattern '{pattern.GetType().Name}' should not share its Name with another rule: '{pattern.Name}'"); + + ruleIdsObserved.Add(pattern.Id); + ruleNamesObserved.Add(pattern.Name); + } + } + + [TestMethod] + public void WellKnownRegexPatterns_MonikerRuleIdsAndNamesMatchDeclared() { using var assertionScope = new AssertionScope(); - var patterns = new List(); + var patterns = GetAllPatterns(); - patterns.AddRange(WellKnownRegexPatterns.DataClassification); - patterns.AddRange(WellKnownRegexPatterns.PreciselyClassifiedSecurityKeys); - patterns.AddRange(WellKnownRegexPatterns.UnclassifiedPotentialSecurityKeys); + foreach (RegexPattern pattern in patterns) + { + foreach (string example in pattern.GenerateTruePositiveExamples()) + { + var detection = pattern.GetDetections(example, generateCrossCompanyCorrelatingIds: false).FirstOrDefault(); + Assert.AreNotEqual(default, detection); + + string preciseMatch = example.Substring(detection.Start, detection.Length); + + var idAndName = pattern.GetMatchIdAndName(preciseMatch); + + Assert.AreEqual(pattern.Id, idAndName.Item1, + $"Pattern '{pattern.GetType().Name}' id did not match 'GetMatchIdAndName' result"); + + Assert.AreEqual(pattern.Name, idAndName.Item2, + $"Pattern '{pattern.GetType().Name}' name did not match 'GetMatchIdAndName' result"); + } + } + } + + [TestMethod] + public void WellKnownRegexPatterns_GetMatchMonikerHardenedForOutOfOrderExecution() + { + using var assertionScope = new AssertionScope(); + + var patterns = GetAllPatterns(); var masker = new SecretMasker(patterns, generateCorrelatingIds: true, RE2RegexEngine.Instance); - foreach (var pattern in patterns) + foreach (RegexPattern pattern in patterns) { foreach (string example in pattern.GenerateTruePositiveExamples()) { @@ -56,27 +102,14 @@ public void WellKnownRegexPatterns_GetMatchMonikerHardenedForOutOfOrderExecution // is no longer null post-detection. string moniker = pattern.GetMatchMoniker(example); - var detection = masker.DetectSecrets(example).FirstOrDefault((d) => d.Id == pattern.Id); - - // Currently, some rules are tuned not to double-fire, i.e., they determine - // whether a separate rule might identify a pattern and, if so, the rule - // drops the result. This is a problematic design. For one thing, we don't - // to see if the more precise rule is enabled. This gives the appearance of - // false negatives in these low-level checks. This is a subtle topic - // potentially arguing for redesign of the engine or our test expectations. - bool result = - detection != default || - pattern.Name == nameof(Unclassified32CharacterString) || - pattern.Name == nameof(Unclassified16ByteHexadecimalString); + var detection = masker.DetectSecrets(example).FirstOrDefault(); + bool result = detection != default; result.Should().BeTrue(because: $"pattern '{pattern.GetType().Name}' should match '{example}'"); - if (moniker == null) - { - string matched = example.Substring(detection.Start, detection.End - detection.Start); - moniker = pattern.GetMatchMoniker(matched); - moniker.Should().NotBeNull(because: $"'{matched}' should produce a non-null moniker for {pattern.GetType().Name}' test data"); - } + string matched = example.Substring(detection.Start, detection.End - detection.Start); + moniker = pattern.GetMatchMoniker(matched); + moniker.Should().NotBeNull(because: $"'{matched}' should produce a non-null moniker for {pattern.GetType().Name}' test data"); } } } @@ -268,8 +301,23 @@ public void WellKnownRegexPatterns_EnsureAllPatternsHaveCorrectCaptureGroups() groupNames.Length.Should().Be(1, because: $"Pattern '{pattern.GetType().Name}' should not have more than one capture group"); groupNames[0].Should().Be("refine", because: $"Pattern '{pattern.GetType().Name}' capture group should be named 'refine'"); + + pattern.Pattern.Should() + .Contain("(?P", + because: $"Pattern '{pattern.GetType().Name}' should contain an RE2-compatible '(?P...)' named capture group"); } } } + + private static List GetAllPatterns() + { + var patterns = new List(); + + patterns.AddRange(WellKnownRegexPatterns.DataClassification); + patterns.AddRange(WellKnownRegexPatterns.PreciselyClassifiedSecurityKeys); + patterns.AddRange(WellKnownRegexPatterns.UnclassifiedPotentialSecurityKeys); + + return patterns; + } } } diff --git a/version.json b/version.json index cd3edf04..60ae3040 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "$schema": "https://raw.githubusercontent.com/dotnet/Nerdbank.GitVersioning/master/src/NerdBank.GitVersioning/version.schema.json", - "version": "1.15.0", + "version": "1.16.0", "publicReleaseRefSpec": [ "^refs/heads/main$", "^refs/heads/release/v\\d+\\.\\d+\\.\\d+$"