Add fingerprint support to the WAF (#7436)

Add fingerprint support in the WAF
DataDog · Aug 28, 2024 · 01d9133 · 01d9133
1 parent f152e54
commit 01d9133
Show file tree

Hide file tree

Showing 10 changed files with 185 additions and 20 deletions.
diff --git a/dd-java-agent/appsec/build.gradle b/dd-java-agent/appsec/build.gradle
@@ -15,7 +15,7 @@ dependencies {
   implementation project(':internal-api')
   implementation project(':communication')
   implementation project(':telemetry')
-  implementation group: 'io.sqreen', name: 'libsqreen', version: '10.1.0'
+  implementation group: 'io.sqreen', name: 'libsqreen', version: '11.0.0'
   implementation libs.moshi
 
   testImplementation libs.bytebuddy

diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/config/AppSecConfigServiceImpl.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/config/AppSecConfigServiceImpl.java
@@ -9,11 +9,14 @@
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_DD_RULES;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_EXCLUSIONS;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_EXCLUSION_DATA;
+import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_HEADER_FINGERPRINT;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_IP_BLOCKING;
+import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_NETWORK_FINGERPRINT;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_RASP_SQLI;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_REQUEST_BLOCKING;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_TRUSTED_IPS;
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_USER_BLOCKING;
+import static datadog.remoteconfig.Capabilities.CAPABILITY_ENDPOINT_FINGERPRINT;
 
 import com.datadog.appsec.AppSecSystem;
 import com.datadog.appsec.api.security.ApiSecurityRequestSampler;
@@ -104,7 +107,12 @@ private void subscribeConfigurationPoller() {
             | CAPABILITY_ASM_CUSTOM_RULES
             | CAPABILITY_ASM_CUSTOM_BLOCKING_RESPONSE
             | CAPABILITY_ASM_TRUSTED_IPS
-            | CAPABILITY_ASM_RASP_SQLI);
+            | CAPABILITY_ASM_RASP_SQLI
+            | CAPABILITY_ENDPOINT_FINGERPRINT
+            // TODO enable when usr.id and usr.session_id addresses are added
+            // | CAPABILITY_ASM_SESSION_FINGERPRINT
+            | CAPABILITY_ASM_NETWORK_FINGERPRINT
+            | CAPABILITY_ASM_HEADER_FINGERPRINT);
   }
 
   private void subscribeRulesAndData() {
@@ -345,7 +353,12 @@ public void close() {
             | CAPABILITY_ASM_TRUSTED_IPS
             | CAPABILITY_ASM_API_SECURITY_SAMPLE_RATE
             | CAPABILITY_ASM_RASP_SQLI
-            | CAPABILITY_ASM_AUTO_USER_INSTRUM_MODE);
+            | CAPABILITY_ASM_AUTO_USER_INSTRUM_MODE
+            | CAPABILITY_ENDPOINT_FINGERPRINT
+            // TODO enable when usr.id and usr.session_id addresses are added
+            // | CAPABILITY_ASM_SESSION_FINGERPRINT
+            | CAPABILITY_ASM_NETWORK_FINGERPRINT
+            | CAPABILITY_ASM_HEADER_FINGERPRINT);
     this.configurationPoller.removeListeners(Product.ASM_DD);
     this.configurationPoller.removeListeners(Product.ASM_DATA);
     this.configurationPoller.removeListeners(Product.ASM);

diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/AppSecRequestContext.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/AppSecRequestContext.java
@@ -1,5 +1,7 @@
 package com.datadog.appsec.gateway;
 
+import static java.util.Collections.emptySet;
+
 import com.datadog.appsec.event.data.Address;
 import com.datadog.appsec.event.data.DataBundle;
 import com.datadog.appsec.report.AppSecEvent;
@@ -104,7 +106,7 @@ public class AppSecRequestContext implements DataBundle, Closeable {
   private boolean convertedReqBodyPublished;
   private boolean respDataPublished;
   private boolean pathParamsPublished;
-  private Map<String, String> apiSchemas;
+  private Map<String, String> derivatives;
 
   private final AtomicBoolean rateLimited = new AtomicBoolean(false);
   private volatile boolean throttled;
@@ -499,24 +501,29 @@ StackTraceCollection transferStackTracesCollection() {
     }
   }
 
-  public void reportApiSchemas(Map<String, String> schemas) {
-    if (schemas == null || schemas.isEmpty()) return;
+  public void reportDerivatives(Map<String, String> data) {
+    if (data == null || data.isEmpty()) return;
 
-    if (apiSchemas == null) {
-      apiSchemas = schemas;
+    if (derivatives == null) {
+      derivatives = data;
     } else {
-      apiSchemas.putAll(schemas);
+      derivatives.putAll(data);
     }
   }
 
-  boolean commitApiSchemas(TraceSegment traceSegment) {
-    if (traceSegment == null || apiSchemas == null) {
+  boolean commitDerivatives(TraceSegment traceSegment) {
+    if (traceSegment == null || derivatives == null) {
       return false;
     }
-    apiSchemas.forEach(traceSegment::setTagTop);
+    derivatives.forEach(traceSegment::setTagTop);
     return true;
   }
 
+  // Mainly used for testing and logging
+  Set<String> getDerivativeKeys() {
+    return derivatives == null ? emptySet() : new HashSet<>(derivatives.keySet());
+  }
+
   public boolean isThrottled(RateLimiter rateLimiter) {
     if (rateLimiter != null && rateLimited.compareAndSet(false, true)) {
       throttled = rateLimiter.isThrottled();

diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java
@@ -481,9 +481,9 @@ private NoopFlow onRequestEnded(RequestContext ctx_, IGSpanInfo spanInfo) {
         // Report minimum set of collected request headers
         writeRequestHeaders(traceSeg, DEFAULT_REQUEST_HEADERS_ALLOW_LIST, ctx.getRequestHeaders());
       }
-      // If extracted any Api Schemas - commit them
-      if (!ctx.commitApiSchemas(traceSeg)) {
-        log.debug("Unable to commit, api security schemas and will be skipped");
+      // If extracted any derivatives - commit them
+      if (!ctx.commitDerivatives(traceSeg)) {
+        log.debug("Unable to commit, derivatives will be skipped {}", ctx.getDerivativeKeys());
       }
 
       if (ctx.isBlocked()) {

diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/powerwaf/PowerWAFModule.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/powerwaf/PowerWAFModule.java
@@ -512,8 +512,8 @@ public void onDataAvailable(
         }
       }
 
-      if (resultWithData != null && resultWithData.schemas != null) {
-        reqCtx.reportApiSchemas(resultWithData.schemas);
+      if (resultWithData.derivatives != null) {
+        reqCtx.reportDerivatives(resultWithData.derivatives);
       }
     }
 

diff --git a/...sec/src/test/groovy/com/datadog/appsec/config/AppSecConfigServiceImplSpecification.groovy b/...sec/src/test/groovy/com/datadog/appsec/config/AppSecConfigServiceImplSpecification.groovy
@@ -23,11 +23,14 @@ import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_CUSTOM_RULES
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_DD_RULES
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_EXCLUSIONS
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_EXCLUSION_DATA
+import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_HEADER_FINGERPRINT
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_IP_BLOCKING
+import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_NETWORK_FINGERPRINT
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_RASP_SQLI
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_REQUEST_BLOCKING
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_TRUSTED_IPS
 import static datadog.remoteconfig.Capabilities.CAPABILITY_ASM_USER_BLOCKING
+import static datadog.remoteconfig.Capabilities.CAPABILITY_ENDPOINT_FINGERPRINT
 import static datadog.remoteconfig.PollingHinterNoop.NOOP
 import static datadog.trace.api.UserIdCollectionMode.ANONYMIZATION
 import static datadog.trace.api.UserIdCollectionMode.DISABLED
@@ -261,7 +264,11 @@ class AppSecConfigServiceImplSpecification extends DDSpecification {
       | CAPABILITY_ASM_CUSTOM_RULES
       | CAPABILITY_ASM_CUSTOM_BLOCKING_RESPONSE
       | CAPABILITY_ASM_TRUSTED_IPS
-      | CAPABILITY_ASM_RASP_SQLI)
+      | CAPABILITY_ASM_RASP_SQLI
+      | CAPABILITY_ENDPOINT_FINGERPRINT
+      // | CAPABILITY_ASM_SESSION_FINGERPRINT
+      | CAPABILITY_ASM_NETWORK_FINGERPRINT
+      | CAPABILITY_ASM_HEADER_FINGERPRINT)
     0 * _._
     initialWafConfig.get() != null
 
@@ -406,7 +413,11 @@ class AppSecConfigServiceImplSpecification extends DDSpecification {
       | CAPABILITY_ASM_CUSTOM_RULES
       | CAPABILITY_ASM_CUSTOM_BLOCKING_RESPONSE
       | CAPABILITY_ASM_TRUSTED_IPS
-      | CAPABILITY_ASM_RASP_SQLI)
+      | CAPABILITY_ASM_RASP_SQLI
+      | CAPABILITY_ENDPOINT_FINGERPRINT
+      // | CAPABILITY_ASM_SESSION_FINGERPRINT
+      | CAPABILITY_ASM_NETWORK_FINGERPRINT
+      | CAPABILITY_ASM_HEADER_FINGERPRINT)
     0 * _._
 
     when:
@@ -474,7 +485,11 @@ class AppSecConfigServiceImplSpecification extends DDSpecification {
       | CAPABILITY_ASM_TRUSTED_IPS
       | CAPABILITY_ASM_API_SECURITY_SAMPLE_RATE
       | CAPABILITY_ASM_RASP_SQLI
-      | CAPABILITY_ASM_AUTO_USER_INSTRUM_MODE,)
+      | CAPABILITY_ASM_AUTO_USER_INSTRUM_MODE
+      | CAPABILITY_ENDPOINT_FINGERPRINT
+      // | CAPABILITY_ASM_SESSION_FINGERPRINT
+      | CAPABILITY_ASM_NETWORK_FINGERPRINT
+      | CAPABILITY_ASM_HEADER_FINGERPRINT)
     4 * poller.removeListeners(_)
     1 * poller.removeConfigurationEndListener(_)
     1 * poller.stop()

diff --git a/...agent/appsec/src/test/groovy/com/datadog/appsec/gateway/GatewayBridgeSpecification.groovy b/...agent/appsec/src/test/groovy/com/datadog/appsec/gateway/GatewayBridgeSpecification.groovy
@@ -924,4 +924,20 @@ class GatewayBridgeSpecification extends DDSpecification {
     'appsec.events.users.login.failure.track' | true
     'appsec.another.unrelated.tag'            | false
   }
+
+  void 'fingerprints are set in the span after a request'() {
+    given:
+    final mockAppSecCtx = new AppSecRequestContext(derivatives: ['_dd.appsec.fp.http.endpoint': 'xyz'])
+    final mockCtx = Stub(RequestContext) {
+      getData(RequestContextSlot.APPSEC) >> mockAppSecCtx
+      getTraceSegment() >> traceSegment
+    }
+    final spanInfo = Stub(AgentSpan)
+
+    when:
+    requestEndedCB.apply(mockCtx, spanInfo)
+
+    then:
+    1 * traceSegment.setTagTop('_dd.appsec.fp.http.endpoint', 'xyz')
+  }
 }
diff --git a/...ent/appsec/src/test/groovy/com/datadog/appsec/powerwaf/PowerWAFModuleSpecification.groovy b/...ent/appsec/src/test/groovy/com/datadog/appsec/powerwaf/PowerWAFModuleSpecification.groovy
@@ -1486,6 +1486,30 @@ class PowerWAFModuleSpecification extends DDSpecification {
     })
   }
 
+  void 'fingerprint support'() {
+    given:
+    final flow = Mock(ChangeableFlow)
+    setupWithStubConfigService 'fingerprint_config.json'
+    dataListener = pwafModule.dataSubscriptions.first()
+    ctx.closeAdditive()
+    final bundle = MapDataBundle.ofDelegate([
+      (KnownAddresses.WAF_CONTEXT_PROCESSOR): [fingerprint: true],
+      (KnownAddresses.REQUEST_METHOD): 'GET',
+      (KnownAddresses.REQUEST_URI_RAW): 'http://localhost:8080/test',
+      (KnownAddresses.REQUEST_BODY_OBJECT): [:],
+      (KnownAddresses.REQUEST_QUERY): [name: ['test']],
+      (KnownAddresses.HEADERS_NO_COOKIES): new CaseInsensitiveMap<List<String>>(['user-agent': ['Arachni/v1.5.1']])
+    ])
+
+    when:
+    dataListener.onDataAvailable(flow, ctx, bundle, gwCtx)
+    ctx.closeAdditive()
+
+    then:
+    1 * flow.setAction({ it.blocking })
+    ctx.derivativeKeys.contains('_dd.appsec.fp.http.endpoint')
+  }
+
   private Map<String, Object> getDefaultConfig() {
     def service = new StubAppSecConfigService()
     service.init()

diff --git a/dd-java-agent/appsec/src/test/resources/fingerprint_config.json b/dd-java-agent/appsec/src/test/resources/fingerprint_config.json
@@ -0,0 +1,86 @@
+{
+  "version": "2.2",
+  "metadata": {
+    "rules_version": "1.8.0"
+  },
+  "rules": [
+    {
+      "id": "arachni_rule",
+      "name": "Arachni",
+      "tags": {
+        "type": "security_scanner",
+        "category": "attack_attempt"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "user-agent"
+                ]
+              }
+            ],
+            "regex": "^Arachni\\/v"
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": [],
+      "on_match": ["block"]
+    }
+  ],
+  "processors": [
+    {
+      "id": "processor-001",
+      "generator": "http_endpoint_fingerprint",
+      "conditions": [
+        {
+          "operator": "equals",
+          "parameters": {
+            "inputs": [
+              {
+                "address": "waf.context.processor",
+                "key_path": [
+                  "fingerprint"
+                ]
+              }
+            ],
+            "value": true,
+            "type": "boolean"
+          }
+        }
+      ],
+      "parameters": {
+        "mappings": [
+          {
+            "method": [
+              {
+                "address": "server.request.method"
+              }
+            ],
+            "uri_raw": [
+              {
+                "address": "server.request.uri.raw"
+              }
+            ],
+            "body": [
+              {
+                "address": "server.request.body"
+              }
+            ],
+            "query": [
+              {
+                "address": "server.request.query"
+              }
+            ],
+            "output": "_dd.appsec.fp.http.endpoint"
+          }
+        ]
+      },
+      "evaluate": true,
+      "output": true
+    }
+  ]
+}
diff --git a/remote-config/remote-config-api/src/main/java/datadog/remoteconfig/Capabilities.java b/remote-config/remote-config-api/src/main/java/datadog/remoteconfig/Capabilities.java
@@ -33,4 +33,8 @@ public interface Capabilities {
   long CAPABILITY_APM_TRACING_SAMPLE_RULES = 1 << 29;
   long CAPABILITY_CSM_ACTIVATION = 1 << 30;
   long CAPABILITY_ASM_AUTO_USER_INSTRUM_MODE = 1L << 31;
+  long CAPABILITY_ENDPOINT_FINGERPRINT = 1L << 32;
+  long CAPABILITY_ASM_SESSION_FINGERPRINT = 1L << 33;
+  long CAPABILITY_ASM_NETWORK_FINGERPRINT = 1L << 34;
+  long CAPABILITY_ASM_HEADER_FINGERPRINT = 1L << 35;
 }