Extend nmap capability in discovery node (#1090)

Author: noursaidi

misc/discoverynode/README.md

@@ -159,9 +159,21 @@ journalctl -f -xeu udmi_discovery
 
 ```
 
+## Integration Test
+
+```
+misc/discoverynode/testing/integration/integration_test.sh
+```
+
+This will load a docker container, however with a dedicated testing wrapper 
+`misc/discoverynode/src/tests/test_integration.py` 
+as the entry point.
+
+This can be used to validate that discovery actually discovers, message formats, etc.
+
 ## E2E Test
 
-TODO
+Refer to Github CI test. 
 
 ## Test Suite
 

misc/discoverynode/src/tests/test_integration.py

@@ -23,7 +23,6 @@ def test_bacnet_integration():
   test_config["mqtt"] = dict(device_id="THUNDERBIRD-2")
   test_config["bacnet"] = dict(ip=None, port=None, interface=None)
   test_config["udmi"] = {"discovery": dict(ipv4=False,vendor=False,ether=False,bacnet=True)}
-
   # Container for storing all discovery messages
   messages = []
 
@@ -53,15 +52,53 @@ def test_bacnet_integration():
       print(message.to_json())
       print("----")
 
-    return
-    assert False, "failing so messages are printed to terminal"
-
-    expected_ethmacs = set(d["ether"] for d in docker_config.values())
-    seen_ethmac_toplevel = set(m.families["ether"].addr for m in messages if "ether" in m.families)
 
-    expected_bacnet_ids = set(str(d["bacnet_id"]) for d in docker_config.values())
+   
+    expected_bacnet_ids = set(["1"])
     seen_bacnet_ids_toplevel = set(m.addr for m in messages if m.family == "bacnet")
 
-    # subset because passive scan will find the gateway and device itself
-    assert seen_ethmac_toplevel == expected_ethmacs + 2
     assert expected_bacnet_ids == seen_bacnet_ids_toplevel 
+
+
+def test_nmap():
+  
+  # This is the "config.json" which is passed to `main.py` typically
+  test_config = collections.defaultdict()
+  test_config["mqtt"] = dict(device_id="THUNDERBIRD-2")
+  test_config["mqtt"] = dict(device_id="THUNDERBIRD-2")
+  test_config["bacnet"] = dict(ip=None, port=None, interface=None)
+  test_config["udmi"] = {"discovery": dict(ipv4=False,vendor=False,ether=True,bacnet=False)}
+  test_config["nmap"] = dict(targets=["192.168.12.1"])
+
+  # Container for storing all discovery messages
+  messages = []
+
+  with (
+      mock.patch.object(
+          udmi.core.UDMICore, "publish_discovery", new=messages.append
+      ) as published_discovery,
+  ):
+    mock_mqtt_client = mock.MagicMock()
+    udmi_client = udmi.core.UDMICore(publisher=mock_mqtt_client, topic_prefix="notneeded", config=test_config)
+    
+    # Start discovery
+    udmi_client.config_handler(
+        json.dumps({
+            "timestamp": timestamp_now(),
+            "discovery": {
+                "families": {
+                    "ether": {"generation": timestamp_now()}
+                }
+            },
+        })
+    )
+
+    time.sleep(30)
+    
+    print(len(messages))
+    for message in messages:
+      print(message.to_json())
+      print("----")
+    
+
+    assert False, "failing so messages are printed to terminal"

misc/discoverynode/src/udmi/discovery/nmap.py

@@ -7,6 +7,7 @@
 import udmi.discovery.utils.nmap as nmap
 import udmi.schema.discovery_event
 import udmi.schema.state
+import dataclasses
 
 
 class NmapBannerScan(discovery.DiscoveryController):
@@ -42,7 +43,10 @@ def nmap_runner(self):
             "/usr/bin/nmap",
             "--script",
             "banner",
-            "127.0.0.1",
+            "-p-",
+            "-T4",
+            "-A",
+            self.target_ips[0],
             "-oX",
             OUTPUT_FILE,
             "--stats-every",
@@ -71,8 +75,8 @@ def nmap_runner(self):
           generation=self.generation,
           family=self.family,
           addr=host.ip,
-          families={
-              "port": {p.port_number: {"banner": p.banner} for p in host.ports}
+          refs={
+              f"{p.port_number}": {"adjunct": dataclasses.asdict(p)} for p in host.ports
           },
       )
-      self.publish(event.to_json())
+      self.publish(event)

misc/discoverynode/src/udmi/discovery/utils/nmap.py

@@ -17,6 +17,7 @@ class NmapPort:
   port_number: int
   sort_index: int = dataclasses.field(init=False, repr=False)
   service: str | None = None
+  version: str | None = None 
   state: str | None = None
   protocol: str | None = None
   product: str | None = None
@@ -66,6 +67,7 @@ def results_reader(nmap_file: str) -> Generator[NmapHost, None, None]:
         port.product = service.attrib.get('product')
         if service.attrib['method'] == 'probed':
           port.service = service.attrib['name']
+          port.version = service.attrib.get('version')
 
       host.ports.append(port)
 

misc/discoverynode/testing/docker/bacnet_device/Dockerfile

@@ -10,8 +10,11 @@ COPY requirements.txt /requirements.txt
 RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.txt
 
 # Copy the virtualenv into a distroless image
-FROM gcr.io/distroless/python3-debian12:debug
+FROM debian:12
+RUN apt-get update && \
+    apt-get install --no-install-suggests --no-install-recommends --yes python3 coreutils netcat-openbsd ssh openssh-server
 COPY --from=build-venv /venv /venv
 COPY . /app
 WORKDIR /app
-ENTRYPOINT [ "/venv/bin/python3", "bacnet_server.py" ]
+ENTRYPOINT ["/app/entrypoint.sh"]
+

misc/discoverynode/testing/docker/bacnet_device/entrypoint.sh

@@ -0,0 +1,33 @@
+#!/bin/bash -e
+
+echo Starting FTP 21514 and open default 20,21
+nc -nvlt -p 20 &
+nc -nvlt -p 21 &
+(while true; do echo -e "220 ProFTPD 1.3.5e Server (Debian) $(hostname)" | nc -l -w 1 21514; done) &
+
+echo Starting SMTP 1256 and open default 25, 465, 587
+nc -nvlt -p 25 &
+nc -nvlt -p 465 &
+nc -nvlt -p 587 &
+(while true; do echo -e "220 $(hostname) ESMTP Postfix (Ubuntu)" | nc -l -w 1 1256; done) &
+
+echo Starting IMAP 5361 and open default ports 143, 993
+nc -nvlt -p 143 &
+nc -nvlt -p 993 &
+(while true; do echo -e "* OK [CAPABILITY IMAP4rev1 LITERAL+ SASL-IR LOGIN-REFERRALS ID ENABLE IDLE STARTTLS AUTH=PLAIN] Dovecot (Ubuntu) ready.\r\n" \
+    | nc -l -w 1 5361; done) &
+
+echo Starting POP3 23451 and open default 110, 995
+nc -nvlt -p 110 &
+nc -nvlt -p 995 &
+(while true; do echo -ne "+OK POP3 Server ready\r\n" | nc -l -w 1 23451; done) &
+
+echo starting TFTP UDP 69
+(while true; do echo -ne "\0\x05\0\0\x07\0" | nc -u -l -w 1 69; done) &
+
+
+mkdir /var/run/sshd
+chmod 0755 /var/run/sshd
+/usr/sbin/sshd
+
+/venv/bin/python3 bacnet_server.py

misc/discoverynode/testing/docker/discovery_node/Dockerfile

@@ -10,7 +10,9 @@ COPY requirements.txt /requirements.txt
 RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.txt
 
 # Copy the virtualenv into a distroless image
-FROM gcr.io/distroless/python3-debian12:debug
+FROM debian:12-slim
+RUN apt-get update && \
+    apt-get install --no-install-suggests --no-install-recommends --yes python3 coreutils nmap
 COPY --from=build-venv /venv /venv
 COPY . /app
 WORKDIR /app