[LibOS] Add dummy ancillary data support for sockets

Previously, Gramine returned `-ENOSYS` on `recvmsg()`/`sendmsg()` and
similar syscalls if it detected the use of `struct msghdr` ancillary
data (`msg_control` field). This commit is the first step in adding
proper ancillary data support.

Currently no ancillary data type is truly supported:
- on `sendmsg()`:
  - unknown/unsupported types force an error,
  - SCM_RIGHTS/SCM_CREDENTIALS are ignored in TCP/UDP sockets;
- on `recvmsg()`: `msg_controllen` is set to zero to indicate there is
  no ancillary data added to the received network packet.

Signed-off-by: Dmitrii Kuvaiskii <dmitrii.kuvaiskii@intel.com>

common/include/linux_socket.h

@@ -55,6 +55,8 @@ struct cmsghdr {
 #define CMSG_LEN(len)   (CMSG_ALIGN(sizeof(struct cmsghdr)) + (len))
 
 #define SCM_RIGHTS 1
+#define SCM_CREDENTIALS 2
+#define SCM_SECURITY 3
 
 #define AF_UNSPEC 0
 #define AF_UNIX 1
@@ -96,9 +98,14 @@ struct cmsghdr {
 #define SO_RCVTIMEO 20
 #define SO_SNDTIMEO 21
 #define SO_ACCEPTCONN 30
+#define SO_MARK 36
+#define SO_TIMESTAMPING_OLD 37
 #define SO_PROTOCOL 38
 #define SO_DOMAIN 39
 
+#define SO_TXTIME 61
+#define SCM_TXTIME SO_TXTIME
+
 /* TCP options. */
 #define TCP_NODELAY 1       /* Turn off Nagle's algorithm */
 #define TCP_CORK 3          /* Never send partially complete segments */

libos/include/libos_socket.h

@@ -80,6 +80,8 @@ struct libos_sock_ops {
      * \param      handle             A handle.
      * \param      iov                An array of buffers to write from.
      * \param      iov_len            The length of \p iov.
+     * \param      msg_control        An ancillary data buffer.
+     * \param      msg_controllen     The length of \p msg_control.
      * \param[out] out_size           On success contains the number of bytes sent.
      * \param      addr               An address to send to. May be NULL. It's up to
      *                                the implementation to decide what to do with it (which might
@@ -88,15 +90,19 @@ struct libos_sock_ops {
      * \param      force_nonblocking  If `true` this request should not block. Otherwise just use
      *                                whatever mode the handle is in.
      */
-    int (*send)(struct libos_handle* handle, struct iovec* iov, size_t iov_len, size_t* out_size,
-                void* addr, size_t addrlen, bool force_nonblocking);
+    int (*send)(struct libos_handle* handle, struct iovec* iov, size_t iov_len, void* msg_control,
+                size_t msg_controllen, size_t* out_size, void* addr, size_t addrlen,
+                bool force_nonblocking);
 
     /*!
      * \brief Receive continuous data into an array of buffers.
      *
      * \param         handle             A handle.
      * \param         iov                An array of buffers to read to.
      * \param         iov_len            The length of \p iov.
+     * \param         msg_control        An ancillary data buffer to populate.
+     * \param[in,out] msg_controllen     The length of \p msg_control. On success updated to the
+     *                                   actual length of the received ancillary data.
      * \param[out]    out_total_size     On success contains the number of bytes received (STREAM)
      *                                   or the datagram size (DGRAM), which might be bigger than
      *                                   the total size of buffers in \p iov array.
@@ -108,8 +114,9 @@ struct libos_sock_ops {
      * \param         force_nonblocking  If `true` this request should not block. Otherwise just use
      *                                   whatever mode the handle is in.
      */
-    int (*recv)(struct libos_handle* handle, struct iovec* iov, size_t iov_len,
-                size_t* out_total_size, void* addr, size_t* addrlen, bool force_nonblocking);
+    int (*recv)(struct libos_handle* handle, struct iovec* iov, size_t iov_len, void* msg_control,
+                size_t* msg_controllen, size_t* out_total_size, void* addr, size_t* addrlen,
+                bool force_nonblocking);
 };
 
 struct libos_handle* get_new_socket_handle(int family, int type, int protocol,
@@ -118,7 +125,9 @@ struct libos_handle* get_new_socket_handle(int family, int type, int protocol,
 extern struct libos_sock_ops sock_unix_ops;
 extern struct libos_sock_ops sock_ip_ops;
 
-ssize_t do_recvmsg(struct libos_handle* handle, struct iovec* iov, size_t iov_len, void* addr,
-                   size_t* addrlen, unsigned int* flags);
-ssize_t do_sendmsg(struct libos_handle* handle, struct iovec* iov, size_t iov_len, void* addr,
-                   size_t addrlen, unsigned int flags);
+ssize_t do_recvmsg(struct libos_handle* handle, struct iovec* iov, size_t iov_len,
+                   void* msg_control, size_t* msg_controllen, void* addr, size_t* addrlen,
+                   unsigned int* flags);
+ssize_t do_sendmsg(struct libos_handle* handle, struct iovec* iov, size_t iov_len,
+                   void* msg_control, size_t msg_controllen, void* addr, size_t addrlen,
+                   unsigned int flags);

libos/src/fs/socket/fs.c

@@ -36,7 +36,8 @@ static ssize_t read(struct libos_handle* handle, void* buf, size_t size, file_of
         .iov_len = size,
     };
     unsigned int flags = 0;
-    return do_recvmsg(handle, &iov, /*iov_len=*/1, /*addr=*/NULL, /*addrlen=*/NULL, &flags);
+    return do_recvmsg(handle, &iov, /*iov_len=*/1, /*msg_control=*/NULL, /*msg_controllen=*/NULL,
+                      /*addr=*/NULL, /*addrlen=*/NULL, &flags);
 }
 
 static ssize_t write(struct libos_handle* handle, const void* buf, size_t size, file_off_t* pos) {
@@ -45,20 +46,23 @@ static ssize_t write(struct libos_handle* handle, const void* buf, size_t size,
         .iov_base = (void*)buf,
         .iov_len = size,
     };
-    return do_sendmsg(handle, &iov, /*iov_len=*/1, /*addr=*/NULL, /*addrlen=*/0, /*flags=*/0);
+    return do_sendmsg(handle, &iov, /*iov_len=*/1, /*msg_control=*/NULL, /*msg_controllen=*/0,
+                      /*addr=*/NULL, /*addrlen=*/0, /*flags=*/0);
 }
 
 static ssize_t readv(struct libos_handle* handle, struct iovec* iov, size_t iov_len,
                      file_off_t* pos) {
     __UNUSED(pos);
     unsigned int flags = 0;
-    return do_recvmsg(handle, iov, iov_len, /*addr=*/NULL, /*addrlen=*/NULL, &flags);
+    return do_recvmsg(handle, iov, iov_len, /*msg_control=*/NULL, /*msg_controllen=*/NULL,
+                      /*addr=*/NULL, /*addrlen=*/NULL, &flags);
 }
 
 static ssize_t writev(struct libos_handle* handle, struct iovec* iov, size_t iov_len,
                       file_off_t* pos) {
     __UNUSED(pos);
-    return do_sendmsg(handle, iov, iov_len, /*addr=*/NULL, /*addrlen=*/0, /*flags=*/0);
+    return do_sendmsg(handle, iov, iov_len, /*msg_control=*/NULL, /*msg_controllen=*/0,
+                      /*addr=*/NULL, /*addrlen=*/0, /*flags=*/0);
 }
 
 static int hstat(struct libos_handle* handle, struct stat* stat) {

libos/src/net/ip.c

@@ -11,6 +11,7 @@
 
 #include "libos_fs.h"
 #include "libos_socket.h"
+#include "linux_socket.h"
 #include "pal.h"
 #include "socket_utils.h"
 
@@ -662,13 +663,52 @@ static int getsockopt(struct libos_handle* handle, int level, int optname, void*
     }
 }
 
-static int send(struct libos_handle* handle, struct iovec* iov, size_t iov_len, size_t* out_size,
-                void* addr, size_t addrlen, bool force_nonblocking) {
+static int send(struct libos_handle* handle, struct iovec* iov, size_t iov_len, void* msg_control,
+                size_t msg_controllen, size_t* out_size, void* addr, size_t addrlen,
+                bool force_nonblocking) {
     assert(handle->type == TYPE_SOCK);
 
     struct libos_sock_handle* sock = &handle->info.sock;
     struct sockaddr_storage sock_addr;
 
+    struct cmsghdr* cmsg = (struct cmsghdr*)msg_control;
+    size_t rest_msg_controllen = msg_controllen;
+    while (cmsg && rest_msg_controllen >= sizeof(struct cmsghdr)) {
+        if (cmsg->cmsg_len < sizeof(struct cmsghdr) || cmsg->cmsg_len > rest_msg_controllen)
+            return -EINVAL;
+
+        if (cmsg->cmsg_level != SOL_SOCKET) {
+            /*
+             * Currently don't support:
+             * - SOL_UDP:  UDP_SEGMENT
+             * - SOL_IPV6: IPV6_PKTINFO
+             * - SOL_IP:   IP_RETOPTS, IP_PKTINFO, IP_TTL, IP_TOS
+             *
+             * Note that there are no cmsgs for TCP (SOL_TCP) in Linux (as of v6.0).
+             */
+            return -EINVAL;
+        }
+
+        switch (cmsg->cmsg_type) {
+            /* currently don't support below SOL_SOCKET types */
+            case SO_MARK:
+            case SO_TIMESTAMPING_OLD:
+            case SCM_TXTIME:
+                return -EINVAL;
+
+            /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX, simply ignored */
+            case SCM_RIGHTS:
+            case SCM_CREDENTIALS:
+                break;
+
+            default:
+                return -EINVAL;
+        }
+
+        rest_msg_controllen -= CMSG_ALIGN(cmsg->cmsg_len);
+        cmsg = (struct cmsghdr*)((char*)cmsg + CMSG_ALIGN(cmsg->cmsg_len));
+    }
+
     switch (sock->type) {
         case SOCK_STREAM:
             /* TCP sockets ignore destination address - they must have been connected. */
@@ -709,8 +749,9 @@ static int send(struct libos_handle* handle, struct iovec* iov, size_t iov_len,
     return ret;
 }
 
-static int recv(struct libos_handle* handle, struct iovec* iov, size_t iov_len,
-                size_t* out_total_size, void* addr, size_t* addrlen, bool force_nonblocking) {
+static int recv(struct libos_handle* handle, struct iovec* iov, size_t iov_len, void* msg_control,
+                size_t* msg_controllen, size_t* out_total_size, void* addr, size_t* addrlen,
+                bool force_nonblocking) {
     assert(handle->type == TYPE_SOCK);
 
     switch (handle->info.sock.type) {
@@ -731,6 +772,21 @@ static int recv(struct libos_handle* handle, struct iovec* iov, size_t iov_len,
     if (ret < 0) {
         return pal_to_unix_errno(ret);
     }
+
+    if (msg_control && msg_controllen) {
+        /*
+         * Currently don't support:
+         * - SOL_TCP:    TCP_CM_INQ
+         * - SOL_SOCKET: SO_TIMESTAMPNS_NEW, SO_TIMESTAMPNS_OLD, SO_TIMESTAMP_NEW, SO_TIMESTAMP_OLD
+         * - SOL_IPV6:   IPV6_PKTINFO
+         * - SOL_IP:     IP_RETOPTS, IP_RECVOPTS, IP_PKTINFO, IP_TTL, IP_TOS, IP_RECVFRAGSIZE,
+         *               IP_CHECKSUM, SCM_SECURITY, IP_ORIGDSTADDR, IP_RECVERR
+         *
+         *  Note that SCM_RIGHTS and SCM_CREDENTIALS are not possible on TCP/UDP sockets.
+         */
+        *msg_controllen = 0;
+    }
+
     if (addr) {
         struct sockaddr_storage linux_addr;
         size_t linux_addr_len = sizeof(linux_addr);

libos/src/net/unix.c

@@ -19,6 +19,7 @@
 #include "libos_fs.h"
 #include "libos_internal.h"
 #include "libos_socket.h"
+#include "linux_socket.h"
 #include "pal.h"
 
 /*!
@@ -399,8 +400,9 @@ static int maybe_force_nonblocking_wrapper(bool force_nonblocking, struct libos_
     return ret;
 }
 
-static int send(struct libos_handle* handle, struct iovec* iov, size_t iov_len, size_t* out_size,
-                void* addr, size_t addrlen, bool force_nonblocking) {
+static int send(struct libos_handle* handle, struct iovec* iov, size_t iov_len, void* msg_control,
+                size_t msg_controllen, size_t* out_size, void* addr, size_t addrlen,
+                bool force_nonblocking) {
     __UNUSED(addr);
     __UNUSED(addrlen);
 
@@ -409,6 +411,30 @@ static int send(struct libos_handle* handle, struct iovec* iov, size_t iov_len,
         BUG();
     }
 
+    struct cmsghdr* cmsg = (struct cmsghdr*)msg_control;
+    size_t rest_msg_controllen = msg_controllen;
+    while (cmsg && rest_msg_controllen >= sizeof(struct cmsghdr)) {
+        if (cmsg->cmsg_len < sizeof(struct cmsghdr) || cmsg->cmsg_len > rest_msg_controllen)
+            return -EINVAL;
+
+        if (cmsg->cmsg_level != SOL_SOCKET) {
+            /* Linux ignores non-SOL-SOCKET cmsgs instead of erroring out, let's do the same */
+            continue;
+        }
+
+        switch (cmsg->cmsg_type) {
+            /* TODO: implement SCM_RIGHTS and SCM_CREDENTIALS */
+            case SCM_RIGHTS:
+            case SCM_CREDENTIALS:
+                return -ENOSYS;
+            default:
+                return -EINVAL;
+        }
+
+        rest_msg_controllen -= CMSG_ALIGN(cmsg->cmsg_len);
+        cmsg = (struct cmsghdr*)((char*)cmsg + CMSG_ALIGN(cmsg->cmsg_len));
+    }
+
     PAL_HANDLE pal_handle = __atomic_load_n(&handle->info.sock.pal_handle, __ATOMIC_ACQUIRE);
     if (!pal_handle) {
         return -ENOTCONN;
@@ -449,8 +475,9 @@ static int send(struct libos_handle* handle, struct iovec* iov, size_t iov_len,
     return 0;
 }
 
-static int recv(struct libos_handle* handle, struct iovec* iov, size_t iov_len, size_t* out_size,
-                void* addr, size_t* addrlen, bool force_nonblocking) {
+static int recv(struct libos_handle* handle, struct iovec* iov, size_t iov_len, void* msg_control,
+                size_t* msg_controllen, size_t* out_size, void* addr, size_t* addrlen,
+                bool force_nonblocking) {
     __UNUSED(addr);
     __UNUSED(addrlen);
 
@@ -500,6 +527,12 @@ static int recv(struct libos_handle* handle, struct iovec* iov, size_t iov_len,
         *out_size = size;
     }
     free(backing_buf);
+
+    if (msg_control && msg_controllen) {
+        /* TODO: implement SCM_RIGHTS and SCM_CREDENTIALS (if sent by app) */
+        *msg_controllen = 0;
+    }
+
     return ret;
 }