Support for TCP/UDP checksum offloading with partial checksums

src/core/inet_chksum.c

@@ -606,3 +606,144 @@ lwip_chksum_copy(void *dst, const void *src, u16_t len)
   return LWIP_CHKSUM(dst, len);
 }
 #endif /* (LWIP_CHKSUM_COPY_ALGORITHM == 1) */
+
+#if LWIP_CHECKSUM_PARTIAL
+#if LWIP_IPV4
+/*
+ * inet_chksum_pseudohdr
+ *
+ * Computes a partial IPV4/IPv6 internet checksum that covers only the
+ * protocol pseudo header (UDP, TCP). The parameters `proto`, `proto_len`
+ * can be set to `0x0` and `src` and `dest` can be set to `NULL`.
+ * This enables computing a partial checksum of the pseudo header.
+ * Skipped fields can be added later with `ip_chksum_pseudohdr_add16()`.
+ *
+ * @param proto ip protocol (pseudo header)
+ * @param proto_len length of the ip data part (pseudo header)
+ * @param src source IPv4 address (pseudo header)
+ * @param dest destination IPv4 address (pseudo header)
+ * @return checksum (as u16_t) to be saved directly in the protocol header
+ */
+u16_t
+inet_chksum_pseudohdr(u8_t proto, u16_t proto_len,
+                      const ip4_addr_t *src, const ip4_addr_t *dest)
+{
+  u32_t acc;
+  u32_t addr;
+
+  acc  = (u32_t)lwip_htons((u16_t)proto);
+  acc += (u32_t)lwip_htons(proto_len);
+
+  if (src) {
+    addr = ip4_addr_get_u32(src);
+    acc += (addr & 0xffffUL);
+    acc += ((addr >> 16) & 0xffffUL);
+  }
+  if (dest) {
+    addr = ip4_addr_get_u32(dest);
+    acc += (addr & 0xffffUL);
+    acc += ((addr >> 16) & 0xffffUL);
+  }
+
+  /* Fold 32-bit sum to 16 bits
+     calling this twice is probably faster than if statements... */
+  acc = FOLD_U32T(acc);
+  acc = FOLD_U32T(acc);
+  return (acc & 0xffffUL);
+}
+#endif /* LWIP_IPV4 */
+
+#if LWIP_IPV6
+/*
+ * ip6_chksum_pseudohdr
+ *
+ * Computes a partial IPv6 internet checksum that covers only the
+ * protocol pseudo header (UDP, TCP). The parameters `proto`, `proto_len`
+ * can be set to `0x0` and `src` and `dest` can be set to `NULL`.
+ * This enables computing a partial checksum of the pseudo header.
+ * Skipped fields can be added later with `ip_chksum_pseudohdr_add16()`.
+ *
+ * @param proto ip protocol (pseudo header)
+ * @param proto_len length of the ip data part (pseudo header)
+ * @param src source IPv6 address (pseudo header)
+ * @param dest destination IPv6 address (pseudo header)
+ * @return checksum (as u16_t) to be saved directly in the protocol header
+ */
+u16_t
+ip6_chksum_pseudohdr(u8_t proto, u16_t proto_len,
+                     const ip6_addr_t *src, const ip6_addr_t *dest)
+{
+  u32_t acc;
+  u32_t addr;
+  u8_t addr_part;
+
+  acc  = (u32_t)lwip_htons((u16_t)proto);
+  acc += (u32_t)lwip_htons(proto_len);
+
+  if (src) {
+    for (addr_part = 0; addr_part < 4; addr_part++) {
+      addr = src->addr[addr_part];
+      acc += (addr & 0xffffUL);
+      acc += ((addr >> 16) & 0xffffUL);
+    }
+  }
+  if (dest) {
+    for (addr_part = 0; addr_part < 4; addr_part++) {
+      addr = dest->addr[addr_part];
+      acc += (addr & 0xffffUL);
+      acc += ((addr >> 16) & 0xffffUL);
+    }
+  }
+
+  /* Fold 32-bit sum to 16 bits
+     calling this twice is probably faster than if statements... */
+  acc = FOLD_U32T(acc);
+  acc = FOLD_U32T(acc);
+  return (acc & 0xffffUL);
+}
+#endif /* LWIP_IPV6 */
+
+/*
+ * ip_chksum_pseudohdr
+ *
+ * Computes a partial IPV4/IPv6 internet checksum that covers only the
+ * protocol pseudo header (UDP, TCP). The parameters `proto`, `proto_len`
+ * can be set to `0x0` and `src` and `dest` can be set to `NULL`.
+ * This enables computing a partial checksum of the pseudo header.
+ * Skipped fields can be added later with `ip_chksum_pseudohdr_add16()`.
+ *
+ * @param proto ip protocol (pseudo header)
+ * @param proto_len length of the ip data part (pseudo header)
+ * @param src source ip address (pseudo header)
+ * @param dest destination ip address (pseudo header)
+ * @return checksum (as u16_t) to be saved directly in the protocol header
+ */
+u16_t
+ip_chksum_pseudohdr(u8_t proto, u16_t proto_len,
+                    const ip_addr_t *src, const ip_addr_t *dest)
+{
+#if LWIP_IPV6
+  if (src && IP_IS_V6(src)) {
+    const ip6_addr_t *src6;
+    const ip6_addr_t *dest6;
+
+    src6  = (src  != NULL) ? ip_2_ip6(src)  : NULL;
+    dest6 = (dest != NULL) ? ip_2_ip6(dest) : NULL;
+    return ip6_chksum_pseudohdr(proto, proto_len, src6, dest6);
+  }
+#endif /* LWIP_IPV6 */
+#if LWIP_IPV4 && LWIP_IPV6
+  else
+#endif /* LWIP_IPV4 && LWIP_IPV6 */
+#if LWIP_IPV4
+  {
+    const ip4_addr_t *src4;
+    const ip4_addr_t *dest4;
+
+    src4  = (src  != NULL) ? ip_2_ip4(src)  : NULL;
+    dest4 = (dest != NULL) ? ip_2_ip4(dest) : NULL;
+    return inet_chksum_pseudohdr(proto, proto_len, src4, dest4);
+  }
+#endif /* LWIP_IPV4 */
+}
+#endif /* LWIP_CHECKSUM_PARTIAL */

src/core/pbuf.c

@@ -523,6 +523,11 @@ pbuf_add_header_impl(struct pbuf *p, size_t header_size_increment, u8_t force)
   p->len = (u16_t)(p->len + increment_magnitude);
   p->tot_len = (u16_t)(p->tot_len + increment_magnitude);
 
+#if LWIP_CHECKSUM_PARTIAL
+  if ((p->flags & PBUF_FLAG_CSUM_PARTIAL)
+      && ((p->csum_start != 0x0) || (p->csum_offset != 0x0)))
+    p->csum_start = (s32_t)(p->csum_start + increment_magnitude);
+#endif /* LWIP_CHECKSUM_PARTIAL */
 
   return 0;
 }
@@ -606,6 +611,12 @@ pbuf_remove_header(struct pbuf *p, size_t header_size_decrement)
   p->len = (u16_t)(p->len - increment_magnitude);
   p->tot_len = (u16_t)(p->tot_len - increment_magnitude);
 
+#if LWIP_CHECKSUM_PARTIAL
+  if ((p->flags & PBUF_FLAG_CSUM_PARTIAL)
+      && ((p->csum_start != 0x0) || (p->csum_offset != 0x0)))
+    p->csum_start = (s32_t)(p->csum_start - increment_magnitude);
+#endif /* LWIP_CHECKSUM_PARTIAL */
+
   LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_remove_header: old %p new %p (%"U16_F")\n",
               (void *)payload, (void *)p->payload, increment_magnitude));
 
@@ -857,6 +868,9 @@ pbuf_cat(struct pbuf *h, struct pbuf *t)
   LWIP_ERROR("(h != NULL) && (t != NULL) (programmer violates API)",
              ((h != NULL) && (t != NULL)), return;);
 
+  /* header flags can only be set on heading pbuf */
+  LWIP_ASSERT("header flags set in tailing pbuf of a chain", !(t->flags & PBUF_HDR_FLAGS_MASK));
+
   /* proceed to last pbuf of chain */
   for (p = h; p->next != NULL; p = p->next) {
     /* add total length of second chain to all totals of first chain */
@@ -960,11 +974,26 @@ pbuf_dechain(struct pbuf *p)
 err_t
 pbuf_copy(struct pbuf *p_to, const struct pbuf *p_from)
 {
+  err_t status;
   LWIP_DEBUGF(PBUF_DEBUG | LWIP_DBG_TRACE, ("pbuf_copy(%p, %p)\n",
               (const void *)p_to, (const void *)p_from));
 
   LWIP_ERROR("pbuf_copy: invalid source", p_from != NULL, return ERR_ARG;);
-  return pbuf_copy_partial_pbuf(p_to, p_from, p_from->tot_len, 0);
+
+  status = pbuf_copy_partial_pbuf(p_to, p_from, p_from->tot_len, 0);
+  if (status != ERR_OK)
+    goto out;
+
+  /* copy over DATA_VALID and CSUM_PARTIAL flag */
+  p_to->flags |= (p_from->flags & PBUF_HDR_FLAGS_MASK);
+#if LWIP_CHECKSUM_PARTIAL
+  if (p_from->flags & PBUF_FLAG_CSUM_PARTIAL) {
+    p_to->csum_start = p_from->csum_start;
+    p_to->csum_offset = p_from->csum_offset;
+  }
+#endif /* LWIP_CHECKSUM_PARTIAL */
+out:
+  return status;
 }
 
 /**

src/core/tcp_in.c

@@ -158,6 +158,42 @@ tcp_input(struct pbuf *p, struct netif *inp)
 
 #if CHECKSUM_CHECK_TCP
   IF__NETIF_CHECKSUM_ENABLED(inp, NETIF_CHECKSUM_CHECK_TCP) {
+#if CHECKSUM_SKIPVALID_TCP
+    if ((!LWIP_CHECKSUM_CTRL_PER_NETIF ||
+         NETIF_CHECKSUM_ENABLED(inp, NETIF_CHECKSUM_SKIPVALID_TCP)) &&
+	(p->flags & PBUF_FLAG_DATA_VALID)) {
+      /* We received a packet marked as valid, skip checks */
+      LWIP_DEBUGF(TCP_INPUT_DEBUG, ("tcp_input: packet marked as valid, skip checksum validation\n"));
+      goto chkvalid;
+    }
+#endif /* CHECKSUM_SKIPVALID_TCP */
+#if CHECKSUM_PARTIAL_TCP
+    if ((!LWIP_CHECKSUM_CTRL_PER_NETIF ||
+         NETIF_CHECKSUM_ENABLED(inp, NETIF_CHECKSUM_PARTIAL_TCP)) &&
+        (p->flags & PBUF_FLAG_CSUM_PARTIAL)) {
+      /* We received a packet marked with incomplete checksum */
+      LWIP_DEBUGF(TCP_INPUT_DEBUG,
+                  ("tcp_input: received packet with partial checksum (csum_start=%"S32_F", csum_offset=%"U16_F")\n",
+                   p->csum_start, p->csum_offset));
+
+      /* If csum_start and csum_offset are given, check that they
+       * point to the checksum field of the TCP header
+       */
+      if (((p->csum_start != 0x0) || (p->csum_offset != 0x0))
+          && (((p->csum_start + p->csum_offset) != TCPH_CHKSUM_OFFSET))) {
+           LWIP_DEBUGF(TCP_INPUT_DEBUG,
+                       ("tcp_input: packet discarded due to invalid checksum location\n"));
+           TCP_STATS_INC(tcp.chkerr);
+           goto dropped;
+        }
+
+      /* We assume that an incomplete checksummed packet results from
+       * an in-memory communication that can be found in virtualized
+       * environments, like, host-to-guest, guest-to-host, guest-to-guest
+       */
+      goto chkvalid;
+    }
+#endif /* CHECKSUM_PARTIAL_TCP */
     /* Verify TCP checksum. */
     u16_t chksum = ip_chksum_pseudo(p, IP_PROTO_TCP, p->tot_len,
                                     ip_current_src_addr(), ip_current_dest_addr());
@@ -169,6 +205,9 @@ tcp_input(struct pbuf *p, struct netif *inp)
       goto dropped;
     }
   }
+#if CHECKSUM_SKIPVALID_TCP || CHECKSUM_PARTIAL_TCP
+chkvalid:
+#endif /* CHECKSUM_SKIPVALID_TCP || CHECKSUM_PARTIAL_TCP */
 #endif /* CHECKSUM_CHECK_TCP */
 
   /* sanity-check header length */
@@ -675,6 +714,11 @@ tcp_listen_input(struct tcp_pcb_listen *pcb)
     /* Set up the new PCB. */
     ip_addr_copy(npcb->local_ip, *ip_current_dest_addr());
     ip_addr_copy(npcb->remote_ip, *ip_current_src_addr());
+#if TCP_CHECKSUM_PARTIAL
+    /* We will set proto_len later for each segment that we will send */
+    npcb->chksum_base = ip_chksum_pseudohdr(IP_PROTO_TCP, 0x0,
+                                            &npcb->local_ip, &npcb->remote_ip);
+#endif /* TCP_CHECKSUM_PARTIAL */
     npcb->local_port = pcb->local_port;
     npcb->remote_port = tcphdr->src;
     npcb->state = SYN_RCVD;

src/core/tcp_out.c

@@ -1597,11 +1597,30 @@ tcp_output_segment(struct tcp_seg *seg, struct tcp_pcb *pcb, struct netif *netif
     }
 #endif /* TCP_CHECKSUM_ON_COPY_SANITY_CHECK */
 #else /* TCP_CHECKSUM_ON_COPY */
-    seg->tcphdr->chksum = ip_chksum_pseudo(seg->p, IP_PROTO_TCP,
-                                           seg->p->tot_len, &pcb->local_ip, &pcb->remote_ip);
+#if TCP_CHECKSUM_PARTIAL
+    IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_PARTIAL_TCP) {
+      u32_t acc = pcb->chksum_base;
+
+      /* add proto length to pre-computed pseudo header checksum */
+      acc += lwip_htons(seg->p->tot_len);
+      acc = FOLD_U32T(acc);
+      acc = FOLD_U32T(acc);
+      seg->tcphdr->chksum = (u16_t)(acc & 0xffffUL);
+      seg->p->flags |= PBUF_FLAG_CSUM_PARTIAL;
+      seg->p->csum_start = 0;
+      seg->p->csum_offset = TCPH_CHKSUM_OFFSET;
+    } else
+#endif /* TCP_CHECKSUM_PARTIAL */
+    {
+      seg->tcphdr->chksum = ip_chksum_pseudo(seg->p, IP_PROTO_TCP,
+                                             seg->p->tot_len, &pcb->local_ip, &pcb->remote_ip);
+    }
 #endif /* TCP_CHECKSUM_ON_COPY */
   }
 #endif /* CHECKSUM_GEN_TCP */
+#if TCP_CHECKSUM_PARTIAL
+  seg->p->flags |= PBUF_FLAG_DATA_VALID;
+#endif /* TCP_CHECKSUM_PARTIAL */
   TCP_STATS_INC(tcp.xmit);
 
   NETIF_SET_HINTS(netif, &(pcb->netif_hints));
@@ -1934,9 +1953,21 @@ tcp_output_control_segment(const struct tcp_pcb *pcb, struct pbuf *p,
     u8_t ttl, tos;
 #if CHECKSUM_GEN_TCP
     IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_GEN_TCP) {
-      struct tcp_hdr *tcphdr = (struct tcp_hdr *)p->payload;
-      tcphdr->chksum = ip_chksum_pseudo(p, IP_PROTO_TCP, p->tot_len,
-                                        src, dst);
+#if TCP_CHECKSUM_PARTIAL
+      IF__NETIF_CHECKSUM_ENABLED(netif, NETIF_CHECKSUM_PARTIAL_TCP) {
+        struct tcp_hdr *tcphdr = (struct tcp_hdr *)p->payload;
+        tcphdr->chksum = ip_chksum_pseudohdr(IP_PROTO_TCP, p->tot_len,
+					     src, dst);
+        p->flags |= PBUF_FLAG_CSUM_PARTIAL | PBUF_FLAG_DATA_VALID;
+        p->csum_start = 0;
+        p->csum_offset = TCPH_CHKSUM_OFFSET;
+      } else
+#endif /* TCP_CHECKSUM_PARTIAL */
+      {
+        struct tcp_hdr *tcphdr = (struct tcp_hdr *)p->payload;
+        tcphdr->chksum = ip_chksum_pseudo(p, IP_PROTO_TCP, p->tot_len,
+                                          src, dst);
+      }
     }
 #endif
     if (pcb != NULL) {