* new PAYLOAD_TYPE_MULTIPART

* experimental double ACK's (at each hop), sent 300ms apart (direct mode only)

examples/simple_room_server/main.cpp

@@ -576,15 +576,18 @@ protected:
 
         uint32_t delay_millis;
         if (send_ack) {
-          mesh::Packet* ack = createAck(ack_hash);
+          if (client->out_path_len < 0) {
-          if (ack) {
+            mesh::Packet* ack = createAck(ack_hash);
-            if (client->out_path_len < 0) {
+            if (ack) sendFlood(ack, TXT_ACK_DELAY);
-              sendFlood(ack, TXT_ACK_DELAY);
+            delay_millis = TXT_ACK_DELAY + REPLY_DELAY_MILLIS;
-            } else {
+          } else {
-              sendDirect(ack, client->out_path, client->out_path_len, TXT_ACK_DELAY);
+            mesh::Packet* a1 = createMultiAck(ack_hash, 1);
-            }
+            if (a1) sendDirect(a1, client->out_path, client->out_path_len, TXT_ACK_DELAY);
+
+            mesh::Packet* a2 = createAck(ack_hash);
+            if (a2) sendDirect(a2, client->out_path, client->out_path_len, TXT_ACK_DELAY + 300);
+            delay_millis = TXT_ACK_DELAY + REPLY_DELAY_MILLIS + 300;
           }
-          delay_millis = TXT_ACK_DELAY + REPLY_DELAY_MILLIS;
         } else {
           delay_millis = 0;
         }

src/Mesh.cpp

@@ -67,22 +67,22 @@ DispatcherAction Mesh::onRecvPacket(Packet* pkt) {
 
   if (pkt->isRouteDirect() && pkt->path_len >= PATH_HASH_SIZE) {
     if (self_id.isHashMatch(pkt->path) && allowPacketForward(pkt)) {
-      if (_tables->hasSeen(pkt)) return ACTION_RELEASE;  // don't retransmit!
+      if (pkt->getPayloadType() == PAYLOAD_TYPE_MULTIPART) {
-
+        return forwardMultipartDirect(pkt);
-      // remove our hash from 'path', then re-broadcast
+      } else if (pkt->getPayloadType() == PAYLOAD_TYPE_ACK) {
-      pkt->path_len -= PATH_HASH_SIZE;
+        if (!_tables->hasSeen(pkt)) {  // don't retransmit!
-  #if 0
+          removeSelfFromPath(pkt);
-      memcpy(pkt->path, &pkt->path[PATH_HASH_SIZE], pkt->path_len);
+          routeDirectRecvAcks(pkt, 0);
-  #elif PATH_HASH_SIZE == 1
+        }
-      for (int k = 0; k < pkt->path_len; k++) {  // shuffle bytes by 1
+        return ACTION_RELEASE;
-        pkt->path[k] = pkt->path[k + 1];
       }
-  #else
-      #error "need path remove impl"
-  #endif
 
-      uint32_t d = getDirectRetransmitDelay(pkt);
+      if (!_tables->hasSeen(pkt)) {
-      return ACTION_RETRANSMIT_DELAYED(0, d);  // Routed traffic is HIGHEST priority 
+        removeSelfFromPath(pkt);
+
+        uint32_t d = getDirectRetransmitDelay(pkt);
+        return ACTION_RETRANSMIT_DELAYED(0, d);  // Routed traffic is HIGHEST priority 
+      }
     }
     return ACTION_RELEASE;   // this node is NOT the next hop (OR this packet has already been forwarded), so discard.
   }
@@ -99,6 +99,7 @@ DispatcherAction Mesh::onRecvPacket(Packet* pkt) {
       } else if (!_tables->hasSeen(pkt)) {
         onAckRecv(pkt, ack_crc);
         action = routeRecvPacket(pkt);
+        // routeRecvAcks(pkt, 0);  // experimental, double Acks in flood mode(?)
       }
       break;
     }
@@ -261,6 +262,33 @@ DispatcherAction Mesh::onRecvPacket(Packet* pkt) {
       }
       break;
     }
+    case PAYLOAD_TYPE_MULTIPART:
+      if (pkt->payload_len > 2) {
+        uint8_t remaining = pkt->payload[0] >> 4;  // num of packets in this multipart sequence still to be sent
+        uint8_t type = pkt->payload[0] & 0x0F;
+
+        if (type == PAYLOAD_TYPE_ACK && pkt->payload_len >= 5) {    // a multipart ACK
+          Packet tmp;
+          tmp.header = pkt->header;
+          tmp.path_len = pkt->path_len;
+          memcpy(tmp.path, pkt->path, pkt->path_len);
+          tmp.payload_len = pkt->payload_len - 1;
+          memcpy(tmp.payload, &pkt->payload[1], tmp.payload_len);
+
+          if (!_tables->hasSeen(&tmp)) {
+            uint32_t ack_crc;
+            memcpy(&ack_crc, tmp.payload, 4);
+
+            onAckRecv(&tmp, ack_crc);
+            // routeRecvAcks(&tmp, ((uint32_t)remaining) * 600);  // expect multipart ACK 300ms apart (x2)
+            //action = routeRecvPacket(&tmp);  // NOTE: currently not needed, as multipart ACKs not sent Flood
+          }
+        } else {
+          // FUTURE: other multipart types??
+        }
+      }
+      break;
+
     default:
       MESH_DEBUG_PRINTLN("%s Mesh::onRecvPacket(): unknown payload type, header: %d", getLogDateTime(), (int) pkt->header);
       // Don't flood route unknown packet types!   action = routeRecvPacket(pkt);
@@ -269,6 +297,20 @@ DispatcherAction Mesh::onRecvPacket(Packet* pkt) {
   return action;
 }
 
+void Mesh::removeSelfFromPath(Packet* pkt) {
+  // remove our hash from 'path'
+  pkt->path_len -= PATH_HASH_SIZE;
+#if 0
+  memcpy(pkt->path, &pkt->path[PATH_HASH_SIZE], pkt->path_len);
+#elif PATH_HASH_SIZE == 1
+  for (int k = 0; k < pkt->path_len; k++) {  // shuffle bytes by 1
+    pkt->path[k] = pkt->path[k + 1];
+  }
+#else
+  #error "need path remove impl"
+#endif
+}
+
 DispatcherAction Mesh::routeRecvPacket(Packet* packet) {
   if (packet->isRouteFlood() && !packet->isMarkedDoNotRetransmit()
     && packet->path_len + PATH_HASH_SIZE <= MAX_PATH_SIZE && allowPacketForward(packet)) {
@@ -282,6 +324,82 @@ DispatcherAction Mesh::routeRecvPacket(Packet* packet) {
   return ACTION_RELEASE;
 }
 
+#if 0
+void Mesh::routeRecvAcks(Packet* packet, uint32_t delay_millis) {
+  if (packet->isRouteFlood() && !packet->isMarkedDoNotRetransmit()
+    && packet->path_len + PATH_HASH_SIZE <= MAX_PATH_SIZE && allowPacketForward(packet)) {
+    // append this node's hash to 'path'
+    packet->path_len += self_id.copyHashTo(&packet->path[packet->path_len]);
+
+    uint32_t crc;
+    memcpy(&crc, packet->payload, 4);
+
+    delay_millis += getRetransmitDelay(packet);
+    auto a1 = createMultiAck(crc, 1);
+    if (a1) {
+      memcpy(a1->path, packet->path, a1->path_len = packet->path_len);
+      a1->header &= ~PH_ROUTE_MASK;
+      a1->header |= ROUTE_TYPE_FLOOD;
+      sendPacket(a1, 1, delay_millis);
+    }
+
+    delay_millis += 300;
+    auto a2 = createAck(crc);
+    if (a2) {
+      memcpy(a2->path, packet->path, a2->path_len = packet->path_len);
+      a2->header &= ~PH_ROUTE_MASK;
+      a2->header |= ROUTE_TYPE_FLOOD;
+      sendPacket(a2, 1, delay_millis);
+    }
+  }
+}
+#endif
+
+DispatcherAction Mesh::forwardMultipartDirect(Packet* pkt) {
+  uint8_t remaining = pkt->payload[0] >> 4;  // num of packets in this multipart sequence still to be sent
+  uint8_t type = pkt->payload[0] & 0x0F;
+
+  if (type == PAYLOAD_TYPE_ACK && pkt->payload_len >= 5) {    // a multipart ACK
+    Packet tmp;
+    tmp.header = pkt->header;
+    tmp.path_len = pkt->path_len;
+    memcpy(tmp.path, pkt->path, pkt->path_len);
+    tmp.payload_len = pkt->payload_len - 1;
+    memcpy(tmp.payload, &pkt->payload[1], tmp.payload_len);
+
+    if (!_tables->hasSeen(&tmp)) {   // don't retransmit!
+      removeSelfFromPath(&tmp);
+      routeDirectRecvAcks(&tmp, ((uint32_t)remaining) * 600);  // expect multipart ACKs 300ms apart (x2)
+    }
+  }
+  return ACTION_RELEASE;
+}
+
+void Mesh::routeDirectRecvAcks(Packet* packet, uint32_t delay_millis) {
+  if (!packet->isMarkedDoNotRetransmit()) {
+    uint32_t crc;
+    memcpy(&crc, packet->payload, 4);
+
+    delay_millis += getDirectRetransmitDelay(packet);
+    auto a1 = createMultiAck(crc, 1);
+    if (a1) {
+      memcpy(a1->path, packet->path, a1->path_len = packet->path_len);
+      a1->header &= ~PH_ROUTE_MASK;
+      a1->header |= ROUTE_TYPE_DIRECT;
+      sendPacket(a1, 0, delay_millis);
+    }
+
+    delay_millis += 300;
+    auto a2 = createAck(crc);
+    if (a2) {
+      memcpy(a2->path, packet->path, a2->path_len = packet->path_len);
+      a2->header &= ~PH_ROUTE_MASK;
+      a2->header |= ROUTE_TYPE_DIRECT;
+      sendPacket(a2, 0, delay_millis);
+    }
+  }
+}
+
 Packet* Mesh::createAdvert(const LocalIdentity& id, const uint8_t* app_data, size_t app_data_len) {
   if (app_data_len > MAX_ADVERT_DATA_SIZE) return NULL;
 
@@ -449,6 +567,21 @@ Packet* Mesh::createAck(uint32_t ack_crc) {
   return packet;
 }
 
+Packet* Mesh::createMultiAck(uint32_t ack_crc, uint8_t remaining) {
+  Packet* packet = obtainNewPacket();
+  if (packet == NULL) {
+    MESH_DEBUG_PRINTLN("%s Mesh::createMultiAck(): error, packet pool empty", getLogDateTime());
+    return NULL;
+  }
+  packet->header = (PAYLOAD_TYPE_MULTIPART << PH_TYPE_SHIFT);  // ROUTE_TYPE_* set later
+
+  packet->payload[0] = (remaining << 4) | PAYLOAD_TYPE_ACK;
+  memcpy(&packet->payload[1], &ack_crc, 4);
+  packet->payload_len = 5;
+
+  return packet;
+}
+
 Packet* Mesh::createRawData(const uint8_t* data, size_t len) {
   if (len > sizeof(Packet::payload)) return NULL;  // invalid arg
 

src/Mesh.h

@@ -28,6 +28,11 @@ class Mesh : public Dispatcher {
   RNG* _rng;
   MeshTables* _tables;
 
+  void removeSelfFromPath(Packet* packet);
+  void routeDirectRecvAcks(Packet* packet, uint32_t delay_millis);
+  //void routeRecvAcks(Packet* packet, uint32_t delay_millis);
+  DispatcherAction forwardMultipartDirect(Packet* pkt);
+
 protected:
   DispatcherAction onRecvPacket(Packet* pkt) override;
 
@@ -165,6 +170,7 @@ public:
   Packet* createAnonDatagram(uint8_t type, const LocalIdentity& sender, const Identity& dest, const uint8_t* secret, const uint8_t* data, size_t data_len);
   Packet* createGroupDatagram(uint8_t type, const GroupChannel& channel, const uint8_t* data, size_t data_len);
   Packet* createAck(uint32_t ack_crc);
+  Packet* createMultiAck(uint32_t ack_crc, uint8_t remaining);
   Packet* createPathReturn(const uint8_t* dest_hash, const uint8_t* secret, const uint8_t* path, uint8_t path_len, uint8_t extra_type, const uint8_t*extra, size_t extra_len);
   Packet* createPathReturn(const Identity& dest, const uint8_t* secret, const uint8_t* path, uint8_t path_len, uint8_t extra_type, const uint8_t*extra, size_t extra_len);
   Packet* createRawData(const uint8_t* data, size_t len);

src/Packet.h

@@ -26,6 +26,7 @@ namespace mesh {
 #define PAYLOAD_TYPE_ANON_REQ    0x07    // generic request (prefixed with dest_hash, ephemeral pub_key, MAC) (enc data: ...)
 #define PAYLOAD_TYPE_PATH        0x08    // returned path (prefixed with dest/src hashes, MAC) (enc data: path, extra)
 #define PAYLOAD_TYPE_TRACE       0x09    // trace a path, collecting SNI for each hop
+#define PAYLOAD_TYPE_MULTIPART   0x0A    // packet is one of a set of packets
 //...
 #define PAYLOAD_TYPE_RAW_CUSTOM   0x0F    // custom packet as raw bytes, for applications with custom encryption, payloads, etc
 

src/helpers/BaseChatMesh.cpp

@@ -31,6 +31,19 @@ mesh::Packet* BaseChatMesh::createSelfAdvert(const char* name, double lat, doubl
   return createAdvert(self_id, app_data, app_data_len);
 }
 
+void BaseChatMesh::sendAckTo(const ContactInfo& dest, uint32_t ack_hash) {
+  if (dest.out_path_len < 0) {
+    mesh::Packet* ack = createAck(ack_hash);
+    if (ack) sendFlood(ack, TXT_ACK_DELAY);
+  } else {
+    mesh::Packet* a1 = createMultiAck(ack_hash, 1);
+    if (a1) sendDirect(a1, dest.out_path, dest.out_path_len, TXT_ACK_DELAY);
+
+    mesh::Packet* a2 = createAck(ack_hash);
+    if (a2) sendDirect(a2, dest.out_path, dest.out_path_len, TXT_ACK_DELAY + 300);
+  }
+}
+
 void BaseChatMesh::onAdvertRecv(mesh::Packet* packet, const mesh::Identity& id, uint32_t timestamp, const uint8_t* app_data, size_t app_data_len) {
   AdvertDataParser parser(app_data, app_data_len);
   if (!(parser.isValid() && parser.hasName())) {
@@ -152,14 +165,7 @@ void BaseChatMesh::onPeerDataRecv(mesh::Packet* packet, uint8_t type, int sender
                                                 PAYLOAD_TYPE_ACK, (uint8_t *) &ack_hash, 4);
         if (path) sendFlood(path, TXT_ACK_DELAY);
       } else {
-        mesh::Packet* ack = createAck(ack_hash);
+        sendAckTo(from, ack_hash);
-        if (ack) {
-          if (from.out_path_len < 0) {
-            sendFlood(ack, TXT_ACK_DELAY);
-          } else {
-            sendDirect(ack, from.out_path, from.out_path_len, TXT_ACK_DELAY);
-          }
-        }
       }
     } else if (flags == TXT_TYPE_CLI_DATA) {
       onCommandDataRecv(from, packet, timestamp, (const char *) &data[5]);  // let UI know
@@ -185,14 +191,7 @@ void BaseChatMesh::onPeerDataRecv(mesh::Packet* packet, uint8_t type, int sender
                                                 PAYLOAD_TYPE_ACK, (uint8_t *) &ack_hash, 4);
         if (path) sendFlood(path, TXT_ACK_DELAY);
       } else {
-        mesh::Packet* ack = createAck(ack_hash);
+        sendAckTo(from, ack_hash);
-        if (ack) {
-          if (from.out_path_len < 0) {
-            sendFlood(ack, TXT_ACK_DELAY);
-          } else {
-            sendDirect(ack, from.out_path, from.out_path_len, TXT_ACK_DELAY);
-          }
-        }
       }
     } else {
       MESH_DEBUG_PRINTLN("onPeerDataRecv: unsupported message type: %u", (uint32_t) flags);

src/helpers/BaseChatMesh.h

@@ -72,6 +72,7 @@ class BaseChatMesh : public mesh::Mesh {
   ConnectionInfo connections[MAX_CONNECTIONS];
 
   mesh::Packet* composeMsgPacket(const ContactInfo& recipient, uint32_t timestamp, uint8_t attempt, const char *text, uint32_t& expected_ack);
+  void sendAckTo(const ContactInfo& dest, uint32_t ack_hash);
 
 protected:
   BaseChatMesh(mesh::Radio& radio, mesh::MillisecondClock& ms, mesh::RNG& rng, mesh::RTCClock& rtc, mesh::PacketManager& mgr, mesh::MeshTables& tables)