fix interleaved fwd-tsn cleanup.

netinet/sctp_indata.c

@@ -3733,20 +3733,28 @@ struct sctp_tmit_chunk *
 sctp_try_advance_peer_ack_point(struct sctp_tcb *stcb,
     struct sctp_association *asoc)
 {
-	struct sctp_tmit_chunk *tp1, *tp2, *a_adv = NULL;
+	struct sctp_tmit_chunk *top_snd, *tp1, *tp2, *a_adv = NULL;
 	struct timeval now;
 	int now_filled = 0;
 
 	if (asoc->prsctp_supported == 0) {
 		return (NULL);
 	}
+	top_snd = TAILQ_FIRST(&asoc->send_queue);
 	TAILQ_FOREACH_SAFE(tp1, &asoc->sent_queue, sctp_next, tp2) {
 		if (tp1->sent != SCTP_FORWARD_TSN_SKIP &&
 		    tp1->sent != SCTP_DATAGRAM_RESEND &&
 		    tp1->sent != SCTP_DATAGRAM_NR_ACKED) {
 			/* no chance to advance, out of here */
 			break;
 		}
+		if (top_snd && SCTP_TSN_GT(tp1->rec.data.tsn, top_snd->rec.data.tsn)) {
+			/* 
+			 * We can't advance further, there is one ahead of us on
+			 * the not-yet sent queue (send_queue).
+			 */
+			break;
+		}
 		if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_TRY_ADVANCE) {
 			if ((tp1->sent == SCTP_FORWARD_TSN_SKIP) ||
 			    (tp1->sent == SCTP_DATAGRAM_NR_ACKED)) {
@@ -4537,16 +4545,20 @@ sctp_handle_sack(struct mbuf *m, int offset_seg, int offset_dup,
 		 * no way, we have not even sent this TSN out yet.
 		 * Peer is hopelessly messed up with us.
 		 */
-		SCTP_PRINTF("NEW cum_ack:%x send_s:%x is smaller or equal\n",
+		SCTP_PRINTF("NEW cum_ack:%u send_s:%u is smaller or equal\n",
 			    cum_ack, send_s);
 		if (tp1) {
-			SCTP_PRINTF("Got send_s from tsn:%x + 1 of tp1: %p\n",
-				    tp1->rec.data.tsn, (void *)tp1);
+			SCTP_PRINTF("Got send_s from tp:%p tsn:%u sid:%u mid:%u\n",
+				    (void *)tp1,
+				    tp1->rec.data.tsn,
+				    tp1->rec.data.sid,
+				    tp1->rec.data.mid);
 		}
+
 	hopeless_peer:
 		*abort_now = 1;
 		/* XXX */
-		snprintf(msg, sizeof(msg), "Cum ack %8.8x greater or equal than TSN %8.8x",
+		snprintf(msg, sizeof(msg), "Cum ack %u greater or equal than TSN %u",
 			 cum_ack, send_s);
 		op_err = sctp_generate_cause(SCTP_CAUSE_PROTOCOL_VIOLATION, msg);
 		stcb->sctp_ep->last_abort_code = SCTP_FROM_SCTP_INDATA + SCTP_LOC_25;

netinet/sctp_input.c

@@ -273,13 +273,19 @@ sctp_is_there_unsent_data(struct sctp_tcb *stcb, int so_locked
 				 * time through when we took all the data
 				 * the sender_all_done was not set.
 				 */
-				if (sp->put_last_out == 0) {
+
+
+				if ((sp->put_last_out == 0) &&
+				    (sp->discard_rest == 0)) {
 					SCTP_PRINTF("Gak, put out entire msg with NO end!-1\n");
-					SCTP_PRINTF("sender_done:%d len:%d msg_comp:%d put_last_out:%d\n",
+					SCTP_PRINTF("sd:%d len:%d mc:%d pol:%d sid:%u dis:%d\n",
 					            sp->sender_all_done,
 					            sp->length,
 					            sp->msg_is_complete,
-					            sp->put_last_out);
+					            sp->put_last_out,
+						    sp->sid,
+						    sp->discard_rest
+						);
 				}
 				atomic_subtract_int(&stcb->asoc.stream_queue_cnt, 1);
 				TAILQ_REMOVE(&stcb->asoc.strmout[i].outqueue, sp, next);

netinet/sctp_output.c

@@ -8190,8 +8190,7 @@ sctp_move_to_outqueue(struct sctp_tcb *stcb,
 	}
 	asoc->chunks_on_out_queue++;
 	strq->chunks_on_queues++;
-	TAILQ_INSERT_TAIL(&asoc->send_queue, chk, sctp_next);
-	asoc->send_queue_cnt++;
+	sctp_place_chunk_in_queue(&asoc->send_queue, chk, &asoc->send_queue_cnt);
 out_of:
 	if (send_lock_up) {
 		SCTP_TCB_SEND_UNLOCK(stcb);

netinet/sctp_pcb.c

@@ -5884,7 +5884,9 @@ sctp_free_assoc(struct sctp_inpcb *inp, struct sctp_tcb *stcb, int from_inpcbfre
 #ifdef INVARIANTS
 	for (i = 0; i < stcb->asoc.streamoutcnt; i++) {
 		if (stcb->asoc.strmout[i].chunks_on_queues > 0) {
-			panic("%u chunks left for stream %u.", stcb->asoc.strmout[i].chunks_on_queues, i);
+			panic("stcb:%p %u chunks left for stream %u.",
+			      stcb,
+			      stcb->asoc.strmout[i].chunks_on_queues, i);
 		}
 	}
 #endif

netinet/sctp_timer.c

@@ -554,7 +554,8 @@ sctp_mark_all_for_resend(struct sctp_tcb *stcb,
 	TAILQ_FOREACH_SAFE(chk, &stcb->asoc.sent_queue, sctp_next, nchk) {
 		if (SCTP_TSN_GE(stcb->asoc.last_acked_seq, chk->rec.data.tsn)) {
 			/* Strange case our list got out of order? */
-			SCTP_PRINTF("Our list is out of order? last_acked:%x chk:%x\n",
+			SCTP_PRINTF("Our list is out of order? stcb:%p chk:%p last_acked:%u chk-tsn:%u\n",
+				    stcb, chk, 
 			            (unsigned int)stcb->asoc.last_acked_seq, (unsigned int)chk->rec.data.tsn);
 			recovery_cnt++;
 #ifdef INVARIANTS

netinet/sctputil.c

@@ -5112,22 +5112,64 @@ sctp_free_bufspace(struct sctp_tcb *stcb, struct sctp_association *asoc,
 
 #endif
 
+void
+sctp_place_chunk_in_queue(struct sctpchunk_listhead *q, struct sctp_tmit_chunk *chk,  unsigned int *cnt)
+{
+	/*
+	 * Place the chunk on the sent_queue (even though it
+	 * has not been sent). This is for cleanup and fwd
+	 * tsn this way we get the correct TSN to forward
+	 * to and the cum-ack arrival will cause the freeing.
+	 */
+	struct sctp_tmit_chunk *tp1;
+	int inserted = 0;
+
+	if (cnt)
+		*cnt = (*cnt) + 1;
+	tp1 = TAILQ_LAST(q, sctpchunk_listhead);
+	if (tp1 && SCTP_TSN_GT(tp1->rec.data.tsn, chk->rec.data.tsn)) {
+		/*
+		 * We normally insert at the tail (not hitting this
+		 * if block). However when we have a higher one on
+		 * the end, then we must go backwards through the
+		 * list from the tail until we find one smaller or
+		 * we hit the head.
+		 */
+		TAILQ_FOREACH_REVERSE(tp1, q, sctpchunk_listhead, sctp_next) {
+			if (SCTP_TSN_GT(tp1->rec.data.tsn, chk->rec.data.tsn)) {
+			    TAILQ_INSERT_AFTER(q, tp1, chk, sctp_next);
+			    inserted = 1;
+			    break;
+			}
+		}
+		if (inserted == 0) {
+			/* Its the lowest */
+			TAILQ_INSERT_HEAD(q, chk, sctp_next);
+			inserted = 1;
+		}
+	}
+	if (inserted == 0)
+		TAILQ_INSERT_TAIL(q, chk, sctp_next);
+
+}
+
 int
-sctp_release_pr_sctp_chunk(struct sctp_tcb *stcb, struct sctp_tmit_chunk *tp1,
+sctp_release_pr_sctp_chunk(struct sctp_tcb *stcb, struct sctp_tmit_chunk *schk,
 			   uint8_t sent, int so_locked
 #if !defined(__APPLE__) && !defined(SCTP_SO_LOCK_TESTING)
 			   SCTP_UNUSED
 #endif
 	)
 {
 	struct sctp_stream_out *strq;
-	struct sctp_tmit_chunk *chk = NULL, *tp2;
+	struct sctp_tmit_chunk *chk = NULL, *tp2, *tp1;
 	struct sctp_stream_queue_pending *sp;
 	uint32_t mid;
 	uint16_t sid;
-	uint8_t foundeom = 0;
+	uint8_t unordered, foundeom = 0;
 	int ret_sz = 0;
 	int notdone;
+	int cnt  = 0;
 	int do_wakeup_routine = 0;
 
 #if defined(__APPLE__)
@@ -5137,24 +5179,40 @@ sctp_release_pr_sctp_chunk(struct sctp_tcb *stcb, struct sctp_tmit_chunk *tp1,
 		sctp_unlock_assert(SCTP_INP_SO(stcb->sctp_ep));
 	}
 #endif
-	sid = tp1->rec.data.sid;
-	mid = tp1->rec.data.mid;
-	if (sent || !(tp1->rec.data.rcv_flags & SCTP_DATA_FIRST_FRAG)) {
+	sid = schk->rec.data.sid;
+	mid = schk->rec.data.mid;
+	if (schk->rec.data.rcv_flags & SCTP_DATA_UNORDERED)
+		unordered = 1;
+	else
+		unordered = 0;
+	SCTP_PRINTF("PR-SCTP sid:%u mid:%u unordered:%u\n",
+	       sid, mid, unordered);
+	if (sent || !(schk->rec.data.rcv_flags & SCTP_DATA_FIRST_FRAG)) {
 		stcb->asoc.abandoned_sent[0]++;
-		stcb->asoc.abandoned_sent[PR_SCTP_POLICY(tp1->flags)]++;
+		stcb->asoc.abandoned_sent[PR_SCTP_POLICY(schk->flags)]++;
 		stcb->asoc.strmout[sid].abandoned_sent[0]++;
 #if defined(SCTP_DETAILED_STR_STATS)
-		stcb->asoc.strmout[sid].abandoned_sent[PR_SCTP_POLICY(tp1->flags)]++;
+		stcb->asoc.strmout[sid].abandoned_sent[PR_SCTP_POLICY(schk->flags)]++;
 #endif
 	} else {
 		stcb->asoc.abandoned_unsent[0]++;
-		stcb->asoc.abandoned_unsent[PR_SCTP_POLICY(tp1->flags)]++;
+		stcb->asoc.abandoned_unsent[PR_SCTP_POLICY(schk->flags)]++;
 		stcb->asoc.strmout[sid].abandoned_unsent[0]++;
 #if defined(SCTP_DETAILED_STR_STATS)
-		stcb->asoc.strmout[sid].abandoned_unsent[PR_SCTP_POLICY(tp1->flags)]++;
+		stcb->asoc.strmout[sid].abandoned_unsent[PR_SCTP_POLICY(schk->flags)]++;
 #endif
 	}
-	do {
+	TAILQ_FOREACH(tp1, &stcb->asoc.sent_queue, sctp_next) {
+		if ((!SCTP_MID_EQ(stcb->asoc.idata_supported, tp1->rec.data.mid, mid)) ||
+		    (tp1->rec.data.sid != sid)) {
+			/* Not the same sid/mid */
+			continue;
+		}
+		if ((unordered && ((tp1->rec.data.rcv_flags & SCTP_DATA_UNORDERED) == 0)) ||
+		    ((unordered == 0) && (tp1->rec.data.rcv_flags & SCTP_DATA_UNORDERED))) {
+			/* Not the right ordering */
+			continue;
+		}
 		ret_sz += tp1->book_size;
 		if (tp1->data != NULL) {
 			if (tp1->sent < SCTP_DATAGRAM_RESEND) {
@@ -5179,9 +5237,12 @@ sctp_release_pr_sctp_chunk(struct sctp_tcb *stcb, struct sctp_tmit_chunk *tp1,
 			}
 		}
 		tp1->sent = SCTP_FORWARD_TSN_SKIP;
+		SCTP_PRINTF("Sent queue Index:%d marked TSN %u to skip\n",
+		       cnt, tp1->rec.data.tsn);
+		cnt++;
 		if ((tp1->rec.data.rcv_flags & SCTP_DATA_NOT_FRAG) ==
 		    SCTP_DATA_NOT_FRAG) {
-			/* not frag'ed we ae done   */
+			/* not frag'ed we are done   */
 			notdone = 0;
 			foundeom = 1;
 		} else if (tp1->rec.data.rcv_flags & SCTP_DATA_LAST_FRAG) {
@@ -5194,18 +5255,24 @@ sctp_release_pr_sctp_chunk(struct sctp_tcb *stcb, struct sctp_tmit_chunk *tp1,
 			 * it
 			 */
 			notdone = 1;
-			tp1 = TAILQ_NEXT(tp1, sctp_next);
 		}
-	} while (tp1 && notdone);
+	}
 	if (foundeom == 0) {
 		/*
 		 * The multi-part message was scattered across the send and
 		 * sent queue.
 		 */
+		cnt = 0;
 		TAILQ_FOREACH_SAFE(tp1, &stcb->asoc.send_queue, sctp_next, tp2) {
 			if ((tp1->rec.data.sid != sid) ||
 			    (!SCTP_MID_EQ(stcb->asoc.idata_supported, tp1->rec.data.mid, mid))) {
-				break;
+				/* Not the right mid or sid  */
+				continue;
+			}
+			if ((unordered && ((tp1->rec.data.rcv_flags & SCTP_DATA_UNORDERED) == 0)) ||
+			    ((unordered == 0) && (tp1->rec.data.rcv_flags & SCTP_DATA_UNORDERED))) {
+				/* Not the right ordering */
+				continue;
 			}
 			/* save to chk in case we have some on stream out
 			 * queue. If so and we have an un-transmitted one
@@ -5230,12 +5297,12 @@ sctp_release_pr_sctp_chunk(struct sctp_tcb *stcb, struct sctp_tmit_chunk *tp1,
 			}
 			do_wakeup_routine = 1;
 			tp1->sent = SCTP_FORWARD_TSN_SKIP;
+			SCTP_PRINTF("Send queue Index:%d marked TSN %u to skip -- reinsert\n",
+			       cnt, tp1->rec.data.tsn);
+			cnt++;
 			TAILQ_REMOVE(&stcb->asoc.send_queue, tp1, sctp_next);
-			/* on to the sent queue so we can wait for it to be passed by. */
-			TAILQ_INSERT_TAIL(&stcb->asoc.sent_queue, tp1,
-					  sctp_next);
 			stcb->asoc.send_queue_cnt--;
-			stcb->asoc.sent_queue_cnt++;
+			sctp_place_chunk_in_queue(&stcb->asoc.sent_queue, tp1,  &stcb->asoc.sent_queue_cnt);
 		}
 	}
 	if (foundeom == 0) {
@@ -5254,65 +5321,6 @@ sctp_release_pr_sctp_chunk(struct sctp_tcb *stcb, struct sctp_tmit_chunk *tp1,
 			 * would have been sent with the LAST
 			 * bit.
 			 */
-			if (chk == NULL) {
-				/* Yep, we have to */
-				sctp_alloc_a_chunk(stcb, chk);
-				if (chk == NULL) {
-					/* we are hosed. All we can
-					 * do is nothing.. which will
-					 * cause an abort if the peer is
-					 * paying attention.
-					 */
-					goto oh_well;
-				}
-				memset(chk, 0, sizeof(*chk));
-				chk->rec.data.rcv_flags = 0;
-				chk->sent = SCTP_FORWARD_TSN_SKIP;
-				chk->asoc = &stcb->asoc;
-				if (stcb->asoc.idata_supported == 0) {
-					if (sp->sinfo_flags & SCTP_UNORDERED) {
-						chk->rec.data.mid = 0;
-					} else {
-						chk->rec.data.mid = strq->next_mid_ordered;
-					}
-				} else {
-					if (sp->sinfo_flags & SCTP_UNORDERED) {
-						chk->rec.data.mid = strq->next_mid_unordered;
-					} else {
-						chk->rec.data.mid = strq->next_mid_ordered;
-					}
-				}
-				chk->rec.data.sid = sp->sid;
-				chk->rec.data.ppid = sp->ppid;
-				chk->rec.data.context = sp->context;
-				chk->flags = sp->act_flags;
-				chk->whoTo = NULL;
-#if defined(__FreeBSD__) || defined(__Panda__)
-				chk->rec.data.tsn = atomic_fetchadd_int(&stcb->asoc.sending_seq, 1);
-#else
-				chk->rec.data.tsn = stcb->asoc.sending_seq++;
-#endif
-				strq->chunks_on_queues++;
-				TAILQ_INSERT_TAIL(&stcb->asoc.sent_queue, chk, sctp_next);
-				stcb->asoc.sent_queue_cnt++;
-				stcb->asoc.pr_sctp_cnt++;
-			}
-			chk->rec.data.rcv_flags |= SCTP_DATA_LAST_FRAG;
-			if (sp->sinfo_flags & SCTP_UNORDERED) {
-				chk->rec.data.rcv_flags |= SCTP_DATA_UNORDERED;
-			}
-			if (stcb->asoc.idata_supported == 0) {
-				if ((sp->sinfo_flags & SCTP_UNORDERED) == 0) {
-					strq->next_mid_ordered++;
-				}
-			} else {
-				if (sp->sinfo_flags & SCTP_UNORDERED) {
-					strq->next_mid_unordered++;
-				} else {
-					strq->next_mid_ordered++;
-				}
-			}
-		oh_well:
 			if (sp->data) {
 				/* Pull any data to free up the SB and
 				 * allow sender to "add more" while we
@@ -5327,6 +5335,17 @@ sctp_release_pr_sctp_chunk(struct sctp_tcb *stcb, struct sctp_tmit_chunk *tp1,
 				sp->tail_mbuf = NULL;
 				sp->length = 0;
 			}
+			if (stcb->asoc.idata_supported == 0) {
+				if ((sp->sinfo_flags & SCTP_UNORDERED) == 0) {
+					strq->next_mid_ordered++;
+				}
+			} else {
+				if (sp->sinfo_flags & SCTP_UNORDERED) {
+					strq->next_mid_unordered++;
+				} else {
+					strq->next_mid_ordered++;
+				}
+			}
 		}
 		SCTP_TCB_SEND_UNLOCK(stcb);
 	}

netinet/sctputil.h

@@ -274,6 +274,10 @@ int sctp_cmpaddr(struct sockaddr *, struct sockaddr *);
 
 void sctp_print_address(struct sockaddr *);
 
+void
+sctp_place_chunk_in_queue(struct sctpchunk_listhead *q,
+			  struct sctp_tmit_chunk *chk,
+			  unsigned int *cnt);
 int
 sctp_release_pr_sctp_chunk(struct sctp_tcb *, struct sctp_tmit_chunk *,
     uint8_t, int