lsquic_send_ctl.h revision 26e8f082
1/* Copyright (c) 2017 - 2021 LiteSpeed Technologies Inc. See LICENSE. */ 2#ifndef LSQUIC_SEND_CTL_H 3#define LSQUIC_SEND_CTL_H 1 4 5#include <sys/queue.h> 6 7#include "lsquic_types.h" 8 9#ifndef LSQUIC_SEND_STATS 10# define LSQUIC_SEND_STATS 1 11#endif 12 13TAILQ_HEAD(lsquic_packets_tailq, lsquic_packet_out); 14 15struct lsquic_packet_out; 16struct ack_info; 17struct lsquic_alarmset; 18struct lsquic_engine_public; 19struct lsquic_conn_public; 20struct network_path; 21struct ver_neg; 22enum pns; 23struct to_coal; 24 25enum buf_packet_type { BPT_HIGHEST_PRIO, BPT_OTHER_PRIO, }; 26 27struct buf_packet_q 28{ 29 struct lsquic_packets_tailq bpq_packets; 30 unsigned bpq_count; 31}; 32 33enum send_ctl_flags { 34 SC_TCID0 = (1 << 0), 35 SC_NSTP = (1 << 2), 36 SC_PACE = (1 << 3), 37 SC_SCHED_TICK = (1 << 4), 38 SC_BUFFER_STREAM= (1 << 5), 39 SC_WAS_QUIET = (1 << 6), 40 SC_IETF = (1 << 7), 41#define SCBIT_LOST_ACK_SHIFT 8 42 SC_LOST_ACK_INIT= 1 << 8, 43 SC_LOST_ACK_HSK = SC_LOST_ACK_INIT << PNS_HSK, 44 SC_LOST_ACK_APP = SC_LOST_ACK_INIT << PNS_APP, 45 SC_1RTT_ACKED = 1 << 11, 46 SC_APP_LIMITED = 1 << 12, 47 SC_ECN = 1 << 13, 48 SC_QL_BITS = 1 << 14, 49 SC_SANITY_CHECK = 1 << 15, 50 SC_CIDLEN = 1 << 16, /* sc_cidlen is set */ 51 SC_POISON = 1 << 17, /* poisoned packet exists */ 52 SC_CLEANUP_BBR = 1 << 18, 53 SC_ACK_RECV_INIT= 1 << 19, 54 SC_ACK_RECV_HSK = SC_ACK_RECV_INIT << PNS_HSK, 55 SC_ACK_RECV_APP = SC_ACK_RECV_INIT << PNS_APP, 56 SC_ROUGH_RTT = 1 << 22, 57#if LSQUIC_DEVEL 58 SC_DYN_PTHRESH = 1 << 31u, /* dynamic packet threshold enabled */ 59#endif 60}; 61 62typedef struct lsquic_send_ctl { 63 /* The first section consists of struct members which are used in the 64 * time-critical lsquic_send_ctl_got_ack() in the approximate order 65 * of usage. 66 */ 67 lsquic_senhist_t sc_senhist; 68 enum send_ctl_flags sc_flags; 69 enum ecn sc_ecn; 70 unsigned sc_n_stop_waiting; 71 struct lsquic_packets_tailq sc_unacked_packets[N_PNS]; 72 lsquic_packno_t sc_largest_acked_packno; 73 lsquic_time_t sc_largest_acked_sent_time; 74 lsquic_time_t sc_last_sent_time; 75 lsquic_time_t sc_last_rto_time; 76 int (*sc_can_send)(struct lsquic_send_ctl *); 77 unsigned sc_bytes_unacked_retx; 78 unsigned sc_bytes_scheduled; 79 struct adaptive_cc sc_adaptive_cc; 80 const struct cong_ctl_if *sc_ci; 81 void *sc_cong_ctl; 82 struct lsquic_engine_public *sc_enpub; 83 unsigned sc_bytes_unacked_all; 84 unsigned sc_n_in_flight_all; 85 unsigned sc_n_in_flight_retx; 86 unsigned sc_n_consec_rtos; 87 unsigned sc_n_hsk; 88 unsigned sc_n_tlp; 89 enum quic_ft_bit sc_retx_frames; 90 struct lsquic_alarmset *sc_alset; 91 92 /* Second section: everything else. */ 93 struct lsquic_packets_tailq sc_scheduled_packets, 94 sc_0rtt_stash, 95 sc_lost_packets; 96 struct buf_packet_q sc_buffered_packets[BPT_OTHER_PRIO + 1]; 97 const struct ver_neg *sc_ver_neg; 98 struct lsquic_conn_public *sc_conn_pub; 99 struct pacer sc_pacer; 100 lsquic_packno_t sc_cur_packno; 101 lsquic_packno_t sc_largest_sent_at_cutback; 102 lsquic_packno_t sc_max_rtt_packno; 103 /* sc_largest_ack2ed is the packet number sent by peer that we acked and 104 * we know that our ACK was received by peer. This is used to determine 105 * the receive history cutoff point for the purposes of generating ACK 106 * frames in the absense of STOP_WAITING frames. Used when NSTP option 107 * is set. (The "ack2ed" is odd enough to not be confused with anything 108 * else and it is not insanely long.) 109 */ 110 lsquic_packno_t sc_largest_ack2ed[N_PNS]; 111 /* sc_largest_acked is the largest packet number in PNS_APP packet number 112 * space sent by peer for which we generated (not necessarily sent) an ACK. 113 * This information is used to drop stale ACK frames from packets in 114 * buffered queues. 115 */ 116 /* XXX We have both sc_largest_acked_packno and sc_largest_acked. Rename 117 * the latter to make the code more readable. 118 */ 119 lsquic_packno_t sc_largest_acked; 120 lsquic_time_t sc_loss_to; 121 uint64_t sc_ecn_total_acked[N_PNS]; 122 uint64_t sc_ecn_ce_cnt[N_PNS]; 123 struct 124 { 125 lsquic_stream_id_t stream_id; 126 enum buf_packet_type packet_type; 127 } sc_cached_bpt; 128 unsigned sc_next_limit; 129 unsigned sc_n_scheduled; 130 enum packno_bits sc_max_packno_bits; 131#if LSQUIC_SEND_STATS 132 struct { 133 unsigned n_total_sent, 134 n_resent, 135 n_delayed; 136 } sc_stats; 137#endif 138 unsigned char *sc_token; 139 size_t sc_token_sz; 140 unsigned sc_retry_count; 141 unsigned sc_rt_count; /* Count round trips */ 142 lsquic_packno_t sc_cur_rt_end; 143 lsquic_packno_t sc_gap; 144 unsigned sc_loss_count; /* Used to set loss bit */ 145 unsigned sc_square_count;/* Used to set square bit */ 146 unsigned sc_reord_thresh; 147 signed char sc_cidlen; /* For debug purposes */ 148} lsquic_send_ctl_t; 149 150void 151lsquic_send_ctl_init (lsquic_send_ctl_t *, struct lsquic_alarmset *, 152 struct lsquic_engine_public *, const struct ver_neg *, 153 struct lsquic_conn_public *, enum send_ctl_flags); 154 155int 156lsquic_send_ctl_sent_packet (lsquic_send_ctl_t *, struct lsquic_packet_out *); 157 158void 159lsquic_send_ctl_mtu_not_sent (struct lsquic_send_ctl *ctl, 160 struct lsquic_packet_out *); 161 162int 163lsquic_send_ctl_got_ack (lsquic_send_ctl_t *, const struct ack_info *, 164 lsquic_time_t, lsquic_time_t); 165 166lsquic_packno_t 167lsquic_send_ctl_smallest_unacked (lsquic_send_ctl_t *ctl); 168 169int 170lsquic_send_ctl_have_unacked_stream_frames (const lsquic_send_ctl_t *); 171 172int 173lsquic_send_ctl_have_unacked_retx_data (const struct lsquic_send_ctl *); 174 175void 176lsquic_send_ctl_cleanup (lsquic_send_ctl_t *); 177 178int 179lsquic_send_ctl_can_send (lsquic_send_ctl_t *ctl); 180 181void 182lsquic_send_ctl_scheduled_one (lsquic_send_ctl_t *, struct lsquic_packet_out *); 183 184void 185lsquic_send_ctl_delayed_one (lsquic_send_ctl_t *, struct lsquic_packet_out *); 186 187struct lsquic_packet_out * 188lsquic_send_ctl_next_packet_to_send (struct lsquic_send_ctl *, 189 const struct to_coal *); 190 191int 192lsquic_send_ctl_next_packet_to_send_predict (struct lsquic_send_ctl *); 193 194void 195lsquic_send_ctl_expire_all (lsquic_send_ctl_t *ctl); 196 197#define lsquic_send_ctl_n_in_flight(ctl) (+(ctl)->sc_n_in_flight) 198 199#define lsquic_send_ctl_n_scheduled(ctl) (+(ctl)->sc_n_scheduled) 200 201#define lsquic_send_ctl_largest_ack2ed(ctl, pns) \ 202 (+(ctl)->sc_largest_ack2ed[pns]) 203 204void 205lsquic_send_ctl_do_sanity_check (const struct lsquic_send_ctl *ctl); 206 207#ifndef NDEBUG 208#define lsquic_send_ctl_sanity_check(ctl) do { \ 209 if ((ctl)->sc_flags & SC_SANITY_CHECK) \ 210 lsquic_send_ctl_do_sanity_check(ctl); \ 211} while (0) 212#else 213#define lsquic_send_ctl_sanity_check(ctl) 214#endif 215 216int 217lsquic_send_ctl_have_outgoing_stream_frames (const lsquic_send_ctl_t *); 218 219int 220lsquic_send_ctl_have_outgoing_retx_frames (const lsquic_send_ctl_t *); 221 222struct lsquic_packet_out * 223lsquic_send_ctl_last_scheduled (struct lsquic_send_ctl *, enum packnum_space, 224 const struct network_path *, int); 225 226struct lsquic_packet_out * 227lsquic_send_ctl_new_packet_out (lsquic_send_ctl_t *, unsigned, 228 enum packnum_space, const struct network_path *); 229 230struct lsquic_packet_out * 231lsquic_send_ctl_get_writeable_packet (lsquic_send_ctl_t *, enum packnum_space, 232 unsigned need_at_least, const struct network_path *, int, int *is_err); 233 234struct lsquic_packet_out * 235lsquic_send_ctl_get_packet_for_stream (lsquic_send_ctl_t *, 236 unsigned need_at_least, const struct network_path *, 237 const struct lsquic_stream *); 238 239struct lsquic_packet_out * 240lsquic_send_ctl_get_packet_for_crypto (struct lsquic_send_ctl *ctl, 241 unsigned need_at_least, enum packnum_space, const struct network_path *); 242 243unsigned 244lsquic_send_ctl_reschedule_packets (lsquic_send_ctl_t *); 245 246#define lsquic_send_ctl_lost_ack(ctl) \ 247 (((ctl)->sc_flags & (SC_LOST_ACK_INIT|SC_LOST_ACK_HSK|SC_LOST_ACK_APP)) \ 248 >> SCBIT_LOST_ACK_SHIFT) 249 250#define lsquic_send_ctl_scheduled_ack(ctl, pns, acked) do { \ 251 (ctl)->sc_flags &= ~(SC_LOST_ACK_INIT << pns); \ 252 if (PNS_APP == pns) \ 253 (ctl)->sc_largest_acked = acked; \ 254} while (0) 255 256void 257lsquic_send_ctl_set_tcid0 (lsquic_send_ctl_t *, int); 258 259#define lsquic_send_ctl_turn_nstp_on(ctl) ((ctl)->sc_flags |= SC_NSTP) 260 261void 262lsquic_send_ctl_elide_stream_frames (lsquic_send_ctl_t *, lsquic_stream_id_t); 263 264int 265lsquic_send_ctl_squeeze_sched (lsquic_send_ctl_t *); 266 267#define lsquic_send_ctl_maybe_squeeze_sched(ctl) ( \ 268 (ctl)->sc_n_scheduled && lsquic_send_ctl_squeeze_sched(ctl) \ 269) 270 271/* Same return value as for squeezing, but without actual squeezing. */ 272int 273lsquic_send_ctl_have_delayed_packets (const lsquic_send_ctl_t *ctl); 274 275void 276lsquic_send_ctl_reset_packnos (lsquic_send_ctl_t *); 277 278void 279lsquic_send_ctl_ack_to_front (struct lsquic_send_ctl *, unsigned n_acks); 280 281#define lsquic_send_ctl_n_stop_waiting(ctl) \ 282 (+(ctl)->sc_n_stop_waiting) 283 284#define lsquic_send_ctl_n_stop_waiting_reset(ctl) do { \ 285 (ctl)->sc_n_stop_waiting = 0; \ 286} while (0) 287 288void 289lsquic_send_ctl_drop_scheduled (lsquic_send_ctl_t *); 290 291#define lsquic_send_ctl_tick_in(ctl, now) do { \ 292 if ((ctl)->sc_flags & SC_PACE) \ 293 { \ 294 (ctl)->sc_flags |= SC_SCHED_TICK; \ 295 lsquic_pacer_tick_in(&(ctl)->sc_pacer, now); \ 296 } \ 297 (ctl)->sc_flags &= ~SC_APP_LIMITED; \ 298} while (0) 299 300#define lsquic_send_ctl_tick_out(ctl) do { \ 301 if ((ctl)->sc_flags & SC_PACE) \ 302 lsquic_pacer_tick_out(&(ctl)->sc_pacer); \ 303} while (0) 304 305#define lsquic_send_ctl_next_pacer_time(ctl) ( \ 306 ((ctl)->sc_flags & SC_PACE) \ 307 && lsquic_pacer_delayed(&(ctl)->sc_pacer) \ 308 ? lsquic_pacer_next_sched(&(ctl)->sc_pacer) \ 309 : 0 ) 310 311enum packno_bits 312lsquic_send_ctl_packno_bits (struct lsquic_send_ctl *, enum packnum_space); 313 314int 315lsquic_send_ctl_schedule_buffered (lsquic_send_ctl_t *, enum buf_packet_type); 316 317#define lsquic_send_ctl_has_buffered(ctl) ( \ 318 TAILQ_FIRST(&(ctl)->sc_buffered_packets[BPT_HIGHEST_PRIO].bpq_packets) \ 319 || TAILQ_FIRST(&(ctl)->sc_buffered_packets[BPT_OTHER_PRIO].bpq_packets )) 320 321#define lsquic_send_ctl_has_buffered_high(ctl) ( \ 322 !TAILQ_EMPTY(&(ctl)->sc_buffered_packets[BPT_HIGHEST_PRIO].bpq_packets)) 323 324#define lsquic_send_ctl_invalidate_bpt_cache(ctl) do { \ 325 (ctl)->sc_cached_bpt.stream_id = UINT64_MAX; \ 326} while (0) 327 328#ifndef NDEBUG 329enum packno_bits 330lsquic_send_ctl_guess_packno_bits (struct lsquic_send_ctl *); 331 332int 333lsquic_send_ctl_schedule_stream_packets_immediately (struct lsquic_send_ctl *); 334 335enum buf_packet_type 336lsquic_send_ctl_determine_bpt (struct lsquic_send_ctl *, 337 const struct lsquic_stream *); 338 339enum packno_bits 340lsquic_send_ctl_calc_packno_bits (struct lsquic_send_ctl *); 341 342void 343lsquic_send_ctl_set_max_bpq_count (unsigned); 344#endif 345 346size_t 347lsquic_send_ctl_mem_used (const struct lsquic_send_ctl *); 348 349#define lsquic_send_ctl_set_buffer_stream_packets(ctl, b) do { \ 350 (ctl)->sc_flags &= ~SC_BUFFER_STREAM; \ 351 (ctl)->sc_flags |= -!!(b) & SC_BUFFER_STREAM; \ 352} while (0) 353 354int 355lsquic_send_ctl_turn_on_fin (struct lsquic_send_ctl *, 356 const struct lsquic_stream *); 357 358int 359lsquic_send_ctl_pacer_blocked (struct lsquic_send_ctl *); 360 361#define lsquic_send_ctl_incr_pack_sz(ctl, packet, delta) do { \ 362 (packet)->po_data_sz += (delta); \ 363 if ((packet)->po_flags & PO_SCHED) \ 364 (ctl)->sc_bytes_scheduled += (delta); \ 365 lsquic_send_ctl_sanity_check(ctl); \ 366} while (0) 367 368int 369lsquic_send_ctl_sched_is_blocked (struct lsquic_send_ctl *); 370 371void 372lsquic_send_ctl_verneg_done (struct lsquic_send_ctl *); 373 374int 375lsquic_send_ctl_retry (struct lsquic_send_ctl *, const unsigned char *, size_t); 376 377int 378lsquic_send_ctl_set_token (struct lsquic_send_ctl *, 379 const unsigned char *token, size_t token_sz); 380 381void 382lsquic_send_ctl_empty_pns (struct lsquic_send_ctl *, enum packnum_space); 383 384void 385lsquic_send_ctl_maybe_calc_rough_rtt (struct lsquic_send_ctl *, 386 enum packnum_space); 387 388void 389lsquic_send_ctl_repath (struct lsquic_send_ctl *ctl, 390 const struct network_path *old, const struct network_path *new, 391 int keep_path_properties); 392 393void 394lsquic_send_ctl_cancel_path_verification (struct lsquic_send_ctl *, 395 const struct network_path *); 396 397void 398lsquic_send_ctl_resize (struct lsquic_send_ctl *); 399 400void 401lsquic_send_ctl_return_enc_data (struct lsquic_send_ctl *); 402 403#define lsquic_send_ctl_1rtt_acked(ctl) ((ctl)->sc_flags & SC_1RTT_ACKED) 404 405void 406lsquic_send_ctl_maybe_app_limited (struct lsquic_send_ctl *, 407 const struct network_path *); 408 409#define lsquic_send_ctl_do_ql_bits(ctl) do { \ 410 (ctl)->sc_flags |= SC_QL_BITS; \ 411} while (0) 412 413void 414lsquic_send_ctl_cidlen_change (struct lsquic_send_ctl *, 415 unsigned orig_cid_len, unsigned new_cid_len); 416 417void 418lsquic_send_ctl_begin_optack_detection (struct lsquic_send_ctl *); 419 420void 421lsquic_send_ctl_path_validated (struct lsquic_send_ctl *); 422 423/* Has immediately sendable packets */ 424#define lsquic_send_ctl_has_sendable(ctl_) \ 425 (lsquic_send_ctl_n_scheduled(ctl_) > 0 \ 426 && lsquic_send_ctl_next_packet_to_send_predict(ctl_)) 427 428#define lsquic_send_ctl_in_recovery(ctl_) ((ctl_)->sc_largest_acked_packno \ 429 && (ctl_)->sc_largest_acked_packno <= (ctl_)->sc_largest_sent_at_cutback) 430 431#define send_ctl_in_recovery lsquic_send_ctl_in_recovery 432 433int 434lsquic_send_ctl_can_send_probe (const struct lsquic_send_ctl *, 435 const struct network_path *); 436 437#define lsquic_send_ctl_ecn_turned_on(ctl_) ((ctl_)->sc_ecn != ECN_NOT_ECT) 438 439void 440lsquic_send_ctl_disable_ecn (struct lsquic_send_ctl *); 441 442struct send_ctl_state 443{ 444 struct pacer pacer; 445 struct ack_state ack_state; 446 unsigned buf_counts[BPT_OTHER_PRIO + 1]; 447}; 448 449void 450lsquic_send_ctl_snapshot (struct lsquic_send_ctl *, struct send_ctl_state *); 451 452void 453lsquic_send_ctl_rollback (struct lsquic_send_ctl *, struct send_ctl_state *, 454 const struct iovec *, size_t); 455 456void 457lsquic_send_ctl_0rtt_to_1rtt (struct lsquic_send_ctl *); 458 459void 460lsquic_send_ctl_stash_0rtt_packets (struct lsquic_send_ctl *); 461 462#endif 463