/* Copyright (c) 2017 - 2022 LiteSpeed Technologies Inc. See LICENSE. */ #ifndef LSQUIC_SEND_CTL_H #define LSQUIC_SEND_CTL_H 1 #include #include "lsquic_types.h" #ifndef LSQUIC_SEND_STATS # define LSQUIC_SEND_STATS 1 #endif TAILQ_HEAD(lsquic_packets_tailq, lsquic_packet_out); struct lsquic_packet_out; struct ack_info; struct lsquic_alarmset; struct lsquic_engine_public; struct lsquic_conn_public; struct network_path; struct ver_neg; enum pns; struct to_coal; enum buf_packet_type { BPT_HIGHEST_PRIO, BPT_OTHER_PRIO, }; struct buf_packet_q { struct lsquic_packets_tailq bpq_packets; unsigned bpq_count; }; enum send_ctl_flags { SC_TCID0 = (1 << 0), SC_NSTP = (1 << 2), SC_PACE = (1 << 3), SC_SCHED_TICK = (1 << 4), SC_BUFFER_STREAM= (1 << 5), SC_WAS_QUIET = (1 << 6), SC_IETF = (1 << 7), #define SCBIT_LOST_ACK_SHIFT 8 SC_LOST_ACK_INIT= 1 << 8, SC_LOST_ACK_HSK = SC_LOST_ACK_INIT << PNS_HSK, SC_LOST_ACK_APP = SC_LOST_ACK_INIT << PNS_APP, SC_1RTT_ACKED = 1 << 11, SC_APP_LIMITED = 1 << 12, SC_ECN = 1 << 13, SC_QL_BITS = 1 << 14, SC_SANITY_CHECK = 1 << 15, SC_CIDLEN = 1 << 16, /* sc_cidlen is set */ SC_POISON = 1 << 17, /* poisoned packet exists */ SC_CLEANUP_BBR = 1 << 18, SC_ACK_RECV_INIT= 1 << 19, SC_ACK_RECV_HSK = SC_ACK_RECV_INIT << PNS_HSK, SC_ACK_RECV_APP = SC_ACK_RECV_INIT << PNS_APP, SC_ROUGH_RTT = 1 << 22, #if LSQUIC_DEVEL SC_DYN_PTHRESH = 1 << 31u, /* dynamic packet threshold enabled */ #endif }; typedef struct lsquic_send_ctl { /* The first section consists of struct members which are used in the * time-critical lsquic_send_ctl_got_ack() in the approximate order * of usage. */ lsquic_senhist_t sc_senhist; enum send_ctl_flags sc_flags; enum ecn sc_ecn; unsigned sc_n_stop_waiting; struct lsquic_packets_tailq sc_unacked_packets[N_PNS]; lsquic_packno_t sc_largest_acked_packno; lsquic_time_t sc_largest_acked_sent_time; lsquic_time_t sc_last_sent_time; lsquic_time_t sc_last_rto_time; int (*sc_can_send)(struct lsquic_send_ctl *); unsigned sc_bytes_unacked_retx; unsigned sc_bytes_scheduled; struct adaptive_cc sc_adaptive_cc; const struct cong_ctl_if *sc_ci; void *sc_cong_ctl; struct lsquic_engine_public *sc_enpub; unsigned sc_bytes_unacked_all; unsigned sc_n_in_flight_all; unsigned sc_n_in_flight_retx; unsigned sc_n_consec_rtos; unsigned sc_n_hsk; unsigned sc_n_tlp; enum quic_ft_bit sc_retx_frames; struct lsquic_alarmset *sc_alset; /* Second section: everything else. */ struct lsquic_packets_tailq sc_scheduled_packets, sc_0rtt_stash, sc_lost_packets; struct buf_packet_q sc_buffered_packets[BPT_OTHER_PRIO + 1]; const struct ver_neg *sc_ver_neg; struct lsquic_conn_public *sc_conn_pub; struct pacer sc_pacer; lsquic_packno_t sc_cur_packno; lsquic_packno_t sc_largest_sent_at_cutback; lsquic_packno_t sc_max_rtt_packno; /* sc_largest_ack2ed is the packet number sent by peer that we acked and * we know that our ACK was received by peer. This is used to determine * the receive history cutoff point for the purposes of generating ACK * frames in the absense of STOP_WAITING frames. Used when NSTP option * is set. (The "ack2ed" is odd enough to not be confused with anything * else and it is not insanely long.) */ lsquic_packno_t sc_largest_ack2ed[N_PNS]; /* sc_largest_acked is the largest packet number in PNS_APP packet number * space sent by peer for which we generated (not necessarily sent) an ACK. * This information is used to drop stale ACK frames from packets in * buffered queues. */ /* XXX We have both sc_largest_acked_packno and sc_largest_acked. Rename * the latter to make the code more readable. */ lsquic_packno_t sc_largest_acked; lsquic_time_t sc_loss_to; uint64_t sc_ecn_total_acked[N_PNS]; uint64_t sc_ecn_ce_cnt[N_PNS]; struct { lsquic_stream_id_t stream_id; enum buf_packet_type packet_type; } sc_cached_bpt; unsigned sc_next_limit; unsigned sc_n_scheduled; enum packno_bits sc_max_packno_bits; #if LSQUIC_SEND_STATS struct { unsigned n_total_sent, n_resent, n_delayed; } sc_stats; #endif unsigned char *sc_token; size_t sc_token_sz; unsigned sc_retry_count; unsigned sc_rt_count; /* Count round trips */ lsquic_packno_t sc_cur_rt_end; lsquic_packno_t sc_gap; unsigned sc_loss_count; /* Used to set loss bit */ unsigned sc_square_count;/* Used to set square bit */ unsigned sc_reord_thresh; signed char sc_cidlen; /* For debug purposes */ } lsquic_send_ctl_t; void lsquic_send_ctl_init (lsquic_send_ctl_t *, struct lsquic_alarmset *, struct lsquic_engine_public *, const struct ver_neg *, struct lsquic_conn_public *, enum send_ctl_flags); int lsquic_send_ctl_sent_packet (lsquic_send_ctl_t *, struct lsquic_packet_out *); void lsquic_send_ctl_mtu_not_sent (struct lsquic_send_ctl *ctl, struct lsquic_packet_out *); int lsquic_send_ctl_got_ack (lsquic_send_ctl_t *, const struct ack_info *, lsquic_time_t, lsquic_time_t); lsquic_packno_t lsquic_send_ctl_smallest_unacked (lsquic_send_ctl_t *ctl); int lsquic_send_ctl_have_unacked_stream_frames (const lsquic_send_ctl_t *); int lsquic_send_ctl_have_unacked_retx_data (const struct lsquic_send_ctl *); void lsquic_send_ctl_cleanup (lsquic_send_ctl_t *); int lsquic_send_ctl_can_send (lsquic_send_ctl_t *ctl); void lsquic_send_ctl_scheduled_one (lsquic_send_ctl_t *, struct lsquic_packet_out *); void lsquic_send_ctl_delayed_one (lsquic_send_ctl_t *, struct lsquic_packet_out *); struct lsquic_packet_out * lsquic_send_ctl_next_packet_to_send (struct lsquic_send_ctl *, const struct to_coal *); int lsquic_send_ctl_next_packet_to_send_predict (struct lsquic_send_ctl *); void lsquic_send_ctl_expire_all (lsquic_send_ctl_t *ctl); #define lsquic_send_ctl_n_in_flight(ctl) (+(ctl)->sc_n_in_flight) #define lsquic_send_ctl_n_scheduled(ctl) (+(ctl)->sc_n_scheduled) #define lsquic_send_ctl_largest_ack2ed(ctl, pns) \ (+(ctl)->sc_largest_ack2ed[pns]) void lsquic_send_ctl_do_sanity_check (const struct lsquic_send_ctl *ctl); #ifndef NDEBUG #define lsquic_send_ctl_sanity_check(ctl) do { \ if ((ctl)->sc_flags & SC_SANITY_CHECK) \ lsquic_send_ctl_do_sanity_check(ctl); \ } while (0) #else #define lsquic_send_ctl_sanity_check(ctl) #endif int lsquic_send_ctl_have_outgoing_stream_frames (const lsquic_send_ctl_t *); int lsquic_send_ctl_have_outgoing_retx_frames (const lsquic_send_ctl_t *); struct lsquic_packet_out * lsquic_send_ctl_last_scheduled (struct lsquic_send_ctl *, enum packnum_space, const struct network_path *, int); struct lsquic_packet_out * lsquic_send_ctl_new_packet_out (lsquic_send_ctl_t *, unsigned, enum packnum_space, const struct network_path *); struct lsquic_packet_out * lsquic_send_ctl_get_writeable_packet (lsquic_send_ctl_t *, enum packnum_space, unsigned need_at_least, const struct network_path *, int, int *is_err); struct lsquic_packet_out * lsquic_send_ctl_get_packet_for_stream (lsquic_send_ctl_t *, unsigned need_at_least, const struct network_path *, const struct lsquic_stream *); struct lsquic_packet_out * lsquic_send_ctl_get_packet_for_crypto (struct lsquic_send_ctl *ctl, unsigned need_at_least, enum packnum_space, const struct network_path *); unsigned lsquic_send_ctl_reschedule_packets (lsquic_send_ctl_t *); #define lsquic_send_ctl_lost_ack(ctl) \ (((ctl)->sc_flags & (SC_LOST_ACK_INIT|SC_LOST_ACK_HSK|SC_LOST_ACK_APP)) \ >> SCBIT_LOST_ACK_SHIFT) #define lsquic_send_ctl_scheduled_ack(ctl, pns, acked) do { \ (ctl)->sc_flags &= ~(SC_LOST_ACK_INIT << pns); \ if (PNS_APP == pns) \ (ctl)->sc_largest_acked = acked; \ } while (0) void lsquic_send_ctl_set_tcid0 (lsquic_send_ctl_t *, int); #define lsquic_send_ctl_turn_nstp_on(ctl) ((ctl)->sc_flags |= SC_NSTP) void lsquic_send_ctl_elide_stream_frames (lsquic_send_ctl_t *, lsquic_stream_id_t); int lsquic_send_ctl_squeeze_sched (lsquic_send_ctl_t *); #define lsquic_send_ctl_maybe_squeeze_sched(ctl) ( \ (ctl)->sc_n_scheduled && lsquic_send_ctl_squeeze_sched(ctl) \ ) /* Same return value as for squeezing, but without actual squeezing. */ int lsquic_send_ctl_have_delayed_packets (const lsquic_send_ctl_t *ctl); void lsquic_send_ctl_reset_packnos (lsquic_send_ctl_t *); void lsquic_send_ctl_ack_to_front (struct lsquic_send_ctl *, unsigned n_acks); #define lsquic_send_ctl_n_stop_waiting(ctl) \ (+(ctl)->sc_n_stop_waiting) #define lsquic_send_ctl_n_stop_waiting_reset(ctl) do { \ (ctl)->sc_n_stop_waiting = 0; \ } while (0) void lsquic_send_ctl_drop_scheduled (lsquic_send_ctl_t *); #define lsquic_send_ctl_tick_in(ctl, now) do { \ if ((ctl)->sc_flags & SC_PACE) \ { \ (ctl)->sc_flags |= SC_SCHED_TICK; \ lsquic_pacer_tick_in(&(ctl)->sc_pacer, now); \ } \ (ctl)->sc_flags &= ~SC_APP_LIMITED; \ } while (0) #define lsquic_send_ctl_tick_out(ctl) do { \ if ((ctl)->sc_flags & SC_PACE) \ lsquic_pacer_tick_out(&(ctl)->sc_pacer); \ } while (0) #define lsquic_send_ctl_next_pacer_time(ctl) ( \ ((ctl)->sc_flags & SC_PACE) \ && lsquic_pacer_delayed(&(ctl)->sc_pacer) \ ? lsquic_pacer_next_sched(&(ctl)->sc_pacer) \ : 0 ) enum packno_bits lsquic_send_ctl_packno_bits (struct lsquic_send_ctl *, enum packnum_space); int lsquic_send_ctl_schedule_buffered (lsquic_send_ctl_t *, enum buf_packet_type); #define lsquic_send_ctl_has_buffered(ctl) ( \ TAILQ_FIRST(&(ctl)->sc_buffered_packets[BPT_HIGHEST_PRIO].bpq_packets) \ || TAILQ_FIRST(&(ctl)->sc_buffered_packets[BPT_OTHER_PRIO].bpq_packets )) #define lsquic_send_ctl_has_buffered_high(ctl) ( \ !TAILQ_EMPTY(&(ctl)->sc_buffered_packets[BPT_HIGHEST_PRIO].bpq_packets)) #define lsquic_send_ctl_invalidate_bpt_cache(ctl) do { \ (ctl)->sc_cached_bpt.stream_id = UINT64_MAX; \ } while (0) #ifndef NDEBUG enum packno_bits lsquic_send_ctl_guess_packno_bits (struct lsquic_send_ctl *); int lsquic_send_ctl_schedule_stream_packets_immediately (struct lsquic_send_ctl *); enum buf_packet_type lsquic_send_ctl_determine_bpt (struct lsquic_send_ctl *, const struct lsquic_stream *); enum packno_bits lsquic_send_ctl_calc_packno_bits (struct lsquic_send_ctl *); void lsquic_send_ctl_set_max_bpq_count (unsigned); #endif size_t lsquic_send_ctl_mem_used (const struct lsquic_send_ctl *); #define lsquic_send_ctl_set_buffer_stream_packets(ctl, b) do { \ (ctl)->sc_flags &= ~SC_BUFFER_STREAM; \ (ctl)->sc_flags |= -!!(b) & SC_BUFFER_STREAM; \ } while (0) int lsquic_send_ctl_turn_on_fin (struct lsquic_send_ctl *, const struct lsquic_stream *); int lsquic_send_ctl_pacer_blocked (struct lsquic_send_ctl *); #define lsquic_send_ctl_incr_pack_sz(ctl, packet, delta) do { \ (packet)->po_data_sz += (delta); \ if ((packet)->po_flags & PO_SCHED) \ (ctl)->sc_bytes_scheduled += (delta); \ lsquic_send_ctl_sanity_check(ctl); \ } while (0) int lsquic_send_ctl_sched_is_blocked (struct lsquic_send_ctl *); void lsquic_send_ctl_verneg_done (struct lsquic_send_ctl *); int lsquic_send_ctl_retry (struct lsquic_send_ctl *, const unsigned char *, size_t); int lsquic_send_ctl_set_token (struct lsquic_send_ctl *, const unsigned char *token, size_t token_sz); void lsquic_send_ctl_empty_pns (struct lsquic_send_ctl *, enum packnum_space); void lsquic_send_ctl_maybe_calc_rough_rtt (struct lsquic_send_ctl *, enum packnum_space); void lsquic_send_ctl_repath (struct lsquic_send_ctl *ctl, const struct network_path *old, const struct network_path *new, int keep_path_properties); void lsquic_send_ctl_cancel_path_verification (struct lsquic_send_ctl *, const struct network_path *); void lsquic_send_ctl_resize (struct lsquic_send_ctl *); void lsquic_send_ctl_return_enc_data (struct lsquic_send_ctl *); #define lsquic_send_ctl_1rtt_acked(ctl) ((ctl)->sc_flags & SC_1RTT_ACKED) void lsquic_send_ctl_maybe_app_limited (struct lsquic_send_ctl *, const struct network_path *); #define lsquic_send_ctl_do_ql_bits(ctl) do { \ (ctl)->sc_flags |= SC_QL_BITS; \ } while (0) void lsquic_send_ctl_cidlen_change (struct lsquic_send_ctl *, unsigned orig_cid_len, unsigned new_cid_len); void lsquic_send_ctl_begin_optack_detection (struct lsquic_send_ctl *); void lsquic_send_ctl_path_validated (struct lsquic_send_ctl *); /* Has immediately sendable packets */ #define lsquic_send_ctl_has_sendable(ctl_) \ (lsquic_send_ctl_n_scheduled(ctl_) > 0 \ && lsquic_send_ctl_next_packet_to_send_predict(ctl_)) #define lsquic_send_ctl_in_recovery(ctl_) ((ctl_)->sc_largest_acked_packno \ && (ctl_)->sc_largest_acked_packno <= (ctl_)->sc_largest_sent_at_cutback) #define send_ctl_in_recovery lsquic_send_ctl_in_recovery int lsquic_send_ctl_can_send_probe (const struct lsquic_send_ctl *, const struct network_path *); #define lsquic_send_ctl_ecn_turned_on(ctl_) ((ctl_)->sc_ecn != ECN_NOT_ECT) void lsquic_send_ctl_disable_ecn (struct lsquic_send_ctl *); struct send_ctl_state { struct pacer pacer; struct ack_state ack_state; unsigned buf_counts[BPT_OTHER_PRIO + 1]; }; void lsquic_send_ctl_snapshot (struct lsquic_send_ctl *, struct send_ctl_state *); void lsquic_send_ctl_rollback (struct lsquic_send_ctl *, struct send_ctl_state *, const struct iovec *, size_t); void lsquic_send_ctl_0rtt_to_1rtt (struct lsquic_send_ctl *); void lsquic_send_ctl_stash_0rtt_packets (struct lsquic_send_ctl *); #endif