1/* Copyright (c) 2017 - 2022 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#ifndef WIN32 17#include <sys/time.h> 18#include <netinet/in.h> 19#include <sys/types.h> 20#include <sys/stat.h> 21#include <fcntl.h> 22#include <unistd.h> 23#include <netdb.h> 24#endif 25 26#ifndef NDEBUG 27#include <sys/types.h> 28#endif 29 30#if defined(WIN32) || defined(NDEBUG) 31#define CAN_LOSE_PACKETS 0 32#else 33#define CAN_LOSE_PACKETS 1 34#endif 35 36#if CAN_LOSE_PACKETS 37#include <regex.h> /* For code that loses packets */ 38#endif 39 40#if LOG_PACKET_CHECKSUM 41#include <zlib.h> 42#endif 43 44#include <openssl/aead.h> 45 46#include "lsquic.h" 47#include "lsquic_types.h" 48#include "lsquic_int_types.h" 49#include "lsquic_sizes.h" 50#include "lsquic_parse_common.h" 51#include "lsquic_parse.h" 52#include "lsquic_packet_in.h" 53#include "lsquic_packet_out.h" 54#include "lsquic_senhist.h" 55#include "lsquic_rtt.h" 56#include "lsquic_cubic.h" 57#include "lsquic_pacer.h" 58#include "lsquic_bw_sampler.h" 59#include "lsquic_minmax.h" 60#include "lsquic_bbr.h" 61#include "lsquic_adaptive_cc.h" 62#include "lsquic_set.h" 63#include "lsquic_conn_flow.h" 64#include "lsquic_sfcw.h" 65#include "lsquic_hash.h" 66#include "lsquic_conn.h" 67#include "lsquic_full_conn.h" 68#include "lsquic_util.h" 69#include "lsquic_qtags.h" 70#include "lsquic_enc_sess.h" 71#include "lsquic_mm.h" 72#include "lsquic_engine_public.h" 73#include "lsquic_eng_hist.h" 74#include "lsquic_ev_log.h" 75#include "lsquic_version.h" 76#include "lsquic_pr_queue.h" 77#include "lsquic_mini_conn.h" 78#include "lsquic_trechist.h" 79#include "lsquic_mini_conn_ietf.h" 80#include "lsquic_stock_shi.h" 81#include "lsquic_purga.h" 82#include "lsquic_tokgen.h" 83#include "lsquic_attq.h" 84#include "lsquic_min_heap.h" 85#include "lsquic_http1x_if.h" 86#include "lsquic_handshake.h" 87#include "lsquic_crand.h" 88#include "lsquic_ietf.h" 89#include "lsquic_handshake.h" 90 91#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 92#include "lsquic_logger.h" 93 94#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 95#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 96#endif 97 98#if LSQUIC_DEBUG_NEXT_ADV_TICK || LSQUIC_CONN_STATS 99#include "lsquic_alarmset.h" 100#endif 101 102#define MIN(a, b) ((a) < (b) ? (a) : (b)) 103#define MAX(a, b) ((a) > (b) ? (a) : (b)) 104 105/* The batch of outgoing packets grows and shrinks dynamically */ 106/* Batch sizes do not have to be powers of two */ 107#define MAX_OUT_BATCH_SIZE 1024 108#define MIN_OUT_BATCH_SIZE 4 109#define INITIAL_OUT_BATCH_SIZE 32 110 111struct out_batch 112{ 113 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 114 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 115 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 116 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 117 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 118}; 119 120typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 121 122static void 123process_connections (struct lsquic_engine *engine, conn_iter_f iter, 124 lsquic_time_t now); 125 126static void 127engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 128 129static lsquic_conn_t * 130engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 131 enum lsquic_conn_flags flag); 132 133static void 134force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 135 136#if LSQUIC_CONN_STATS 137static void 138update_busy_detector (struct lsquic_engine *, struct lsquic_conn *, int); 139#endif 140 141#if LSQUIC_COUNT_ENGINE_CALLS 142#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 143#else 144#define ENGINE_CALLS_INCR(e) 145#endif 146 147/* Nested calls to some LSQUIC functions are not supported. Functions that 148 * iterate over connections cannot be nested. 149 */ 150#define ENGINE_IN(e) do { \ 151 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 152 (e)->pub.enp_flags |= ENPUB_PROC; \ 153 ENGINE_CALLS_INCR(e); \ 154} while (0) 155 156#define ENGINE_OUT(e) do { \ 157 assert((e)->pub.enp_flags & ENPUB_PROC); \ 158 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 159} while (0) 160 161/* A connection can be referenced from one of six places: 162 * 163 * 1. A hash is used to find connections in order to dispatch an incoming 164 * packet. Connections can be hashed by CIDs or by address. In the 165 * former case, each connection has one or more mappings in the hash 166 * table. IETF QUIC connections have up to eight (in our implementation) 167 * source CIDs and each of those would have a mapping. In client mode, 168 * depending on QUIC versions and options selected, it is may be 169 * necessary to hash connections by address, in which case incoming 170 * packets are delivered to connections based on the address. 171 * 172 * 2. Outgoing queue. 173 * 174 * 3. Tickable queue 175 * 176 * 4. Advisory Tick Time queue. 177 * 178 * 5. Closing connections queue. This is a transient queue -- it only 179 * exists for the duration of process_connections() function call. 180 * 181 * 6. Ticked connections queue. Another transient queue, similar to (5). 182 * 183 * The idea is to destroy the connection when it is no longer referenced. 184 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 185 * that case, the connection is referenced from two places: (2) and (5). 186 * After its packets are sent, it is only referenced in (5), and at the 187 * end of the function call, when it is removed from (5), reference count 188 * goes to zero and the connection is destroyed. If not all packets can 189 * be sent, at the end of the function call, the connection is referenced 190 * by (2) and will only be removed once all outgoing packets have been 191 * sent. 192 */ 193#define CONN_REF_FLAGS (LSCONN_HASHED \ 194 |LSCONN_HAS_OUTGOING \ 195 |LSCONN_TICKABLE \ 196 |LSCONN_TICKED \ 197 |LSCONN_CLOSING \ 198 |LSCONN_ATTQ) 199 200 201 202 203struct cid_update_batch 204{ 205 lsquic_cids_update_f cub_update_cids; 206 void *cub_update_ctx; 207 unsigned cub_count; 208 lsquic_cid_t cub_cids[20]; 209 void *cub_peer_ctxs[20]; 210}; 211 212static void 213cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 214 215 216struct lsquic_engine 217{ 218 struct lsquic_engine_public pub; 219 enum { 220 ENG_SERVER = LSENG_SERVER, 221 ENG_HTTP = LSENG_HTTP, 222 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 223 ENG_PAST_DEADLINE 224 = (1 << 8), /* Previous call to a processing 225 * function went past time threshold. 226 */ 227 ENG_CONNS_BY_ADDR 228 = (1 << 9), /* Connections are hashed by address */ 229#ifndef NDEBUG 230 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 231#endif 232#if CAN_LOSE_PACKETS 233 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 234#endif 235#ifndef NDEBUG 236 ENG_DTOR = (1 << 26), /* Engine destructor */ 237#endif 238 } flags; 239 lsquic_packets_out_f packets_out; 240 void *packets_out_ctx; 241 lsquic_cids_update_f report_new_scids; 242 lsquic_cids_update_f report_live_scids; 243 lsquic_cids_update_f report_old_scids; 244 void *scids_ctx; 245 struct lsquic_hash *conns_hash; 246 struct min_heap conns_tickable; 247 struct min_heap conns_out; 248 struct eng_hist history; 249 unsigned batch_size; 250 unsigned min_batch_size, max_batch_size; 251 struct lsquic_conn *curr_conn; 252 struct pr_queue *pr_queue; 253 struct attq *attq; 254 /* Track time last time a packet was sent to give new connections 255 * priority lower than that of existing connections. 256 */ 257 lsquic_time_t last_sent; 258#if CAN_LOSE_PACKETS 259 regex_t lose_packets_re; 260 const char *lose_packets_str; 261#endif 262 unsigned n_conns; 263 lsquic_time_t deadline; 264 lsquic_time_t resume_sending_at; 265 unsigned mini_conns_count; 266 struct lsquic_purga *purga; 267#if LSQUIC_CONN_STATS 268 struct { 269 unsigned conns; 270 } stats; 271 struct conn_stats conn_stats_sum; 272 FILE *stats_fh; 273#endif 274 struct cid_update_batch new_scids; 275 struct out_batch out_batch; 276#if LSQUIC_COUNT_ENGINE_CALLS 277 unsigned long n_engine_calls; 278#endif 279#if LSQUIC_DEBUG_NEXT_ADV_TICK 280 uintptr_t last_logged_conn; 281 unsigned last_logged_ae_why; 282 int last_tick_diff; 283#endif 284 struct crand crand; 285 EVP_AEAD_CTX retry_aead_ctx[N_IETF_RETRY_VERSIONS]; 286#if LSQUIC_CONN_STATS 287 struct { 288 uint16_t immed_ticks; /* bitmask */ 289#define MAX_IMMED_TICKS UINT16_MAX 290 struct lsquic_conn *last_conn, /* from last call */ 291 *pin_conn, /* last connection with packet in */ 292 *current; /* currently busy connection */ 293 lsquic_time_t last_log; 294 } busy; 295#endif 296}; 297 298 299void 300lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 301 unsigned flags) 302{ 303 memset(settings, 0, sizeof(*settings)); 304 settings->es_versions = LSQUIC_DF_VERSIONS; 305 if (flags & ENG_SERVER) 306 { 307 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 308 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 309 settings->es_init_max_data 310 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 311 settings->es_init_max_stream_data_bidi_remote 312 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 313 settings->es_init_max_stream_data_bidi_local 314 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 315 settings->es_init_max_stream_data_uni 316 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 317 settings->es_init_max_streams_uni 318 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 319 settings->es_ping_period = 0; 320 settings->es_noprogress_timeout 321 = LSQUIC_DF_NOPROGRESS_TIMEOUT_SERVER; 322 } 323 else 324 { 325 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 326 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 327 settings->es_init_max_data 328 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 329 settings->es_init_max_stream_data_bidi_remote 330 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 331 settings->es_init_max_stream_data_bidi_local 332 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 333 settings->es_init_max_stream_data_uni 334 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 335 settings->es_init_max_streams_uni 336 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 337 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 338 settings->es_noprogress_timeout 339 = LSQUIC_DF_NOPROGRESS_TIMEOUT_CLIENT; 340 } 341 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 342 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 343 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 344 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 345 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 346 settings->es_max_header_list_size 347 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 348 settings->es_ua = LSQUIC_DF_UA; 349 settings->es_ecn = LSQUIC_DF_ECN; 350 351 settings->es_pdmd = QTAG_X509; 352 settings->es_aead = QTAG_AESG; 353 settings->es_kexs = QTAG_C255; 354 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 355 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 356 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 357 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 358 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 359 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 360 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 361 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 362 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 363 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 364 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 365 settings->es_sttl = LSQUIC_DF_STTL; 366 settings->es_init_max_streams_bidi 367 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 368 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 369 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 370 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 371 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 372 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 373 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 374 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 375 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 376 settings->es_spin = LSQUIC_DF_SPIN; 377 settings->es_delayed_acks = LSQUIC_DF_DELAYED_ACKS; 378 settings->es_timestamps = LSQUIC_DF_TIMESTAMPS; 379 settings->es_grease_quic_bit = LSQUIC_DF_GREASE_QUIC_BIT; 380 settings->es_mtu_probe_timer = LSQUIC_DF_MTU_PROBE_TIMER; 381 settings->es_dplpmtud = LSQUIC_DF_DPLPMTUD; 382 settings->es_cc_algo = LSQUIC_DF_CC_ALGO; 383 settings->es_cc_rtt_thresh = LSQUIC_DF_CC_RTT_THRESH; 384 settings->es_optimistic_nat = LSQUIC_DF_OPTIMISTIC_NAT; 385 settings->es_ext_http_prio = LSQUIC_DF_EXT_HTTP_PRIO; 386 settings->es_ptpc_periodicity= LSQUIC_DF_PTPC_PERIODICITY; 387 settings->es_ptpc_max_packtol= LSQUIC_DF_PTPC_MAX_PACKTOL; 388 settings->es_ptpc_dyn_target = LSQUIC_DF_PTPC_DYN_TARGET; 389 settings->es_ptpc_target = LSQUIC_DF_PTPC_TARGET; 390 settings->es_ptpc_prop_gain = LSQUIC_DF_PTPC_PROP_GAIN; 391 settings->es_ptpc_int_gain = LSQUIC_DF_PTPC_INT_GAIN; 392 settings->es_ptpc_err_thresh = LSQUIC_DF_PTPC_ERR_THRESH; 393 settings->es_ptpc_err_divisor= LSQUIC_DF_PTPC_ERR_DIVISOR; 394 settings->es_delay_onclose = LSQUIC_DF_DELAY_ONCLOSE; 395 settings->es_check_tp_sanity = LSQUIC_DF_CHECK_TP_SANITY; 396} 397 398 399/* Note: if returning an error, err_buf must be valid if non-NULL */ 400int 401lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 402 unsigned flags, 403 char *err_buf, size_t err_buf_sz) 404{ 405 if (settings->es_cfcw < LSQUIC_MIN_FCW || 406 settings->es_sfcw < LSQUIC_MIN_FCW) 407 { 408 if (err_buf) 409 snprintf(err_buf, err_buf_sz, "%s", 410 "flow control window set too low"); 411 return -1; 412 } 413 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 414 { 415 if (err_buf) 416 snprintf(err_buf, err_buf_sz, "%s", 417 "No supported QUIC versions specified"); 418 return -1; 419 } 420 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 421 { 422 if (err_buf) 423 snprintf(err_buf, err_buf_sz, "%s", 424 "one or more unsupported QUIC version is specified"); 425 return -1; 426 } 427 if (flags & ENG_SERVER) 428 { 429 if (settings->es_handshake_to > 430 MAX_MINI_CONN_LIFESPAN_IN_USEC) 431 { 432 if (err_buf) 433 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 434 " usec is too large. The maximum for server is %u usec", 435 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 436 return -1; 437 } 438 } 439 if (settings->es_idle_timeout > 600) 440 { 441 if (err_buf) 442 snprintf(err_buf, err_buf_sz, "%s", 443 "The maximum value of idle timeout is 600 seconds"); 444 return -1; 445 } 446 if (settings->es_scid_len > MAX_CID_LEN) 447 { 448 if (err_buf) 449 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 450 "bytes long; it must be between 0 and %u.", 451 settings->es_scid_len, MAX_CID_LEN); 452 return -1; 453 } 454 455 if (settings->es_cc_algo > 3) 456 { 457 if (err_buf) 458 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 459 "algorithm value %u", settings->es_cc_algo); 460 return -1; 461 } 462 463 if (!(settings->es_ql_bits >= 0 && settings->es_ql_bits <= 2)) 464 { 465 if (err_buf) 466 snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ", 467 settings->es_ql_bits); 468 return -1; 469 } 470 471 if (!(settings->es_spin == 0 || settings->es_spin == 1)) 472 { 473 if (err_buf) 474 snprintf(err_buf, err_buf_sz, "Invalid spin value %d", 475 settings->es_spin); 476 return -1; 477 } 478 479 if (settings->es_mtu_probe_timer && settings->es_mtu_probe_timer < 1000) 480 { 481 if (err_buf) 482 snprintf(err_buf, err_buf_sz, "mtu probe timer is too small: " 483 "%u ms", settings->es_mtu_probe_timer); 484 return -1; 485 } 486 487 if (settings->es_max_batch_size > MAX_OUT_BATCH_SIZE) 488 { 489 if (err_buf) 490 snprintf(err_buf, err_buf_sz, "max batch size is greater than " 491 "the allowed maximum of %u", (unsigned) MAX_OUT_BATCH_SIZE); 492 return -1; 493 } 494 495 return 0; 496} 497 498 499static void 500free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 501{ 502 free(packet_data); 503} 504 505 506static void * 507malloc_buf (void *ctx, void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, unsigned short size, char is_ipv6) 508{ 509 return malloc(size); 510} 511 512 513static const struct lsquic_packout_mem_if stock_pmi = 514{ 515 malloc_buf, free_packet, free_packet, 516}; 517 518 519static int 520hash_conns_by_addr (const struct lsquic_engine *engine) 521{ 522 if (engine->flags & ENG_SERVER) 523 return 0; 524 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 525 return 1; 526 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 527 && engine->pub.enp_settings.es_support_tcid0) 528 return 1; 529 if (engine->pub.enp_settings.es_scid_len == 0) 530 return 1; 531 return 0; 532} 533 534 535lsquic_engine_t * 536lsquic_engine_new (unsigned flags, 537 const struct lsquic_engine_api *api) 538{ 539 lsquic_engine_t *engine; 540 size_t alpn_len; 541 unsigned i; 542 char err_buf[100]; 543 544 if (!api->ea_packets_out) 545 { 546 LSQ_ERROR("packets_out callback is not specified"); 547 return NULL; 548 } 549 550 if (!api->ea_stream_if) 551 { 552 LSQ_ERROR("stream interface is not specified"); 553 return NULL; 554 } 555 556 if (!(flags & LSENG_HTTP) && api->ea_alpn) 557 { 558 alpn_len = strlen(api->ea_alpn); 559 if (alpn_len < 1 || alpn_len > 255) 560 { 561 LSQ_ERROR("ALPN string length invalid: %zd bytes", alpn_len); 562 return NULL; 563 } 564 } 565 else 566 alpn_len = 0; 567 568 if (api->ea_settings && 569 0 != lsquic_engine_check_settings(api->ea_settings, flags, 570 err_buf, sizeof(err_buf))) 571 { 572 LSQ_ERROR("cannot create engine: %s", err_buf); 573 return NULL; 574 } 575 576 engine = calloc(1, sizeof(*engine)); 577 if (!engine) 578 return NULL; 579 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 580 { 581 free(engine); 582 return NULL; 583 } 584 if (api->ea_settings) 585 engine->pub.enp_settings = *api->ea_settings; 586 else 587 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 588 int tag_buf_len; 589 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 590 sizeof(engine->pub.enp_ver_tags_buf), 591 engine->pub.enp_settings.es_versions); 592 if (tag_buf_len <= 0) 593 { 594 LSQ_ERROR("cannot generate version tags buffer"); 595 free(engine); 596 return NULL; 597 } 598 engine->pub.enp_ver_tags_len = tag_buf_len; 599 engine->pub.enp_flags = ENPUB_CAN_SEND; 600 engine->pub.enp_stream_if = api->ea_stream_if; 601 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 602 603 engine->flags = flags; 604#ifndef NDEBUG 605 engine->flags |= ENG_COALESCE; 606#endif 607 engine->packets_out = api->ea_packets_out; 608 engine->packets_out_ctx = api->ea_packets_out_ctx; 609 engine->report_new_scids = api->ea_new_scids; 610 engine->report_live_scids = api->ea_live_scids; 611 engine->report_old_scids = api->ea_old_scids; 612 engine->scids_ctx = api->ea_cids_update_ctx; 613 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 614 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 615 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 616 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 617 618 if (api->ea_generate_scid) 619 { 620 engine->pub.enp_generate_scid = api->ea_generate_scid; 621 engine->pub.enp_gen_scid_ctx = api->ea_gen_scid_ctx; 622 } 623 else 624 engine->pub.enp_generate_scid = lsquic_generate_scid; 625 626 if (api->ea_shi) 627 { 628 engine->pub.enp_shi = api->ea_shi; 629 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 630 } 631 else 632 { 633 engine->pub.enp_shi = &stock_shi; 634 engine->pub.enp_shi_ctx = lsquic_stock_shared_hash_new(); 635 if (!engine->pub.enp_shi_ctx) 636 { 637 free(engine); 638 return NULL; 639 } 640 } 641 if (api->ea_hsi_if) 642 { 643 engine->pub.enp_hsi_if = api->ea_hsi_if; 644 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 645 } 646 else 647 { 648 engine->pub.enp_hsi_if = lsquic_http1x_if; 649 engine->pub.enp_hsi_ctx = NULL; 650 } 651 if (api->ea_pmi) 652 { 653 engine->pub.enp_pmi = api->ea_pmi; 654 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 655 } 656 else 657 { 658 engine->pub.enp_pmi = &stock_pmi; 659 engine->pub.enp_pmi_ctx = NULL; 660 } 661 engine->pub.enp_verify_cert = api->ea_verify_cert; 662 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 663 engine->pub.enp_engine = engine; 664 if (hash_conns_by_addr(engine)) 665 engine->flags |= ENG_CONNS_BY_ADDR; 666 engine->conns_hash = lsquic_hash_create(); 667 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 668 if (!engine->pub.enp_tokgen) 669 return NULL; 670 if (engine->flags & ENG_SERVER) 671 for (i = 0; i < sizeof(engine->pub.enp_quic_ctx_sz) 672 / sizeof(engine->pub.enp_quic_ctx_sz[0]); ++i) 673 { 674 int sz = lsquic_enc_sess_ietf_gen_quic_ctx( 675 &engine->pub.enp_settings, 676 i == 0 ? LSQVER_ID27 : LSQVER_ID29, 677 engine->pub.enp_quic_ctx_buf[i], 678 sizeof(engine->pub.enp_quic_ctx_buf)); 679 if (sz < 0) 680 { 681 free(engine); 682 return NULL; 683 } 684 engine->pub.enp_quic_ctx_sz[i] = (unsigned) sz; 685 } 686 engine->pub.enp_crand = &engine->crand; 687 if (engine->pub.enp_settings.es_noprogress_timeout) 688 engine->pub.enp_noprog_timeout 689 = engine->pub.enp_settings.es_noprogress_timeout * 1000000; 690 engine->pub.enp_mtu_probe_timer = 1000 691 * (engine->pub.enp_settings.es_mtu_probe_timer 692 ? engine->pub.enp_settings.es_mtu_probe_timer 693 : LSQUIC_DF_MTU_PROBE_TIMER); 694 if (flags & ENG_SERVER) 695 { 696 engine->pr_queue = lsquic_prq_create( 697 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 698 &engine->pub); 699 if (!engine->pr_queue) 700 { 701 lsquic_tg_destroy(engine->pub.enp_tokgen); 702 return NULL; 703 } 704 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 705 engine->report_old_scids, engine->scids_ctx); 706 if (!engine->purga) 707 { 708 lsquic_tg_destroy(engine->pub.enp_tokgen); 709 lsquic_prq_destroy(engine->pr_queue); 710 return NULL; 711 } 712 } 713 engine->attq = lsquic_attq_create(); 714 eng_hist_init(&engine->history); 715 if (engine->pub.enp_settings.es_max_batch_size) 716 { 717 engine->max_batch_size = engine->pub.enp_settings.es_max_batch_size; 718 engine->min_batch_size = MIN(4, engine->max_batch_size); 719 engine->batch_size = MAX(engine->max_batch_size / 4, 720 engine->min_batch_size); 721 } 722 else 723 { 724 engine->min_batch_size = MIN_OUT_BATCH_SIZE; 725 engine->max_batch_size = MAX_OUT_BATCH_SIZE; 726 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 727 } 728 if (engine->pub.enp_settings.es_honor_prst) 729 { 730 engine->pub.enp_srst_hash = lsquic_hash_create(); 731 if (!engine->pub.enp_srst_hash) 732 { 733 lsquic_engine_destroy(engine); 734 return NULL; 735 } 736 } 737 if ((flags & LSENG_SERVER) && 0 != lsquic_init_gquic_crypto(&engine->pub)) 738 { 739 lsquic_engine_destroy(engine); 740 return NULL; 741 } 742 743 if (alpn_len) 744 { 745 engine->pub.enp_alpn = malloc(alpn_len + 1); 746 if (!engine->pub.enp_alpn) 747 { 748 lsquic_engine_destroy(engine); 749 return NULL; 750 } 751 engine->pub.enp_alpn[0] = alpn_len; 752 memcpy(engine->pub.enp_alpn + 1, api->ea_alpn, alpn_len); 753 } 754 755 if (flags & LSENG_HTTP) 756 engine->pub.enp_flags |= ENPUB_HTTP; 757 758#ifndef NDEBUG 759 { 760 const char *env; 761 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 762#if CAN_LOSE_PACKETS 763 if (env) 764 { 765 if (0 != regcomp(&engine->lose_packets_re, env, 766 REG_EXTENDED|REG_NOSUB)) 767 { 768 LSQ_ERROR("could not compile lost packet regex `%s'", env); 769 return NULL; 770 } 771 engine->flags |= ENG_LOSE_PACKETS; 772 engine->lose_packets_str = env; 773 LSQ_WARN("will lose packets that match the following regex: %s", 774 env); 775 } 776#endif 777 env = getenv("LSQUIC_COALESCE"); 778 if (env) 779 { 780 engine->flags &= ~ENG_COALESCE; 781 if (atoi(env)) 782 { 783 engine->flags |= ENG_COALESCE; 784 LSQ_NOTICE("will coalesce packets"); 785 } 786 else 787 LSQ_NOTICE("will not coalesce packets"); 788 } 789 } 790#endif 791#if LSQUIC_CONN_STATS 792 engine->stats_fh = api->ea_stats_fh; 793#endif 794 for (i = 0; i < sizeof(engine->retry_aead_ctx) 795 / sizeof(engine->retry_aead_ctx[0]); ++i) 796 if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx[i], 797 EVP_aead_aes_128_gcm(), lsquic_retry_key_buf[i], 798 IETF_RETRY_KEY_SZ, 16, NULL)) 799 { 800 LSQ_ERROR("could not initialize retry AEAD ctx #%u", i); 801 lsquic_engine_destroy(engine); 802 return NULL; 803 } 804 engine->pub.enp_retry_aead_ctx = engine->retry_aead_ctx; 805 806 LSQ_INFO("instantiated engine"); 807 return engine; 808} 809 810 811#if LOG_PACKET_CHECKSUM 812static void 813log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 814 const unsigned char *buf, size_t bufsz) 815{ 816 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 817 (uint32_t) crc32(0, buf, bufsz)); 818} 819 820 821#endif 822 823 824static void 825grow_batch_size (struct lsquic_engine *engine) 826{ 827 engine->batch_size = MIN(engine->batch_size * 2, engine->max_batch_size); 828} 829 830 831static void 832shrink_batch_size (struct lsquic_engine *engine) 833{ 834 engine->batch_size = MAX(engine->batch_size / 2, engine->min_batch_size); 835} 836 837 838struct cce_cid_iter 839{ 840 const struct lsquic_conn *conn; 841 unsigned todo, n; 842}; 843 844 845static struct conn_cid_elem * 846cce_iter_next (struct cce_cid_iter *citer) 847{ 848 struct conn_cid_elem *cce; 849 850 while (citer->todo) 851 if (citer->todo & (1 << citer->n)) 852 { 853 citer->todo &= ~(1 << citer->n); 854 cce = &citer->conn->cn_cces[ citer->n++ ]; 855 if (!(cce->cce_flags & CCE_PORT)) 856 return cce; 857 } 858 else 859 ++citer->n; 860 861 return NULL; 862} 863 864 865static struct conn_cid_elem * 866cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 867{ 868 citer->conn = conn; 869 citer->todo = conn->cn_cces_mask; 870 citer->n = 0; 871 return cce_iter_next(citer); 872} 873 874 875#if LSQUIC_CONN_STATS 876void 877update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 878{ 879 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 880 const unsigned long *src; 881 const struct conn_stats *stats; 882 unsigned i; 883 884 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 885 { 886 ++engine->stats.conns; 887 src = (unsigned long *) stats; 888 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 889 dst[i] += src[i]; 890 } 891} 892 893 894#endif 895 896 897/* Wrapper to make sure important things occur before the connection is 898 * really destroyed. 899 */ 900static void 901destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 902 lsquic_time_t now) 903{ 904 struct cce_cid_iter citer; 905 const struct conn_cid_elem *cce; 906 lsquic_time_t drain_time; 907 struct purga_el *puel; 908 909 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 910 if (engine->purga 911 /* Blacklist all CIDs except for promoted mini connections */ 912 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 913 != (LSCONN_MINI|LSCONN_PROMOTED)) 914 { 915 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 916 && conn->cn_if->ci_drain_time && 917 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 918 { 919 for (cce = cce_iter_first(&citer, conn); cce; 920 cce = cce_iter_next(&citer)) 921 { 922 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 923 lsquic_conn_get_peer_ctx(conn, NULL), 924 PUTY_CONN_DRAIN, now); 925 if (puel) 926 puel->puel_time = now + drain_time; 927 } 928 } 929 else 930 { 931 for (cce = cce_iter_first(&citer, conn); cce; 932 cce = cce_iter_next(&citer)) 933 { 934 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 935 lsquic_conn_get_peer_ctx(conn, NULL), 936 PUTY_CONN_DELETED, now); 937 if (puel) 938 { 939 puel->puel_time = now; 940 puel->puel_count = 0; 941 } 942 } 943 } 944 } 945#if LSQUIC_CONN_STATS 946 update_stats_sum(engine, conn); 947 if (engine->busy.last_conn == conn) 948 engine->busy.last_conn = NULL; 949 if (engine->busy.current == conn) 950 { 951 char cidstr[MAX_CID_LEN * 2 + 1]; 952 lsquic_logger_log1(LSQ_LOG_NOTICE, LSQLM_CONN_STATS, 953 "busy connection %s is destroyed", 954 (lsquic_cid2str(lsquic_conn_log_cid(conn), cidstr), cidstr)); 955 engine->busy.current = NULL; 956 engine->busy.last_log = 0; 957 } 958 if (engine->busy.pin_conn == conn) 959 engine->busy.pin_conn = NULL; 960#endif 961 --engine->n_conns; 962 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 963 conn->cn_if->ci_destroy(conn); 964} 965 966 967static int 968maybe_grow_conn_heaps (struct lsquic_engine *engine) 969{ 970 struct min_heap_elem *els; 971 unsigned count; 972 973 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 974 return 0; /* Nothing to do */ 975 976 if (lsquic_mh_nalloc(&engine->conns_tickable)) 977 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 978 else 979 count = 8; 980 981 els = malloc(sizeof(els[0]) * count); 982 if (!els) 983 { 984 LSQ_ERROR("%s: malloc failed", __func__); 985 return -1; 986 } 987 988 LSQ_DEBUG("grew heaps to %u elements", count / 2); 989 memcpy(&els[0], engine->conns_tickable.mh_elems, 990 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 991 memcpy(&els[count / 2], engine->conns_out.mh_elems, 992 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 993 free(engine->conns_tickable.mh_elems); 994 engine->conns_tickable.mh_elems = els; 995 engine->conns_out.mh_elems = &els[count / 2]; 996 engine->conns_tickable.mh_nalloc = count / 2; 997 engine->conns_out.mh_nalloc = count / 2; 998 return 0; 999} 1000 1001 1002static void 1003remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 1004 unsigned todo) 1005{ 1006 unsigned n; 1007 1008 for (n = 0; todo; todo &= ~(1 << n++)) 1009 if ((todo & (1 << n)) && 1010 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 1011 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 1012} 1013 1014 1015static void 1016remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 1017{ 1018 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 1019} 1020 1021 1022static void 1023cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 1024 1025 1026static int 1027insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1028 void *peer_ctx) 1029{ 1030 struct conn_cid_elem *cce; 1031 unsigned todo, done, n; 1032 1033 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 1034 if (todo & (1 << n)) 1035 { 1036 cce = &conn->cn_cces[n]; 1037 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 1038 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 1039 cce->cce_cid.len, conn, &cce->cce_hash_el)) 1040 done |= 1 << n; 1041 else 1042 goto err; 1043 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 1044 { 1045 cce->cce_flags |= CCE_REG; 1046 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 1047 } 1048 } 1049 1050 return 0; 1051 1052 err: 1053 remove_cces_from_hash(engine->conns_hash, conn, done); 1054 return -1; 1055} 1056 1057 1058static lsquic_conn_t * 1059new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 1060 lsquic_time_t now) 1061{ 1062 const lsquic_cid_t *cid; 1063 server_conn_ctor_f ctor; 1064 lsquic_conn_t *conn; 1065 unsigned flags; 1066 if (0 != maybe_grow_conn_heaps(engine)) 1067 return NULL; 1068 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1069 1070 if (mini_conn->cn_flags & LSCONN_IETF) 1071 ctor = lsquic_ietf_full_conn_server_new; 1072 else 1073 ctor = lsquic_gquic_full_conn_server_new; 1074 1075 conn = ctor(&engine->pub, flags, mini_conn); 1076 if (!conn) 1077 { 1078 /* Otherwise, full_conn_server_new prints its own warnings */ 1079 if (ENOMEM == errno) 1080 { 1081 cid = lsquic_conn_log_cid(mini_conn); 1082 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 1083 CID_BITS(cid), strerror(errno)); 1084 } 1085 return NULL; 1086 } 1087 ++engine->n_conns; 1088 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 1089 { 1090 cid = lsquic_conn_log_cid(conn); 1091 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1092 CID_BITS(cid)); 1093 destroy_conn(engine, conn, now); 1094 return NULL; 1095 } 1096 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1097 conn->cn_flags |= LSCONN_HASHED; 1098 return conn; 1099} 1100 1101 1102static enum 1103{ 1104 VER_NOT_SPECIFIED, 1105 VER_SUPPORTED, 1106 VER_UNSUPPORTED, 1107} 1108 1109 1110version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 1111 enum lsquic_version *pversion) 1112{ 1113 lsquic_ver_tag_t ver_tag; 1114 enum lsquic_version version; 1115 1116 if (!packet_in->pi_quic_ver) 1117 { 1118 LSQ_DEBUG("packet does not specify version"); 1119 return VER_NOT_SPECIFIED; 1120 } 1121 1122 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 1123 version = lsquic_tag2ver(ver_tag); 1124 if (version < N_LSQVER) 1125 { 1126 if (engine->pub.enp_settings.es_versions & (1 << version)) 1127 { 1128 LSQ_DEBUG("client-supplied version %s is supported", 1129 lsquic_ver2str[version]); 1130 *pversion = version; 1131 return VER_SUPPORTED; 1132 } 1133 else 1134 LSQ_DEBUG("client-supplied version %s is not supported", 1135 lsquic_ver2str[version]); 1136 } 1137 else 1138 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 1139 ver_tag); 1140 1141 return VER_UNSUPPORTED; 1142} 1143 1144 1145static void 1146schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 1147 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 1148 const struct sockaddr *sa_peer, void *peer_ctx) 1149{ 1150 assert(engine->pr_queue); 1151 if (0 == lsquic_prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 1152 sa_local, sa_peer)) 1153 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 1154 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 1155 else 1156 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 1157} 1158 1159 1160static unsigned short 1161sa2port (const struct sockaddr *sa) 1162{ 1163 if (sa->sa_family == AF_INET) 1164 { 1165 struct sockaddr_in *const sa4 = (void *) sa; 1166 return sa4->sin_port; 1167 } 1168 else 1169 { 1170 struct sockaddr_in6 *const sa6 = (void *) sa; 1171 return sa6->sin6_port; 1172 } 1173} 1174 1175 1176static struct lsquic_hash_elem * 1177find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 1178{ 1179 unsigned short port; 1180 1181 port = sa2port(sa); 1182 return lsquic_hash_find(hash, &port, sizeof(port)); 1183} 1184 1185 1186/* When connections are identified by the local address, we need to drop 1187 * packets that use DCIDs that do not correspond to any of SCIDs. This 1188 * can happen when peer retires a SCID. This mimics the normal behavior, 1189 * when connections are looked up in engine->conns_hash by ID: when there 1190 * is no match, the packet is dropped. 1191 */ 1192static int 1193dcid_checks_out (const struct lsquic_conn *conn, const lsquic_cid_t *dcid) 1194{ 1195 const struct conn_cid_elem *cce; 1196 1197 if (LSQUIC_CIDS_EQ(CN_SCID(conn), dcid)) 1198 return 1; 1199 1200 /* Slow check for those rare cases */ 1201 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1202 if ((conn->cn_cces_mask & (1 << (cce - conn->cn_cces))) 1203 && !(cce->cce_flags & CCE_PORT) 1204 && LSQUIC_CIDS_EQ(&cce->cce_cid, dcid)) 1205 { 1206 LSQ_DEBUG("connection checks out"); 1207 return 1; 1208 } 1209 1210 return 0; 1211} 1212 1213 1214static lsquic_conn_t * 1215find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1216 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 1217{ 1218 struct lsquic_hash_elem *el; 1219 lsquic_conn_t *conn; 1220 1221 if (engine->flags & ENG_CONNS_BY_ADDR) 1222 { 1223 el = find_conn_by_addr(engine->conns_hash, sa_local); 1224 if ((packet_in->pi_flags & PI_CONN_ID) 1225 && !dcid_checks_out(lsquic_hashelem_getdata(el), 1226 &packet_in->pi_conn_id)) 1227 { 1228 LSQ_DEBUGC("DCID matches no SCID in connection %"CID_FMT": drop it", 1229 CID_BITS(&packet_in->pi_conn_id)); 1230 return NULL; 1231 } 1232 } 1233 else if (packet_in->pi_flags & PI_CONN_ID) 1234 el = lsquic_hash_find(engine->conns_hash, 1235 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1236 else 1237 { 1238 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1239 return NULL; 1240 } 1241 1242 if (!el) 1243 return NULL; 1244 1245 conn = lsquic_hashelem_getdata(el); 1246 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1247 if ((engine->flags & ENG_CONNS_BY_ADDR) 1248 && !(conn->cn_flags & LSCONN_IETF) 1249 && (packet_in->pi_flags & PI_CONN_ID) 1250 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1251 { 1252 LSQ_DEBUG("connection IDs do not match"); 1253 return NULL; 1254 } 1255 1256 return conn; 1257} 1258 1259 1260static const char *const puty2str[] = { 1261 [PUTY_CONN_DELETED] = "deleted", 1262 [PUTY_CONN_DRAIN] = "being drained", 1263 [PUTY_CID_RETIRED] = "retired", 1264}; 1265 1266 1267static lsquic_conn_t * 1268find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1269 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1270 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1271{ 1272 struct lsquic_hash_elem *el; 1273 struct purga_el *puel; 1274 lsquic_conn_t *conn; 1275 1276 if (!(packet_in->pi_flags & PI_CONN_ID)) 1277 { 1278 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1279 return NULL; 1280 } 1281 el = lsquic_hash_find(engine->conns_hash, 1282 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1283 1284 if (el) 1285 { 1286 conn = lsquic_hashelem_getdata(el); 1287 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1288 return conn; 1289 } 1290 1291 if (engine->flags & ENG_COOLDOWN) 1292 { /* Do not create incoming connections during cooldown */ 1293 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1294 return NULL; 1295 } 1296 1297 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1298 { 1299 LSQ_DEBUG("reached limit of %u inchoate connections", 1300 engine->pub.enp_settings.es_max_inchoate); 1301 return NULL; 1302 } 1303 1304 1305 if (engine->purga 1306 && (puel = lsquic_purga_contains(engine->purga, 1307 &packet_in->pi_conn_id), puel)) 1308 { 1309 switch (puel->puel_type) 1310 { 1311 case PUTY_CID_RETIRED: 1312 case PUTY_CONN_DRAIN: 1313 if (puel->puel_time > packet_in->pi_received) 1314 { 1315 LSQ_DEBUGC("CID %"CID_FMT" is %s for another %"PRIu64 1316 "usec, ignore packet", CID_BITS(&packet_in->pi_conn_id), 1317 puty2str[puel->puel_type], 1318 puel->puel_time - packet_in->pi_received); 1319 return NULL; 1320 } 1321 LSQ_DEBUGC("CID %"CID_FMT" is no longer %s", 1322 CID_BITS(&packet_in->pi_conn_id), puty2str[puel->puel_type]); 1323 break; 1324 case PUTY_CONN_DELETED: 1325 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1326 CID_BITS(&packet_in->pi_conn_id)); 1327 if (puel->puel_time < packet_in->pi_received) 1328 { 1329 if (puel->puel_count < 4) 1330 { 1331 puel->puel_time = packet_in->pi_received 1332 /* Exponential back-off */ 1333 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1334 ++puel->puel_count; 1335 goto maybe_send_prst; 1336 } 1337 else 1338 break; 1339 } 1340 return NULL; 1341 default: 1342 assert(0); 1343 return NULL; 1344 } 1345 } 1346 1347 if (engine->pub.enp_settings.es_send_prst 1348 && !(packet_in->pi_flags & PI_GQUIC) 1349 && HETY_NOT_SET == packet_in->pi_header_type) 1350 goto maybe_send_prst; 1351 1352 if (0 != maybe_grow_conn_heaps(engine)) 1353 return NULL; 1354 1355 const struct parse_funcs *pf; 1356 enum lsquic_version version; 1357 switch (version_matches(engine, packet_in, &version)) 1358 { 1359 case VER_UNSUPPORTED: 1360 if (engine->flags & ENG_SERVER) 1361 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1362 sa_local, sa_peer, peer_ctx); 1363 return NULL; 1364 case VER_NOT_SPECIFIED: 1365 maybe_send_prst: 1366 if ((engine->flags & ENG_SERVER) && 1367 engine->pub.enp_settings.es_send_prst) 1368 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1369 sa_local, sa_peer, peer_ctx); 1370 return NULL; 1371 case VER_SUPPORTED: 1372 pf = select_pf_by_ver(version); 1373 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1374 break; 1375 } 1376 1377 1378 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1379 { 1380 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1381 sa_peer->sa_family == AF_INET, NULL, packet_in_size); 1382 } 1383 else 1384 { 1385 conn = lsquic_mini_conn_new(&engine->pub, packet_in, version); 1386 } 1387 if (!conn) 1388 return NULL; 1389 ++engine->mini_conns_count; 1390 ++engine->n_conns; 1391 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1392 { 1393 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1394 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1395 CID_BITS(cid)); 1396 destroy_conn(engine, conn, packet_in->pi_received); 1397 return NULL; 1398 } 1399 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1400 conn->cn_flags |= LSCONN_HASHED; 1401 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1402 conn->cn_last_sent = engine->last_sent; 1403 return conn; 1404} 1405 1406 1407lsquic_conn_t * 1408lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1409 const lsquic_cid_t *cid) 1410{ 1411 struct lsquic_hash_elem *el; 1412 lsquic_conn_t *conn = NULL; 1413 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1414 1415 if (el) 1416 conn = lsquic_hashelem_getdata(el); 1417 return conn; 1418} 1419 1420 1421#if !defined(NDEBUG) && __GNUC__ 1422__attribute__((weak)) 1423#endif 1424void 1425lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1426 lsquic_conn_t *conn) 1427{ 1428 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1429 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1430 { 1431 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1432 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1433 engine_incref_conn(conn, LSCONN_TICKABLE); 1434 } 1435} 1436 1437 1438void 1439lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1440 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1441{ 1442 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1443 if (conn->cn_flags & LSCONN_TICKABLE) 1444 { 1445 /* Optimization: no need to add the connection to the Advisory Tick 1446 * Time Queue: it is about to be ticked, after which it its next tick 1447 * time may be queried again. 1448 */; 1449 } 1450 else if (conn->cn_flags & LSCONN_ATTQ) 1451 { 1452 if (lsquic_conn_adv_time(conn) != tick_time) 1453 { 1454 lsquic_attq_remove(engine->attq, conn); 1455 if (0 != lsquic_attq_add(engine->attq, conn, tick_time, why)) 1456 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1457 } 1458 } 1459 else if (0 == lsquic_attq_add(engine->attq, conn, tick_time, why)) 1460 engine_incref_conn(conn, LSCONN_ATTQ); 1461} 1462 1463 1464static struct lsquic_conn * 1465find_conn_by_srst (struct lsquic_engine *engine, 1466 const struct lsquic_packet_in *packet_in) 1467{ 1468 struct lsquic_hash_elem *el; 1469 struct lsquic_conn *conn; 1470 1471 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1472 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1473 return NULL; 1474 1475 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1476 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1477 IQUIC_SRESET_TOKEN_SZ); 1478 if (!el) 1479 return NULL; 1480 1481 conn = lsquic_hashelem_getdata(el); 1482 return conn; 1483} 1484 1485 1486/* Return 0 if packet is being processed by a real connection (mini or full), 1487 * otherwise return 1. 1488 */ 1489static int 1490process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1491 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1492 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1493{ 1494 lsquic_conn_t *conn; 1495 const unsigned char *packet_in_data; 1496 1497 if (lsquic_packet_in_is_gquic_prst(packet_in) 1498 && !engine->pub.enp_settings.es_honor_prst) 1499 { 1500 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1501 LSQ_DEBUG("public reset packet: discarding"); 1502 return 1; 1503 } 1504 1505 if (engine->flags & ENG_SERVER) 1506 { 1507 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1508 sa_peer, peer_ctx, packet_in_size); 1509 if (!engine->curr_conn) 1510 engine->curr_conn = conn; 1511 } 1512 else 1513 conn = find_conn(engine, packet_in, ppstate, sa_local); 1514 1515 if (!conn) 1516 { 1517 if (engine->pub.enp_settings.es_honor_prst 1518 && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */ 1519 && !(packet_in->pi_flags & PI_GQUIC) 1520 && engine->pub.enp_srst_hash 1521 && (conn = find_conn_by_srst(engine, packet_in))) 1522 { 1523 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1524 CID_BITS(lsquic_conn_log_cid(conn))); 1525 conn->cn_if->ci_stateless_reset(conn); 1526 if (!(conn->cn_flags & LSCONN_TICKABLE) 1527 && conn->cn_if->ci_is_tickable(conn)) 1528 { 1529 lsquic_mh_insert(&engine->conns_tickable, conn, 1530 conn->cn_last_ticked); 1531 engine_incref_conn(conn, LSCONN_TICKABLE); 1532 } 1533 /* Even though the connection processes this packet, we return 1534 * 1 so that the caller does not add reset packet's random 1535 * bytes to the list of valid CIDs. 1536 */ 1537 } 1538 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1539 return 1; 1540 } 1541 1542 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1543 { 1544 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1545 engine_incref_conn(conn, LSCONN_TICKABLE); 1546 } 1547 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1548 sa_local, sa_peer); 1549 lsquic_packet_in_upref(packet_in); 1550#if LOG_PACKET_CHECKSUM 1551 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1552 packet_in->pi_data_sz); 1553#endif 1554 /* Note on QLog: 1555 * For the PACKET_RX QLog event, we are interested in logging these things: 1556 * - raw packet (however it comes in, encrypted or not) 1557 * - frames (list of frame names) 1558 * - packet type and number 1559 * - packet rx timestamp 1560 * 1561 * Since only some of these items are available at this code 1562 * juncture, we will wait until after the packet has been 1563 * decrypted (if necessary) and parsed to call the log functions. 1564 * 1565 * Once the PACKET_RX event is finally logged, the timestamp 1566 * will come from packet_in->pi_received. For correct sequential 1567 * ordering of QLog events, be sure to process the QLogs downstream. 1568 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1569 */ 1570 packet_in_data = packet_in->pi_data; 1571 packet_in_size = packet_in->pi_data_sz; 1572 conn->cn_if->ci_packet_in(conn, packet_in); 1573#if LSQUIC_CONN_STATS 1574 engine->busy.pin_conn = conn; 1575#endif 1576 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1577 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1578 return 0; 1579} 1580 1581 1582void 1583lsquic_engine_destroy (lsquic_engine_t *engine) 1584{ 1585 struct lsquic_hash_elem *el; 1586 lsquic_conn_t *conn; 1587 unsigned i; 1588 1589 LSQ_DEBUG("destroying engine"); 1590#ifndef NDEBUG 1591 engine->flags |= ENG_DTOR; 1592#endif 1593 1594 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1595 { 1596 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1597 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1598 } 1599 1600 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1601 { 1602 assert(conn->cn_flags & LSCONN_TICKABLE); 1603 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1604 } 1605 1606 for (el = lsquic_hash_first(engine->conns_hash); el; 1607 el = lsquic_hash_next(engine->conns_hash)) 1608 { 1609 conn = lsquic_hashelem_getdata(el); 1610 force_close_conn(engine, conn); 1611 } 1612 lsquic_hash_destroy(engine->conns_hash); 1613 1614 while ((conn = lsquic_attq_pop(engine->attq, UINT64_MAX))) 1615 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1616 1617 assert(0 == engine->n_conns); 1618 assert(0 == engine->mini_conns_count); 1619 if (engine->pr_queue) 1620 lsquic_prq_destroy(engine->pr_queue); 1621 if (engine->purga) 1622 lsquic_purga_destroy(engine->purga); 1623 lsquic_attq_destroy(engine->attq); 1624 1625 assert(0 == lsquic_mh_count(&engine->conns_out)); 1626 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1627 if (engine->pub.enp_shi == &stock_shi) 1628 lsquic_stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1629 lsquic_mm_cleanup(&engine->pub.enp_mm); 1630 free(engine->conns_tickable.mh_elems); 1631#if CAN_LOSE_PACKETS 1632 if (engine->flags & ENG_LOSE_PACKETS) 1633 regfree(&engine->lose_packets_re); 1634#endif 1635 if (engine->pub.enp_tokgen) 1636 lsquic_tg_destroy(engine->pub.enp_tokgen); 1637 if (engine->flags & LSENG_SERVER) 1638 lsquic_cleanup_gquic_crypto(&engine->pub); 1639#if LSQUIC_CONN_STATS 1640 if (engine->stats_fh) 1641 { 1642 const struct conn_stats *const stats = &engine->conn_stats_sum; 1643 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1644 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1645 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1646 fprintf(engine->stats_fh, "In:\n"); 1647 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1648 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1649 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1650 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1651 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1652 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1653 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1654 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1655 stats->in.headers_comp, stats->in.headers_uncomp, 1656 stats->in.headers_uncomp ? 1657 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1658 : 0); 1659 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1660 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1661 fprintf(engine->stats_fh, " ACK frames merged: %lu\n", stats->in.n_acks_merged); 1662 fprintf(engine->stats_fh, "Out:\n"); 1663 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1664 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1665 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1666 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1667 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1668 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1669 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1670 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1671 stats->out.headers_comp, stats->out.headers_uncomp, 1672 stats->out.headers_uncomp ? 1673 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1674 : 0); 1675 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1676 } 1677#endif 1678 if (engine->pub.enp_srst_hash) 1679 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1680#if LSQUIC_COUNT_ENGINE_CALLS 1681 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1682#endif 1683 for (i = 0; i < sizeof(engine->retry_aead_ctx) 1684 / sizeof(engine->retry_aead_ctx[0]); ++i) 1685 EVP_AEAD_CTX_cleanup(&engine->pub.enp_retry_aead_ctx[i]); 1686 free(engine->pub.enp_alpn); 1687 free(engine); 1688} 1689 1690 1691static struct conn_cid_elem * 1692find_free_cce (struct lsquic_conn *conn) 1693{ 1694 struct conn_cid_elem *cce; 1695 1696 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1697 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1698 return cce; 1699 1700 return NULL; 1701} 1702 1703 1704static int 1705add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1706 const struct sockaddr *local_sa, void *peer_ctx) 1707{ 1708 struct conn_cid_elem *cce; 1709 1710 if (engine->flags & ENG_CONNS_BY_ADDR) 1711 { 1712 cce = find_free_cce(conn); 1713 if (!cce) 1714 { 1715 LSQ_ERROR("cannot find free CCE"); 1716 return -1; 1717 } 1718 cce->cce_port = sa2port(local_sa); 1719 cce->cce_flags = CCE_PORT; 1720 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1721 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1722 { 1723 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1724 return 0; 1725 } 1726 else 1727 return -1; 1728 1729 } 1730 else 1731 return insert_conn_into_hash(engine, conn, peer_ctx); 1732} 1733 1734 1735lsquic_conn_t * 1736lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1737 const struct sockaddr *local_sa, 1738 const struct sockaddr *peer_sa, 1739 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1740 const char *hostname, unsigned short base_plpmtu, 1741 const unsigned char *sess_resume, size_t sess_resume_len, 1742 const unsigned char *token, size_t token_sz) 1743{ 1744 lsquic_conn_t *conn; 1745 unsigned flags, versions; 1746 int is_ipv4; 1747 1748 ENGINE_CALLS_INCR(engine); 1749 1750 if (engine->flags & ENG_SERVER) 1751 { 1752 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1753 goto err; 1754 } 1755 1756 if (engine->flags & ENG_CONNS_BY_ADDR 1757 && find_conn_by_addr(engine->conns_hash, local_sa)) 1758 { 1759 LSQ_ERROR("cannot have more than one connection on the same port"); 1760 goto err; 1761 } 1762 1763 if (0 != maybe_grow_conn_heaps(engine)) 1764 return NULL; 1765 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1766 is_ipv4 = peer_sa->sa_family == AF_INET; 1767 if (sess_resume && sess_resume_len) 1768 { 1769 version = lsquic_sess_resume_version(sess_resume, sess_resume_len); 1770 if (version >= N_LSQVER) 1771 { 1772 LSQ_INFO("session resumption version is bad, won't use"); 1773 sess_resume = NULL; 1774 sess_resume_len = 0; 1775 } 1776 } 1777 if (version >= N_LSQVER) 1778 { 1779 if (version > N_LSQVER) 1780 LSQ_WARN("invalid version specified, engine will pick"); 1781 versions = engine->pub.enp_settings.es_versions; 1782 } 1783 else 1784 versions = 1u << version; 1785 if (versions & LSQUIC_IETF_VERSIONS) 1786 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1787 flags, hostname, base_plpmtu, 1788 is_ipv4, sess_resume, sess_resume_len, token, token_sz, peer_ctx); 1789 else 1790 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1791 flags, hostname, base_plpmtu, is_ipv4, 1792 sess_resume, sess_resume_len); 1793 if (!conn) 1794 goto err; 1795 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1796 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1797 lsquic_ver2str[conn->cn_version]); 1798 ++engine->n_conns; 1799 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1800 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1801 { 1802 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1803 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1804 CID_BITS(cid)); 1805 destroy_conn(engine, conn, lsquic_time_now()); 1806 goto err; 1807 } 1808 assert(!(conn->cn_flags & 1809 (CONN_REF_FLAGS 1810 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1811 callbacks */ 1812 ))); 1813 conn->cn_flags |= LSCONN_HASHED; 1814 if (!(conn->cn_flags & LSCONN_TICKABLE)) 1815 { 1816 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1817 engine_incref_conn(conn, LSCONN_TICKABLE); 1818 } 1819 lsquic_conn_set_ctx(conn, conn_ctx); 1820 conn->cn_if->ci_client_call_on_new(conn); 1821 end: 1822 return conn; 1823 err: 1824 conn = NULL; 1825 goto end; 1826} 1827 1828 1829static void 1830remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1831{ 1832 remove_all_cces_from_hash(engine->conns_hash, conn); 1833 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1834} 1835 1836 1837static void 1838refflags2str (enum lsquic_conn_flags flags, char s[7]) 1839{ 1840 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1841 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1842 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1843 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1844 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1845 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1846 *s = '\0'; 1847} 1848 1849 1850static void 1851engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1852{ 1853 char str[2][7]; 1854 assert(flag & CONN_REF_FLAGS); 1855 assert(!(conn->cn_flags & flag)); 1856 conn->cn_flags |= flag; 1857 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1858 CID_BITS(lsquic_conn_log_cid(conn)), 1859 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1860 (refflags2str(conn->cn_flags, str[1]), str[1])); 1861} 1862 1863 1864static lsquic_conn_t * 1865engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1866 enum lsquic_conn_flags flags) 1867{ 1868 char str[2][7]; 1869 lsquic_time_t now; 1870 assert(flags & CONN_REF_FLAGS); 1871 assert(conn->cn_flags & flags); 1872#ifndef NDEBUG 1873 if (flags & LSCONN_CLOSING) 1874 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1875#endif 1876 conn->cn_flags &= ~flags; 1877 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1878 CID_BITS(lsquic_conn_log_cid(conn)), 1879 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1880 (refflags2str(conn->cn_flags, str[1]), str[1])); 1881 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1882 { 1883 now = lsquic_time_now(); 1884 if (conn->cn_flags & LSCONN_MINI) 1885 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1886 else 1887 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1888 destroy_conn(engine, conn, now); 1889 return NULL; 1890 } 1891 else 1892 return conn; 1893} 1894 1895 1896/* This is not a general-purpose function. Only call from engine dtor. */ 1897static void 1898force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1899{ 1900 assert(engine->flags & ENG_DTOR); 1901 const enum lsquic_conn_flags flags = conn->cn_flags; 1902 assert(conn->cn_flags & CONN_REF_FLAGS); 1903 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1904 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1905 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1906 if (flags & LSCONN_ATTQ) 1907 { 1908 lsquic_attq_remove(engine->attq, conn); 1909 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1910 } 1911 if (flags & LSCONN_HASHED) 1912 remove_conn_from_hash(engine, conn); 1913} 1914 1915 1916/* Iterator for tickable connections (those on the Tickable Queue). Before 1917 * a connection is returned, it is removed from the Advisory Tick Time queue 1918 * if necessary. 1919 */ 1920static lsquic_conn_t * 1921conn_iter_next_tickable (struct lsquic_engine *engine) 1922{ 1923 lsquic_conn_t *conn; 1924 1925 if (engine->flags & ENG_SERVER) 1926 while (1) 1927 { 1928 conn = lsquic_mh_pop(&engine->conns_tickable); 1929 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1930 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1931 else 1932 break; 1933 } 1934 else 1935 conn = lsquic_mh_pop(&engine->conns_tickable); 1936 1937 if (conn) 1938 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1939 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1940 { 1941 lsquic_attq_remove(engine->attq, conn); 1942 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1943 } 1944 1945 return conn; 1946} 1947 1948 1949static void 1950cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1951 void *update_ctx) 1952{ 1953 cub->cub_update_cids = update; 1954 cub->cub_update_ctx = update_ctx; 1955 cub->cub_count = 0; 1956} 1957 1958 1959static void 1960cub_flush (struct cid_update_batch *cub) 1961{ 1962 if (cub->cub_count > 0 && cub->cub_update_cids) 1963 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1964 cub->cub_cids, cub->cub_count); 1965 cub->cub_count = 0; 1966} 1967 1968 1969static void 1970cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1971{ 1972 cub->cub_cids [ cub->cub_count ] = *cid; 1973 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1974 ++cub->cub_count; 1975 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1976 cub_flush(cub); 1977} 1978 1979 1980/* Process registered CIDs */ 1981static void 1982cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1983{ 1984 struct cce_cid_iter citer; 1985 struct conn_cid_elem *cce; 1986 void *peer_ctx; 1987 1988 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1989 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1990 if (cce->cce_flags & CCE_REG) 1991 cub_add(cub, &cce->cce_cid, peer_ctx); 1992} 1993 1994 1995static void 1996drop_all_mini_conns (lsquic_engine_t *engine) 1997{ 1998 struct lsquic_hash_elem *el; 1999 lsquic_conn_t *conn; 2000 struct cid_update_batch cub; 2001 2002 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2003 2004 for (el = lsquic_hash_first(engine->conns_hash); el; 2005 el = lsquic_hash_next(engine->conns_hash)) 2006 { 2007 conn = lsquic_hashelem_getdata(el); 2008 if (conn->cn_flags & LSCONN_MINI) 2009 { 2010 /* If promoted, why is it still in this hash? */ 2011 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 2012 if (!(conn->cn_flags & LSCONN_PROMOTED)) 2013 cub_add_cids_from_cces(&cub, conn); 2014 remove_conn_from_hash(engine, conn); 2015 } 2016 } 2017 2018 cub_flush(&cub); 2019} 2020 2021 2022void 2023lsquic_engine_process_conns (lsquic_engine_t *engine) 2024{ 2025 lsquic_conn_t *conn; 2026 lsquic_time_t now; 2027 2028#if LSQUIC_CONN_STATS 2029 if (engine->busy.pin_conn) 2030 { 2031 update_busy_detector(engine, engine->busy.pin_conn, 1); 2032 engine->busy.pin_conn = NULL; 2033 } 2034#endif 2035 2036 ENGINE_IN(engine); 2037 2038 now = lsquic_time_now(); 2039 while ((conn = lsquic_attq_pop(engine->attq, now))) 2040 { 2041 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 2042 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 2043 { 2044 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2045 engine_incref_conn(conn, LSCONN_TICKABLE); 2046 } 2047 } 2048 2049 process_connections(engine, conn_iter_next_tickable, now); 2050 ENGINE_OUT(engine); 2051} 2052 2053 2054static void 2055release_or_return_enc_data (struct lsquic_engine *engine, 2056 void (*pmi_rel_or_ret) (void *, void *, void *, char), 2057 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 2058{ 2059 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 2060 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 2061 packet_out->po_flags &= ~PO_ENCRYPTED; 2062 packet_out->po_enc_data = NULL; 2063} 2064 2065 2066static void 2067release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 2068 struct lsquic_packet_out *packet_out) 2069{ 2070 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 2071 conn, packet_out); 2072} 2073 2074 2075static void 2076return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 2077 struct lsquic_packet_out *packet_out) 2078{ 2079 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 2080 conn, packet_out); 2081} 2082 2083 2084static int 2085copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 2086 struct lsquic_packet_out *packet_out) 2087{ 2088 int ipv6; 2089 2090 ipv6 = NP_IS_IPv6(packet_out->po_path); 2091 if (packet_out->po_flags & PO_ENCRYPTED) 2092 { 2093 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 2094 && packet_out->po_data_sz == packet_out->po_enc_data_sz 2095 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 2096 packet_out->po_data_sz)) 2097 return 0; 2098 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 2099 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 2100 goto copy; 2101 return_enc_data(engine, conn, packet_out); 2102 } 2103 2104 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 2105 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 2106 conn->cn_conn_ctx, packet_out->po_data_sz, ipv6); 2107 if (!packet_out->po_enc_data) 2108 { 2109 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 2110 "of size %hu", packet_out->po_data_sz); 2111 return -1; 2112 } 2113 2114 copy: 2115 memcpy(packet_out->po_enc_data, packet_out->po_data, 2116 packet_out->po_data_sz); 2117 packet_out->po_enc_data_sz = packet_out->po_data_sz; 2118 packet_out->po_sent_sz = packet_out->po_data_sz; 2119 packet_out->po_flags &= ~PO_IPv6; 2120 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 2121 2122 return 0; 2123} 2124 2125 2126STAILQ_HEAD(conns_stailq, lsquic_conn); 2127TAILQ_HEAD(conns_tailq, lsquic_conn); 2128 2129 2130struct conns_out_iter 2131{ 2132 struct min_heap *coi_heap; 2133 struct pr_queue *coi_prq; 2134 TAILQ_HEAD(, lsquic_conn) coi_active_list, 2135 coi_inactive_list; 2136 lsquic_conn_t *coi_next; 2137#ifndef NDEBUG 2138 lsquic_time_t coi_last_sent; 2139#endif 2140}; 2141 2142 2143static void 2144coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 2145{ 2146 iter->coi_heap = &engine->conns_out; 2147 iter->coi_prq = engine->pr_queue; 2148 iter->coi_next = NULL; 2149 TAILQ_INIT(&iter->coi_active_list); 2150 TAILQ_INIT(&iter->coi_inactive_list); 2151#ifndef NDEBUG 2152 iter->coi_last_sent = 0; 2153#endif 2154} 2155 2156 2157static lsquic_conn_t * 2158coi_next (struct conns_out_iter *iter) 2159{ 2160 lsquic_conn_t *conn; 2161 2162 if (lsquic_mh_count(iter->coi_heap) > 0) 2163 { 2164 conn = lsquic_mh_pop(iter->coi_heap); 2165 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2166 conn->cn_flags |= LSCONN_COI_ACTIVE; 2167#ifndef NDEBUG 2168 if (iter->coi_last_sent) 2169 assert(iter->coi_last_sent <= conn->cn_last_sent); 2170 iter->coi_last_sent = conn->cn_last_sent; 2171#endif 2172 return conn; 2173 } 2174 else if (iter->coi_prq && (conn = lsquic_prq_next_conn(iter->coi_prq))) 2175 { 2176 return conn; 2177 } 2178 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 2179 { 2180 iter->coi_prq = NULL; /* Save function call in previous conditional */ 2181 conn = iter->coi_next; 2182 if (!conn) 2183 conn = TAILQ_FIRST(&iter->coi_active_list); 2184 if (conn) 2185 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 2186 return conn; 2187 } 2188 else 2189 return NULL; 2190} 2191 2192 2193static void 2194coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2195{ 2196 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2197 { 2198 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 2199 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2200 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2201 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 2202 conn->cn_flags |= LSCONN_COI_INACTIVE; 2203 } 2204} 2205 2206 2207static void 2208coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2209{ 2210 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 2211 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2212 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2213 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2214 conn->cn_flags |= LSCONN_COI_ACTIVE; 2215} 2216 2217 2218static void 2219coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 2220{ 2221 lsquic_conn_t *conn; 2222 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 2223 { 2224 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2225 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2226 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 2227 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 2228 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 2229 else /* Closed connection gets one shot at sending packets */ 2230 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2231 } 2232 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 2233 { 2234 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2235 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2236 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2237 } 2238} 2239 2240 2241#if CAN_LOSE_PACKETS 2242static void 2243lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 2244 unsigned n) 2245{ 2246 const lsquic_cid_t *cid; 2247 struct iovec *iov; 2248 unsigned i; 2249 char packno_str[22]; 2250 2251 for (i = 0; i < n; ++i) 2252 { 2253 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 2254 batch->packets[i]->po_packno); 2255 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 2256 { 2257 for (iov = batch->outs[i].iov; iov < 2258 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 2259 batch->outs[i].iov->iov_len -= 1; 2260 cid = lsquic_conn_log_cid(batch->conns[i]); 2261 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 2262 CID_BITS(cid)); 2263 } 2264 } 2265} 2266 2267 2268#endif 2269 2270 2271#ifdef NDEBUG 2272#define CONST_BATCH const 2273#else 2274#define CONST_BATCH 2275#endif 2276 2277 2278struct send_batch_ctx { 2279 struct conns_stailq *closed_conns; 2280 struct conns_tailq *ticked_conns; 2281 struct conns_out_iter *conns_iter; 2282 CONST_BATCH struct out_batch *batch; 2283}; 2284 2285 2286static void 2287close_conn_immediately (struct lsquic_engine *engine, 2288 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2289{ 2290 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2291 if (!(conn->cn_flags & LSCONN_CLOSING)) 2292 { 2293 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2294 engine_incref_conn(conn, LSCONN_CLOSING); 2295 if (conn->cn_flags & LSCONN_HASHED) 2296 remove_conn_from_hash(engine, conn); 2297 } 2298 if (conn->cn_flags & LSCONN_TICKED) 2299 { 2300 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2301 engine_decref_conn(engine, conn, LSCONN_TICKED); 2302 } 2303} 2304 2305 2306static void 2307close_conn_on_send_error (struct lsquic_engine *engine, 2308 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2309{ 2310 const struct out_batch *batch = sb_ctx->batch; 2311 struct lsquic_conn *const conn = batch->conns[n]; 2312 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2313 2314 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2315 "src: %s; dst: %s; errno: %d", 2316 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2317 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2318 CID_BITS(lsquic_conn_log_cid(conn)), 2319 SA2STR(batch->outs[n].local_sa, buf[0]), 2320 SA2STR(batch->outs[n].dest_sa, buf[1]), 2321 e_val); 2322 if (conn->cn_flags & LSCONN_EVANESCENT) 2323 lsquic_prq_drop(conn); 2324 else 2325 close_conn_immediately(engine, sb_ctx, conn); 2326} 2327 2328 2329static void 2330apply_hp (struct conns_out_iter *iter) 2331{ 2332 struct lsquic_conn *conn; 2333 2334 TAILQ_FOREACH(conn, &iter->coi_active_list, cn_next_out) 2335 if (conn->cn_esf_c->esf_flush_encryption && conn->cn_enc_session) 2336 conn->cn_esf_c->esf_flush_encryption(conn->cn_enc_session); 2337 TAILQ_FOREACH(conn, &iter->coi_inactive_list, cn_next_out) 2338 if (conn->cn_esf_c->esf_flush_encryption && conn->cn_enc_session) 2339 conn->cn_esf_c->esf_flush_encryption(conn->cn_enc_session); 2340} 2341 2342 2343#if LSQUIC_CONN_STATS 2344static void 2345update_batch_size_stats (struct lsquic_engine *engine, unsigned batch_size) 2346{ 2347 struct batch_size_stats *const stats = &engine->pub.enp_batch_size_stats; 2348 2349 ++stats->count; 2350 if (batch_size > stats->max) 2351 stats->max = batch_size; 2352 if (batch_size < stats->min || 0 == stats->min) 2353 stats->min = batch_size; 2354 if (stats->avg) 2355 stats->avg = ((float) batch_size - stats->avg) * 0.4 + stats->avg; 2356 else 2357 stats->avg = (float) batch_size; 2358} 2359 2360 2361#endif 2362 2363 2364static unsigned 2365send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2366 unsigned n_to_send) 2367{ 2368 int n_sent, i, e_val; 2369 lsquic_time_t now; 2370 unsigned off, skip; 2371 size_t count; 2372 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2373 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2374 2375#if LSQUIC_CONN_STATS 2376 update_batch_size_stats(engine, n_to_send); 2377#endif 2378 2379 apply_hp(sb_ctx->conns_iter); 2380#if CAN_LOSE_PACKETS 2381 if (engine->flags & ENG_LOSE_PACKETS) 2382 lose_matching_packets(engine, batch, n_to_send); 2383#endif 2384 skip = 0; 2385 restart_batch: 2386 /* Set sent time before the write to avoid underestimating RTT */ 2387 now = lsquic_time_now(); 2388 for (i = skip; i < (int) (n_to_send - skip); ++i) 2389 { 2390 off = batch->pack_off[i]; 2391 count = batch->outs[i].iovlen; 2392 assert(count > 0); 2393 packet_out = &batch->packets[off]; 2394 end = packet_out + count; 2395 do 2396 (*packet_out)->po_sent = now; 2397 while (++packet_out < end); 2398 } 2399 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs + skip, 2400 n_to_send - skip); 2401 e_val = errno; 2402 if (n_sent < (int) (n_to_send - skip) && e_val != EMSGSIZE) 2403 { 2404 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2405 engine->resume_sending_at = now + 1000000; 2406 LSQ_DEBUG("cannot send packets"); 2407 EV_LOG_GENERIC_EVENT("cannot send packets"); 2408 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2409 close_conn_on_send_error(engine, sb_ctx, 2410 n_sent < 0 ? 0 : n_sent, e_val); 2411 } 2412 if (n_sent >= 0) 2413 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, 2414 n_to_send - skip); 2415 else 2416 { 2417 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2418 n_sent = 0; 2419 } 2420 if (n_sent > 0) 2421 engine->last_sent = now + n_sent; 2422 for (i = skip; i < (int) (skip + n_sent); ++i) 2423 { 2424 eng_hist_inc(&engine->history, now, sl_packets_out); 2425 /* `i' is added to maintain relative order */ 2426 batch->conns[i]->cn_last_sent = now + i; 2427 2428 off = batch->pack_off[i]; 2429 count = batch->outs[i].iovlen; 2430 assert(count > 0); 2431 packet_out = &batch->packets[off]; 2432 end = packet_out + count; 2433 do 2434 { 2435#if LOG_PACKET_CHECKSUM 2436 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2437 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2438 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2439#endif 2440 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2441 *packet_out); 2442 /* Release packet out buffer as soon as the packet is sent 2443 * successfully. If not successfully sent, we hold on to 2444 * this buffer until the packet sending is attempted again 2445 * or until it times out and regenerated. 2446 */ 2447 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2448 release_enc_data(engine, batch->conns[i], *packet_out); 2449 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2450 *packet_out); 2451 } 2452 while (++packet_out < end); 2453 } 2454 if (i < (int) n_to_send && e_val == EMSGSIZE) 2455 { 2456 LSQ_DEBUG("packet #%d could not be sent out for being too large", i); 2457 if (batch->conns[i]->cn_if->ci_packet_too_large 2458 && batch->outs[i].iovlen == 1) 2459 { 2460 off = batch->pack_off[i]; 2461 packet_out = &batch->packets[off]; 2462 batch->conns[i]->cn_if->ci_packet_too_large(batch->conns[i], 2463 *packet_out); 2464 ++i; 2465 if (i < (int) n_to_send) 2466 { 2467 skip = i; 2468 LSQ_DEBUG("restart batch starting at packet #%u", skip); 2469 goto restart_batch; 2470 } 2471 n_sent = n_to_send; 2472 } 2473 else 2474 close_conn_on_send_error(engine, sb_ctx, i, e_val); 2475 } 2476 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2477 for ( ; i < (int) n_to_send; ++i) 2478 { 2479 off = batch->pack_off[i]; 2480 count = batch->outs[i].iovlen; 2481 assert(count > 0); 2482 packet_out = &batch->packets[off]; 2483 end = packet_out + count; 2484 do 2485 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2486 *packet_out); 2487 while (++packet_out < end); 2488 } 2489 /* Return packets to the connection in reverse order so that the packet 2490 * ordering is maintained. 2491 */ 2492 for (i = (int) n_to_send - 1; i >= (int) (skip + n_sent); --i) 2493 { 2494 off = batch->pack_off[i]; 2495 count = batch->outs[i].iovlen; 2496 assert(count > 0); 2497 packet_out = &batch->packets[off + count - 1]; 2498 end = &batch->packets[off - 1]; 2499 do 2500 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2501 *packet_out); 2502 while (--packet_out > end); 2503 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2504 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2505 } 2506 return skip + n_sent; 2507} 2508 2509 2510/* Return 1 if went past deadline, 0 otherwise */ 2511static int 2512check_deadline (lsquic_engine_t *engine) 2513{ 2514 if (engine->pub.enp_settings.es_proc_time_thresh && 2515 lsquic_time_now() > engine->deadline) 2516 { 2517 LSQ_INFO("went past threshold of %u usec, stop sending", 2518 engine->pub.enp_settings.es_proc_time_thresh); 2519 engine->flags |= ENG_PAST_DEADLINE; 2520 return 1; 2521 } 2522 else 2523 return 0; 2524} 2525 2526 2527static size_t 2528iov_size (const struct iovec *iov, const struct iovec *const end) 2529{ 2530 size_t size; 2531 2532 assert(iov < end); 2533 2534 size = 0; 2535 do 2536 size += iov->iov_len; 2537 while (++iov < end); 2538 2539 return size; 2540} 2541 2542 2543static void 2544send_packets_out (struct lsquic_engine *engine, 2545 struct conns_tailq *ticked_conns, 2546 struct conns_stailq *closed_conns) 2547{ 2548 unsigned n, w, n_sent, n_batches_sent; 2549 lsquic_packet_out_t *packet_out; 2550 struct lsquic_packet_out **packet; 2551 lsquic_conn_t *conn; 2552 struct out_batch *const batch = &engine->out_batch; 2553 struct iovec *iov, *packet_iov; 2554 struct conns_out_iter conns_iter; 2555 int shrink, deadline_exceeded; 2556 const struct send_batch_ctx sb_ctx = { 2557 closed_conns, 2558 ticked_conns, 2559 &conns_iter, 2560 &engine->out_batch, 2561 }; 2562 2563 coi_init(&conns_iter, engine); 2564 n_batches_sent = 0; 2565 n_sent = 0, n = 0; 2566 shrink = 0; 2567 deadline_exceeded = 0; 2568 iov = batch->iov; 2569 packet = batch->packets; 2570 2571 while ((conn = coi_next(&conns_iter))) 2572 { 2573 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2574 if (!packet_out) { 2575 /* Evanescent connection always has a packet to send: */ 2576 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2577 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2578 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2579 CID_BITS(lsquic_conn_log_cid(conn))); 2580 coi_deactivate(&conns_iter, conn); 2581 continue; 2582 } 2583 batch->outs[n].iov = packet_iov = iov; 2584 next_coa: 2585 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2586 { 2587 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2588 &engine->pub, conn, packet_out)) 2589 { 2590 case ENCPA_NOMEM: 2591 /* Send what we have and wait for a more opportune moment */ 2592 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2593 goto end_for; 2594 case ENCPA_BADCRYPT: 2595 /* This is pretty bad: close connection immediately */ 2596 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2597 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2598 CID_BITS(lsquic_conn_log_cid(conn))); 2599 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2600 { 2601 close_conn_immediately(engine, &sb_ctx, conn); 2602 coi_deactivate(&conns_iter, conn); 2603 } 2604 continue; 2605 case ENCPA_OK: 2606 break; 2607 } 2608 } 2609 else if ((packet_out->po_flags & PO_NOENCRYPT) 2610 && engine->pub.enp_pmi != &stock_pmi) 2611 { 2612 if (0 != copy_packet(engine, conn, packet_out)) 2613 { 2614 /* Copy can only fail if packet could not be allocated */ 2615 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2616 goto end_for; 2617 } 2618 } 2619 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2620 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2621 if (packet_out->po_flags & PO_ENCRYPTED) 2622 { 2623 iov->iov_base = packet_out->po_enc_data; 2624 iov->iov_len = packet_out->po_enc_data_sz; 2625 } 2626 else 2627 { 2628 iov->iov_base = packet_out->po_data; 2629 iov->iov_len = packet_out->po_data_sz; 2630 } 2631 if (packet_iov == iov) 2632 { 2633 batch->pack_off[n] = packet - batch->packets; 2634 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2635 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2636 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2637 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2638 batch->outs [n].conn_ctx = conn->cn_conn_ctx; 2639 batch->conns [n] = conn; 2640 } 2641 *packet = packet_out; 2642 ++packet; 2643 ++iov; 2644 if ((conn->cn_flags & LSCONN_IETF) 2645 && ((1 << packet_out->po_header_type) 2646 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2647#ifndef NDEBUG 2648 && (engine->flags & ENG_COALESCE) 2649#endif 2650 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2651 { 2652 const struct to_coal to_coal = { 2653 .prev_packet = packet_out, 2654 .prev_sz_sum = iov_size(packet_iov, iov), 2655 }; 2656 packet_out = conn->cn_if->ci_next_packet_to_send(conn, &to_coal); 2657 if (packet_out) 2658 goto next_coa; 2659 } 2660 batch->outs [n].iovlen = iov - packet_iov; 2661 ++n; 2662 if (n == engine->batch_size 2663 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2664 { 2665 w = send_batch(engine, &sb_ctx, n); 2666 n = 0; 2667 iov = batch->iov; 2668 packet = batch->packets; 2669 ++n_batches_sent; 2670 n_sent += w; 2671 if (w < engine->batch_size) 2672 { 2673 shrink = 1; 2674 break; 2675 } 2676 deadline_exceeded = check_deadline(engine); 2677 if (deadline_exceeded) 2678 break; 2679 grow_batch_size(engine); 2680 } 2681 } 2682 end_for: 2683 2684 if (n > 0) { 2685 w = send_batch(engine, &sb_ctx, n); 2686 n_sent += w; 2687 shrink = w < n; 2688 ++n_batches_sent; 2689 } 2690 2691 if (shrink) 2692 shrink_batch_size(engine); 2693 else if (n_batches_sent > 1) 2694 { 2695 deadline_exceeded = check_deadline(engine); 2696 if (!deadline_exceeded) 2697 grow_batch_size(engine); 2698 } 2699 2700 coi_reheap(&conns_iter, engine); 2701 2702 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2703} 2704 2705 2706int 2707lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2708{ 2709 return lsquic_mh_count(&engine->conns_out) > 0 2710 || (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 2711 ; 2712} 2713 2714 2715static void 2716reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2717{ 2718 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2719 engine->flags &= ~ENG_PAST_DEADLINE; 2720} 2721 2722 2723static void 2724check_tickable_conns_again (struct lsquic_engine *engine) 2725{ 2726 struct lsquic_hash_elem *el; 2727 struct lsquic_conn *conn; 2728 unsigned count; 2729 2730 count = 0; 2731 for (el = lsquic_hash_first(engine->conns_hash); el; 2732 el = lsquic_hash_next(engine->conns_hash)) 2733 { 2734 conn = lsquic_hashelem_getdata(el); 2735 if (!(conn->cn_flags & LSCONN_TICKABLE) 2736 && conn->cn_if->ci_is_tickable(conn)) 2737 { 2738 lsquic_mh_insert(&engine->conns_tickable, conn, 2739 conn->cn_last_ticked); 2740 engine_incref_conn(conn, LSCONN_TICKABLE); 2741 ++count; 2742 } 2743 } 2744 LSQ_DEBUG("%u connection%s tickable again after sending has been " 2745 "re-enabled", count, count == 1 ? " is" : "s are"); 2746} 2747 2748 2749void 2750lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2751{ 2752 lsquic_conn_t *conn; 2753 struct conns_stailq closed_conns; 2754 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2755 struct cid_update_batch cub; 2756 2757 ENGINE_IN(engine); 2758 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2759 STAILQ_INIT(&closed_conns); 2760 reset_deadline(engine, lsquic_time_now()); 2761 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2762 { 2763 LSQ_DEBUG("can send again"); 2764 EV_LOG_GENERIC_EVENT("can send again"); 2765 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2766 check_tickable_conns_again(engine); 2767 } 2768 2769 send_packets_out(engine, &ticked_conns, &closed_conns); 2770 2771 while ((conn = STAILQ_FIRST(&closed_conns))) { 2772 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2773 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2774 cub_add_cids_from_cces(&cub, conn); 2775 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2776 } 2777 2778 cub_flush(&cub); 2779 ENGINE_OUT(engine); 2780} 2781 2782 2783static lsquic_conn_t * 2784next_new_full_conn (struct conns_stailq *new_full_conns) 2785{ 2786 lsquic_conn_t *conn; 2787 2788 conn = STAILQ_FIRST(new_full_conns); 2789 if (conn) 2790 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2791 return conn; 2792} 2793 2794 2795#if LSQUIC_CONN_STATS 2796static void 2797maybe_log_conn_stats (struct lsquic_engine *engine, struct lsquic_conn *conn, 2798 lsquic_time_t now) 2799{ 2800 char cidstr[MAX_CID_LEN * 2 + 1]; 2801 2802 if (!LSQ_LOG_ENABLED_EXT(LSQ_LOG_NOTICE, LSQLM_CONN_STATS)) 2803 return; 2804 2805 if (conn->cn_last_ticked + 1000000 >= now) 2806 { 2807 if (0 == engine->busy.last_log 2808 || engine->busy.last_log + 1000000 - 1000 < now) 2809 { 2810 engine->busy.last_log = now; 2811 conn->cn_if->ci_log_stats(conn); 2812 } 2813 } 2814 else 2815 { 2816 lsquic_logger_log1(LSQ_LOG_NOTICE, LSQLM_CONN_STATS, 2817 "stop logging status for connection %s: no longer busy", 2818 (lsquic_cid2str(lsquic_conn_log_cid(conn), cidstr), cidstr)); 2819 engine->busy.current = NULL; 2820 engine->busy.last_log = 0; 2821 } 2822} 2823 2824 2825#endif 2826 2827 2828static void 2829process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2830 lsquic_time_t now) 2831{ 2832 lsquic_conn_t *conn; 2833 enum tick_st tick_st; 2834 unsigned i, why; 2835 lsquic_time_t next_tick_time; 2836 struct conns_stailq closed_conns; 2837 struct conns_tailq ticked_conns; 2838 struct conns_stailq new_full_conns; 2839 struct cid_update_batch cub_old, cub_live; 2840 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2841 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2842 2843 eng_hist_tick(&engine->history, now); 2844 2845 STAILQ_INIT(&closed_conns); 2846 TAILQ_INIT(&ticked_conns); 2847 reset_deadline(engine, now); 2848 STAILQ_INIT(&new_full_conns); 2849 2850 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2851 && now > engine->resume_sending_at) 2852 { 2853 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2854 "timeout"); 2855 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2856 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2857 } 2858 2859 i = 0; 2860 while ((conn = next_conn(engine)) 2861 || (conn = next_new_full_conn(&new_full_conns))) 2862 { 2863 tick_st = conn->cn_if->ci_tick(conn, now); 2864#if LSQUIC_CONN_STATS 2865 if (conn == engine->busy.current) 2866 maybe_log_conn_stats(engine, conn, now); 2867#endif 2868 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2869 if (tick_st & TICK_PROMOTE) 2870 { 2871 lsquic_conn_t *new_conn; 2872 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2873 "scheduled for promotion"); 2874 assert(conn->cn_flags & LSCONN_MINI); 2875 new_conn = new_full_conn_server(engine, conn, now); 2876 if (new_conn) 2877 { 2878 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2879 new_conn->cn_last_sent = engine->last_sent; 2880 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2881 conn->cn_flags |= LSCONN_PROMOTED; 2882 } 2883 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2884 } 2885 if (tick_st & TICK_SEND) 2886 { 2887 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2888 { 2889 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2890 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2891 } 2892 } 2893 if (tick_st & TICK_CLOSE) 2894 { 2895 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2896 engine_incref_conn(conn, LSCONN_CLOSING); 2897 if (conn->cn_flags & LSCONN_HASHED) 2898 remove_conn_from_hash(engine, conn); 2899 } 2900 else 2901 { 2902 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2903 engine_incref_conn(conn, LSCONN_TICKED); 2904 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2905 && conn->cn_if->ci_report_live(conn, now)) 2906 cub_add_cids_from_cces(&cub_live, conn); 2907 } 2908 } 2909 2910 cub_flush(&engine->new_scids); 2911 2912 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2913 && lsquic_engine_has_unsent_packets(engine)) 2914 send_packets_out(engine, &ticked_conns, &closed_conns); 2915 2916 while ((conn = STAILQ_FIRST(&closed_conns))) { 2917 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2918 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2919 cub_add_cids_from_cces(&cub_old, conn); 2920 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2921 } 2922 2923 while ((conn = TAILQ_FIRST(&ticked_conns))) 2924 { 2925 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2926 engine_decref_conn(engine, conn, LSCONN_TICKED); 2927 if (!(conn->cn_flags & LSCONN_TICKABLE) 2928 && conn->cn_if->ci_is_tickable(conn)) 2929 { 2930 /* Floyd heapification is not faster, don't bother. */ 2931 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2932 engine_incref_conn(conn, LSCONN_TICKABLE); 2933 } 2934 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2935 { 2936 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2937 if (next_tick_time) 2938 { 2939 if (0 == lsquic_attq_add(engine->attq, conn, next_tick_time, 2940 why)) 2941 engine_incref_conn(conn, LSCONN_ATTQ); 2942 } 2943 else 2944 /* In all other cases, the idle timeout would make the next 2945 * tick time non-zero: 2946 */ 2947 assert((conn->cn_flags & LSCONN_IETF) 2948 && engine->pub.enp_settings.es_idle_timeout == 0); 2949 } 2950 } 2951 2952 cub_flush(&cub_live); 2953 cub_flush(&cub_old); 2954} 2955 2956 2957static void 2958maybe_count_garbage (struct lsquic_engine *engine, size_t garbage_sz) 2959{ 2960 /* This is not very pretty (action at a distance via engine->curr_conn), 2961 * but it's the cheapest I can come up with to handle the "count garbage 2962 * toward amplification limit" requirement in 2963 * [draft-ietf-quic-transport-28] Section 8.1. 2964 */ 2965 if (engine->curr_conn && engine->curr_conn->cn_if->ci_count_garbage) 2966 engine->curr_conn->cn_if->ci_count_garbage(engine->curr_conn, 2967 garbage_sz); 2968} 2969 2970 2971/* Return 0 if packet is being processed by a real connection, 1 if the 2972 * packet was processed, but not by a connection, and -1 on error. 2973 */ 2974int 2975lsquic_engine_packet_in (lsquic_engine_t *engine, 2976 const unsigned char *packet_in_data, size_t packet_in_size, 2977 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2978 void *peer_ctx, int ecn) 2979{ 2980 const unsigned char *const packet_begin = packet_in_data; 2981 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2982 struct packin_parse_state ppstate; 2983 lsquic_packet_in_t *packet_in; 2984 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2985 int is_server, unsigned cid_len, struct packin_parse_state *); 2986 unsigned n_zeroes; 2987 int s, is_ietf; 2988 lsquic_cid_t cid; 2989 2990 ENGINE_CALLS_INCR(engine); 2991 2992 if (engine->flags & ENG_SERVER) 2993 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2994 else if (engine->flags & ENG_CONNS_BY_ADDR) 2995 { 2996 struct lsquic_hash_elem *el; 2997 const struct lsquic_conn *conn; 2998 el = find_conn_by_addr(engine->conns_hash, sa_local); 2999 if (!el) 3000 return -1; 3001 conn = lsquic_hashelem_getdata(el); 3002 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 3003 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 3004 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 3005 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 3006 else if (conn->cn_version == LSQVER_050) 3007 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 3008 else 3009 { 3010 assert(conn->cn_version == LSQVER_046); 3011 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 3012 } 3013 } 3014 else 3015 parse_packet_in_begin = lsquic_parse_packet_in_begin; 3016 3017 engine->curr_conn = NULL; 3018 n_zeroes = 0; 3019 is_ietf = 0; 3020#ifdef _MSC_VER 3021 s = 0; 3022 cid.len = 0; 3023 cid.idbuf[0] = 0; 3024#endif 3025 do 3026 { 3027 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 3028 if (!packet_in) 3029 return -1; 3030 /* Library does not modify packet_in_data, it is not referenced after 3031 * this function returns and subsequent release of pi_data is guarded 3032 * by PI_OWN_DATA flag. 3033 */ 3034 packet_in->pi_data = (unsigned char *) packet_in_data; 3035 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 3036 engine->flags & ENG_SERVER, 3037 engine->pub.enp_settings.es_scid_len, &ppstate)) 3038 { 3039 LSQ_DEBUG("Cannot parse incoming packet's header"); 3040 maybe_count_garbage(engine, packet_end - packet_in_data); 3041 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 3042 s = 1; 3043 break; 3044 } 3045 3046 /* [draft-ietf-quic-transport-30] Section 12.2: 3047 * " Receivers SHOULD ignore any subsequent packets with a different 3048 * " Destination Connection ID than the first packet in the datagram. 3049 */ 3050 if (is_ietf && packet_in_data > packet_begin) 3051 { 3052 if (!((packet_in->pi_flags & (PI_GQUIC|PI_CONN_ID)) == PI_CONN_ID 3053 && LSQUIC_CIDS_EQ(&packet_in->pi_dcid, &cid))) 3054 { 3055 packet_in_data += packet_in->pi_data_sz; 3056 maybe_count_garbage(engine, packet_in->pi_data_sz); 3057 continue; 3058 } 3059 } 3060 3061 is_ietf = 0 == (packet_in->pi_flags & PI_GQUIC); 3062 packet_in_data += packet_in->pi_data_sz; 3063 if (is_ietf && packet_in_data < packet_end) 3064 { 3065 cid = packet_in->pi_dcid; 3066 if (packet_begin == packet_in->pi_data) /* Only log once: */ 3067 LSQ_DEBUGC("received coalesced datagram of %zd bytes for " 3068 "connection %"CID_FMT, packet_in_size, CID_BITS(&cid)); 3069 } 3070 packet_in->pi_received = lsquic_time_now(); 3071 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 3072 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 3073 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 3074 peer_ctx, packet_in_size); 3075 n_zeroes += s == 0; 3076 } 3077 while (0 == s && packet_in_data < packet_end); 3078 3079 return n_zeroes > 0 ? 0 : s; 3080} 3081 3082 3083#if __GNUC__ && !defined(NDEBUG) 3084__attribute__((weak)) 3085#endif 3086unsigned 3087lsquic_engine_quic_versions (const lsquic_engine_t *engine) 3088{ 3089 return engine->pub.enp_settings.es_versions; 3090} 3091 3092 3093void 3094lsquic_engine_cooldown (lsquic_engine_t *engine) 3095{ 3096 struct lsquic_hash_elem *el; 3097 lsquic_conn_t *conn; 3098 3099 if (engine->flags & ENG_COOLDOWN) 3100 /* AFAICT, there is no harm in calling this function more than once, 3101 * but log it just in case, as it may indicate an error in the caller. 3102 */ 3103 LSQ_INFO("cooldown called again"); 3104 engine->flags |= ENG_COOLDOWN; 3105 LSQ_INFO("entering cooldown mode"); 3106 if (engine->flags & ENG_SERVER) 3107 drop_all_mini_conns(engine); 3108 for (el = lsquic_hash_first(engine->conns_hash); el; 3109 el = lsquic_hash_next(engine->conns_hash)) 3110 { 3111 conn = lsquic_hashelem_getdata(el); 3112 lsquic_conn_going_away(conn); 3113 } 3114} 3115 3116 3117#if LSQUIC_CONN_STATS 3118static void 3119update_busy_detector (struct lsquic_engine *engine, struct lsquic_conn *conn, 3120 int immed) 3121{ 3122 char cidstr[MAX_CID_LEN * 2 + 1]; 3123 3124 if (!(LSQ_LOG_ENABLED_EXT(LSQ_LOG_NOTICE, LSQLM_CONN_STATS) 3125 && conn->cn_if->ci_log_stats)) 3126 return; 3127 3128 if (conn == engine->busy.last_conn) 3129 { 3130 engine->busy.immed_ticks <<= 1u; 3131 engine->busy.immed_ticks |= immed; 3132 if (MAX_IMMED_TICKS == engine->busy.immed_ticks) 3133 { 3134 if (engine->busy.current != conn) 3135 lsquic_logger_log1(LSQ_LOG_NOTICE, LSQLM_CONN_STATS, 3136 "connection %s marked busy: it's had %u immediate ticks " 3137 "in a row", 3138 (lsquic_cid2str(lsquic_conn_log_cid(conn), cidstr), cidstr), 3139 (unsigned) (sizeof(engine->busy.immed_ticks) * 8)); 3140 engine->busy.current = conn; 3141 } 3142 } 3143 else 3144 engine->busy.immed_ticks <<= 1; 3145 3146 engine->busy.last_conn = conn; 3147} 3148 3149 3150#endif 3151 3152 3153int 3154lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 3155{ 3156 const struct attq_elem *next_attq; 3157 lsquic_time_t now, next_time; 3158#if LSQUIC_DEBUG_NEXT_ADV_TICK || LSQUIC_CONN_STATS 3159 struct lsquic_conn *conn; 3160#endif 3161#if LSQUIC_DEBUG_NEXT_ADV_TICK 3162 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 3163#endif 3164 3165 ENGINE_CALLS_INCR(engine); 3166 3167 if ((engine->flags & ENG_PAST_DEADLINE) 3168 && lsquic_mh_count(&engine->conns_out)) 3169 { 3170#if LSQUIC_DEBUG_NEXT_ADV_TICK 3171 conn = lsquic_mh_peek(&engine->conns_out); 3172 engine->last_logged_conn = 0; 3173 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 3174 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 3175 lsquic_mh_count(&engine->conns_out), 3176 lsquic_mh_count(&engine->conns_out) != 1, "s", 3177 CID_BITS(lsquic_conn_log_cid(conn))); 3178#endif 3179#if LSQUIC_CONN_STATS 3180 conn = lsquic_mh_peek(&engine->conns_out); 3181 update_busy_detector(engine, conn, 1); 3182#endif 3183 *diff = 0; 3184 return 1; 3185 } 3186 3187 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 3188 && engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 3189 { 3190#if LSQUIC_DEBUG_NEXT_ADV_TICK 3191 engine->last_logged_conn = 0; 3192 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 3193#endif 3194 *diff = 0; 3195 return 1; 3196 } 3197 3198 if (lsquic_mh_count(&engine->conns_tickable)) 3199 { 3200#if LSQUIC_DEBUG_NEXT_ADV_TICK 3201 conn = lsquic_mh_peek(&engine->conns_tickable); 3202 engine->last_logged_conn = 0; 3203 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 3204 "connection%.*s (%"CID_FMT" first)", 3205 lsquic_mh_count(&engine->conns_tickable), 3206 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 3207 CID_BITS(lsquic_conn_log_cid(conn))); 3208#endif 3209#if LSQUIC_CONN_STATS 3210 conn = lsquic_mh_peek(&engine->conns_tickable); 3211 update_busy_detector(engine, conn, 1); 3212#endif 3213 *diff = 0; 3214 return 1; 3215 } 3216 3217 next_attq = lsquic_attq_next(engine->attq); 3218 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 3219 { 3220 if (next_attq) 3221 next_time = next_attq->ae_adv_time; 3222 else 3223 return 0; 3224 } 3225 else 3226 { 3227 if (next_attq) 3228 { 3229 next_time = next_attq->ae_adv_time; 3230 if (engine->resume_sending_at < next_time) 3231 { 3232 next_time = engine->resume_sending_at; 3233 next_attq = NULL; 3234 } 3235 } 3236 else 3237 next_time = engine->resume_sending_at; 3238 } 3239 3240 now = lsquic_time_now(); 3241 *diff = (int) ((int64_t) next_time - (int64_t) now); 3242#if LSQUIC_DEBUG_NEXT_ADV_TICK 3243 if (next_attq) 3244 { 3245 /* Deduplicate consecutive log messages about the same reason for the 3246 * same connection. 3247 * If diff is always zero or diff reset to a higher value, event is 3248 * still logged. 3249 */ 3250 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 3251 && (uintptr_t) next_attq->ae_conn 3252 == engine->last_logged_conn 3253 && *diff < engine->last_tick_diff)) 3254 { 3255 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 3256 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 3257 engine->last_tick_diff = *diff; 3258 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 3259 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 3260 lsquic_attq_why2str(next_attq->ae_why)); 3261 } 3262 } 3263 else 3264 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 3265#endif 3266 3267#if LSQUIC_CONN_STATS 3268 if (next_attq) 3269 update_busy_detector(engine, next_attq->ae_conn, 3270 /* Immediate if: a) time is now or in the past */ 3271 *diff <= 0 3272 /* b) next event is pacer, which means that the 3273 * connection wants to send, but is prevented 3274 * by the pacer from doing so. 3275 */ 3276 || next_attq->ae_why == AEW_PACER 3277 /* c) next event is to retransmit data (meaning 3278 * that there is data in flight) and the 3279 * time is small, which implies a small RTT. 3280 */ 3281 || (next_attq->ae_why == N_AEWS + AL_RETX_APP 3282 && *diff < 5000)); 3283#endif 3284 3285 return 1; 3286} 3287 3288 3289unsigned 3290lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 3291{ 3292 lsquic_time_t now; 3293 ENGINE_CALLS_INCR(engine); 3294 now = lsquic_time_now(); 3295 if (from_now < 0) 3296 now -= from_now; 3297 else 3298 now += from_now; 3299 return lsquic_attq_count_before(engine->attq, now); 3300} 3301 3302 3303int 3304lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 3305 struct lsquic_conn *conn, unsigned cce_idx) 3306{ 3307 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 3308 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 3309 void *peer_ctx; 3310 3311 assert(cce_idx < conn->cn_n_cces); 3312 assert(conn->cn_cces_mask & (1 << cce_idx)); 3313 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 3314 3315 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 3316 cce->cce_cid.len, conn, &cce->cce_hash_el)) 3317 { 3318 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 3319 CID_BITS(&cce->cce_cid)); 3320 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 3321 cce->cce_flags |= CCE_REG; 3322 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 3323 return 0; 3324 } 3325 else 3326 { 3327 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 3328 CID_BITS(&cce->cce_cid)); 3329 return -1; 3330 } 3331} 3332 3333 3334void 3335lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 3336 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now, 3337 lsquic_time_t drain_time) 3338{ 3339 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 3340 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 3341 struct purga_el *puel; 3342 void *peer_ctx; 3343 3344 assert(cce_idx < conn->cn_n_cces); 3345 3346 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 3347 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 3348 3349 if (engine->purga) 3350 { 3351 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 3352 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 3353 PUTY_CID_RETIRED, now); 3354 if (puel) 3355 puel->puel_time = now + drain_time; 3356 } 3357 conn->cn_cces_mask &= ~(1u << cce_idx); 3358 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 3359} 3360 3361 3362