lsquic_engine.c revision 3a537672
1/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#ifndef WIN32 17#include <sys/time.h> 18#include <netinet/in.h> 19#include <sys/types.h> 20#include <sys/stat.h> 21#include <fcntl.h> 22#include <unistd.h> 23#include <netdb.h> 24#endif 25 26#ifndef NDEBUG 27#include <sys/types.h> 28#endif 29 30#if defined(WIN32) || defined(NDEBUG) 31#define CAN_LOSE_PACKETS 0 32#else 33#define CAN_LOSE_PACKETS 1 34#endif 35 36#if CAN_LOSE_PACKETS 37#include <regex.h> /* For code that loses packets */ 38#endif 39 40#if LOG_PACKET_CHECKSUM 41#include <zlib.h> 42#endif 43 44#include <openssl/aead.h> 45 46#include "lsquic.h" 47#include "lsquic_types.h" 48#include "lsquic_int_types.h" 49#include "lsquic_sizes.h" 50#include "lsquic_parse_common.h" 51#include "lsquic_parse.h" 52#include "lsquic_packet_in.h" 53#include "lsquic_packet_out.h" 54#include "lsquic_senhist.h" 55#include "lsquic_rtt.h" 56#include "lsquic_cubic.h" 57#include "lsquic_pacer.h" 58#include "lsquic_bw_sampler.h" 59#include "lsquic_minmax.h" 60#include "lsquic_bbr.h" 61#include "lsquic_send_ctl.h" 62#include "lsquic_set.h" 63#include "lsquic_conn_flow.h" 64#include "lsquic_sfcw.h" 65#include "lsquic_hash.h" 66#include "lsquic_conn.h" 67#include "lsquic_full_conn.h" 68#include "lsquic_util.h" 69#include "lsquic_qtags.h" 70#include "lsquic_enc_sess.h" 71#include "lsquic_mm.h" 72#include "lsquic_engine_public.h" 73#include "lsquic_eng_hist.h" 74#include "lsquic_ev_log.h" 75#include "lsquic_version.h" 76#include "lsquic_pr_queue.h" 77#include "lsquic_mini_conn.h" 78#include "lsquic_mini_conn_ietf.h" 79#include "lsquic_stock_shi.h" 80#include "lsquic_purga.h" 81#include "lsquic_tokgen.h" 82#include "lsquic_attq.h" 83#include "lsquic_min_heap.h" 84#include "lsquic_http1x_if.h" 85#include "lsquic_handshake.h" 86#include "lsquic_crand.h" 87#include "lsquic_ietf.h" 88 89#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 90#include "lsquic_logger.h" 91 92#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 93#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 94#endif 95 96#if LSQUIC_DEBUG_NEXT_ADV_TICK 97#include "lsquic_alarmset.h" 98#endif 99 100#define MIN(a, b) ((a) < (b) ? (a) : (b)) 101 102/* The batch of outgoing packets grows and shrinks dynamically */ 103#define MAX_OUT_BATCH_SIZE 1024 104#define MIN_OUT_BATCH_SIZE 4 105#define INITIAL_OUT_BATCH_SIZE 32 106 107struct out_batch 108{ 109 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 110 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 111 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 112 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 113 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 114}; 115 116typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 117 118static void 119process_connections (struct lsquic_engine *engine, conn_iter_f iter, 120 lsquic_time_t now); 121 122static void 123engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 124 125static lsquic_conn_t * 126engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 127 enum lsquic_conn_flags flag); 128 129static void 130force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 131 132#if LSQUIC_COUNT_ENGINE_CALLS 133#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 134#else 135#define ENGINE_CALLS_INCR(e) 136#endif 137 138/* Nested calls to some LSQUIC functions are not supported. Functions that 139 * iterate over connections cannot be nested. 140 */ 141#define ENGINE_IN(e) do { \ 142 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 143 (e)->pub.enp_flags |= ENPUB_PROC; \ 144 ENGINE_CALLS_INCR(e); \ 145} while (0) 146 147#define ENGINE_OUT(e) do { \ 148 assert((e)->pub.enp_flags & ENPUB_PROC); \ 149 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 150} while (0) 151 152/* A connection can be referenced from one of six places: 153 * 154 * 1. A hash is used to find connections in order to dispatch an incoming 155 * packet. Connections can be hashed by CIDs or by address. In the 156 * former case, each connection has one or more mappings in the hash 157 * table. IETF QUIC connections have up to eight (in our implementation) 158 * source CIDs and each of those would have a mapping. In client mode, 159 * depending on QUIC versions and options selected, it is may be 160 * necessary to hash connections by address, in which case incoming 161 * packets are delivered to connections based on the address. 162 * 163 * 2. Outgoing queue. 164 * 165 * 3. Tickable queue 166 * 167 * 4. Advisory Tick Time queue. 168 * 169 * 5. Closing connections queue. This is a transient queue -- it only 170 * exists for the duration of process_connections() function call. 171 * 172 * 6. Ticked connections queue. Another transient queue, similar to (5). 173 * 174 * The idea is to destroy the connection when it is no longer referenced. 175 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 176 * that case, the connection is referenced from two places: (2) and (5). 177 * After its packets are sent, it is only referenced in (5), and at the 178 * end of the function call, when it is removed from (5), reference count 179 * goes to zero and the connection is destroyed. If not all packets can 180 * be sent, at the end of the function call, the connection is referenced 181 * by (2) and will only be removed once all outgoing packets have been 182 * sent. 183 */ 184#define CONN_REF_FLAGS (LSCONN_HASHED \ 185 |LSCONN_HAS_OUTGOING \ 186 |LSCONN_TICKABLE \ 187 |LSCONN_TICKED \ 188 |LSCONN_CLOSING \ 189 |LSCONN_ATTQ) 190 191 192 193 194struct cid_update_batch 195{ 196 lsquic_cids_update_f cub_update_cids; 197 void *cub_update_ctx; 198 unsigned cub_count; 199 lsquic_cid_t cub_cids[20]; 200 void *cub_peer_ctxs[20]; 201}; 202 203static void 204cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 205 206 207struct lsquic_engine 208{ 209 struct lsquic_engine_public pub; 210 enum { 211 ENG_SERVER = LSENG_SERVER, 212 ENG_HTTP = LSENG_HTTP, 213 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 214 ENG_PAST_DEADLINE 215 = (1 << 8), /* Previous call to a processing 216 * function went past time threshold. 217 */ 218 ENG_CONNS_BY_ADDR 219 = (1 << 9), /* Connections are hashed by address */ 220#ifndef NDEBUG 221 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 222#endif 223#if CAN_LOSE_PACKETS 224 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 225#endif 226#ifndef NDEBUG 227 ENG_DTOR = (1 << 26), /* Engine destructor */ 228#endif 229 } flags; 230 lsquic_packets_out_f packets_out; 231 void *packets_out_ctx; 232 lsquic_cids_update_f report_new_scids; 233 lsquic_cids_update_f report_live_scids; 234 lsquic_cids_update_f report_old_scids; 235 void *scids_ctx; 236 struct lsquic_hash *conns_hash; 237 struct min_heap conns_tickable; 238 struct min_heap conns_out; 239 struct eng_hist history; 240 unsigned batch_size; 241 struct lsquic_conn *curr_conn; 242 struct pr_queue *pr_queue; 243 struct attq *attq; 244 /* Track time last time a packet was sent to give new connections 245 * priority lower than that of existing connections. 246 */ 247 lsquic_time_t last_sent; 248#if CAN_LOSE_PACKETS 249 regex_t lose_packets_re; 250 const char *lose_packets_str; 251#endif 252 unsigned n_conns; 253 lsquic_time_t deadline; 254 lsquic_time_t resume_sending_at; 255 unsigned mini_conns_count; 256 struct lsquic_purga *purga; 257#if LSQUIC_CONN_STATS 258 struct { 259 unsigned conns; 260 } stats; 261 struct conn_stats conn_stats_sum; 262 FILE *stats_fh; 263#endif 264 struct cid_update_batch new_scids; 265 struct out_batch out_batch; 266#if LSQUIC_COUNT_ENGINE_CALLS 267 unsigned long n_engine_calls; 268#endif 269#if LSQUIC_DEBUG_NEXT_ADV_TICK 270 uintptr_t last_logged_conn; 271 unsigned last_logged_ae_why; 272 int last_tick_diff; 273#endif 274 struct crand crand; 275 EVP_AEAD_CTX retry_aead_ctx[N_IETF_RETRY_VERSIONS]; 276}; 277 278 279void 280lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 281 unsigned flags) 282{ 283 memset(settings, 0, sizeof(*settings)); 284 settings->es_versions = LSQUIC_DF_VERSIONS; 285 if (flags & ENG_SERVER) 286 { 287 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 288 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 289 settings->es_init_max_data 290 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 291 settings->es_init_max_stream_data_bidi_remote 292 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 293 settings->es_init_max_stream_data_bidi_local 294 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 295 settings->es_init_max_stream_data_uni 296 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 297 settings->es_init_max_streams_uni 298 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 299 settings->es_ping_period = 0; 300 settings->es_noprogress_timeout 301 = LSQUIC_DF_NOPROGRESS_TIMEOUT_SERVER; 302 } 303 else 304 { 305 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 306 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 307 settings->es_init_max_data 308 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 309 settings->es_init_max_stream_data_bidi_remote 310 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 311 settings->es_init_max_stream_data_bidi_local 312 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 313 settings->es_init_max_stream_data_uni 314 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 315 settings->es_init_max_streams_uni 316 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 317 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 318 settings->es_noprogress_timeout 319 = LSQUIC_DF_NOPROGRESS_TIMEOUT_CLIENT; 320 } 321 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 322 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 323 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 324 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 325 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 326 settings->es_max_header_list_size 327 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 328 settings->es_ua = LSQUIC_DF_UA; 329 settings->es_ecn = LSQUIC_DF_ECN; 330 331 settings->es_pdmd = QTAG_X509; 332 settings->es_aead = QTAG_AESG; 333 settings->es_kexs = QTAG_C255; 334 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 335 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 336 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 337 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 338 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 339 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 340 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 341 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 342 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 343 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 344 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 345 settings->es_sttl = LSQUIC_DF_STTL; 346 settings->es_init_max_streams_bidi 347 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 348 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 349 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 350 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 351 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 352 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 353 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 354 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 355 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 356 settings->es_spin = LSQUIC_DF_SPIN; 357 settings->es_delayed_acks = LSQUIC_DF_DELAYED_ACKS; 358 settings->es_timestamps = LSQUIC_DF_TIMESTAMPS; 359 settings->es_grease_quic_bit = LSQUIC_DF_GREASE_QUIC_BIT; 360 settings->es_mtu_probe_timer = LSQUIC_DF_MTU_PROBE_TIMER; 361 settings->es_dplpmtud = LSQUIC_DF_DPLPMTUD; 362} 363 364 365/* Note: if returning an error, err_buf must be valid if non-NULL */ 366int 367lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 368 unsigned flags, 369 char *err_buf, size_t err_buf_sz) 370{ 371 if (settings->es_cfcw < LSQUIC_MIN_FCW || 372 settings->es_sfcw < LSQUIC_MIN_FCW) 373 { 374 if (err_buf) 375 snprintf(err_buf, err_buf_sz, "%s", 376 "flow control window set too low"); 377 return -1; 378 } 379 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 380 { 381 if (err_buf) 382 snprintf(err_buf, err_buf_sz, "%s", 383 "No supported QUIC versions specified"); 384 return -1; 385 } 386 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 387 { 388 if (err_buf) 389 snprintf(err_buf, err_buf_sz, "%s", 390 "one or more unsupported QUIC version is specified"); 391 return -1; 392 } 393 if (flags & ENG_SERVER) 394 { 395 if (settings->es_handshake_to > 396 MAX_MINI_CONN_LIFESPAN_IN_USEC) 397 { 398 if (err_buf) 399 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 400 " usec is too large. The maximum for server is %u usec", 401 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 402 return -1; 403 } 404 } 405 if (settings->es_idle_timeout > 600) 406 { 407 if (err_buf) 408 snprintf(err_buf, err_buf_sz, "%s", 409 "The maximum value of idle timeout is 600 seconds"); 410 return -1; 411 } 412 if (settings->es_scid_len > MAX_CID_LEN) 413 { 414 if (err_buf) 415 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 416 "bytes long; it must be between 0 and %u.", 417 settings->es_scid_len, MAX_CID_LEN); 418 return -1; 419 } 420 421 if (settings->es_cc_algo > 2) 422 { 423 if (err_buf) 424 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 425 "algorithm value %u", settings->es_cc_algo); 426 return -1; 427 } 428 429 if (!(settings->es_ql_bits >= 0 && settings->es_ql_bits <= 2)) 430 { 431 if (err_buf) 432 snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ", 433 settings->es_ql_bits); 434 return -1; 435 } 436 437 if (!(settings->es_spin == 0 || settings->es_spin == 1)) 438 { 439 if (err_buf) 440 snprintf(err_buf, err_buf_sz, "Invalid spin value %d", 441 settings->es_spin); 442 return -1; 443 } 444 445 if (settings->es_mtu_probe_timer && settings->es_mtu_probe_timer < 1000) 446 { 447 if (err_buf) 448 snprintf(err_buf, err_buf_sz, "mtu probe timer is too small: " 449 "%u ms", settings->es_mtu_probe_timer); 450 return -1; 451 } 452 453 return 0; 454} 455 456 457static void 458free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 459{ 460 free(packet_data); 461} 462 463 464static void * 465malloc_buf (void *ctx, void *conn_ctx, unsigned short size, char is_ipv6) 466{ 467 return malloc(size); 468} 469 470 471static const struct lsquic_packout_mem_if stock_pmi = 472{ 473 malloc_buf, free_packet, free_packet, 474}; 475 476 477static int 478hash_conns_by_addr (const struct lsquic_engine *engine) 479{ 480 if (engine->flags & ENG_SERVER) 481 return 0; 482 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 483 return 1; 484 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 485 && engine->pub.enp_settings.es_support_tcid0) 486 return 1; 487 if (engine->pub.enp_settings.es_scid_len == 0) 488 return 1; 489 return 0; 490} 491 492 493lsquic_engine_t * 494lsquic_engine_new (unsigned flags, 495 const struct lsquic_engine_api *api) 496{ 497 lsquic_engine_t *engine; 498 size_t alpn_len; 499 unsigned i; 500 char err_buf[100]; 501 502 if (!api->ea_packets_out) 503 { 504 LSQ_ERROR("packets_out callback is not specified"); 505 return NULL; 506 } 507 508 if (!api->ea_stream_if) 509 { 510 LSQ_ERROR("stream interface is not specified"); 511 return NULL; 512 } 513 514 if (!(flags & LSENG_HTTP) && api->ea_alpn) 515 { 516 alpn_len = strlen(api->ea_alpn); 517 if (alpn_len < 1 || alpn_len > 255) 518 { 519 LSQ_ERROR("ALPN string length invalid: %zd bytes", alpn_len); 520 return NULL; 521 } 522 } 523 else 524 alpn_len = 0; 525 526 if (api->ea_settings && 527 0 != lsquic_engine_check_settings(api->ea_settings, flags, 528 err_buf, sizeof(err_buf))) 529 { 530 LSQ_ERROR("cannot create engine: %s", err_buf); 531 return NULL; 532 } 533 534 engine = calloc(1, sizeof(*engine)); 535 if (!engine) 536 return NULL; 537 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 538 { 539 free(engine); 540 return NULL; 541 } 542 if (api->ea_settings) 543 engine->pub.enp_settings = *api->ea_settings; 544 else 545 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 546 int tag_buf_len; 547 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 548 sizeof(engine->pub.enp_ver_tags_buf), 549 engine->pub.enp_settings.es_versions); 550 if (tag_buf_len <= 0) 551 { 552 LSQ_ERROR("cannot generate version tags buffer"); 553 free(engine); 554 return NULL; 555 } 556 engine->pub.enp_ver_tags_len = tag_buf_len; 557 engine->pub.enp_flags = ENPUB_CAN_SEND; 558 engine->pub.enp_stream_if = api->ea_stream_if; 559 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 560 561 engine->flags = flags; 562#ifndef NDEBUG 563 engine->flags |= ENG_COALESCE; 564#endif 565 engine->packets_out = api->ea_packets_out; 566 engine->packets_out_ctx = api->ea_packets_out_ctx; 567 engine->report_new_scids = api->ea_new_scids; 568 engine->report_live_scids = api->ea_live_scids; 569 engine->report_old_scids = api->ea_old_scids; 570 engine->scids_ctx = api->ea_cids_update_ctx; 571 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 572 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 573 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 574 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 575 if (api->ea_shi) 576 { 577 engine->pub.enp_shi = api->ea_shi; 578 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 579 } 580 else 581 { 582 engine->pub.enp_shi = &stock_shi; 583 engine->pub.enp_shi_ctx = lsquic_stock_shared_hash_new(); 584 if (!engine->pub.enp_shi_ctx) 585 { 586 free(engine); 587 return NULL; 588 } 589 } 590 if (api->ea_hsi_if) 591 { 592 engine->pub.enp_hsi_if = api->ea_hsi_if; 593 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 594 } 595 else 596 { 597 engine->pub.enp_hsi_if = lsquic_http1x_if; 598 engine->pub.enp_hsi_ctx = NULL; 599 } 600 if (api->ea_pmi) 601 { 602 engine->pub.enp_pmi = api->ea_pmi; 603 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 604 } 605 else 606 { 607 engine->pub.enp_pmi = &stock_pmi; 608 engine->pub.enp_pmi_ctx = NULL; 609 } 610 engine->pub.enp_verify_cert = api->ea_verify_cert; 611 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 612 engine->pub.enp_kli = api->ea_keylog_if; 613 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 614 engine->pub.enp_engine = engine; 615 if (hash_conns_by_addr(engine)) 616 engine->flags |= ENG_CONNS_BY_ADDR; 617 engine->conns_hash = lsquic_hash_create(); 618 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 619 if (!engine->pub.enp_tokgen) 620 return NULL; 621 engine->pub.enp_crand = &engine->crand; 622 if (engine->pub.enp_settings.es_noprogress_timeout) 623 engine->pub.enp_noprog_timeout 624 = engine->pub.enp_settings.es_noprogress_timeout * 1000000; 625 engine->pub.enp_mtu_probe_timer = 1000 626 * (engine->pub.enp_settings.es_mtu_probe_timer 627 ? engine->pub.enp_settings.es_mtu_probe_timer 628 : LSQUIC_DF_MTU_PROBE_TIMER); 629 if (flags & ENG_SERVER) 630 { 631 engine->pr_queue = lsquic_prq_create( 632 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 633 &engine->pub); 634 if (!engine->pr_queue) 635 { 636 lsquic_tg_destroy(engine->pub.enp_tokgen); 637 return NULL; 638 } 639 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 640 engine->report_old_scids, engine->scids_ctx); 641 if (!engine->purga) 642 { 643 lsquic_tg_destroy(engine->pub.enp_tokgen); 644 lsquic_prq_destroy(engine->pr_queue); 645 return NULL; 646 } 647 } 648 engine->attq = lsquic_attq_create(); 649 eng_hist_init(&engine->history); 650 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 651 if (engine->pub.enp_settings.es_honor_prst) 652 { 653 engine->pub.enp_srst_hash = lsquic_hash_create(); 654 if (!engine->pub.enp_srst_hash) 655 { 656 lsquic_engine_destroy(engine); 657 return NULL; 658 } 659 } 660 661 if (alpn_len) 662 { 663 engine->pub.enp_alpn = malloc(alpn_len + 1); 664 if (!engine->pub.enp_alpn) 665 { 666 lsquic_engine_destroy(engine); 667 return NULL; 668 } 669 engine->pub.enp_alpn[0] = alpn_len; 670 memcpy(engine->pub.enp_alpn + 1, api->ea_alpn, alpn_len); 671 } 672 673 if (flags & LSENG_HTTP) 674 engine->pub.enp_flags |= ENPUB_HTTP; 675 676#ifndef NDEBUG 677 { 678 const char *env; 679 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 680#if CAN_LOSE_PACKETS 681 if (env) 682 { 683 if (0 != regcomp(&engine->lose_packets_re, env, 684 REG_EXTENDED|REG_NOSUB)) 685 { 686 LSQ_ERROR("could not compile lost packet regex `%s'", env); 687 return NULL; 688 } 689 engine->flags |= ENG_LOSE_PACKETS; 690 engine->lose_packets_str = env; 691 LSQ_WARN("will lose packets that match the following regex: %s", 692 env); 693 } 694#endif 695 env = getenv("LSQUIC_COALESCE"); 696 if (env) 697 { 698 engine->flags &= ~ENG_COALESCE; 699 if (atoi(env)) 700 { 701 engine->flags |= ENG_COALESCE; 702 LSQ_NOTICE("will coalesce packets"); 703 } 704 else 705 LSQ_NOTICE("will not coalesce packets"); 706 } 707 } 708#endif 709#if LSQUIC_CONN_STATS 710 engine->stats_fh = api->ea_stats_fh; 711#endif 712 for (i = 0; i < sizeof(engine->retry_aead_ctx) 713 / sizeof(engine->retry_aead_ctx[0]); ++i) 714 if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx[i], 715 EVP_aead_aes_128_gcm(), lsquic_retry_key_buf[i], 716 IETF_RETRY_KEY_SZ, 16, NULL)) 717 { 718 LSQ_ERROR("could not initialize retry AEAD ctx #%u", i); 719 lsquic_engine_destroy(engine); 720 return NULL; 721 } 722 engine->pub.enp_retry_aead_ctx = engine->retry_aead_ctx; 723 724 LSQ_INFO("instantiated engine"); 725 return engine; 726} 727 728 729#if LOG_PACKET_CHECKSUM 730static void 731log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 732 const unsigned char *buf, size_t bufsz) 733{ 734 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 735 (uint32_t) crc32(0, buf, bufsz)); 736} 737 738 739#endif 740 741 742static void 743grow_batch_size (struct lsquic_engine *engine) 744{ 745 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 746} 747 748 749static void 750shrink_batch_size (struct lsquic_engine *engine) 751{ 752 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 753} 754 755 756struct cce_cid_iter 757{ 758 const struct lsquic_conn *conn; 759 unsigned todo, n; 760}; 761 762 763static struct conn_cid_elem * 764cce_iter_next (struct cce_cid_iter *citer) 765{ 766 struct conn_cid_elem *cce; 767 768 while (citer->todo) 769 if (citer->todo & (1 << citer->n)) 770 { 771 citer->todo &= ~(1 << citer->n); 772 cce = &citer->conn->cn_cces[ citer->n++ ]; 773 if (!(cce->cce_flags & CCE_PORT)) 774 return cce; 775 } 776 else 777 ++citer->n; 778 779 return NULL; 780} 781 782 783static struct conn_cid_elem * 784cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 785{ 786 citer->conn = conn; 787 citer->todo = conn->cn_cces_mask; 788 citer->n = 0; 789 return cce_iter_next(citer); 790} 791 792 793#if LSQUIC_CONN_STATS 794void 795update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 796{ 797 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 798 const unsigned long *src; 799 const struct conn_stats *stats; 800 unsigned i; 801 802 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 803 { 804 ++engine->stats.conns; 805 src = (unsigned long *) stats; 806 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 807 dst[i] += src[i]; 808 } 809} 810 811 812#endif 813 814 815/* Wrapper to make sure important things occur before the connection is 816 * really destroyed. 817 */ 818static void 819destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 820 lsquic_time_t now) 821{ 822 struct cce_cid_iter citer; 823 const struct conn_cid_elem *cce; 824 lsquic_time_t drain_time; 825 struct purga_el *puel; 826 827 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 828 if (engine->purga 829 /* Blacklist all CIDs except for promoted mini connections */ 830 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 831 != (LSCONN_MINI|LSCONN_PROMOTED)) 832 { 833 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 834 && conn->cn_if->ci_drain_time && 835 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 836 { 837 for (cce = cce_iter_first(&citer, conn); cce; 838 cce = cce_iter_next(&citer)) 839 { 840 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 841 lsquic_conn_get_peer_ctx(conn, NULL), 842 PUTY_CONN_DRAIN, now); 843 if (puel) 844 puel->puel_time = now + drain_time; 845 } 846 } 847 else 848 { 849 for (cce = cce_iter_first(&citer, conn); cce; 850 cce = cce_iter_next(&citer)) 851 { 852 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 853 lsquic_conn_get_peer_ctx(conn, NULL), 854 PUTY_CONN_DELETED, now); 855 if (puel) 856 { 857 puel->puel_time = now; 858 puel->puel_count = 0; 859 } 860 } 861 } 862 } 863#if LSQUIC_CONN_STATS 864 update_stats_sum(engine, conn); 865#endif 866 --engine->n_conns; 867 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 868 conn->cn_if->ci_destroy(conn); 869} 870 871 872static int 873maybe_grow_conn_heaps (struct lsquic_engine *engine) 874{ 875 struct min_heap_elem *els; 876 unsigned count; 877 878 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 879 return 0; /* Nothing to do */ 880 881 if (lsquic_mh_nalloc(&engine->conns_tickable)) 882 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 883 else 884 count = 8; 885 886 els = malloc(sizeof(els[0]) * count); 887 if (!els) 888 { 889 LSQ_ERROR("%s: malloc failed", __func__); 890 return -1; 891 } 892 893 LSQ_DEBUG("grew heaps to %u elements", count / 2); 894 memcpy(&els[0], engine->conns_tickable.mh_elems, 895 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 896 memcpy(&els[count / 2], engine->conns_out.mh_elems, 897 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 898 free(engine->conns_tickable.mh_elems); 899 engine->conns_tickable.mh_elems = els; 900 engine->conns_out.mh_elems = &els[count / 2]; 901 engine->conns_tickable.mh_nalloc = count / 2; 902 engine->conns_out.mh_nalloc = count / 2; 903 return 0; 904} 905 906 907static void 908remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 909 unsigned todo) 910{ 911 unsigned n; 912 913 for (n = 0; todo; todo &= ~(1 << n++)) 914 if ((todo & (1 << n)) && 915 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 916 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 917} 918 919 920static void 921remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 922{ 923 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 924} 925 926 927static void 928cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 929 930 931static int 932insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 933 void *peer_ctx) 934{ 935 struct conn_cid_elem *cce; 936 unsigned todo, done, n; 937 938 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 939 if (todo & (1 << n)) 940 { 941 cce = &conn->cn_cces[n]; 942 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 943 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 944 cce->cce_cid.len, conn, &cce->cce_hash_el)) 945 done |= 1 << n; 946 else 947 goto err; 948 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 949 { 950 cce->cce_flags |= CCE_REG; 951 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 952 } 953 } 954 955 return 0; 956 957 err: 958 remove_cces_from_hash(engine->conns_hash, conn, done); 959 return -1; 960} 961 962 963static lsquic_conn_t * 964new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 965 lsquic_time_t now) 966{ 967 const lsquic_cid_t *cid; 968 server_conn_ctor_f ctor; 969 lsquic_conn_t *conn; 970 unsigned flags; 971 if (0 != maybe_grow_conn_heaps(engine)) 972 return NULL; 973 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 974 975 if (mini_conn->cn_flags & LSCONN_IETF) 976 ctor = lsquic_ietf_full_conn_server_new; 977 else 978 ctor = lsquic_gquic_full_conn_server_new; 979 980 conn = ctor(&engine->pub, flags, mini_conn); 981 if (!conn) 982 { 983 /* Otherwise, full_conn_server_new prints its own warnings */ 984 if (ENOMEM == errno) 985 { 986 cid = lsquic_conn_log_cid(mini_conn); 987 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 988 CID_BITS(cid), strerror(errno)); 989 } 990 return NULL; 991 } 992 ++engine->n_conns; 993 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 994 { 995 cid = lsquic_conn_log_cid(conn); 996 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 997 CID_BITS(cid)); 998 destroy_conn(engine, conn, now); 999 return NULL; 1000 } 1001 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1002 conn->cn_flags |= LSCONN_HASHED; 1003 return conn; 1004} 1005 1006 1007static enum 1008{ 1009 VER_NOT_SPECIFIED, 1010 VER_SUPPORTED, 1011 VER_UNSUPPORTED, 1012} 1013 1014 1015version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 1016 enum lsquic_version *pversion) 1017{ 1018 lsquic_ver_tag_t ver_tag; 1019 enum lsquic_version version; 1020 1021 if (!packet_in->pi_quic_ver) 1022 { 1023 LSQ_DEBUG("packet does not specify version"); 1024 return VER_NOT_SPECIFIED; 1025 } 1026 1027 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 1028 version = lsquic_tag2ver(ver_tag); 1029 if (version < N_LSQVER) 1030 { 1031 if (engine->pub.enp_settings.es_versions & (1 << version)) 1032 { 1033 LSQ_DEBUG("client-supplied version %s is supported", 1034 lsquic_ver2str[version]); 1035 *pversion = version; 1036 return VER_SUPPORTED; 1037 } 1038 else 1039 LSQ_DEBUG("client-supplied version %s is not supported", 1040 lsquic_ver2str[version]); 1041 } 1042 else 1043 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 1044 ver_tag); 1045 1046 return VER_UNSUPPORTED; 1047} 1048 1049 1050static void 1051schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 1052 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 1053 const struct sockaddr *sa_peer, void *peer_ctx) 1054{ 1055 assert(engine->pr_queue); 1056 if (0 == lsquic_prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 1057 sa_local, sa_peer)) 1058 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 1059 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 1060 else 1061 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 1062} 1063 1064 1065static unsigned short 1066sa2port (const struct sockaddr *sa) 1067{ 1068 if (sa->sa_family == AF_INET) 1069 { 1070 struct sockaddr_in *const sa4 = (void *) sa; 1071 return sa4->sin_port; 1072 } 1073 else 1074 { 1075 struct sockaddr_in6 *const sa6 = (void *) sa; 1076 return sa6->sin6_port; 1077 } 1078} 1079 1080 1081static struct lsquic_hash_elem * 1082find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 1083{ 1084 unsigned short port; 1085 1086 port = sa2port(sa); 1087 return lsquic_hash_find(hash, &port, sizeof(port)); 1088} 1089 1090 1091static lsquic_conn_t * 1092find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1093 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 1094{ 1095 struct lsquic_hash_elem *el; 1096 lsquic_conn_t *conn; 1097 1098 if (engine->flags & ENG_CONNS_BY_ADDR) 1099 el = find_conn_by_addr(engine->conns_hash, sa_local); 1100 else if (packet_in->pi_flags & PI_CONN_ID) 1101 el = lsquic_hash_find(engine->conns_hash, 1102 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1103 else 1104 { 1105 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1106 return NULL; 1107 } 1108 1109 if (!el) 1110 return NULL; 1111 1112 conn = lsquic_hashelem_getdata(el); 1113 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1114 if ((engine->flags & ENG_CONNS_BY_ADDR) 1115 && !(conn->cn_flags & LSCONN_IETF) 1116 && (packet_in->pi_flags & PI_CONN_ID) 1117 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1118 { 1119 LSQ_DEBUG("connection IDs do not match"); 1120 return NULL; 1121 } 1122 1123 return conn; 1124} 1125 1126 1127static lsquic_conn_t * 1128find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1129 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1130 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1131{ 1132 struct lsquic_hash_elem *el; 1133 struct purga_el *puel; 1134 lsquic_conn_t *conn; 1135 1136 if (!(packet_in->pi_flags & PI_CONN_ID)) 1137 { 1138 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1139 return NULL; 1140 } 1141 el = lsquic_hash_find(engine->conns_hash, 1142 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1143 1144 if (el) 1145 { 1146 conn = lsquic_hashelem_getdata(el); 1147 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1148 return conn; 1149 } 1150 1151 if (engine->flags & ENG_COOLDOWN) 1152 { /* Do not create incoming connections during cooldown */ 1153 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1154 return NULL; 1155 } 1156 1157 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1158 { 1159 LSQ_DEBUG("reached limit of %u inchoate connections", 1160 engine->pub.enp_settings.es_max_inchoate); 1161 return NULL; 1162 } 1163 1164 1165 if (engine->purga 1166 && (puel = lsquic_purga_contains(engine->purga, 1167 &packet_in->pi_conn_id), puel)) 1168 { 1169 switch (puel->puel_type) 1170 { 1171 case PUTY_CID_RETIRED: 1172 LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet", 1173 CID_BITS(&packet_in->pi_conn_id)); 1174 return NULL; 1175 case PUTY_CONN_DRAIN: 1176 LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64, 1177 puel->puel_time, packet_in->pi_received); 1178 if (puel->puel_time > packet_in->pi_received) 1179 { 1180 LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet", 1181 CID_BITS(&packet_in->pi_conn_id)); 1182 return NULL; 1183 } 1184 LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted", 1185 CID_BITS(&packet_in->pi_conn_id)); 1186 puel->puel_type = PUTY_CONN_DELETED; 1187 puel->puel_count = 0; 1188 puel->puel_time = 0; 1189 /* fall-through */ 1190 case PUTY_CONN_DELETED: 1191 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1192 CID_BITS(&packet_in->pi_conn_id)); 1193 if (puel->puel_time < packet_in->pi_received) 1194 { 1195 puel->puel_time = packet_in->pi_received 1196 /* Exponential back-off */ 1197 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1198 ++puel->puel_count; 1199 goto maybe_send_prst; 1200 } 1201 return NULL; 1202 default: 1203 assert(0); 1204 return NULL; 1205 } 1206 } 1207 1208 if (engine->pub.enp_settings.es_send_prst 1209 && !(packet_in->pi_flags & PI_GQUIC) 1210 && HETY_NOT_SET == packet_in->pi_header_type) 1211 goto maybe_send_prst; 1212 1213 if (0 != maybe_grow_conn_heaps(engine)) 1214 return NULL; 1215 1216 const struct parse_funcs *pf; 1217 enum lsquic_version version; 1218 switch (version_matches(engine, packet_in, &version)) 1219 { 1220 case VER_UNSUPPORTED: 1221 if (engine->flags & ENG_SERVER) 1222 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1223 sa_local, sa_peer, peer_ctx); 1224 return NULL; 1225 case VER_NOT_SPECIFIED: 1226 maybe_send_prst: 1227 if ((engine->flags & ENG_SERVER) && 1228 engine->pub.enp_settings.es_send_prst) 1229 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1230 sa_local, sa_peer, peer_ctx); 1231 return NULL; 1232 case VER_SUPPORTED: 1233 pf = select_pf_by_ver(version); 1234 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1235 break; 1236 } 1237 1238 1239 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1240 { 1241 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1242 sa_peer->sa_family == AF_INET, NULL, packet_in_size); 1243 } 1244 else 1245 { 1246 conn = lsquic_mini_conn_new(&engine->pub, packet_in, version); 1247 } 1248 if (!conn) 1249 return NULL; 1250 ++engine->mini_conns_count; 1251 ++engine->n_conns; 1252 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1253 { 1254 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1255 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1256 CID_BITS(cid)); 1257 destroy_conn(engine, conn, packet_in->pi_received); 1258 return NULL; 1259 } 1260 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1261 conn->cn_flags |= LSCONN_HASHED; 1262 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1263 conn->cn_last_sent = engine->last_sent; 1264 return conn; 1265} 1266 1267 1268lsquic_conn_t * 1269lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1270 const lsquic_cid_t *cid) 1271{ 1272 struct lsquic_hash_elem *el; 1273 lsquic_conn_t *conn = NULL; 1274 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1275 1276 if (el) 1277 conn = lsquic_hashelem_getdata(el); 1278 return conn; 1279} 1280 1281 1282#if !defined(NDEBUG) && __GNUC__ 1283__attribute__((weak)) 1284#endif 1285void 1286lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1287 lsquic_conn_t *conn) 1288{ 1289 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1290 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1291 { 1292 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1293 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1294 engine_incref_conn(conn, LSCONN_TICKABLE); 1295 } 1296} 1297 1298 1299void 1300lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1301 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1302{ 1303 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1304 if (conn->cn_flags & LSCONN_TICKABLE) 1305 { 1306 /* Optimization: no need to add the connection to the Advisory Tick 1307 * Time Queue: it is about to be ticked, after which it its next tick 1308 * time may be queried again. 1309 */; 1310 } 1311 else if (conn->cn_flags & LSCONN_ATTQ) 1312 { 1313 if (lsquic_conn_adv_time(conn) != tick_time) 1314 { 1315 lsquic_attq_remove(engine->attq, conn); 1316 if (0 != lsquic_attq_add(engine->attq, conn, tick_time, why)) 1317 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1318 } 1319 } 1320 else if (0 == lsquic_attq_add(engine->attq, conn, tick_time, why)) 1321 engine_incref_conn(conn, LSCONN_ATTQ); 1322} 1323 1324 1325static struct lsquic_conn * 1326find_conn_by_srst (struct lsquic_engine *engine, 1327 const struct lsquic_packet_in *packet_in) 1328{ 1329 struct lsquic_hash_elem *el; 1330 struct lsquic_conn *conn; 1331 1332 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1333 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1334 return NULL; 1335 1336 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1337 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1338 IQUIC_SRESET_TOKEN_SZ); 1339 if (!el) 1340 return NULL; 1341 1342 conn = lsquic_hashelem_getdata(el); 1343 return conn; 1344} 1345 1346 1347/* Return 0 if packet is being processed by a real connection (mini or full), 1348 * otherwise return 1. 1349 */ 1350static int 1351process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1352 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1353 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1354{ 1355 lsquic_conn_t *conn; 1356 const unsigned char *packet_in_data; 1357 1358 if (lsquic_packet_in_is_gquic_prst(packet_in) 1359 && !engine->pub.enp_settings.es_honor_prst) 1360 { 1361 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1362 LSQ_DEBUG("public reset packet: discarding"); 1363 return 1; 1364 } 1365 1366 if (engine->flags & ENG_SERVER) 1367 { 1368 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1369 sa_peer, peer_ctx, packet_in_size); 1370 if (!engine->curr_conn) 1371 engine->curr_conn = conn; 1372 } 1373 else 1374 conn = find_conn(engine, packet_in, ppstate, sa_local); 1375 1376 if (!conn) 1377 { 1378 if (engine->pub.enp_settings.es_honor_prst 1379 && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */ 1380 && !(packet_in->pi_flags & PI_GQUIC) 1381 && engine->pub.enp_srst_hash 1382 && (conn = find_conn_by_srst(engine, packet_in))) 1383 { 1384 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1385 CID_BITS(lsquic_conn_log_cid(conn))); 1386 conn->cn_if->ci_stateless_reset(conn); 1387 if (!(conn->cn_flags & LSCONN_TICKABLE) 1388 && conn->cn_if->ci_is_tickable(conn)) 1389 { 1390 lsquic_mh_insert(&engine->conns_tickable, conn, 1391 conn->cn_last_ticked); 1392 engine_incref_conn(conn, LSCONN_TICKABLE); 1393 } 1394 /* Even though the connection processes this packet, we return 1395 * 1 so that the caller does not add reset packet's random 1396 * bytes to the list of valid CIDs. 1397 */ 1398 } 1399 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1400 return 1; 1401 } 1402 1403 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1404 { 1405 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1406 engine_incref_conn(conn, LSCONN_TICKABLE); 1407 } 1408 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1409 sa_local, sa_peer); 1410 lsquic_packet_in_upref(packet_in); 1411#if LOG_PACKET_CHECKSUM 1412 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1413 packet_in->pi_data_sz); 1414#endif 1415 /* Note on QLog: 1416 * For the PACKET_RX QLog event, we are interested in logging these things: 1417 * - raw packet (however it comes in, encrypted or not) 1418 * - frames (list of frame names) 1419 * - packet type and number 1420 * - packet rx timestamp 1421 * 1422 * Since only some of these items are available at this code 1423 * juncture, we will wait until after the packet has been 1424 * decrypted (if necessary) and parsed to call the log functions. 1425 * 1426 * Once the PACKET_RX event is finally logged, the timestamp 1427 * will come from packet_in->pi_received. For correct sequential 1428 * ordering of QLog events, be sure to process the QLogs downstream. 1429 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1430 */ 1431 packet_in_data = packet_in->pi_data; 1432 packet_in_size = packet_in->pi_data_sz; 1433 conn->cn_if->ci_packet_in(conn, packet_in); 1434 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1435 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1436 return 0; 1437} 1438 1439 1440void 1441lsquic_engine_destroy (lsquic_engine_t *engine) 1442{ 1443 struct lsquic_hash_elem *el; 1444 lsquic_conn_t *conn; 1445 unsigned i; 1446 1447 LSQ_DEBUG("destroying engine"); 1448#ifndef NDEBUG 1449 engine->flags |= ENG_DTOR; 1450#endif 1451 1452 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1453 { 1454 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1455 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1456 } 1457 1458 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1459 { 1460 assert(conn->cn_flags & LSCONN_TICKABLE); 1461 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1462 } 1463 1464 for (el = lsquic_hash_first(engine->conns_hash); el; 1465 el = lsquic_hash_next(engine->conns_hash)) 1466 { 1467 conn = lsquic_hashelem_getdata(el); 1468 force_close_conn(engine, conn); 1469 } 1470 lsquic_hash_destroy(engine->conns_hash); 1471 1472 while ((conn = lsquic_attq_pop(engine->attq, UINT64_MAX))) 1473 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1474 1475 assert(0 == engine->n_conns); 1476 assert(0 == engine->mini_conns_count); 1477 if (engine->pr_queue) 1478 lsquic_prq_destroy(engine->pr_queue); 1479 if (engine->purga) 1480 lsquic_purga_destroy(engine->purga); 1481 lsquic_attq_destroy(engine->attq); 1482 1483 assert(0 == lsquic_mh_count(&engine->conns_out)); 1484 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1485 if (engine->pub.enp_shi == &stock_shi) 1486 lsquic_stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1487 lsquic_mm_cleanup(&engine->pub.enp_mm); 1488 free(engine->conns_tickable.mh_elems); 1489#if CAN_LOSE_PACKETS 1490 if (engine->flags & ENG_LOSE_PACKETS) 1491 regfree(&engine->lose_packets_re); 1492#endif 1493 if (engine->pub.enp_tokgen) 1494 lsquic_tg_destroy(engine->pub.enp_tokgen); 1495#if LSQUIC_CONN_STATS 1496 if (engine->stats_fh) 1497 { 1498 const struct conn_stats *const stats = &engine->conn_stats_sum; 1499 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1500 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1501 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1502 fprintf(engine->stats_fh, "In:\n"); 1503 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1504 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1505 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1506 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1507 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1508 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1509 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1510 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1511 stats->in.headers_comp, stats->in.headers_uncomp, 1512 stats->in.headers_uncomp ? 1513 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1514 : 0); 1515 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1516 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1517 fprintf(engine->stats_fh, " ACK frames merged to new: %lu\n", stats->in.n_acks_merged[0]); 1518 fprintf(engine->stats_fh, " ACK frames merged to old: %lu\n", stats->in.n_acks_merged[1]); 1519 fprintf(engine->stats_fh, "Out:\n"); 1520 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1521 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1522 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1523 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1524 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1525 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1526 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1527 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1528 stats->out.headers_comp, stats->out.headers_uncomp, 1529 stats->out.headers_uncomp ? 1530 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1531 : 0); 1532 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1533 } 1534#endif 1535 if (engine->pub.enp_srst_hash) 1536 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1537#if LSQUIC_COUNT_ENGINE_CALLS 1538 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1539#endif 1540 for (i = 0; i < sizeof(engine->retry_aead_ctx) 1541 / sizeof(engine->retry_aead_ctx[0]); ++i) 1542 EVP_AEAD_CTX_cleanup(&engine->pub.enp_retry_aead_ctx[i]); 1543 free(engine->pub.enp_alpn); 1544 free(engine); 1545} 1546 1547 1548static struct conn_cid_elem * 1549find_free_cce (struct lsquic_conn *conn) 1550{ 1551 struct conn_cid_elem *cce; 1552 1553 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1554 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1555 return cce; 1556 1557 return NULL; 1558} 1559 1560 1561static int 1562add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1563 const struct sockaddr *local_sa, void *peer_ctx) 1564{ 1565 struct conn_cid_elem *cce; 1566 1567 if (engine->flags & ENG_CONNS_BY_ADDR) 1568 { 1569 cce = find_free_cce(conn); 1570 if (!cce) 1571 { 1572 LSQ_ERROR("cannot find free CCE"); 1573 return -1; 1574 } 1575 cce->cce_port = sa2port(local_sa); 1576 cce->cce_flags = CCE_PORT; 1577 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1578 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1579 { 1580 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1581 return 0; 1582 } 1583 else 1584 return -1; 1585 1586 } 1587 else 1588 return insert_conn_into_hash(engine, conn, peer_ctx); 1589} 1590 1591 1592lsquic_conn_t * 1593lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1594 const struct sockaddr *local_sa, 1595 const struct sockaddr *peer_sa, 1596 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1597 const char *hostname, unsigned short base_plpmtu, 1598 const unsigned char *sess_resume, size_t sess_resume_len, 1599 const unsigned char *token, size_t token_sz) 1600{ 1601 lsquic_conn_t *conn; 1602 unsigned flags, versions; 1603 int is_ipv4; 1604 1605 ENGINE_CALLS_INCR(engine); 1606 1607 if (engine->flags & ENG_SERVER) 1608 { 1609 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1610 goto err; 1611 } 1612 1613 if (engine->flags & ENG_CONNS_BY_ADDR 1614 && find_conn_by_addr(engine->conns_hash, local_sa)) 1615 { 1616 LSQ_ERROR("cannot have more than one connection on the same port"); 1617 goto err; 1618 } 1619 1620 if (0 != maybe_grow_conn_heaps(engine)) 1621 return NULL; 1622 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1623 is_ipv4 = peer_sa->sa_family == AF_INET; 1624 if (sess_resume && sess_resume_len) 1625 { 1626 version = lsquic_sess_resume_version(sess_resume, sess_resume_len); 1627 if (version >= N_LSQVER) 1628 { 1629 LSQ_INFO("session resumption version is bad, won't use"); 1630 sess_resume = NULL; 1631 sess_resume_len = 0; 1632 } 1633 } 1634 if (version >= N_LSQVER) 1635 { 1636 if (version > N_LSQVER) 1637 LSQ_WARN("invalid version specified, engine will pick"); 1638 versions = engine->pub.enp_settings.es_versions; 1639 } 1640 else 1641 versions = 1u << version; 1642 if (versions & LSQUIC_IETF_VERSIONS) 1643 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1644 flags, hostname, base_plpmtu, 1645 is_ipv4, sess_resume, sess_resume_len, token, token_sz); 1646 else 1647 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1648 flags, hostname, base_plpmtu, is_ipv4, 1649 sess_resume, sess_resume_len); 1650 if (!conn) 1651 goto err; 1652 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1653 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1654 lsquic_ver2str[conn->cn_version]); 1655 ++engine->n_conns; 1656 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1657 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1658 { 1659 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1660 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1661 CID_BITS(cid)); 1662 destroy_conn(engine, conn, lsquic_time_now()); 1663 goto err; 1664 } 1665 assert(!(conn->cn_flags & 1666 (CONN_REF_FLAGS 1667 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1668 callbacks */ 1669 ))); 1670 conn->cn_flags |= LSCONN_HASHED; 1671 if (!(conn->cn_flags & LSCONN_TICKABLE)) 1672 { 1673 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1674 engine_incref_conn(conn, LSCONN_TICKABLE); 1675 } 1676 lsquic_conn_set_ctx(conn, conn_ctx); 1677 conn->cn_if->ci_client_call_on_new(conn); 1678 end: 1679 return conn; 1680 err: 1681 conn = NULL; 1682 goto end; 1683} 1684 1685 1686static void 1687remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1688{ 1689 remove_all_cces_from_hash(engine->conns_hash, conn); 1690 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1691} 1692 1693 1694static void 1695refflags2str (enum lsquic_conn_flags flags, char s[7]) 1696{ 1697 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1698 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1699 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1700 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1701 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1702 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1703 *s = '\0'; 1704} 1705 1706 1707static void 1708engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1709{ 1710 char str[2][7]; 1711 assert(flag & CONN_REF_FLAGS); 1712 assert(!(conn->cn_flags & flag)); 1713 conn->cn_flags |= flag; 1714 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1715 CID_BITS(lsquic_conn_log_cid(conn)), 1716 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1717 (refflags2str(conn->cn_flags, str[1]), str[1])); 1718} 1719 1720 1721static lsquic_conn_t * 1722engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1723 enum lsquic_conn_flags flags) 1724{ 1725 char str[2][7]; 1726 lsquic_time_t now; 1727 assert(flags & CONN_REF_FLAGS); 1728 assert(conn->cn_flags & flags); 1729#ifndef NDEBUG 1730 if (flags & LSCONN_CLOSING) 1731 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1732#endif 1733 conn->cn_flags &= ~flags; 1734 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1735 CID_BITS(lsquic_conn_log_cid(conn)), 1736 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1737 (refflags2str(conn->cn_flags, str[1]), str[1])); 1738 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1739 { 1740 now = lsquic_time_now(); 1741 if (conn->cn_flags & LSCONN_MINI) 1742 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1743 else 1744 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1745 destroy_conn(engine, conn, now); 1746 return NULL; 1747 } 1748 else 1749 return conn; 1750} 1751 1752 1753/* This is not a general-purpose function. Only call from engine dtor. */ 1754static void 1755force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1756{ 1757 assert(engine->flags & ENG_DTOR); 1758 const enum lsquic_conn_flags flags = conn->cn_flags; 1759 assert(conn->cn_flags & CONN_REF_FLAGS); 1760 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1761 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1762 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1763 if (flags & LSCONN_ATTQ) 1764 { 1765 lsquic_attq_remove(engine->attq, conn); 1766 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1767 } 1768 if (flags & LSCONN_HASHED) 1769 remove_conn_from_hash(engine, conn); 1770} 1771 1772 1773/* Iterator for tickable connections (those on the Tickable Queue). Before 1774 * a connection is returned, it is removed from the Advisory Tick Time queue 1775 * if necessary. 1776 */ 1777static lsquic_conn_t * 1778conn_iter_next_tickable (struct lsquic_engine *engine) 1779{ 1780 lsquic_conn_t *conn; 1781 1782 if (engine->flags & ENG_SERVER) 1783 while (1) 1784 { 1785 conn = lsquic_mh_pop(&engine->conns_tickable); 1786 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1787 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1788 else 1789 break; 1790 } 1791 else 1792 conn = lsquic_mh_pop(&engine->conns_tickable); 1793 1794 if (conn) 1795 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1796 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1797 { 1798 lsquic_attq_remove(engine->attq, conn); 1799 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1800 } 1801 1802 return conn; 1803} 1804 1805 1806static void 1807cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1808 void *update_ctx) 1809{ 1810 cub->cub_update_cids = update; 1811 cub->cub_update_ctx = update_ctx; 1812 cub->cub_count = 0; 1813} 1814 1815 1816static void 1817cub_flush (struct cid_update_batch *cub) 1818{ 1819 if (cub->cub_count > 0 && cub->cub_update_cids) 1820 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1821 cub->cub_cids, cub->cub_count); 1822 cub->cub_count = 0; 1823} 1824 1825 1826static void 1827cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1828{ 1829 cub->cub_cids [ cub->cub_count ] = *cid; 1830 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1831 ++cub->cub_count; 1832 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1833 cub_flush(cub); 1834} 1835 1836 1837/* Process registered CIDs */ 1838static void 1839cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1840{ 1841 struct cce_cid_iter citer; 1842 struct conn_cid_elem *cce; 1843 void *peer_ctx; 1844 1845 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1846 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1847 if (cce->cce_flags & CCE_REG) 1848 cub_add(cub, &cce->cce_cid, peer_ctx); 1849} 1850 1851 1852static void 1853drop_all_mini_conns (lsquic_engine_t *engine) 1854{ 1855 struct lsquic_hash_elem *el; 1856 lsquic_conn_t *conn; 1857 struct cid_update_batch cub; 1858 1859 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1860 1861 for (el = lsquic_hash_first(engine->conns_hash); el; 1862 el = lsquic_hash_next(engine->conns_hash)) 1863 { 1864 conn = lsquic_hashelem_getdata(el); 1865 if (conn->cn_flags & LSCONN_MINI) 1866 { 1867 /* If promoted, why is it still in this hash? */ 1868 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1869 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1870 cub_add_cids_from_cces(&cub, conn); 1871 remove_conn_from_hash(engine, conn); 1872 } 1873 } 1874 1875 cub_flush(&cub); 1876} 1877 1878 1879void 1880lsquic_engine_process_conns (lsquic_engine_t *engine) 1881{ 1882 lsquic_conn_t *conn; 1883 lsquic_time_t now; 1884 1885 ENGINE_IN(engine); 1886 1887 now = lsquic_time_now(); 1888 while ((conn = lsquic_attq_pop(engine->attq, now))) 1889 { 1890 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1891 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 1892 { 1893 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1894 engine_incref_conn(conn, LSCONN_TICKABLE); 1895 } 1896 } 1897 1898 process_connections(engine, conn_iter_next_tickable, now); 1899 ENGINE_OUT(engine); 1900} 1901 1902 1903static void 1904release_or_return_enc_data (struct lsquic_engine *engine, 1905 void (*pmi_rel_or_ret) (void *, void *, void *, char), 1906 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 1907{ 1908 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1909 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 1910 packet_out->po_flags &= ~PO_ENCRYPTED; 1911 packet_out->po_enc_data = NULL; 1912} 1913 1914 1915static void 1916release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1917 struct lsquic_packet_out *packet_out) 1918{ 1919 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 1920 conn, packet_out); 1921} 1922 1923 1924static void 1925return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1926 struct lsquic_packet_out *packet_out) 1927{ 1928 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 1929 conn, packet_out); 1930} 1931 1932 1933static int 1934copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 1935 struct lsquic_packet_out *packet_out) 1936{ 1937 int ipv6; 1938 1939 ipv6 = NP_IS_IPv6(packet_out->po_path); 1940 if (packet_out->po_flags & PO_ENCRYPTED) 1941 { 1942 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1943 && packet_out->po_data_sz == packet_out->po_enc_data_sz 1944 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 1945 packet_out->po_data_sz)) 1946 return 0; 1947 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1948 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 1949 goto copy; 1950 return_enc_data(engine, conn, packet_out); 1951 } 1952 1953 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 1954 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1955 packet_out->po_data_sz, ipv6); 1956 if (!packet_out->po_enc_data) 1957 { 1958 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 1959 "of size %hu", packet_out->po_data_sz); 1960 return -1; 1961 } 1962 1963 copy: 1964 memcpy(packet_out->po_enc_data, packet_out->po_data, 1965 packet_out->po_data_sz); 1966 packet_out->po_enc_data_sz = packet_out->po_data_sz; 1967 packet_out->po_sent_sz = packet_out->po_data_sz; 1968 packet_out->po_flags &= ~PO_IPv6; 1969 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 1970 1971 return 0; 1972} 1973 1974 1975STAILQ_HEAD(conns_stailq, lsquic_conn); 1976TAILQ_HEAD(conns_tailq, lsquic_conn); 1977 1978 1979struct conns_out_iter 1980{ 1981 struct min_heap *coi_heap; 1982 struct pr_queue *coi_prq; 1983 TAILQ_HEAD(, lsquic_conn) coi_active_list, 1984 coi_inactive_list; 1985 lsquic_conn_t *coi_next; 1986#ifndef NDEBUG 1987 lsquic_time_t coi_last_sent; 1988#endif 1989}; 1990 1991 1992static void 1993coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 1994{ 1995 iter->coi_heap = &engine->conns_out; 1996 iter->coi_prq = engine->pr_queue; 1997 iter->coi_next = NULL; 1998 TAILQ_INIT(&iter->coi_active_list); 1999 TAILQ_INIT(&iter->coi_inactive_list); 2000#ifndef NDEBUG 2001 iter->coi_last_sent = 0; 2002#endif 2003} 2004 2005 2006static lsquic_conn_t * 2007coi_next (struct conns_out_iter *iter) 2008{ 2009 lsquic_conn_t *conn; 2010 2011 if (lsquic_mh_count(iter->coi_heap) > 0) 2012 { 2013 conn = lsquic_mh_pop(iter->coi_heap); 2014 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2015 conn->cn_flags |= LSCONN_COI_ACTIVE; 2016#ifndef NDEBUG 2017 if (iter->coi_last_sent) 2018 assert(iter->coi_last_sent <= conn->cn_last_sent); 2019 iter->coi_last_sent = conn->cn_last_sent; 2020#endif 2021 return conn; 2022 } 2023 else if (iter->coi_prq && (conn = lsquic_prq_next_conn(iter->coi_prq))) 2024 { 2025 return conn; 2026 } 2027 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 2028 { 2029 iter->coi_prq = NULL; /* Save function call in previous conditional */ 2030 conn = iter->coi_next; 2031 if (!conn) 2032 conn = TAILQ_FIRST(&iter->coi_active_list); 2033 if (conn) 2034 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 2035 return conn; 2036 } 2037 else 2038 return NULL; 2039} 2040 2041 2042static void 2043coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2044{ 2045 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2046 { 2047 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 2048 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2049 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2050 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 2051 conn->cn_flags |= LSCONN_COI_INACTIVE; 2052 } 2053} 2054 2055 2056static void 2057coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2058{ 2059 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 2060 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2061 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2062 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2063 conn->cn_flags |= LSCONN_COI_ACTIVE; 2064} 2065 2066 2067static void 2068coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 2069{ 2070 lsquic_conn_t *conn; 2071 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 2072 { 2073 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2074 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2075 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 2076 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 2077 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 2078 else /* Closed connection gets one shot at sending packets */ 2079 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2080 } 2081 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 2082 { 2083 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2084 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2085 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2086 } 2087} 2088 2089 2090#if CAN_LOSE_PACKETS 2091static void 2092lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 2093 unsigned n) 2094{ 2095 const lsquic_cid_t *cid; 2096 struct iovec *iov; 2097 unsigned i; 2098 char packno_str[22]; 2099 2100 for (i = 0; i < n; ++i) 2101 { 2102 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 2103 batch->packets[i]->po_packno); 2104 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 2105 { 2106 for (iov = batch->outs[i].iov; iov < 2107 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 2108 batch->outs[i].iov->iov_len -= 1; 2109 cid = lsquic_conn_log_cid(batch->conns[i]); 2110 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 2111 CID_BITS(cid)); 2112 } 2113 } 2114} 2115 2116 2117#endif 2118 2119 2120#ifdef NDEBUG 2121#define CONST_BATCH const 2122#else 2123#define CONST_BATCH 2124#endif 2125 2126 2127struct send_batch_ctx { 2128 struct conns_stailq *closed_conns; 2129 struct conns_tailq *ticked_conns; 2130 struct conns_out_iter *conns_iter; 2131 CONST_BATCH struct out_batch *batch; 2132}; 2133 2134 2135static void 2136close_conn_immediately (struct lsquic_engine *engine, 2137 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2138{ 2139 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2140 if (!(conn->cn_flags & LSCONN_CLOSING)) 2141 { 2142 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2143 engine_incref_conn(conn, LSCONN_CLOSING); 2144 if (conn->cn_flags & LSCONN_HASHED) 2145 remove_conn_from_hash(engine, conn); 2146 } 2147 if (conn->cn_flags & LSCONN_TICKED) 2148 { 2149 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2150 engine_decref_conn(engine, conn, LSCONN_TICKED); 2151 } 2152} 2153 2154 2155static void 2156close_conn_on_send_error (struct lsquic_engine *engine, 2157 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2158{ 2159 const struct out_batch *batch = sb_ctx->batch; 2160 struct lsquic_conn *const conn = batch->conns[n]; 2161 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2162 2163 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2164 "src: %s; dst: %s; errno: %d", 2165 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2166 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2167 CID_BITS(lsquic_conn_log_cid(conn)), 2168 SA2STR(batch->outs[n].local_sa, buf[0]), 2169 SA2STR(batch->outs[n].dest_sa, buf[1]), 2170 e_val); 2171 if (conn->cn_flags & LSCONN_EVANESCENT) 2172 lsquic_prq_drop(conn); 2173 else 2174 close_conn_immediately(engine, sb_ctx, conn); 2175} 2176 2177 2178static void 2179apply_hp (struct conns_out_iter *iter) 2180{ 2181 struct lsquic_conn *conn; 2182 2183 TAILQ_FOREACH(conn, &iter->coi_active_list, cn_next_out) 2184 if (conn->cn_esf_c->esf_flush_encryption && conn->cn_enc_session) 2185 conn->cn_esf_c->esf_flush_encryption(conn->cn_enc_session); 2186 TAILQ_FOREACH(conn, &iter->coi_inactive_list, cn_next_out) 2187 if (conn->cn_esf_c->esf_flush_encryption && conn->cn_enc_session) 2188 conn->cn_esf_c->esf_flush_encryption(conn->cn_enc_session); 2189} 2190 2191 2192static unsigned 2193send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2194 unsigned n_to_send) 2195{ 2196 int n_sent, i, e_val; 2197 lsquic_time_t now; 2198 unsigned off, skip; 2199 size_t count; 2200 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2201 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2202 2203 apply_hp(sb_ctx->conns_iter); 2204#if CAN_LOSE_PACKETS 2205 if (engine->flags & ENG_LOSE_PACKETS) 2206 lose_matching_packets(engine, batch, n_to_send); 2207#endif 2208 skip = 0; 2209 restart_batch: 2210 /* Set sent time before the write to avoid underestimating RTT */ 2211 now = lsquic_time_now(); 2212 for (i = skip; i < (int) (n_to_send - skip); ++i) 2213 { 2214 off = batch->pack_off[i]; 2215 count = batch->outs[i].iovlen; 2216 assert(count > 0); 2217 packet_out = &batch->packets[off]; 2218 end = packet_out + count; 2219 do 2220 (*packet_out)->po_sent = now; 2221 while (++packet_out < end); 2222 } 2223 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs + skip, 2224 n_to_send - skip); 2225 e_val = errno; 2226 if (n_sent < (int) (n_to_send - skip) && e_val != EMSGSIZE) 2227 { 2228 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2229 engine->resume_sending_at = now + 1000000; 2230 LSQ_DEBUG("cannot send packets"); 2231 EV_LOG_GENERIC_EVENT("cannot send packets"); 2232 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2233 close_conn_on_send_error(engine, sb_ctx, 2234 n_sent < 0 ? 0 : n_sent, e_val); 2235 } 2236 if (n_sent >= 0) 2237 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, 2238 n_to_send - skip); 2239 else 2240 { 2241 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2242 n_sent = 0; 2243 } 2244 if (n_sent > 0) 2245 engine->last_sent = now + n_sent; 2246 for (i = skip; i < (int) (skip + n_sent); ++i) 2247 { 2248 eng_hist_inc(&engine->history, now, sl_packets_out); 2249 /* `i' is added to maintain relative order */ 2250 batch->conns[i]->cn_last_sent = now + i; 2251 2252 off = batch->pack_off[i]; 2253 count = batch->outs[i].iovlen; 2254 assert(count > 0); 2255 packet_out = &batch->packets[off]; 2256 end = packet_out + count; 2257 do 2258 { 2259#if LOG_PACKET_CHECKSUM 2260 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2261 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2262 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2263#endif 2264 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2265 *packet_out); 2266 /* Release packet out buffer as soon as the packet is sent 2267 * successfully. If not successfully sent, we hold on to 2268 * this buffer until the packet sending is attempted again 2269 * or until it times out and regenerated. 2270 */ 2271 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2272 release_enc_data(engine, batch->conns[i], *packet_out); 2273 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2274 *packet_out); 2275 } 2276 while (++packet_out < end); 2277 } 2278 if (i < (int) n_to_send && e_val == EMSGSIZE) 2279 { 2280 LSQ_DEBUG("packet #%d could not be sent out for being too large", i); 2281 if (batch->conns[i]->cn_if->ci_packet_too_large 2282 && batch->outs[i].iovlen == 1) 2283 { 2284 off = batch->pack_off[i]; 2285 packet_out = &batch->packets[off]; 2286 batch->conns[i]->cn_if->ci_packet_too_large(batch->conns[i], 2287 *packet_out); 2288 ++i; 2289 if (i < (int) n_to_send) 2290 { 2291 skip = i; 2292 LSQ_DEBUG("restart batch starting at packet #%u", skip); 2293 goto restart_batch; 2294 } 2295 n_sent = n_to_send; 2296 } 2297 else 2298 close_conn_on_send_error(engine, sb_ctx, i, e_val); 2299 } 2300 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2301 for ( ; i < (int) n_to_send; ++i) 2302 { 2303 off = batch->pack_off[i]; 2304 count = batch->outs[i].iovlen; 2305 assert(count > 0); 2306 packet_out = &batch->packets[off]; 2307 end = packet_out + count; 2308 do 2309 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2310 *packet_out); 2311 while (++packet_out < end); 2312 } 2313 /* Return packets to the connection in reverse order so that the packet 2314 * ordering is maintained. 2315 */ 2316 for (i = (int) n_to_send - 1; i >= (int) (skip + n_sent); --i) 2317 { 2318 off = batch->pack_off[i]; 2319 count = batch->outs[i].iovlen; 2320 assert(count > 0); 2321 packet_out = &batch->packets[off + count - 1]; 2322 end = &batch->packets[off - 1]; 2323 do 2324 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2325 *packet_out); 2326 while (--packet_out > end); 2327 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2328 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2329 } 2330 return skip + n_sent; 2331} 2332 2333 2334/* Return 1 if went past deadline, 0 otherwise */ 2335static int 2336check_deadline (lsquic_engine_t *engine) 2337{ 2338 if (engine->pub.enp_settings.es_proc_time_thresh && 2339 lsquic_time_now() > engine->deadline) 2340 { 2341 LSQ_INFO("went past threshold of %u usec, stop sending", 2342 engine->pub.enp_settings.es_proc_time_thresh); 2343 engine->flags |= ENG_PAST_DEADLINE; 2344 return 1; 2345 } 2346 else 2347 return 0; 2348} 2349 2350 2351static size_t 2352iov_size (const struct iovec *iov, const struct iovec *const end) 2353{ 2354 size_t size; 2355 2356 assert(iov < end); 2357 2358 size = 0; 2359 do 2360 size += iov->iov_len; 2361 while (++iov < end); 2362 2363 return size; 2364} 2365 2366 2367static void 2368send_packets_out (struct lsquic_engine *engine, 2369 struct conns_tailq *ticked_conns, 2370 struct conns_stailq *closed_conns) 2371{ 2372 unsigned n, w, n_sent, n_batches_sent; 2373 lsquic_packet_out_t *packet_out; 2374 struct lsquic_packet_out **packet; 2375 lsquic_conn_t *conn; 2376 struct out_batch *const batch = &engine->out_batch; 2377 struct iovec *iov, *packet_iov; 2378 struct conns_out_iter conns_iter; 2379 int shrink, deadline_exceeded; 2380 const struct send_batch_ctx sb_ctx = { 2381 closed_conns, 2382 ticked_conns, 2383 &conns_iter, 2384 &engine->out_batch, 2385 }; 2386 2387 coi_init(&conns_iter, engine); 2388 n_batches_sent = 0; 2389 n_sent = 0, n = 0; 2390 shrink = 0; 2391 deadline_exceeded = 0; 2392 iov = batch->iov; 2393 packet = batch->packets; 2394 2395 while ((conn = coi_next(&conns_iter))) 2396 { 2397 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2398 if (!packet_out) { 2399 /* Evanescent connection always has a packet to send: */ 2400 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2401 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2402 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2403 CID_BITS(lsquic_conn_log_cid(conn))); 2404 coi_deactivate(&conns_iter, conn); 2405 continue; 2406 } 2407 batch->outs[n].iov = packet_iov = iov; 2408 next_coa: 2409 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2410 { 2411 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2412 &engine->pub, conn, packet_out)) 2413 { 2414 case ENCPA_NOMEM: 2415 /* Send what we have and wait for a more opportune moment */ 2416 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2417 goto end_for; 2418 case ENCPA_BADCRYPT: 2419 /* This is pretty bad: close connection immediately */ 2420 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2421 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2422 CID_BITS(lsquic_conn_log_cid(conn))); 2423 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2424 { 2425 close_conn_immediately(engine, &sb_ctx, conn); 2426 coi_deactivate(&conns_iter, conn); 2427 } 2428 continue; 2429 case ENCPA_OK: 2430 break; 2431 } 2432 } 2433 else if ((packet_out->po_flags & PO_NOENCRYPT) 2434 && engine->pub.enp_pmi != &stock_pmi) 2435 { 2436 if (0 != copy_packet(engine, conn, packet_out)) 2437 { 2438 /* Copy can only fail if packet could not be allocated */ 2439 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2440 goto end_for; 2441 } 2442 } 2443 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2444 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2445 if (packet_out->po_flags & PO_ENCRYPTED) 2446 { 2447 iov->iov_base = packet_out->po_enc_data; 2448 iov->iov_len = packet_out->po_enc_data_sz; 2449 } 2450 else 2451 { 2452 iov->iov_base = packet_out->po_data; 2453 iov->iov_len = packet_out->po_data_sz; 2454 } 2455 if (packet_iov == iov) 2456 { 2457 batch->pack_off[n] = packet - batch->packets; 2458 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2459 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2460 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2461 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2462 batch->conns [n] = conn; 2463 } 2464 *packet = packet_out; 2465 ++packet; 2466 ++iov; 2467 if ((conn->cn_flags & LSCONN_IETF) 2468 && ((1 << packet_out->po_header_type) 2469 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2470#ifndef NDEBUG 2471 && (engine->flags & ENG_COALESCE) 2472#endif 2473 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2474 { 2475 const size_t size = iov_size(packet_iov, iov); 2476 packet_out = conn->cn_if->ci_next_packet_to_send(conn, size); 2477 if (packet_out) 2478 goto next_coa; 2479 } 2480 batch->outs [n].iovlen = iov - packet_iov; 2481 ++n; 2482 if (n == engine->batch_size 2483 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2484 { 2485 w = send_batch(engine, &sb_ctx, n); 2486 n = 0; 2487 iov = batch->iov; 2488 packet = batch->packets; 2489 ++n_batches_sent; 2490 n_sent += w; 2491 if (w < engine->batch_size) 2492 { 2493 shrink = 1; 2494 break; 2495 } 2496 deadline_exceeded = check_deadline(engine); 2497 if (deadline_exceeded) 2498 break; 2499 grow_batch_size(engine); 2500 } 2501 } 2502 end_for: 2503 2504 if (n > 0) { 2505 w = send_batch(engine, &sb_ctx, n); 2506 n_sent += w; 2507 shrink = w < n; 2508 ++n_batches_sent; 2509 } 2510 2511 if (shrink) 2512 shrink_batch_size(engine); 2513 else if (n_batches_sent > 1) 2514 { 2515 deadline_exceeded = check_deadline(engine); 2516 if (!deadline_exceeded) 2517 grow_batch_size(engine); 2518 } 2519 2520 coi_reheap(&conns_iter, engine); 2521 2522 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2523} 2524 2525 2526int 2527lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2528{ 2529 return lsquic_mh_count(&engine->conns_out) > 0 2530 || (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 2531 ; 2532} 2533 2534 2535static void 2536reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2537{ 2538 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2539 engine->flags &= ~ENG_PAST_DEADLINE; 2540} 2541 2542 2543static void 2544check_tickable_conns_again (struct lsquic_engine *engine) 2545{ 2546 struct lsquic_hash_elem *el; 2547 struct lsquic_conn *conn; 2548 unsigned count; 2549 2550 count = 0; 2551 for (el = lsquic_hash_first(engine->conns_hash); el; 2552 el = lsquic_hash_next(engine->conns_hash)) 2553 { 2554 conn = lsquic_hashelem_getdata(el); 2555 if (!(conn->cn_flags & LSCONN_TICKABLE) 2556 && conn->cn_if->ci_is_tickable(conn)) 2557 { 2558 lsquic_mh_insert(&engine->conns_tickable, conn, 2559 conn->cn_last_ticked); 2560 engine_incref_conn(conn, LSCONN_TICKABLE); 2561 ++count; 2562 } 2563 } 2564 LSQ_DEBUG("%u connection%s tickable again after sending has been " 2565 "re-enabled", count, count == 1 ? " is" : "s are"); 2566} 2567 2568 2569void 2570lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2571{ 2572 lsquic_conn_t *conn; 2573 struct conns_stailq closed_conns; 2574 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2575 struct cid_update_batch cub; 2576 2577 ENGINE_IN(engine); 2578 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2579 STAILQ_INIT(&closed_conns); 2580 reset_deadline(engine, lsquic_time_now()); 2581 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2582 { 2583 LSQ_DEBUG("can send again"); 2584 EV_LOG_GENERIC_EVENT("can send again"); 2585 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2586 check_tickable_conns_again(engine); 2587 } 2588 2589 send_packets_out(engine, &ticked_conns, &closed_conns); 2590 2591 while ((conn = STAILQ_FIRST(&closed_conns))) { 2592 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2593 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2594 cub_add_cids_from_cces(&cub, conn); 2595 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2596 } 2597 2598 cub_flush(&cub); 2599 ENGINE_OUT(engine); 2600} 2601 2602 2603static lsquic_conn_t * 2604next_new_full_conn (struct conns_stailq *new_full_conns) 2605{ 2606 lsquic_conn_t *conn; 2607 2608 conn = STAILQ_FIRST(new_full_conns); 2609 if (conn) 2610 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2611 return conn; 2612} 2613 2614 2615static void 2616process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2617 lsquic_time_t now) 2618{ 2619 lsquic_conn_t *conn; 2620 enum tick_st tick_st; 2621 unsigned i, why; 2622 lsquic_time_t next_tick_time; 2623 struct conns_stailq closed_conns; 2624 struct conns_tailq ticked_conns; 2625 struct conns_stailq new_full_conns; 2626 struct cid_update_batch cub_old, cub_live; 2627 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2628 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2629 2630 eng_hist_tick(&engine->history, now); 2631 2632 STAILQ_INIT(&closed_conns); 2633 TAILQ_INIT(&ticked_conns); 2634 reset_deadline(engine, now); 2635 STAILQ_INIT(&new_full_conns); 2636 2637 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2638 && now > engine->resume_sending_at) 2639 { 2640 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2641 "timeout"); 2642 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2643 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2644 } 2645 2646 i = 0; 2647 while ((conn = next_conn(engine)) 2648 || (conn = next_new_full_conn(&new_full_conns))) 2649 { 2650 tick_st = conn->cn_if->ci_tick(conn, now); 2651 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2652 if (tick_st & TICK_PROMOTE) 2653 { 2654 lsquic_conn_t *new_conn; 2655 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2656 "scheduled for promotion"); 2657 assert(conn->cn_flags & LSCONN_MINI); 2658 new_conn = new_full_conn_server(engine, conn, now); 2659 if (new_conn) 2660 { 2661 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2662 new_conn->cn_last_sent = engine->last_sent; 2663 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2664 conn->cn_flags |= LSCONN_PROMOTED; 2665 } 2666 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2667 } 2668 if (tick_st & TICK_SEND) 2669 { 2670 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2671 { 2672 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2673 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2674 } 2675 } 2676 if (tick_st & TICK_CLOSE) 2677 { 2678 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2679 engine_incref_conn(conn, LSCONN_CLOSING); 2680 if (conn->cn_flags & LSCONN_HASHED) 2681 remove_conn_from_hash(engine, conn); 2682 } 2683 else 2684 { 2685 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2686 engine_incref_conn(conn, LSCONN_TICKED); 2687 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2688 && conn->cn_if->ci_report_live(conn, now)) 2689 cub_add_cids_from_cces(&cub_live, conn); 2690 } 2691 } 2692 2693 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2694 && lsquic_engine_has_unsent_packets(engine)) 2695 send_packets_out(engine, &ticked_conns, &closed_conns); 2696 2697 while ((conn = STAILQ_FIRST(&closed_conns))) { 2698 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2699 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2700 cub_add_cids_from_cces(&cub_old, conn); 2701 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2702 } 2703 2704 while ((conn = TAILQ_FIRST(&ticked_conns))) 2705 { 2706 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2707 engine_decref_conn(engine, conn, LSCONN_TICKED); 2708 if (!(conn->cn_flags & LSCONN_TICKABLE) 2709 && conn->cn_if->ci_is_tickable(conn)) 2710 { 2711 /* Floyd heapification is not faster, don't bother. */ 2712 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2713 engine_incref_conn(conn, LSCONN_TICKABLE); 2714 } 2715 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2716 { 2717 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2718 if (next_tick_time) 2719 { 2720 if (0 == lsquic_attq_add(engine->attq, conn, next_tick_time, 2721 why)) 2722 engine_incref_conn(conn, LSCONN_ATTQ); 2723 } 2724 else 2725 /* In all other cases, the idle timeout would make the next 2726 * tick time non-zero: 2727 */ 2728 assert((conn->cn_flags & LSCONN_IETF) 2729 && engine->pub.enp_settings.es_idle_timeout == 0); 2730 } 2731 } 2732 2733 cub_flush(&engine->new_scids); 2734 cub_flush(&cub_live); 2735 cub_flush(&cub_old); 2736} 2737 2738 2739static void 2740maybe_count_garbage (struct lsquic_engine *engine, size_t garbage_sz) 2741{ 2742 /* This is not very pretty (action at a distance via engine->curr_conn), 2743 * but it's the cheapest I can come up with to handle the "count garbage 2744 * toward amplification limit" requirement in 2745 * [draft-ietf-quic-transport-28] Section 8.1. 2746 */ 2747 if (engine->curr_conn && engine->curr_conn->cn_if->ci_count_garbage) 2748 engine->curr_conn->cn_if->ci_count_garbage(engine->curr_conn, 2749 garbage_sz); 2750} 2751 2752 2753/* Return 0 if packet is being processed by a real connection, 1 if the 2754 * packet was processed, but not by a connection, and -1 on error. 2755 */ 2756int 2757lsquic_engine_packet_in (lsquic_engine_t *engine, 2758 const unsigned char *packet_in_data, size_t packet_in_size, 2759 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2760 void *peer_ctx, int ecn) 2761{ 2762 const unsigned char *const packet_begin = packet_in_data; 2763 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2764 struct packin_parse_state ppstate; 2765 lsquic_packet_in_t *packet_in; 2766 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2767 int is_server, unsigned cid_len, struct packin_parse_state *); 2768 unsigned n_zeroes; 2769 int s, is_ietf; 2770 lsquic_cid_t cid; 2771 2772 ENGINE_CALLS_INCR(engine); 2773 2774 if (engine->flags & ENG_SERVER) 2775 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2776 else if (engine->flags & ENG_CONNS_BY_ADDR) 2777 { 2778 struct lsquic_hash_elem *el; 2779 const struct lsquic_conn *conn; 2780 el = find_conn_by_addr(engine->conns_hash, sa_local); 2781 if (!el) 2782 return -1; 2783 conn = lsquic_hashelem_getdata(el); 2784 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2785 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2786 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2787 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2788 else if (conn->cn_version == LSQVER_050) 2789 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 2790 else 2791 { 2792#if LSQUIC_USE_Q098 2793 assert(conn->cn_version == LSQVER_046 || conn->cn_version == LSQVER_098); 2794#else 2795 assert(conn->cn_version == LSQVER_046); 2796#endif 2797 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2798 } 2799 } 2800 else 2801 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2802 2803 engine->curr_conn = NULL; 2804 n_zeroes = 0; 2805 is_ietf = 0; 2806#ifdef _MSC_VER 2807 s = 0; 2808 cid.len = 0; 2809 cid.idbuf[0] = 0; 2810#endif 2811 do 2812 { 2813 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 2814 if (!packet_in) 2815 return -1; 2816 /* Library does not modify packet_in_data, it is not referenced after 2817 * this function returns and subsequent release of pi_data is guarded 2818 * by PI_OWN_DATA flag. 2819 */ 2820 packet_in->pi_data = (unsigned char *) packet_in_data; 2821 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 2822 engine->flags & ENG_SERVER, 2823 engine->pub.enp_settings.es_scid_len, &ppstate)) 2824 { 2825 LSQ_DEBUG("Cannot parse incoming packet's header"); 2826 maybe_count_garbage(engine, packet_end - packet_in_data); 2827 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 2828 s = 1; 2829 break; 2830 } 2831 2832 /* [draft-ietf-quic-transport-27] Section 12.2: 2833 * " Receivers SHOULD ignore any subsequent packets with a different 2834 * " Destination Connection ID than the first packet in the datagram. 2835 */ 2836 if (is_ietf && packet_in_data > packet_begin) 2837 { 2838 if (!((packet_in->pi_flags & (PI_GQUIC|PI_CONN_ID)) == PI_CONN_ID 2839 && LSQUIC_CIDS_EQ(&packet_in->pi_dcid, &cid))) 2840 { 2841 packet_in_data += packet_in->pi_data_sz; 2842 maybe_count_garbage(engine, packet_in->pi_data_sz); 2843 continue; 2844 } 2845 } 2846 2847 is_ietf = 0 == (packet_in->pi_flags & PI_GQUIC); 2848 packet_in_data += packet_in->pi_data_sz; 2849 if (is_ietf && packet_in_data < packet_end) 2850 cid = packet_in->pi_dcid; 2851 packet_in->pi_received = lsquic_time_now(); 2852 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 2853 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 2854 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 2855 peer_ctx, packet_in_size); 2856 n_zeroes += s == 0; 2857 } 2858 while (0 == s && packet_in_data < packet_end); 2859 2860 return n_zeroes > 0 ? 0 : s; 2861} 2862 2863 2864#if __GNUC__ && !defined(NDEBUG) 2865__attribute__((weak)) 2866#endif 2867unsigned 2868lsquic_engine_quic_versions (const lsquic_engine_t *engine) 2869{ 2870 return engine->pub.enp_settings.es_versions; 2871} 2872 2873 2874void 2875lsquic_engine_cooldown (lsquic_engine_t *engine) 2876{ 2877 struct lsquic_hash_elem *el; 2878 lsquic_conn_t *conn; 2879 2880 if (engine->flags & ENG_COOLDOWN) 2881 /* AFAICT, there is no harm in calling this function more than once, 2882 * but log it just in case, as it may indicate an error in the caller. 2883 */ 2884 LSQ_INFO("cooldown called again"); 2885 engine->flags |= ENG_COOLDOWN; 2886 LSQ_INFO("entering cooldown mode"); 2887 if (engine->flags & ENG_SERVER) 2888 drop_all_mini_conns(engine); 2889 for (el = lsquic_hash_first(engine->conns_hash); el; 2890 el = lsquic_hash_next(engine->conns_hash)) 2891 { 2892 conn = lsquic_hashelem_getdata(el); 2893 lsquic_conn_going_away(conn); 2894 } 2895} 2896 2897 2898int 2899lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 2900{ 2901 const struct attq_elem *next_attq; 2902 lsquic_time_t now, next_time; 2903#if LSQUIC_DEBUG_NEXT_ADV_TICK 2904 const struct lsquic_conn *conn; 2905 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 2906#endif 2907 2908 ENGINE_CALLS_INCR(engine); 2909 2910 if ((engine->flags & ENG_PAST_DEADLINE) 2911 && lsquic_mh_count(&engine->conns_out)) 2912 { 2913#if LSQUIC_DEBUG_NEXT_ADV_TICK 2914 conn = lsquic_mh_peek(&engine->conns_out); 2915 engine->last_logged_conn = 0; 2916 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 2917 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 2918 lsquic_mh_count(&engine->conns_out), 2919 lsquic_mh_count(&engine->conns_out) != 1, "s", 2920 CID_BITS(lsquic_conn_log_cid(conn))); 2921#endif 2922 *diff = 0; 2923 return 1; 2924 } 2925 2926 if (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 2927 { 2928#if LSQUIC_DEBUG_NEXT_ADV_TICK 2929 engine->last_logged_conn = 0; 2930 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 2931#endif 2932 *diff = 0; 2933 return 1; 2934 } 2935 2936 if (lsquic_mh_count(&engine->conns_tickable)) 2937 { 2938#if LSQUIC_DEBUG_NEXT_ADV_TICK 2939 conn = lsquic_mh_peek(&engine->conns_tickable); 2940 engine->last_logged_conn = 0; 2941 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 2942 "connection%.*s (%"CID_FMT" first)", 2943 lsquic_mh_count(&engine->conns_tickable), 2944 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 2945 CID_BITS(lsquic_conn_log_cid(conn))); 2946#endif 2947 *diff = 0; 2948 return 1; 2949 } 2950 2951 next_attq = lsquic_attq_next(engine->attq); 2952 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 2953 { 2954 if (next_attq) 2955 next_time = next_attq->ae_adv_time; 2956 else 2957 return 0; 2958 } 2959 else 2960 { 2961 if (next_attq) 2962 { 2963 next_time = next_attq->ae_adv_time; 2964 if (engine->resume_sending_at < next_time) 2965 { 2966 next_time = engine->resume_sending_at; 2967 next_attq = NULL; 2968 } 2969 } 2970 else 2971 next_time = engine->resume_sending_at; 2972 } 2973 2974 now = lsquic_time_now(); 2975 *diff = (int) ((int64_t) next_time - (int64_t) now); 2976#if LSQUIC_DEBUG_NEXT_ADV_TICK 2977 if (next_attq) 2978 { 2979 /* Deduplicate consecutive log messages about the same reason for the 2980 * same connection. 2981 * If diff is always zero or diff reset to a higher value, event is 2982 * still logged. 2983 */ 2984 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 2985 && (uintptr_t) next_attq->ae_conn 2986 == engine->last_logged_conn 2987 && *diff < engine->last_tick_diff)) 2988 { 2989 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 2990 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 2991 engine->last_tick_diff = *diff; 2992 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 2993 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 2994 lsquic_attq_why2str(next_attq->ae_why)); 2995 } 2996 } 2997 else 2998 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 2999#endif 3000 return 1; 3001} 3002 3003 3004unsigned 3005lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 3006{ 3007 lsquic_time_t now; 3008 ENGINE_CALLS_INCR(engine); 3009 now = lsquic_time_now(); 3010 if (from_now < 0) 3011 now -= from_now; 3012 else 3013 now += from_now; 3014 return lsquic_attq_count_before(engine->attq, now); 3015} 3016 3017 3018int 3019lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 3020 struct lsquic_conn *conn, unsigned cce_idx) 3021{ 3022 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 3023 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 3024 void *peer_ctx; 3025 3026 assert(cce_idx < conn->cn_n_cces); 3027 assert(conn->cn_cces_mask & (1 << cce_idx)); 3028 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 3029 3030 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 3031 cce->cce_cid.len, conn, &cce->cce_hash_el)) 3032 { 3033 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 3034 CID_BITS(&cce->cce_cid)); 3035 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 3036 cce->cce_flags |= CCE_REG; 3037 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 3038 return 0; 3039 } 3040 else 3041 { 3042 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 3043 CID_BITS(&cce->cce_cid)); 3044 return -1; 3045 } 3046} 3047 3048 3049void 3050lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 3051 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now) 3052{ 3053 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 3054 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 3055 void *peer_ctx; 3056 3057 assert(cce_idx < conn->cn_n_cces); 3058 3059 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 3060 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 3061 3062 if (engine->purga) 3063 { 3064 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 3065 lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 3066 PUTY_CID_RETIRED, now); 3067 } 3068 conn->cn_cces_mask &= ~(1u << cce_idx); 3069 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 3070} 3071 3072 3073