lsquic_engine.c revision fecdd104
1/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#ifndef WIN32 17#include <sys/time.h> 18#include <netinet/in.h> 19#include <sys/types.h> 20#include <sys/stat.h> 21#include <fcntl.h> 22#include <unistd.h> 23#include <netdb.h> 24#endif 25 26#ifndef NDEBUG 27#include <sys/types.h> 28#endif 29 30#if defined(WIN32) || defined(NDEBUG) 31#define CAN_LOSE_PACKETS 0 32#else 33#define CAN_LOSE_PACKETS 1 34#endif 35 36#if CAN_LOSE_PACKETS 37#include <regex.h> /* For code that loses packets */ 38#endif 39 40#if LOG_PACKET_CHECKSUM 41#include <zlib.h> 42#endif 43 44#include <openssl/aead.h> 45 46#include "lsquic.h" 47#include "lsquic_types.h" 48#include "lsquic_int_types.h" 49#include "lsquic_sizes.h" 50#include "lsquic_parse_common.h" 51#include "lsquic_parse.h" 52#include "lsquic_packet_in.h" 53#include "lsquic_packet_out.h" 54#include "lsquic_senhist.h" 55#include "lsquic_rtt.h" 56#include "lsquic_cubic.h" 57#include "lsquic_pacer.h" 58#include "lsquic_bw_sampler.h" 59#include "lsquic_minmax.h" 60#include "lsquic_bbr.h" 61#include "lsquic_adaptive_cc.h" 62#include "lsquic_set.h" 63#include "lsquic_conn_flow.h" 64#include "lsquic_sfcw.h" 65#include "lsquic_hash.h" 66#include "lsquic_conn.h" 67#include "lsquic_send_ctl.h" 68#include "lsquic_full_conn.h" 69#include "lsquic_util.h" 70#include "lsquic_qtags.h" 71#include "lsquic_enc_sess.h" 72#include "lsquic_mm.h" 73#include "lsquic_engine_public.h" 74#include "lsquic_eng_hist.h" 75#include "lsquic_ev_log.h" 76#include "lsquic_version.h" 77#include "lsquic_pr_queue.h" 78#include "lsquic_mini_conn.h" 79#include "lsquic_mini_conn_ietf.h" 80#include "lsquic_stock_shi.h" 81#include "lsquic_purga.h" 82#include "lsquic_tokgen.h" 83#include "lsquic_attq.h" 84#include "lsquic_min_heap.h" 85#include "lsquic_http1x_if.h" 86#include "lsquic_handshake.h" 87#include "lsquic_crand.h" 88#include "lsquic_ietf.h" 89#include "lsquic_handshake.h" 90 91#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 92#include "lsquic_logger.h" 93 94#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 95#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 96#endif 97 98#if LSQUIC_DEBUG_NEXT_ADV_TICK 99#include "lsquic_alarmset.h" 100#endif 101 102#define MIN(a, b) ((a) < (b) ? (a) : (b)) 103 104/* The batch of outgoing packets grows and shrinks dynamically */ 105#define MAX_OUT_BATCH_SIZE 1024 106#define MIN_OUT_BATCH_SIZE 4 107#define INITIAL_OUT_BATCH_SIZE 32 108 109struct out_batch 110{ 111 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 112 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 113 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 114 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 115 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 116}; 117 118typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 119 120static void 121process_connections (struct lsquic_engine *engine, conn_iter_f iter, 122 lsquic_time_t now); 123 124static void 125engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 126 127static lsquic_conn_t * 128engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 129 enum lsquic_conn_flags flag); 130 131static void 132force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 133 134#if LSQUIC_COUNT_ENGINE_CALLS 135#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 136#else 137#define ENGINE_CALLS_INCR(e) 138#endif 139 140/* Nested calls to some LSQUIC functions are not supported. Functions that 141 * iterate over connections cannot be nested. 142 */ 143#define ENGINE_IN(e) do { \ 144 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 145 (e)->pub.enp_flags |= ENPUB_PROC; \ 146 ENGINE_CALLS_INCR(e); \ 147} while (0) 148 149#define ENGINE_OUT(e) do { \ 150 assert((e)->pub.enp_flags & ENPUB_PROC); \ 151 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 152} while (0) 153 154/* A connection can be referenced from one of six places: 155 * 156 * 1. A hash is used to find connections in order to dispatch an incoming 157 * packet. Connections can be hashed by CIDs or by address. In the 158 * former case, each connection has one or more mappings in the hash 159 * table. IETF QUIC connections have up to eight (in our implementation) 160 * source CIDs and each of those would have a mapping. In client mode, 161 * depending on QUIC versions and options selected, it is may be 162 * necessary to hash connections by address, in which case incoming 163 * packets are delivered to connections based on the address. 164 * 165 * 2. Outgoing queue. 166 * 167 * 3. Tickable queue 168 * 169 * 4. Advisory Tick Time queue. 170 * 171 * 5. Closing connections queue. This is a transient queue -- it only 172 * exists for the duration of process_connections() function call. 173 * 174 * 6. Ticked connections queue. Another transient queue, similar to (5). 175 * 176 * The idea is to destroy the connection when it is no longer referenced. 177 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 178 * that case, the connection is referenced from two places: (2) and (5). 179 * After its packets are sent, it is only referenced in (5), and at the 180 * end of the function call, when it is removed from (5), reference count 181 * goes to zero and the connection is destroyed. If not all packets can 182 * be sent, at the end of the function call, the connection is referenced 183 * by (2) and will only be removed once all outgoing packets have been 184 * sent. 185 */ 186#define CONN_REF_FLAGS (LSCONN_HASHED \ 187 |LSCONN_HAS_OUTGOING \ 188 |LSCONN_TICKABLE \ 189 |LSCONN_TICKED \ 190 |LSCONN_CLOSING \ 191 |LSCONN_ATTQ) 192 193 194 195 196struct cid_update_batch 197{ 198 lsquic_cids_update_f cub_update_cids; 199 void *cub_update_ctx; 200 unsigned cub_count; 201 lsquic_cid_t cub_cids[20]; 202 void *cub_peer_ctxs[20]; 203}; 204 205static void 206cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 207 208 209struct lsquic_engine 210{ 211 struct lsquic_engine_public pub; 212 enum { 213 ENG_SERVER = LSENG_SERVER, 214 ENG_HTTP = LSENG_HTTP, 215 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 216 ENG_PAST_DEADLINE 217 = (1 << 8), /* Previous call to a processing 218 * function went past time threshold. 219 */ 220 ENG_CONNS_BY_ADDR 221 = (1 << 9), /* Connections are hashed by address */ 222#ifndef NDEBUG 223 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 224#endif 225#if CAN_LOSE_PACKETS 226 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 227#endif 228#ifndef NDEBUG 229 ENG_DTOR = (1 << 26), /* Engine destructor */ 230#endif 231 } flags; 232 lsquic_packets_out_f packets_out; 233 void *packets_out_ctx; 234 lsquic_cids_update_f report_new_scids; 235 lsquic_cids_update_f report_live_scids; 236 lsquic_cids_update_f report_old_scids; 237 void *scids_ctx; 238 struct lsquic_hash *conns_hash; 239 struct min_heap conns_tickable; 240 struct min_heap conns_out; 241 struct eng_hist history; 242 unsigned batch_size; 243 struct lsquic_conn *curr_conn; 244 struct pr_queue *pr_queue; 245 struct attq *attq; 246 /* Track time last time a packet was sent to give new connections 247 * priority lower than that of existing connections. 248 */ 249 lsquic_time_t last_sent; 250#if CAN_LOSE_PACKETS 251 regex_t lose_packets_re; 252 const char *lose_packets_str; 253#endif 254 unsigned n_conns; 255 lsquic_time_t deadline; 256 lsquic_time_t resume_sending_at; 257 unsigned mini_conns_count; 258 struct lsquic_purga *purga; 259#if LSQUIC_CONN_STATS 260 struct { 261 unsigned conns; 262 } stats; 263 struct conn_stats conn_stats_sum; 264 FILE *stats_fh; 265#endif 266 struct cid_update_batch new_scids; 267 struct out_batch out_batch; 268#if LSQUIC_COUNT_ENGINE_CALLS 269 unsigned long n_engine_calls; 270#endif 271#if LSQUIC_DEBUG_NEXT_ADV_TICK 272 uintptr_t last_logged_conn; 273 unsigned last_logged_ae_why; 274 int last_tick_diff; 275#endif 276 struct crand crand; 277 EVP_AEAD_CTX retry_aead_ctx[N_IETF_RETRY_VERSIONS]; 278}; 279 280 281void 282lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 283 unsigned flags) 284{ 285 memset(settings, 0, sizeof(*settings)); 286 settings->es_versions = LSQUIC_DF_VERSIONS; 287 if (flags & ENG_SERVER) 288 { 289 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 290 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 291 settings->es_init_max_data 292 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 293 settings->es_init_max_stream_data_bidi_remote 294 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 295 settings->es_init_max_stream_data_bidi_local 296 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 297 settings->es_init_max_stream_data_uni 298 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 299 settings->es_init_max_streams_uni 300 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 301 settings->es_ping_period = 0; 302 settings->es_noprogress_timeout 303 = LSQUIC_DF_NOPROGRESS_TIMEOUT_SERVER; 304 } 305 else 306 { 307 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 308 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 309 settings->es_init_max_data 310 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 311 settings->es_init_max_stream_data_bidi_remote 312 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 313 settings->es_init_max_stream_data_bidi_local 314 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 315 settings->es_init_max_stream_data_uni 316 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 317 settings->es_init_max_streams_uni 318 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 319 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 320 settings->es_noprogress_timeout 321 = LSQUIC_DF_NOPROGRESS_TIMEOUT_CLIENT; 322 } 323 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 324 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 325 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 326 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 327 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 328 settings->es_max_header_list_size 329 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 330 settings->es_ua = LSQUIC_DF_UA; 331 settings->es_ecn = LSQUIC_DF_ECN; 332 333 settings->es_pdmd = QTAG_X509; 334 settings->es_aead = QTAG_AESG; 335 settings->es_kexs = QTAG_C255; 336 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 337 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 338 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 339 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 340 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 341 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 342 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 343 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 344 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 345 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 346 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 347 settings->es_sttl = LSQUIC_DF_STTL; 348 settings->es_init_max_streams_bidi 349 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 350 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 351 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 352 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 353 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 354 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 355 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 356 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 357 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 358 settings->es_spin = LSQUIC_DF_SPIN; 359 settings->es_delayed_acks = LSQUIC_DF_DELAYED_ACKS; 360 settings->es_timestamps = LSQUIC_DF_TIMESTAMPS; 361 settings->es_grease_quic_bit = LSQUIC_DF_GREASE_QUIC_BIT; 362 settings->es_mtu_probe_timer = LSQUIC_DF_MTU_PROBE_TIMER; 363 settings->es_dplpmtud = LSQUIC_DF_DPLPMTUD; 364 settings->es_cc_algo = LSQUIC_DF_CC_ALGO; 365 settings->es_cc_rtt_thresh = LSQUIC_DF_CC_RTT_THRESH; 366 settings->es_optimistic_nat = LSQUIC_DF_OPTIMISTIC_NAT; 367} 368 369 370/* Note: if returning an error, err_buf must be valid if non-NULL */ 371int 372lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 373 unsigned flags, 374 char *err_buf, size_t err_buf_sz) 375{ 376 if (settings->es_cfcw < LSQUIC_MIN_FCW || 377 settings->es_sfcw < LSQUIC_MIN_FCW) 378 { 379 if (err_buf) 380 snprintf(err_buf, err_buf_sz, "%s", 381 "flow control window set too low"); 382 return -1; 383 } 384 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 385 { 386 if (err_buf) 387 snprintf(err_buf, err_buf_sz, "%s", 388 "No supported QUIC versions specified"); 389 return -1; 390 } 391 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 392 { 393 if (err_buf) 394 snprintf(err_buf, err_buf_sz, "%s", 395 "one or more unsupported QUIC version is specified"); 396 return -1; 397 } 398 if (flags & ENG_SERVER) 399 { 400 if (settings->es_handshake_to > 401 MAX_MINI_CONN_LIFESPAN_IN_USEC) 402 { 403 if (err_buf) 404 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 405 " usec is too large. The maximum for server is %u usec", 406 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 407 return -1; 408 } 409 } 410 if (settings->es_idle_timeout > 600) 411 { 412 if (err_buf) 413 snprintf(err_buf, err_buf_sz, "%s", 414 "The maximum value of idle timeout is 600 seconds"); 415 return -1; 416 } 417 if (settings->es_scid_len > MAX_CID_LEN) 418 { 419 if (err_buf) 420 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 421 "bytes long; it must be between 0 and %u.", 422 settings->es_scid_len, MAX_CID_LEN); 423 return -1; 424 } 425 426 if (settings->es_cc_algo > 3) 427 { 428 if (err_buf) 429 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 430 "algorithm value %u", settings->es_cc_algo); 431 return -1; 432 } 433 434 if (!(settings->es_ql_bits >= 0 && settings->es_ql_bits <= 2)) 435 { 436 if (err_buf) 437 snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ", 438 settings->es_ql_bits); 439 return -1; 440 } 441 442 if (!(settings->es_spin == 0 || settings->es_spin == 1)) 443 { 444 if (err_buf) 445 snprintf(err_buf, err_buf_sz, "Invalid spin value %d", 446 settings->es_spin); 447 return -1; 448 } 449 450 if (settings->es_mtu_probe_timer && settings->es_mtu_probe_timer < 1000) 451 { 452 if (err_buf) 453 snprintf(err_buf, err_buf_sz, "mtu probe timer is too small: " 454 "%u ms", settings->es_mtu_probe_timer); 455 return -1; 456 } 457 458 return 0; 459} 460 461 462static void 463free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 464{ 465 free(packet_data); 466} 467 468 469static void * 470malloc_buf (void *ctx, void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, unsigned short size, char is_ipv6) 471{ 472 return malloc(size); 473} 474 475 476static const struct lsquic_packout_mem_if stock_pmi = 477{ 478 malloc_buf, free_packet, free_packet, 479}; 480 481 482static int 483hash_conns_by_addr (const struct lsquic_engine *engine) 484{ 485 if (engine->flags & ENG_SERVER) 486 return 0; 487 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 488 return 1; 489 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 490 && engine->pub.enp_settings.es_support_tcid0) 491 return 1; 492 if (engine->pub.enp_settings.es_scid_len == 0) 493 return 1; 494 return 0; 495} 496 497 498lsquic_engine_t * 499lsquic_engine_new (unsigned flags, 500 const struct lsquic_engine_api *api) 501{ 502 lsquic_engine_t *engine; 503 size_t alpn_len; 504 unsigned i; 505 char err_buf[100]; 506 507 if (!api->ea_packets_out) 508 { 509 LSQ_ERROR("packets_out callback is not specified"); 510 return NULL; 511 } 512 513 if (!api->ea_stream_if) 514 { 515 LSQ_ERROR("stream interface is not specified"); 516 return NULL; 517 } 518 519 if (!(flags & LSENG_HTTP) && api->ea_alpn) 520 { 521 alpn_len = strlen(api->ea_alpn); 522 if (alpn_len < 1 || alpn_len > 255) 523 { 524 LSQ_ERROR("ALPN string length invalid: %zd bytes", alpn_len); 525 return NULL; 526 } 527 } 528 else 529 alpn_len = 0; 530 531 if (api->ea_settings && 532 0 != lsquic_engine_check_settings(api->ea_settings, flags, 533 err_buf, sizeof(err_buf))) 534 { 535 LSQ_ERROR("cannot create engine: %s", err_buf); 536 return NULL; 537 } 538 539 engine = calloc(1, sizeof(*engine)); 540 if (!engine) 541 return NULL; 542 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 543 { 544 free(engine); 545 return NULL; 546 } 547 if (api->ea_settings) 548 engine->pub.enp_settings = *api->ea_settings; 549 else 550 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 551 int tag_buf_len; 552 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 553 sizeof(engine->pub.enp_ver_tags_buf), 554 engine->pub.enp_settings.es_versions); 555 if (tag_buf_len <= 0) 556 { 557 LSQ_ERROR("cannot generate version tags buffer"); 558 free(engine); 559 return NULL; 560 } 561 engine->pub.enp_ver_tags_len = tag_buf_len; 562 engine->pub.enp_flags = ENPUB_CAN_SEND; 563 engine->pub.enp_stream_if = api->ea_stream_if; 564 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 565 566 engine->flags = flags; 567#ifndef NDEBUG 568 engine->flags |= ENG_COALESCE; 569#endif 570 engine->packets_out = api->ea_packets_out; 571 engine->packets_out_ctx = api->ea_packets_out_ctx; 572 engine->report_new_scids = api->ea_new_scids; 573 engine->report_live_scids = api->ea_live_scids; 574 engine->report_old_scids = api->ea_old_scids; 575 engine->scids_ctx = api->ea_cids_update_ctx; 576 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 577 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 578 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 579 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 580 581 if (api->ea_generate_scid) 582 engine->pub.enp_generate_scid = api->ea_generate_scid; 583 else 584 engine->pub.enp_generate_scid = lsquic_generate_scid; 585 586 if (api->ea_shi) 587 { 588 engine->pub.enp_shi = api->ea_shi; 589 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 590 } 591 else 592 { 593 engine->pub.enp_shi = &stock_shi; 594 engine->pub.enp_shi_ctx = lsquic_stock_shared_hash_new(); 595 if (!engine->pub.enp_shi_ctx) 596 { 597 free(engine); 598 return NULL; 599 } 600 } 601 if (api->ea_hsi_if) 602 { 603 engine->pub.enp_hsi_if = api->ea_hsi_if; 604 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 605 } 606 else 607 { 608 engine->pub.enp_hsi_if = lsquic_http1x_if; 609 engine->pub.enp_hsi_ctx = NULL; 610 } 611 if (api->ea_pmi) 612 { 613 engine->pub.enp_pmi = api->ea_pmi; 614 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 615 } 616 else 617 { 618 engine->pub.enp_pmi = &stock_pmi; 619 engine->pub.enp_pmi_ctx = NULL; 620 } 621 engine->pub.enp_verify_cert = api->ea_verify_cert; 622 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 623 engine->pub.enp_kli = api->ea_keylog_if; 624 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 625 engine->pub.enp_engine = engine; 626 if (hash_conns_by_addr(engine)) 627 engine->flags |= ENG_CONNS_BY_ADDR; 628 engine->conns_hash = lsquic_hash_create(); 629 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 630 if (!engine->pub.enp_tokgen) 631 return NULL; 632 engine->pub.enp_crand = &engine->crand; 633 if (engine->pub.enp_settings.es_noprogress_timeout) 634 engine->pub.enp_noprog_timeout 635 = engine->pub.enp_settings.es_noprogress_timeout * 1000000; 636 engine->pub.enp_mtu_probe_timer = 1000 637 * (engine->pub.enp_settings.es_mtu_probe_timer 638 ? engine->pub.enp_settings.es_mtu_probe_timer 639 : LSQUIC_DF_MTU_PROBE_TIMER); 640 if (flags & ENG_SERVER) 641 { 642 engine->pr_queue = lsquic_prq_create( 643 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 644 &engine->pub); 645 if (!engine->pr_queue) 646 { 647 lsquic_tg_destroy(engine->pub.enp_tokgen); 648 return NULL; 649 } 650 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 651 engine->report_old_scids, engine->scids_ctx); 652 if (!engine->purga) 653 { 654 lsquic_tg_destroy(engine->pub.enp_tokgen); 655 lsquic_prq_destroy(engine->pr_queue); 656 return NULL; 657 } 658 } 659 engine->attq = lsquic_attq_create(); 660 eng_hist_init(&engine->history); 661 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 662 if (engine->pub.enp_settings.es_honor_prst) 663 { 664 engine->pub.enp_srst_hash = lsquic_hash_create(); 665 if (!engine->pub.enp_srst_hash) 666 { 667 lsquic_engine_destroy(engine); 668 return NULL; 669 } 670 } 671 if ((flags & LSENG_SERVER) && 0 != lsquic_init_gquic_crypto(&engine->pub)) 672 { 673 lsquic_engine_destroy(engine); 674 return NULL; 675 } 676 677 if (alpn_len) 678 { 679 engine->pub.enp_alpn = malloc(alpn_len + 1); 680 if (!engine->pub.enp_alpn) 681 { 682 lsquic_engine_destroy(engine); 683 return NULL; 684 } 685 engine->pub.enp_alpn[0] = alpn_len; 686 memcpy(engine->pub.enp_alpn + 1, api->ea_alpn, alpn_len); 687 } 688 689 if (flags & LSENG_HTTP) 690 engine->pub.enp_flags |= ENPUB_HTTP; 691 692#ifndef NDEBUG 693 { 694 const char *env; 695 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 696#if CAN_LOSE_PACKETS 697 if (env) 698 { 699 if (0 != regcomp(&engine->lose_packets_re, env, 700 REG_EXTENDED|REG_NOSUB)) 701 { 702 LSQ_ERROR("could not compile lost packet regex `%s'", env); 703 return NULL; 704 } 705 engine->flags |= ENG_LOSE_PACKETS; 706 engine->lose_packets_str = env; 707 LSQ_WARN("will lose packets that match the following regex: %s", 708 env); 709 } 710#endif 711 env = getenv("LSQUIC_COALESCE"); 712 if (env) 713 { 714 engine->flags &= ~ENG_COALESCE; 715 if (atoi(env)) 716 { 717 engine->flags |= ENG_COALESCE; 718 LSQ_NOTICE("will coalesce packets"); 719 } 720 else 721 LSQ_NOTICE("will not coalesce packets"); 722 } 723 } 724#endif 725#if LSQUIC_CONN_STATS 726 engine->stats_fh = api->ea_stats_fh; 727#endif 728 for (i = 0; i < sizeof(engine->retry_aead_ctx) 729 / sizeof(engine->retry_aead_ctx[0]); ++i) 730 if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx[i], 731 EVP_aead_aes_128_gcm(), lsquic_retry_key_buf[i], 732 IETF_RETRY_KEY_SZ, 16, NULL)) 733 { 734 LSQ_ERROR("could not initialize retry AEAD ctx #%u", i); 735 lsquic_engine_destroy(engine); 736 return NULL; 737 } 738 engine->pub.enp_retry_aead_ctx = engine->retry_aead_ctx; 739 740 LSQ_INFO("instantiated engine"); 741 return engine; 742} 743 744 745#if LOG_PACKET_CHECKSUM 746static void 747log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 748 const unsigned char *buf, size_t bufsz) 749{ 750 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 751 (uint32_t) crc32(0, buf, bufsz)); 752} 753 754 755#endif 756 757 758static void 759grow_batch_size (struct lsquic_engine *engine) 760{ 761 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 762} 763 764 765static void 766shrink_batch_size (struct lsquic_engine *engine) 767{ 768 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 769} 770 771 772struct cce_cid_iter 773{ 774 const struct lsquic_conn *conn; 775 unsigned todo, n; 776}; 777 778 779static struct conn_cid_elem * 780cce_iter_next (struct cce_cid_iter *citer) 781{ 782 struct conn_cid_elem *cce; 783 784 while (citer->todo) 785 if (citer->todo & (1 << citer->n)) 786 { 787 citer->todo &= ~(1 << citer->n); 788 cce = &citer->conn->cn_cces[ citer->n++ ]; 789 if (!(cce->cce_flags & CCE_PORT)) 790 return cce; 791 } 792 else 793 ++citer->n; 794 795 return NULL; 796} 797 798 799static struct conn_cid_elem * 800cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 801{ 802 citer->conn = conn; 803 citer->todo = conn->cn_cces_mask; 804 citer->n = 0; 805 return cce_iter_next(citer); 806} 807 808 809#if LSQUIC_CONN_STATS 810void 811update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 812{ 813 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 814 const unsigned long *src; 815 const struct conn_stats *stats; 816 unsigned i; 817 818 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 819 { 820 ++engine->stats.conns; 821 src = (unsigned long *) stats; 822 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 823 dst[i] += src[i]; 824 } 825} 826 827 828#endif 829 830 831/* Wrapper to make sure important things occur before the connection is 832 * really destroyed. 833 */ 834static void 835destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 836 lsquic_time_t now) 837{ 838 struct cce_cid_iter citer; 839 const struct conn_cid_elem *cce; 840 lsquic_time_t drain_time; 841 struct purga_el *puel; 842 843 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 844 if (engine->purga 845 /* Blacklist all CIDs except for promoted mini connections */ 846 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 847 != (LSCONN_MINI|LSCONN_PROMOTED)) 848 { 849 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 850 && conn->cn_if->ci_drain_time && 851 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 852 { 853 for (cce = cce_iter_first(&citer, conn); cce; 854 cce = cce_iter_next(&citer)) 855 { 856 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 857 lsquic_conn_get_peer_ctx(conn, NULL), 858 PUTY_CONN_DRAIN, now); 859 if (puel) 860 puel->puel_time = now + drain_time; 861 } 862 } 863 else 864 { 865 for (cce = cce_iter_first(&citer, conn); cce; 866 cce = cce_iter_next(&citer)) 867 { 868 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 869 lsquic_conn_get_peer_ctx(conn, NULL), 870 PUTY_CONN_DELETED, now); 871 if (puel) 872 { 873 puel->puel_time = now; 874 puel->puel_count = 0; 875 } 876 } 877 } 878 } 879#if LSQUIC_CONN_STATS 880 update_stats_sum(engine, conn); 881#endif 882 --engine->n_conns; 883 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 884 conn->cn_if->ci_destroy(conn); 885} 886 887 888static int 889maybe_grow_conn_heaps (struct lsquic_engine *engine) 890{ 891 struct min_heap_elem *els; 892 unsigned count; 893 894 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 895 return 0; /* Nothing to do */ 896 897 if (lsquic_mh_nalloc(&engine->conns_tickable)) 898 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 899 else 900 count = 8; 901 902 els = malloc(sizeof(els[0]) * count); 903 if (!els) 904 { 905 LSQ_ERROR("%s: malloc failed", __func__); 906 return -1; 907 } 908 909 LSQ_DEBUG("grew heaps to %u elements", count / 2); 910 memcpy(&els[0], engine->conns_tickable.mh_elems, 911 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 912 memcpy(&els[count / 2], engine->conns_out.mh_elems, 913 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 914 free(engine->conns_tickable.mh_elems); 915 engine->conns_tickable.mh_elems = els; 916 engine->conns_out.mh_elems = &els[count / 2]; 917 engine->conns_tickable.mh_nalloc = count / 2; 918 engine->conns_out.mh_nalloc = count / 2; 919 return 0; 920} 921 922 923static void 924remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 925 unsigned todo) 926{ 927 unsigned n; 928 929 for (n = 0; todo; todo &= ~(1 << n++)) 930 if ((todo & (1 << n)) && 931 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 932 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 933} 934 935 936static void 937remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 938{ 939 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 940} 941 942 943static void 944cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 945 946 947static int 948insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 949 void *peer_ctx) 950{ 951 struct conn_cid_elem *cce; 952 unsigned todo, done, n; 953 954 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 955 if (todo & (1 << n)) 956 { 957 cce = &conn->cn_cces[n]; 958 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 959 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 960 cce->cce_cid.len, conn, &cce->cce_hash_el)) 961 done |= 1 << n; 962 else 963 goto err; 964 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 965 { 966 cce->cce_flags |= CCE_REG; 967 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 968 } 969 } 970 971 return 0; 972 973 err: 974 remove_cces_from_hash(engine->conns_hash, conn, done); 975 return -1; 976} 977 978 979static lsquic_conn_t * 980new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 981 lsquic_time_t now) 982{ 983 const lsquic_cid_t *cid; 984 server_conn_ctor_f ctor; 985 lsquic_conn_t *conn; 986 unsigned flags; 987 if (0 != maybe_grow_conn_heaps(engine)) 988 return NULL; 989 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 990 991 if (mini_conn->cn_flags & LSCONN_IETF) 992 ctor = lsquic_ietf_full_conn_server_new; 993 else 994 ctor = lsquic_gquic_full_conn_server_new; 995 996 conn = ctor(&engine->pub, flags, mini_conn); 997 if (!conn) 998 { 999 /* Otherwise, full_conn_server_new prints its own warnings */ 1000 if (ENOMEM == errno) 1001 { 1002 cid = lsquic_conn_log_cid(mini_conn); 1003 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 1004 CID_BITS(cid), strerror(errno)); 1005 } 1006 return NULL; 1007 } 1008 ++engine->n_conns; 1009 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 1010 { 1011 cid = lsquic_conn_log_cid(conn); 1012 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1013 CID_BITS(cid)); 1014 destroy_conn(engine, conn, now); 1015 return NULL; 1016 } 1017 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1018 conn->cn_flags |= LSCONN_HASHED; 1019 return conn; 1020} 1021 1022 1023static enum 1024{ 1025 VER_NOT_SPECIFIED, 1026 VER_SUPPORTED, 1027 VER_UNSUPPORTED, 1028} 1029 1030 1031version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 1032 enum lsquic_version *pversion) 1033{ 1034 lsquic_ver_tag_t ver_tag; 1035 enum lsquic_version version; 1036 1037 if (!packet_in->pi_quic_ver) 1038 { 1039 LSQ_DEBUG("packet does not specify version"); 1040 return VER_NOT_SPECIFIED; 1041 } 1042 1043 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 1044 version = lsquic_tag2ver(ver_tag); 1045 if (version < N_LSQVER) 1046 { 1047 if (engine->pub.enp_settings.es_versions & (1 << version)) 1048 { 1049 LSQ_DEBUG("client-supplied version %s is supported", 1050 lsquic_ver2str[version]); 1051 *pversion = version; 1052 return VER_SUPPORTED; 1053 } 1054 else 1055 LSQ_DEBUG("client-supplied version %s is not supported", 1056 lsquic_ver2str[version]); 1057 } 1058 else 1059 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 1060 ver_tag); 1061 1062 return VER_UNSUPPORTED; 1063} 1064 1065 1066static void 1067schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 1068 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 1069 const struct sockaddr *sa_peer, void *peer_ctx) 1070{ 1071 assert(engine->pr_queue); 1072 if (0 == lsquic_prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 1073 sa_local, sa_peer)) 1074 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 1075 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 1076 else 1077 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 1078} 1079 1080 1081static unsigned short 1082sa2port (const struct sockaddr *sa) 1083{ 1084 if (sa->sa_family == AF_INET) 1085 { 1086 struct sockaddr_in *const sa4 = (void *) sa; 1087 return sa4->sin_port; 1088 } 1089 else 1090 { 1091 struct sockaddr_in6 *const sa6 = (void *) sa; 1092 return sa6->sin6_port; 1093 } 1094} 1095 1096 1097static struct lsquic_hash_elem * 1098find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 1099{ 1100 unsigned short port; 1101 1102 port = sa2port(sa); 1103 return lsquic_hash_find(hash, &port, sizeof(port)); 1104} 1105 1106 1107static lsquic_conn_t * 1108find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1109 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 1110{ 1111 struct lsquic_hash_elem *el; 1112 lsquic_conn_t *conn; 1113 1114 if (engine->flags & ENG_CONNS_BY_ADDR) 1115 el = find_conn_by_addr(engine->conns_hash, sa_local); 1116 else if (packet_in->pi_flags & PI_CONN_ID) 1117 el = lsquic_hash_find(engine->conns_hash, 1118 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1119 else 1120 { 1121 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1122 return NULL; 1123 } 1124 1125 if (!el) 1126 return NULL; 1127 1128 conn = lsquic_hashelem_getdata(el); 1129 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1130 if ((engine->flags & ENG_CONNS_BY_ADDR) 1131 && !(conn->cn_flags & LSCONN_IETF) 1132 && (packet_in->pi_flags & PI_CONN_ID) 1133 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1134 { 1135 LSQ_DEBUG("connection IDs do not match"); 1136 return NULL; 1137 } 1138 1139 return conn; 1140} 1141 1142 1143static lsquic_conn_t * 1144find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1145 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1146 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1147{ 1148 struct lsquic_hash_elem *el; 1149 struct purga_el *puel; 1150 lsquic_conn_t *conn; 1151 1152 if (!(packet_in->pi_flags & PI_CONN_ID)) 1153 { 1154 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1155 return NULL; 1156 } 1157 el = lsquic_hash_find(engine->conns_hash, 1158 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1159 1160 if (el) 1161 { 1162 conn = lsquic_hashelem_getdata(el); 1163 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1164 return conn; 1165 } 1166 1167 if (engine->flags & ENG_COOLDOWN) 1168 { /* Do not create incoming connections during cooldown */ 1169 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1170 return NULL; 1171 } 1172 1173 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1174 { 1175 LSQ_DEBUG("reached limit of %u inchoate connections", 1176 engine->pub.enp_settings.es_max_inchoate); 1177 return NULL; 1178 } 1179 1180 1181 if (engine->purga 1182 && (puel = lsquic_purga_contains(engine->purga, 1183 &packet_in->pi_conn_id), puel)) 1184 { 1185 switch (puel->puel_type) 1186 { 1187 case PUTY_CID_RETIRED: 1188 LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet", 1189 CID_BITS(&packet_in->pi_conn_id)); 1190 return NULL; 1191 case PUTY_CONN_DRAIN: 1192 LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64, 1193 puel->puel_time, packet_in->pi_received); 1194 if (puel->puel_time > packet_in->pi_received) 1195 { 1196 LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet", 1197 CID_BITS(&packet_in->pi_conn_id)); 1198 return NULL; 1199 } 1200 LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted", 1201 CID_BITS(&packet_in->pi_conn_id)); 1202 puel->puel_type = PUTY_CONN_DELETED; 1203 puel->puel_count = 0; 1204 puel->puel_time = 0; 1205 /* fall-through */ 1206 case PUTY_CONN_DELETED: 1207 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1208 CID_BITS(&packet_in->pi_conn_id)); 1209 if (puel->puel_time < packet_in->pi_received) 1210 { 1211 puel->puel_time = packet_in->pi_received 1212 /* Exponential back-off */ 1213 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1214 ++puel->puel_count; 1215 goto maybe_send_prst; 1216 } 1217 return NULL; 1218 default: 1219 assert(0); 1220 return NULL; 1221 } 1222 } 1223 1224 if (engine->pub.enp_settings.es_send_prst 1225 && !(packet_in->pi_flags & PI_GQUIC) 1226 && HETY_NOT_SET == packet_in->pi_header_type) 1227 goto maybe_send_prst; 1228 1229 if (0 != maybe_grow_conn_heaps(engine)) 1230 return NULL; 1231 1232 const struct parse_funcs *pf; 1233 enum lsquic_version version; 1234 switch (version_matches(engine, packet_in, &version)) 1235 { 1236 case VER_UNSUPPORTED: 1237 if (engine->flags & ENG_SERVER) 1238 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1239 sa_local, sa_peer, peer_ctx); 1240 return NULL; 1241 case VER_NOT_SPECIFIED: 1242 maybe_send_prst: 1243 if ((engine->flags & ENG_SERVER) && 1244 engine->pub.enp_settings.es_send_prst) 1245 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1246 sa_local, sa_peer, peer_ctx); 1247 return NULL; 1248 case VER_SUPPORTED: 1249 pf = select_pf_by_ver(version); 1250 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1251 break; 1252 } 1253 1254 1255 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1256 { 1257 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1258 sa_peer->sa_family == AF_INET, NULL, packet_in_size); 1259 } 1260 else 1261 { 1262 conn = lsquic_mini_conn_new(&engine->pub, packet_in, version); 1263 } 1264 if (!conn) 1265 return NULL; 1266 ++engine->mini_conns_count; 1267 ++engine->n_conns; 1268 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1269 { 1270 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1271 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1272 CID_BITS(cid)); 1273 destroy_conn(engine, conn, packet_in->pi_received); 1274 return NULL; 1275 } 1276 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1277 conn->cn_flags |= LSCONN_HASHED; 1278 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1279 conn->cn_last_sent = engine->last_sent; 1280 return conn; 1281} 1282 1283 1284lsquic_conn_t * 1285lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1286 const lsquic_cid_t *cid) 1287{ 1288 struct lsquic_hash_elem *el; 1289 lsquic_conn_t *conn = NULL; 1290 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1291 1292 if (el) 1293 conn = lsquic_hashelem_getdata(el); 1294 return conn; 1295} 1296 1297 1298#if !defined(NDEBUG) && __GNUC__ 1299__attribute__((weak)) 1300#endif 1301void 1302lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1303 lsquic_conn_t *conn) 1304{ 1305 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1306 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1307 { 1308 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1309 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1310 engine_incref_conn(conn, LSCONN_TICKABLE); 1311 } 1312} 1313 1314 1315void 1316lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1317 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1318{ 1319 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1320 if (conn->cn_flags & LSCONN_TICKABLE) 1321 { 1322 /* Optimization: no need to add the connection to the Advisory Tick 1323 * Time Queue: it is about to be ticked, after which it its next tick 1324 * time may be queried again. 1325 */; 1326 } 1327 else if (conn->cn_flags & LSCONN_ATTQ) 1328 { 1329 if (lsquic_conn_adv_time(conn) != tick_time) 1330 { 1331 lsquic_attq_remove(engine->attq, conn); 1332 if (0 != lsquic_attq_add(engine->attq, conn, tick_time, why)) 1333 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1334 } 1335 } 1336 else if (0 == lsquic_attq_add(engine->attq, conn, tick_time, why)) 1337 engine_incref_conn(conn, LSCONN_ATTQ); 1338} 1339 1340 1341static struct lsquic_conn * 1342find_conn_by_srst (struct lsquic_engine *engine, 1343 const struct lsquic_packet_in *packet_in) 1344{ 1345 struct lsquic_hash_elem *el; 1346 struct lsquic_conn *conn; 1347 1348 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1349 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1350 return NULL; 1351 1352 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1353 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1354 IQUIC_SRESET_TOKEN_SZ); 1355 if (!el) 1356 return NULL; 1357 1358 conn = lsquic_hashelem_getdata(el); 1359 return conn; 1360} 1361 1362 1363/* Return 0 if packet is being processed by a real connection (mini or full), 1364 * otherwise return 1. 1365 */ 1366static int 1367process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1368 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1369 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1370{ 1371 lsquic_conn_t *conn; 1372 const unsigned char *packet_in_data; 1373 1374 if (lsquic_packet_in_is_gquic_prst(packet_in) 1375 && !engine->pub.enp_settings.es_honor_prst) 1376 { 1377 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1378 LSQ_DEBUG("public reset packet: discarding"); 1379 return 1; 1380 } 1381 1382 if (engine->flags & ENG_SERVER) 1383 { 1384 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1385 sa_peer, peer_ctx, packet_in_size); 1386 if (!engine->curr_conn) 1387 engine->curr_conn = conn; 1388 } 1389 else 1390 conn = find_conn(engine, packet_in, ppstate, sa_local); 1391 1392 if (!conn) 1393 { 1394 if (engine->pub.enp_settings.es_honor_prst 1395 && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */ 1396 && !(packet_in->pi_flags & PI_GQUIC) 1397 && engine->pub.enp_srst_hash 1398 && (conn = find_conn_by_srst(engine, packet_in))) 1399 { 1400 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1401 CID_BITS(lsquic_conn_log_cid(conn))); 1402 conn->cn_if->ci_stateless_reset(conn); 1403 if (!(conn->cn_flags & LSCONN_TICKABLE) 1404 && conn->cn_if->ci_is_tickable(conn)) 1405 { 1406 lsquic_mh_insert(&engine->conns_tickable, conn, 1407 conn->cn_last_ticked); 1408 engine_incref_conn(conn, LSCONN_TICKABLE); 1409 } 1410 /* Even though the connection processes this packet, we return 1411 * 1 so that the caller does not add reset packet's random 1412 * bytes to the list of valid CIDs. 1413 */ 1414 } 1415 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1416 return 1; 1417 } 1418 1419 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1420 { 1421 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1422 engine_incref_conn(conn, LSCONN_TICKABLE); 1423 } 1424 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1425 sa_local, sa_peer); 1426 lsquic_packet_in_upref(packet_in); 1427#if LOG_PACKET_CHECKSUM 1428 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1429 packet_in->pi_data_sz); 1430#endif 1431 /* Note on QLog: 1432 * For the PACKET_RX QLog event, we are interested in logging these things: 1433 * - raw packet (however it comes in, encrypted or not) 1434 * - frames (list of frame names) 1435 * - packet type and number 1436 * - packet rx timestamp 1437 * 1438 * Since only some of these items are available at this code 1439 * juncture, we will wait until after the packet has been 1440 * decrypted (if necessary) and parsed to call the log functions. 1441 * 1442 * Once the PACKET_RX event is finally logged, the timestamp 1443 * will come from packet_in->pi_received. For correct sequential 1444 * ordering of QLog events, be sure to process the QLogs downstream. 1445 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1446 */ 1447 packet_in_data = packet_in->pi_data; 1448 packet_in_size = packet_in->pi_data_sz; 1449 conn->cn_if->ci_packet_in(conn, packet_in); 1450 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1451 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1452 return 0; 1453} 1454 1455 1456void 1457lsquic_engine_destroy (lsquic_engine_t *engine) 1458{ 1459 struct lsquic_hash_elem *el; 1460 lsquic_conn_t *conn; 1461 unsigned i; 1462 1463 LSQ_DEBUG("destroying engine"); 1464#ifndef NDEBUG 1465 engine->flags |= ENG_DTOR; 1466#endif 1467 1468 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1469 { 1470 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1471 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1472 } 1473 1474 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1475 { 1476 assert(conn->cn_flags & LSCONN_TICKABLE); 1477 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1478 } 1479 1480 for (el = lsquic_hash_first(engine->conns_hash); el; 1481 el = lsquic_hash_next(engine->conns_hash)) 1482 { 1483 conn = lsquic_hashelem_getdata(el); 1484 force_close_conn(engine, conn); 1485 } 1486 lsquic_hash_destroy(engine->conns_hash); 1487 1488 while ((conn = lsquic_attq_pop(engine->attq, UINT64_MAX))) 1489 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1490 1491 assert(0 == engine->n_conns); 1492 assert(0 == engine->mini_conns_count); 1493 if (engine->pr_queue) 1494 lsquic_prq_destroy(engine->pr_queue); 1495 if (engine->purga) 1496 lsquic_purga_destroy(engine->purga); 1497 lsquic_attq_destroy(engine->attq); 1498 1499 assert(0 == lsquic_mh_count(&engine->conns_out)); 1500 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1501 if (engine->pub.enp_shi == &stock_shi) 1502 lsquic_stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1503 lsquic_mm_cleanup(&engine->pub.enp_mm); 1504 free(engine->conns_tickable.mh_elems); 1505#if CAN_LOSE_PACKETS 1506 if (engine->flags & ENG_LOSE_PACKETS) 1507 regfree(&engine->lose_packets_re); 1508#endif 1509 if (engine->pub.enp_tokgen) 1510 lsquic_tg_destroy(engine->pub.enp_tokgen); 1511 if (engine->flags & LSENG_SERVER) 1512 lsquic_cleanup_gquic_crypto(&engine->pub); 1513#if LSQUIC_CONN_STATS 1514 if (engine->stats_fh) 1515 { 1516 const struct conn_stats *const stats = &engine->conn_stats_sum; 1517 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1518 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1519 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1520 fprintf(engine->stats_fh, "In:\n"); 1521 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1522 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1523 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1524 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1525 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1526 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1527 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1528 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1529 stats->in.headers_comp, stats->in.headers_uncomp, 1530 stats->in.headers_uncomp ? 1531 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1532 : 0); 1533 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1534 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1535 fprintf(engine->stats_fh, " ACK frames merged to new: %lu\n", stats->in.n_acks_merged[0]); 1536 fprintf(engine->stats_fh, " ACK frames merged to old: %lu\n", stats->in.n_acks_merged[1]); 1537 fprintf(engine->stats_fh, "Out:\n"); 1538 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1539 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1540 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1541 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1542 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1543 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1544 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1545 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1546 stats->out.headers_comp, stats->out.headers_uncomp, 1547 stats->out.headers_uncomp ? 1548 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1549 : 0); 1550 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1551 } 1552#endif 1553 if (engine->pub.enp_srst_hash) 1554 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1555#if LSQUIC_COUNT_ENGINE_CALLS 1556 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1557#endif 1558 for (i = 0; i < sizeof(engine->retry_aead_ctx) 1559 / sizeof(engine->retry_aead_ctx[0]); ++i) 1560 EVP_AEAD_CTX_cleanup(&engine->pub.enp_retry_aead_ctx[i]); 1561 free(engine->pub.enp_alpn); 1562 free(engine); 1563} 1564 1565 1566static struct conn_cid_elem * 1567find_free_cce (struct lsquic_conn *conn) 1568{ 1569 struct conn_cid_elem *cce; 1570 1571 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1572 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1573 return cce; 1574 1575 return NULL; 1576} 1577 1578 1579static int 1580add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1581 const struct sockaddr *local_sa, void *peer_ctx) 1582{ 1583 struct conn_cid_elem *cce; 1584 1585 if (engine->flags & ENG_CONNS_BY_ADDR) 1586 { 1587 cce = find_free_cce(conn); 1588 if (!cce) 1589 { 1590 LSQ_ERROR("cannot find free CCE"); 1591 return -1; 1592 } 1593 cce->cce_port = sa2port(local_sa); 1594 cce->cce_flags = CCE_PORT; 1595 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1596 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1597 { 1598 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1599 return 0; 1600 } 1601 else 1602 return -1; 1603 1604 } 1605 else 1606 return insert_conn_into_hash(engine, conn, peer_ctx); 1607} 1608 1609 1610lsquic_conn_t * 1611lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1612 const struct sockaddr *local_sa, 1613 const struct sockaddr *peer_sa, 1614 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1615 const char *hostname, unsigned short base_plpmtu, 1616 const unsigned char *sess_resume, size_t sess_resume_len, 1617 const unsigned char *token, size_t token_sz) 1618{ 1619 lsquic_conn_t *conn; 1620 unsigned flags, versions; 1621 int is_ipv4; 1622 1623 ENGINE_CALLS_INCR(engine); 1624 1625 if (engine->flags & ENG_SERVER) 1626 { 1627 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1628 goto err; 1629 } 1630 1631 if (engine->flags & ENG_CONNS_BY_ADDR 1632 && find_conn_by_addr(engine->conns_hash, local_sa)) 1633 { 1634 LSQ_ERROR("cannot have more than one connection on the same port"); 1635 goto err; 1636 } 1637 1638 if (0 != maybe_grow_conn_heaps(engine)) 1639 return NULL; 1640 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1641 is_ipv4 = peer_sa->sa_family == AF_INET; 1642 if (sess_resume && sess_resume_len) 1643 { 1644 version = lsquic_sess_resume_version(sess_resume, sess_resume_len); 1645 if (version >= N_LSQVER) 1646 { 1647 LSQ_INFO("session resumption version is bad, won't use"); 1648 sess_resume = NULL; 1649 sess_resume_len = 0; 1650 } 1651 } 1652 if (version >= N_LSQVER) 1653 { 1654 if (version > N_LSQVER) 1655 LSQ_WARN("invalid version specified, engine will pick"); 1656 versions = engine->pub.enp_settings.es_versions; 1657 } 1658 else 1659 versions = 1u << version; 1660 if (versions & LSQUIC_IETF_VERSIONS) 1661 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1662 flags, hostname, base_plpmtu, 1663 is_ipv4, sess_resume, sess_resume_len, token, token_sz); 1664 else 1665 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1666 flags, hostname, base_plpmtu, is_ipv4, 1667 sess_resume, sess_resume_len); 1668 if (!conn) 1669 goto err; 1670 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1671 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1672 lsquic_ver2str[conn->cn_version]); 1673 ++engine->n_conns; 1674 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1675 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1676 { 1677 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1678 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1679 CID_BITS(cid)); 1680 destroy_conn(engine, conn, lsquic_time_now()); 1681 goto err; 1682 } 1683 assert(!(conn->cn_flags & 1684 (CONN_REF_FLAGS 1685 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1686 callbacks */ 1687 ))); 1688 conn->cn_flags |= LSCONN_HASHED; 1689 if (!(conn->cn_flags & LSCONN_TICKABLE)) 1690 { 1691 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1692 engine_incref_conn(conn, LSCONN_TICKABLE); 1693 } 1694 lsquic_conn_set_ctx(conn, conn_ctx); 1695 conn->cn_if->ci_client_call_on_new(conn); 1696 end: 1697 return conn; 1698 err: 1699 conn = NULL; 1700 goto end; 1701} 1702 1703 1704static void 1705remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1706{ 1707 remove_all_cces_from_hash(engine->conns_hash, conn); 1708 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1709} 1710 1711 1712static void 1713refflags2str (enum lsquic_conn_flags flags, char s[7]) 1714{ 1715 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1716 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1717 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1718 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1719 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1720 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1721 *s = '\0'; 1722} 1723 1724 1725static void 1726engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1727{ 1728 char str[2][7]; 1729 assert(flag & CONN_REF_FLAGS); 1730 assert(!(conn->cn_flags & flag)); 1731 conn->cn_flags |= flag; 1732 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1733 CID_BITS(lsquic_conn_log_cid(conn)), 1734 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1735 (refflags2str(conn->cn_flags, str[1]), str[1])); 1736} 1737 1738 1739static lsquic_conn_t * 1740engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1741 enum lsquic_conn_flags flags) 1742{ 1743 char str[2][7]; 1744 lsquic_time_t now; 1745 assert(flags & CONN_REF_FLAGS); 1746 assert(conn->cn_flags & flags); 1747#ifndef NDEBUG 1748 if (flags & LSCONN_CLOSING) 1749 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1750#endif 1751 conn->cn_flags &= ~flags; 1752 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1753 CID_BITS(lsquic_conn_log_cid(conn)), 1754 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1755 (refflags2str(conn->cn_flags, str[1]), str[1])); 1756 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1757 { 1758 now = lsquic_time_now(); 1759 if (conn->cn_flags & LSCONN_MINI) 1760 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1761 else 1762 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1763 destroy_conn(engine, conn, now); 1764 return NULL; 1765 } 1766 else 1767 return conn; 1768} 1769 1770 1771/* This is not a general-purpose function. Only call from engine dtor. */ 1772static void 1773force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1774{ 1775 assert(engine->flags & ENG_DTOR); 1776 const enum lsquic_conn_flags flags = conn->cn_flags; 1777 assert(conn->cn_flags & CONN_REF_FLAGS); 1778 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1779 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1780 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1781 if (flags & LSCONN_ATTQ) 1782 { 1783 lsquic_attq_remove(engine->attq, conn); 1784 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1785 } 1786 if (flags & LSCONN_HASHED) 1787 remove_conn_from_hash(engine, conn); 1788} 1789 1790 1791/* Iterator for tickable connections (those on the Tickable Queue). Before 1792 * a connection is returned, it is removed from the Advisory Tick Time queue 1793 * if necessary. 1794 */ 1795static lsquic_conn_t * 1796conn_iter_next_tickable (struct lsquic_engine *engine) 1797{ 1798 lsquic_conn_t *conn; 1799 1800 if (engine->flags & ENG_SERVER) 1801 while (1) 1802 { 1803 conn = lsquic_mh_pop(&engine->conns_tickable); 1804 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1805 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1806 else 1807 break; 1808 } 1809 else 1810 conn = lsquic_mh_pop(&engine->conns_tickable); 1811 1812 if (conn) 1813 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1814 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1815 { 1816 lsquic_attq_remove(engine->attq, conn); 1817 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1818 } 1819 1820 return conn; 1821} 1822 1823 1824static void 1825cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1826 void *update_ctx) 1827{ 1828 cub->cub_update_cids = update; 1829 cub->cub_update_ctx = update_ctx; 1830 cub->cub_count = 0; 1831} 1832 1833 1834static void 1835cub_flush (struct cid_update_batch *cub) 1836{ 1837 if (cub->cub_count > 0 && cub->cub_update_cids) 1838 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1839 cub->cub_cids, cub->cub_count); 1840 cub->cub_count = 0; 1841} 1842 1843 1844static void 1845cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1846{ 1847 cub->cub_cids [ cub->cub_count ] = *cid; 1848 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1849 ++cub->cub_count; 1850 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1851 cub_flush(cub); 1852} 1853 1854 1855/* Process registered CIDs */ 1856static void 1857cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1858{ 1859 struct cce_cid_iter citer; 1860 struct conn_cid_elem *cce; 1861 void *peer_ctx; 1862 1863 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1864 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1865 if (cce->cce_flags & CCE_REG) 1866 cub_add(cub, &cce->cce_cid, peer_ctx); 1867} 1868 1869 1870static void 1871drop_all_mini_conns (lsquic_engine_t *engine) 1872{ 1873 struct lsquic_hash_elem *el; 1874 lsquic_conn_t *conn; 1875 struct cid_update_batch cub; 1876 1877 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1878 1879 for (el = lsquic_hash_first(engine->conns_hash); el; 1880 el = lsquic_hash_next(engine->conns_hash)) 1881 { 1882 conn = lsquic_hashelem_getdata(el); 1883 if (conn->cn_flags & LSCONN_MINI) 1884 { 1885 /* If promoted, why is it still in this hash? */ 1886 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1887 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1888 cub_add_cids_from_cces(&cub, conn); 1889 remove_conn_from_hash(engine, conn); 1890 } 1891 } 1892 1893 cub_flush(&cub); 1894} 1895 1896 1897void 1898lsquic_engine_process_conns (lsquic_engine_t *engine) 1899{ 1900 lsquic_conn_t *conn; 1901 lsquic_time_t now; 1902 1903 ENGINE_IN(engine); 1904 1905 now = lsquic_time_now(); 1906 while ((conn = lsquic_attq_pop(engine->attq, now))) 1907 { 1908 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1909 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 1910 { 1911 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1912 engine_incref_conn(conn, LSCONN_TICKABLE); 1913 } 1914 } 1915 1916 process_connections(engine, conn_iter_next_tickable, now); 1917 ENGINE_OUT(engine); 1918} 1919 1920 1921static void 1922release_or_return_enc_data (struct lsquic_engine *engine, 1923 void (*pmi_rel_or_ret) (void *, void *, void *, char), 1924 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 1925{ 1926 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1927 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 1928 packet_out->po_flags &= ~PO_ENCRYPTED; 1929 packet_out->po_enc_data = NULL; 1930} 1931 1932 1933static void 1934release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1935 struct lsquic_packet_out *packet_out) 1936{ 1937 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 1938 conn, packet_out); 1939} 1940 1941 1942static void 1943return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1944 struct lsquic_packet_out *packet_out) 1945{ 1946 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 1947 conn, packet_out); 1948} 1949 1950 1951static int 1952copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 1953 struct lsquic_packet_out *packet_out) 1954{ 1955 int ipv6; 1956 1957 ipv6 = NP_IS_IPv6(packet_out->po_path); 1958 if (packet_out->po_flags & PO_ENCRYPTED) 1959 { 1960 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1961 && packet_out->po_data_sz == packet_out->po_enc_data_sz 1962 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 1963 packet_out->po_data_sz)) 1964 return 0; 1965 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1966 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 1967 goto copy; 1968 return_enc_data(engine, conn, packet_out); 1969 } 1970 1971 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 1972 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, conn->conn_ctx, 1973 packet_out->po_data_sz, ipv6); 1974 if (!packet_out->po_enc_data) 1975 { 1976 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 1977 "of size %hu", packet_out->po_data_sz); 1978 return -1; 1979 } 1980 1981 copy: 1982 memcpy(packet_out->po_enc_data, packet_out->po_data, 1983 packet_out->po_data_sz); 1984 packet_out->po_enc_data_sz = packet_out->po_data_sz; 1985 packet_out->po_sent_sz = packet_out->po_data_sz; 1986 packet_out->po_flags &= ~PO_IPv6; 1987 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 1988 1989 return 0; 1990} 1991 1992 1993STAILQ_HEAD(conns_stailq, lsquic_conn); 1994TAILQ_HEAD(conns_tailq, lsquic_conn); 1995 1996 1997struct conns_out_iter 1998{ 1999 struct min_heap *coi_heap; 2000 struct pr_queue *coi_prq; 2001 TAILQ_HEAD(, lsquic_conn) coi_active_list, 2002 coi_inactive_list; 2003 lsquic_conn_t *coi_next; 2004#ifndef NDEBUG 2005 lsquic_time_t coi_last_sent; 2006#endif 2007}; 2008 2009 2010static void 2011coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 2012{ 2013 iter->coi_heap = &engine->conns_out; 2014 iter->coi_prq = engine->pr_queue; 2015 iter->coi_next = NULL; 2016 TAILQ_INIT(&iter->coi_active_list); 2017 TAILQ_INIT(&iter->coi_inactive_list); 2018#ifndef NDEBUG 2019 iter->coi_last_sent = 0; 2020#endif 2021} 2022 2023 2024static lsquic_conn_t * 2025coi_next (struct conns_out_iter *iter) 2026{ 2027 lsquic_conn_t *conn; 2028 2029 if (lsquic_mh_count(iter->coi_heap) > 0) 2030 { 2031 conn = lsquic_mh_pop(iter->coi_heap); 2032 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2033 conn->cn_flags |= LSCONN_COI_ACTIVE; 2034#ifndef NDEBUG 2035 if (iter->coi_last_sent) 2036 assert(iter->coi_last_sent <= conn->cn_last_sent); 2037 iter->coi_last_sent = conn->cn_last_sent; 2038#endif 2039 return conn; 2040 } 2041 else if (iter->coi_prq && (conn = lsquic_prq_next_conn(iter->coi_prq))) 2042 { 2043 return conn; 2044 } 2045 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 2046 { 2047 iter->coi_prq = NULL; /* Save function call in previous conditional */ 2048 conn = iter->coi_next; 2049 if (!conn) 2050 conn = TAILQ_FIRST(&iter->coi_active_list); 2051 if (conn) 2052 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 2053 return conn; 2054 } 2055 else 2056 return NULL; 2057} 2058 2059 2060static void 2061coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2062{ 2063 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2064 { 2065 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 2066 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2067 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2068 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 2069 conn->cn_flags |= LSCONN_COI_INACTIVE; 2070 } 2071} 2072 2073 2074static void 2075coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2076{ 2077 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 2078 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2079 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2080 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2081 conn->cn_flags |= LSCONN_COI_ACTIVE; 2082} 2083 2084 2085static void 2086coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 2087{ 2088 lsquic_conn_t *conn; 2089 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 2090 { 2091 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2092 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2093 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 2094 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 2095 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 2096 else /* Closed connection gets one shot at sending packets */ 2097 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2098 } 2099 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 2100 { 2101 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2102 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2103 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2104 } 2105} 2106 2107 2108#if CAN_LOSE_PACKETS 2109static void 2110lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 2111 unsigned n) 2112{ 2113 const lsquic_cid_t *cid; 2114 struct iovec *iov; 2115 unsigned i; 2116 char packno_str[22]; 2117 2118 for (i = 0; i < n; ++i) 2119 { 2120 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 2121 batch->packets[i]->po_packno); 2122 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 2123 { 2124 for (iov = batch->outs[i].iov; iov < 2125 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 2126 batch->outs[i].iov->iov_len -= 1; 2127 cid = lsquic_conn_log_cid(batch->conns[i]); 2128 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 2129 CID_BITS(cid)); 2130 } 2131 } 2132} 2133 2134 2135#endif 2136 2137 2138#ifdef NDEBUG 2139#define CONST_BATCH const 2140#else 2141#define CONST_BATCH 2142#endif 2143 2144 2145struct send_batch_ctx { 2146 struct conns_stailq *closed_conns; 2147 struct conns_tailq *ticked_conns; 2148 struct conns_out_iter *conns_iter; 2149 CONST_BATCH struct out_batch *batch; 2150}; 2151 2152 2153static void 2154close_conn_immediately (struct lsquic_engine *engine, 2155 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2156{ 2157 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2158 if (!(conn->cn_flags & LSCONN_CLOSING)) 2159 { 2160 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2161 engine_incref_conn(conn, LSCONN_CLOSING); 2162 if (conn->cn_flags & LSCONN_HASHED) 2163 remove_conn_from_hash(engine, conn); 2164 } 2165 if (conn->cn_flags & LSCONN_TICKED) 2166 { 2167 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2168 engine_decref_conn(engine, conn, LSCONN_TICKED); 2169 } 2170} 2171 2172 2173static void 2174close_conn_on_send_error (struct lsquic_engine *engine, 2175 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2176{ 2177 const struct out_batch *batch = sb_ctx->batch; 2178 struct lsquic_conn *const conn = batch->conns[n]; 2179 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2180 2181 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2182 "src: %s; dst: %s; errno: %d", 2183 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2184 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2185 CID_BITS(lsquic_conn_log_cid(conn)), 2186 SA2STR(batch->outs[n].local_sa, buf[0]), 2187 SA2STR(batch->outs[n].dest_sa, buf[1]), 2188 e_val); 2189 if (conn->cn_flags & LSCONN_EVANESCENT) 2190 lsquic_prq_drop(conn); 2191 else 2192 close_conn_immediately(engine, sb_ctx, conn); 2193} 2194 2195 2196static void 2197apply_hp (struct conns_out_iter *iter) 2198{ 2199 struct lsquic_conn *conn; 2200 2201 TAILQ_FOREACH(conn, &iter->coi_active_list, cn_next_out) 2202 if (conn->cn_esf_c->esf_flush_encryption && conn->cn_enc_session) 2203 conn->cn_esf_c->esf_flush_encryption(conn->cn_enc_session); 2204 TAILQ_FOREACH(conn, &iter->coi_inactive_list, cn_next_out) 2205 if (conn->cn_esf_c->esf_flush_encryption && conn->cn_enc_session) 2206 conn->cn_esf_c->esf_flush_encryption(conn->cn_enc_session); 2207} 2208 2209 2210static unsigned 2211send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2212 unsigned n_to_send) 2213{ 2214 int n_sent, i, e_val; 2215 lsquic_time_t now; 2216 unsigned off, skip; 2217 size_t count; 2218 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2219 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2220 2221 apply_hp(sb_ctx->conns_iter); 2222#if CAN_LOSE_PACKETS 2223 if (engine->flags & ENG_LOSE_PACKETS) 2224 lose_matching_packets(engine, batch, n_to_send); 2225#endif 2226 skip = 0; 2227 restart_batch: 2228 /* Set sent time before the write to avoid underestimating RTT */ 2229 now = lsquic_time_now(); 2230 for (i = skip; i < (int) (n_to_send - skip); ++i) 2231 { 2232 off = batch->pack_off[i]; 2233 count = batch->outs[i].iovlen; 2234 assert(count > 0); 2235 packet_out = &batch->packets[off]; 2236 end = packet_out + count; 2237 do 2238 (*packet_out)->po_sent = now; 2239 while (++packet_out < end); 2240 } 2241 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs + skip, 2242 n_to_send - skip); 2243 e_val = errno; 2244 if (n_sent < (int) (n_to_send - skip) && e_val != EMSGSIZE) 2245 { 2246 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2247 engine->resume_sending_at = now + 1000000; 2248 LSQ_DEBUG("cannot send packets"); 2249 EV_LOG_GENERIC_EVENT("cannot send packets"); 2250 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2251 close_conn_on_send_error(engine, sb_ctx, 2252 n_sent < 0 ? 0 : n_sent, e_val); 2253 } 2254 if (n_sent >= 0) 2255 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, 2256 n_to_send - skip); 2257 else 2258 { 2259 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2260 n_sent = 0; 2261 } 2262 if (n_sent > 0) 2263 engine->last_sent = now + n_sent; 2264 for (i = skip; i < (int) (skip + n_sent); ++i) 2265 { 2266 eng_hist_inc(&engine->history, now, sl_packets_out); 2267 /* `i' is added to maintain relative order */ 2268 batch->conns[i]->cn_last_sent = now + i; 2269 2270 off = batch->pack_off[i]; 2271 count = batch->outs[i].iovlen; 2272 assert(count > 0); 2273 packet_out = &batch->packets[off]; 2274 end = packet_out + count; 2275 do 2276 { 2277#if LOG_PACKET_CHECKSUM 2278 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2279 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2280 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2281#endif 2282 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2283 *packet_out); 2284 /* Release packet out buffer as soon as the packet is sent 2285 * successfully. If not successfully sent, we hold on to 2286 * this buffer until the packet sending is attempted again 2287 * or until it times out and regenerated. 2288 */ 2289 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2290 release_enc_data(engine, batch->conns[i], *packet_out); 2291 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2292 *packet_out); 2293 } 2294 while (++packet_out < end); 2295 } 2296 if (i < (int) n_to_send && e_val == EMSGSIZE) 2297 { 2298 LSQ_DEBUG("packet #%d could not be sent out for being too large", i); 2299 if (batch->conns[i]->cn_if->ci_packet_too_large 2300 && batch->outs[i].iovlen == 1) 2301 { 2302 off = batch->pack_off[i]; 2303 packet_out = &batch->packets[off]; 2304 batch->conns[i]->cn_if->ci_packet_too_large(batch->conns[i], 2305 *packet_out); 2306 ++i; 2307 if (i < (int) n_to_send) 2308 { 2309 skip = i; 2310 LSQ_DEBUG("restart batch starting at packet #%u", skip); 2311 goto restart_batch; 2312 } 2313 n_sent = n_to_send; 2314 } 2315 else 2316 close_conn_on_send_error(engine, sb_ctx, i, e_val); 2317 } 2318 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2319 for ( ; i < (int) n_to_send; ++i) 2320 { 2321 off = batch->pack_off[i]; 2322 count = batch->outs[i].iovlen; 2323 assert(count > 0); 2324 packet_out = &batch->packets[off]; 2325 end = packet_out + count; 2326 do 2327 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2328 *packet_out); 2329 while (++packet_out < end); 2330 } 2331 /* Return packets to the connection in reverse order so that the packet 2332 * ordering is maintained. 2333 */ 2334 for (i = (int) n_to_send - 1; i >= (int) (skip + n_sent); --i) 2335 { 2336 off = batch->pack_off[i]; 2337 count = batch->outs[i].iovlen; 2338 assert(count > 0); 2339 packet_out = &batch->packets[off + count - 1]; 2340 end = &batch->packets[off - 1]; 2341 do 2342 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2343 *packet_out); 2344 while (--packet_out > end); 2345 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2346 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2347 } 2348 return skip + n_sent; 2349} 2350 2351 2352/* Return 1 if went past deadline, 0 otherwise */ 2353static int 2354check_deadline (lsquic_engine_t *engine) 2355{ 2356 if (engine->pub.enp_settings.es_proc_time_thresh && 2357 lsquic_time_now() > engine->deadline) 2358 { 2359 LSQ_INFO("went past threshold of %u usec, stop sending", 2360 engine->pub.enp_settings.es_proc_time_thresh); 2361 engine->flags |= ENG_PAST_DEADLINE; 2362 return 1; 2363 } 2364 else 2365 return 0; 2366} 2367 2368 2369static size_t 2370iov_size (const struct iovec *iov, const struct iovec *const end) 2371{ 2372 size_t size; 2373 2374 assert(iov < end); 2375 2376 size = 0; 2377 do 2378 size += iov->iov_len; 2379 while (++iov < end); 2380 2381 return size; 2382} 2383 2384 2385static void 2386send_packets_out (struct lsquic_engine *engine, 2387 struct conns_tailq *ticked_conns, 2388 struct conns_stailq *closed_conns) 2389{ 2390 unsigned n, w, n_sent, n_batches_sent; 2391 lsquic_packet_out_t *packet_out; 2392 struct lsquic_packet_out **packet; 2393 lsquic_conn_t *conn; 2394 struct out_batch *const batch = &engine->out_batch; 2395 struct iovec *iov, *packet_iov; 2396 struct conns_out_iter conns_iter; 2397 int shrink, deadline_exceeded; 2398 const struct send_batch_ctx sb_ctx = { 2399 closed_conns, 2400 ticked_conns, 2401 &conns_iter, 2402 &engine->out_batch, 2403 }; 2404 2405 coi_init(&conns_iter, engine); 2406 n_batches_sent = 0; 2407 n_sent = 0, n = 0; 2408 shrink = 0; 2409 deadline_exceeded = 0; 2410 iov = batch->iov; 2411 packet = batch->packets; 2412 2413 while ((conn = coi_next(&conns_iter))) 2414 { 2415 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2416 if (!packet_out) { 2417 /* Evanescent connection always has a packet to send: */ 2418 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2419 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2420 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2421 CID_BITS(lsquic_conn_log_cid(conn))); 2422 coi_deactivate(&conns_iter, conn); 2423 continue; 2424 } 2425 batch->outs[n].iov = packet_iov = iov; 2426 next_coa: 2427 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2428 { 2429 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2430 &engine->pub, conn, packet_out)) 2431 { 2432 case ENCPA_NOMEM: 2433 /* Send what we have and wait for a more opportune moment */ 2434 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2435 goto end_for; 2436 case ENCPA_BADCRYPT: 2437 /* This is pretty bad: close connection immediately */ 2438 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2439 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2440 CID_BITS(lsquic_conn_log_cid(conn))); 2441 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2442 { 2443 close_conn_immediately(engine, &sb_ctx, conn); 2444 coi_deactivate(&conns_iter, conn); 2445 } 2446 continue; 2447 case ENCPA_OK: 2448 break; 2449 } 2450 } 2451 else if ((packet_out->po_flags & PO_NOENCRYPT) 2452 && engine->pub.enp_pmi != &stock_pmi) 2453 { 2454 if (0 != copy_packet(engine, conn, packet_out)) 2455 { 2456 /* Copy can only fail if packet could not be allocated */ 2457 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2458 goto end_for; 2459 } 2460 } 2461 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2462 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2463 if (packet_out->po_flags & PO_ENCRYPTED) 2464 { 2465 iov->iov_base = packet_out->po_enc_data; 2466 iov->iov_len = packet_out->po_enc_data_sz; 2467 } 2468 else 2469 { 2470 iov->iov_base = packet_out->po_data; 2471 iov->iov_len = packet_out->po_data_sz; 2472 } 2473 if (packet_iov == iov) 2474 { 2475 batch->pack_off[n] = packet - batch->packets; 2476 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2477 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2478 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2479 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2480 batch->outs [n].conn_ctx = conn->conn_ctx; 2481 batch->conns [n] = conn; 2482 } 2483 *packet = packet_out; 2484 ++packet; 2485 ++iov; 2486 if ((conn->cn_flags & LSCONN_IETF) 2487 && ((1 << packet_out->po_header_type) 2488 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2489#ifndef NDEBUG 2490 && (engine->flags & ENG_COALESCE) 2491#endif 2492 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2493 { 2494 const struct to_coal to_coal = { 2495 .prev_packet = packet_out, 2496 .prev_sz_sum = iov_size(packet_iov, iov), 2497 }; 2498 packet_out = conn->cn_if->ci_next_packet_to_send(conn, &to_coal); 2499 if (packet_out) 2500 goto next_coa; 2501 } 2502 batch->outs [n].iovlen = iov - packet_iov; 2503 ++n; 2504 if (n == engine->batch_size 2505 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2506 { 2507 w = send_batch(engine, &sb_ctx, n); 2508 n = 0; 2509 iov = batch->iov; 2510 packet = batch->packets; 2511 ++n_batches_sent; 2512 n_sent += w; 2513 if (w < engine->batch_size) 2514 { 2515 shrink = 1; 2516 break; 2517 } 2518 deadline_exceeded = check_deadline(engine); 2519 if (deadline_exceeded) 2520 break; 2521 grow_batch_size(engine); 2522 } 2523 } 2524 end_for: 2525 2526 if (n > 0) { 2527 w = send_batch(engine, &sb_ctx, n); 2528 n_sent += w; 2529 shrink = w < n; 2530 ++n_batches_sent; 2531 } 2532 2533 if (shrink) 2534 shrink_batch_size(engine); 2535 else if (n_batches_sent > 1) 2536 { 2537 deadline_exceeded = check_deadline(engine); 2538 if (!deadline_exceeded) 2539 grow_batch_size(engine); 2540 } 2541 2542 coi_reheap(&conns_iter, engine); 2543 2544 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2545} 2546 2547 2548int 2549lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2550{ 2551 return lsquic_mh_count(&engine->conns_out) > 0 2552 || (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 2553 ; 2554} 2555 2556 2557static void 2558reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2559{ 2560 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2561 engine->flags &= ~ENG_PAST_DEADLINE; 2562} 2563 2564 2565static void 2566check_tickable_conns_again (struct lsquic_engine *engine) 2567{ 2568 struct lsquic_hash_elem *el; 2569 struct lsquic_conn *conn; 2570 unsigned count; 2571 2572 count = 0; 2573 for (el = lsquic_hash_first(engine->conns_hash); el; 2574 el = lsquic_hash_next(engine->conns_hash)) 2575 { 2576 conn = lsquic_hashelem_getdata(el); 2577 if (!(conn->cn_flags & LSCONN_TICKABLE) 2578 && conn->cn_if->ci_is_tickable(conn)) 2579 { 2580 lsquic_mh_insert(&engine->conns_tickable, conn, 2581 conn->cn_last_ticked); 2582 engine_incref_conn(conn, LSCONN_TICKABLE); 2583 ++count; 2584 } 2585 } 2586 LSQ_DEBUG("%u connection%s tickable again after sending has been " 2587 "re-enabled", count, count == 1 ? " is" : "s are"); 2588} 2589 2590 2591void 2592lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2593{ 2594 lsquic_conn_t *conn; 2595 struct conns_stailq closed_conns; 2596 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2597 struct cid_update_batch cub; 2598 2599 ENGINE_IN(engine); 2600 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2601 STAILQ_INIT(&closed_conns); 2602 reset_deadline(engine, lsquic_time_now()); 2603 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2604 { 2605 LSQ_DEBUG("can send again"); 2606 EV_LOG_GENERIC_EVENT("can send again"); 2607 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2608 check_tickable_conns_again(engine); 2609 } 2610 2611 send_packets_out(engine, &ticked_conns, &closed_conns); 2612 2613 while ((conn = STAILQ_FIRST(&closed_conns))) { 2614 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2615 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2616 cub_add_cids_from_cces(&cub, conn); 2617 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2618 } 2619 2620 cub_flush(&cub); 2621 ENGINE_OUT(engine); 2622} 2623 2624 2625static lsquic_conn_t * 2626next_new_full_conn (struct conns_stailq *new_full_conns) 2627{ 2628 lsquic_conn_t *conn; 2629 2630 conn = STAILQ_FIRST(new_full_conns); 2631 if (conn) 2632 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2633 return conn; 2634} 2635 2636 2637static void 2638process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2639 lsquic_time_t now) 2640{ 2641 lsquic_conn_t *conn; 2642 enum tick_st tick_st; 2643 unsigned i, why; 2644 lsquic_time_t next_tick_time; 2645 struct conns_stailq closed_conns; 2646 struct conns_tailq ticked_conns; 2647 struct conns_stailq new_full_conns; 2648 struct cid_update_batch cub_old, cub_live; 2649 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2650 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2651 2652 eng_hist_tick(&engine->history, now); 2653 2654 STAILQ_INIT(&closed_conns); 2655 TAILQ_INIT(&ticked_conns); 2656 reset_deadline(engine, now); 2657 STAILQ_INIT(&new_full_conns); 2658 2659 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2660 && now > engine->resume_sending_at) 2661 { 2662 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2663 "timeout"); 2664 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2665 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2666 } 2667 2668 i = 0; 2669 while ((conn = next_conn(engine)) 2670 || (conn = next_new_full_conn(&new_full_conns))) 2671 { 2672 tick_st = conn->cn_if->ci_tick(conn, now); 2673 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2674 if (tick_st & TICK_PROMOTE) 2675 { 2676 lsquic_conn_t *new_conn; 2677 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2678 "scheduled for promotion"); 2679 assert(conn->cn_flags & LSCONN_MINI); 2680 new_conn = new_full_conn_server(engine, conn, now); 2681 if (new_conn) 2682 { 2683 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2684 new_conn->cn_last_sent = engine->last_sent; 2685 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2686 conn->cn_flags |= LSCONN_PROMOTED; 2687 } 2688 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2689 } 2690 if (tick_st & TICK_SEND) 2691 { 2692 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2693 { 2694 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2695 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2696 } 2697 } 2698 if (tick_st & TICK_CLOSE) 2699 { 2700 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2701 engine_incref_conn(conn, LSCONN_CLOSING); 2702 if (conn->cn_flags & LSCONN_HASHED) 2703 remove_conn_from_hash(engine, conn); 2704 } 2705 else 2706 { 2707 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2708 engine_incref_conn(conn, LSCONN_TICKED); 2709 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2710 && conn->cn_if->ci_report_live(conn, now)) 2711 cub_add_cids_from_cces(&cub_live, conn); 2712 } 2713 } 2714 2715 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2716 && lsquic_engine_has_unsent_packets(engine)) 2717 send_packets_out(engine, &ticked_conns, &closed_conns); 2718 2719 while ((conn = STAILQ_FIRST(&closed_conns))) { 2720 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2721 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2722 cub_add_cids_from_cces(&cub_old, conn); 2723 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2724 } 2725 2726 while ((conn = TAILQ_FIRST(&ticked_conns))) 2727 { 2728 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2729 engine_decref_conn(engine, conn, LSCONN_TICKED); 2730 if (!(conn->cn_flags & LSCONN_TICKABLE) 2731 && conn->cn_if->ci_is_tickable(conn)) 2732 { 2733 /* Floyd heapification is not faster, don't bother. */ 2734 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2735 engine_incref_conn(conn, LSCONN_TICKABLE); 2736 } 2737 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2738 { 2739 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2740 if (next_tick_time) 2741 { 2742 if (0 == lsquic_attq_add(engine->attq, conn, next_tick_time, 2743 why)) 2744 engine_incref_conn(conn, LSCONN_ATTQ); 2745 } 2746 else 2747 /* In all other cases, the idle timeout would make the next 2748 * tick time non-zero: 2749 */ 2750 assert((conn->cn_flags & LSCONN_IETF) 2751 && engine->pub.enp_settings.es_idle_timeout == 0); 2752 } 2753 } 2754 2755 cub_flush(&engine->new_scids); 2756 cub_flush(&cub_live); 2757 cub_flush(&cub_old); 2758} 2759 2760 2761static void 2762maybe_count_garbage (struct lsquic_engine *engine, size_t garbage_sz) 2763{ 2764 /* This is not very pretty (action at a distance via engine->curr_conn), 2765 * but it's the cheapest I can come up with to handle the "count garbage 2766 * toward amplification limit" requirement in 2767 * [draft-ietf-quic-transport-28] Section 8.1. 2768 */ 2769 if (engine->curr_conn && engine->curr_conn->cn_if->ci_count_garbage) 2770 engine->curr_conn->cn_if->ci_count_garbage(engine->curr_conn, 2771 garbage_sz); 2772} 2773 2774 2775/* Return 0 if packet is being processed by a real connection, 1 if the 2776 * packet was processed, but not by a connection, and -1 on error. 2777 */ 2778int 2779lsquic_engine_packet_in (lsquic_engine_t *engine, 2780 const unsigned char *packet_in_data, size_t packet_in_size, 2781 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2782 void *peer_ctx, int ecn) 2783{ 2784 const unsigned char *const packet_begin = packet_in_data; 2785 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2786 struct packin_parse_state ppstate; 2787 lsquic_packet_in_t *packet_in; 2788 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2789 int is_server, unsigned cid_len, struct packin_parse_state *); 2790 unsigned n_zeroes; 2791 int s, is_ietf; 2792 lsquic_cid_t cid; 2793 2794 ENGINE_CALLS_INCR(engine); 2795 2796 if (engine->flags & ENG_SERVER) 2797 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2798 else if (engine->flags & ENG_CONNS_BY_ADDR) 2799 { 2800 struct lsquic_hash_elem *el; 2801 const struct lsquic_conn *conn; 2802 el = find_conn_by_addr(engine->conns_hash, sa_local); 2803 if (!el) 2804 return -1; 2805 conn = lsquic_hashelem_getdata(el); 2806 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2807 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2808 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2809 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2810 else if (conn->cn_version == LSQVER_050) 2811 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 2812 else 2813 { 2814#if LSQUIC_USE_Q098 2815 assert(conn->cn_version == LSQVER_046 || conn->cn_version == LSQVER_098); 2816#else 2817 assert(conn->cn_version == LSQVER_046); 2818#endif 2819 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2820 } 2821 } 2822 else 2823 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2824 2825 engine->curr_conn = NULL; 2826 n_zeroes = 0; 2827 is_ietf = 0; 2828#ifdef _MSC_VER 2829 s = 0; 2830 cid.len = 0; 2831 cid.idbuf[0] = 0; 2832#endif 2833 do 2834 { 2835 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 2836 if (!packet_in) 2837 return -1; 2838 /* Library does not modify packet_in_data, it is not referenced after 2839 * this function returns and subsequent release of pi_data is guarded 2840 * by PI_OWN_DATA flag. 2841 */ 2842 packet_in->pi_data = (unsigned char *) packet_in_data; 2843 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 2844 engine->flags & ENG_SERVER, 2845 engine->pub.enp_settings.es_scid_len, &ppstate)) 2846 { 2847 LSQ_DEBUG("Cannot parse incoming packet's header"); 2848 maybe_count_garbage(engine, packet_end - packet_in_data); 2849 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 2850 s = 1; 2851 break; 2852 } 2853 2854 /* [draft-ietf-quic-transport-30] Section 12.2: 2855 * " Receivers SHOULD ignore any subsequent packets with a different 2856 * " Destination Connection ID than the first packet in the datagram. 2857 */ 2858 if (is_ietf && packet_in_data > packet_begin) 2859 { 2860 if (!((packet_in->pi_flags & (PI_GQUIC|PI_CONN_ID)) == PI_CONN_ID 2861 && LSQUIC_CIDS_EQ(&packet_in->pi_dcid, &cid))) 2862 { 2863 packet_in_data += packet_in->pi_data_sz; 2864 maybe_count_garbage(engine, packet_in->pi_data_sz); 2865 continue; 2866 } 2867 } 2868 2869 is_ietf = 0 == (packet_in->pi_flags & PI_GQUIC); 2870 packet_in_data += packet_in->pi_data_sz; 2871 if (is_ietf && packet_in_data < packet_end) 2872 cid = packet_in->pi_dcid; 2873 packet_in->pi_received = lsquic_time_now(); 2874 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 2875 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 2876 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 2877 peer_ctx, packet_in_size); 2878 n_zeroes += s == 0; 2879 } 2880 while (0 == s && packet_in_data < packet_end); 2881 2882 return n_zeroes > 0 ? 0 : s; 2883} 2884 2885 2886#if __GNUC__ && !defined(NDEBUG) 2887__attribute__((weak)) 2888#endif 2889unsigned 2890lsquic_engine_quic_versions (const lsquic_engine_t *engine) 2891{ 2892 return engine->pub.enp_settings.es_versions; 2893} 2894 2895 2896void 2897lsquic_engine_cooldown (lsquic_engine_t *engine) 2898{ 2899 struct lsquic_hash_elem *el; 2900 lsquic_conn_t *conn; 2901 2902 if (engine->flags & ENG_COOLDOWN) 2903 /* AFAICT, there is no harm in calling this function more than once, 2904 * but log it just in case, as it may indicate an error in the caller. 2905 */ 2906 LSQ_INFO("cooldown called again"); 2907 engine->flags |= ENG_COOLDOWN; 2908 LSQ_INFO("entering cooldown mode"); 2909 if (engine->flags & ENG_SERVER) 2910 drop_all_mini_conns(engine); 2911 for (el = lsquic_hash_first(engine->conns_hash); el; 2912 el = lsquic_hash_next(engine->conns_hash)) 2913 { 2914 conn = lsquic_hashelem_getdata(el); 2915 lsquic_conn_going_away(conn); 2916 } 2917} 2918 2919 2920int 2921lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 2922{ 2923 const struct attq_elem *next_attq; 2924 lsquic_time_t now, next_time; 2925#if LSQUIC_DEBUG_NEXT_ADV_TICK 2926 const struct lsquic_conn *conn; 2927 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 2928#endif 2929 2930 ENGINE_CALLS_INCR(engine); 2931 2932 if ((engine->flags & ENG_PAST_DEADLINE) 2933 && lsquic_mh_count(&engine->conns_out)) 2934 { 2935#if LSQUIC_DEBUG_NEXT_ADV_TICK 2936 conn = lsquic_mh_peek(&engine->conns_out); 2937 engine->last_logged_conn = 0; 2938 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 2939 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 2940 lsquic_mh_count(&engine->conns_out), 2941 lsquic_mh_count(&engine->conns_out) != 1, "s", 2942 CID_BITS(lsquic_conn_log_cid(conn))); 2943#endif 2944 *diff = 0; 2945 return 1; 2946 } 2947 2948 if (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 2949 { 2950#if LSQUIC_DEBUG_NEXT_ADV_TICK 2951 engine->last_logged_conn = 0; 2952 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 2953#endif 2954 *diff = 0; 2955 return 1; 2956 } 2957 2958 if (lsquic_mh_count(&engine->conns_tickable)) 2959 { 2960#if LSQUIC_DEBUG_NEXT_ADV_TICK 2961 conn = lsquic_mh_peek(&engine->conns_tickable); 2962 engine->last_logged_conn = 0; 2963 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 2964 "connection%.*s (%"CID_FMT" first)", 2965 lsquic_mh_count(&engine->conns_tickable), 2966 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 2967 CID_BITS(lsquic_conn_log_cid(conn))); 2968#endif 2969 *diff = 0; 2970 return 1; 2971 } 2972 2973 next_attq = lsquic_attq_next(engine->attq); 2974 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 2975 { 2976 if (next_attq) 2977 next_time = next_attq->ae_adv_time; 2978 else 2979 return 0; 2980 } 2981 else 2982 { 2983 if (next_attq) 2984 { 2985 next_time = next_attq->ae_adv_time; 2986 if (engine->resume_sending_at < next_time) 2987 { 2988 next_time = engine->resume_sending_at; 2989 next_attq = NULL; 2990 } 2991 } 2992 else 2993 next_time = engine->resume_sending_at; 2994 } 2995 2996 now = lsquic_time_now(); 2997 *diff = (int) ((int64_t) next_time - (int64_t) now); 2998#if LSQUIC_DEBUG_NEXT_ADV_TICK 2999 if (next_attq) 3000 { 3001 /* Deduplicate consecutive log messages about the same reason for the 3002 * same connection. 3003 * If diff is always zero or diff reset to a higher value, event is 3004 * still logged. 3005 */ 3006 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 3007 && (uintptr_t) next_attq->ae_conn 3008 == engine->last_logged_conn 3009 && *diff < engine->last_tick_diff)) 3010 { 3011 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 3012 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 3013 engine->last_tick_diff = *diff; 3014 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 3015 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 3016 lsquic_attq_why2str(next_attq->ae_why)); 3017 } 3018 } 3019 else 3020 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 3021#endif 3022 return 1; 3023} 3024 3025 3026unsigned 3027lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 3028{ 3029 lsquic_time_t now; 3030 ENGINE_CALLS_INCR(engine); 3031 now = lsquic_time_now(); 3032 if (from_now < 0) 3033 now -= from_now; 3034 else 3035 now += from_now; 3036 return lsquic_attq_count_before(engine->attq, now); 3037} 3038 3039 3040int 3041lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 3042 struct lsquic_conn *conn, unsigned cce_idx) 3043{ 3044 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 3045 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 3046 void *peer_ctx; 3047 3048 assert(cce_idx < conn->cn_n_cces); 3049 assert(conn->cn_cces_mask & (1 << cce_idx)); 3050 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 3051 3052 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 3053 cce->cce_cid.len, conn, &cce->cce_hash_el)) 3054 { 3055 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 3056 CID_BITS(&cce->cce_cid)); 3057 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 3058 cce->cce_flags |= CCE_REG; 3059 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 3060 return 0; 3061 } 3062 else 3063 { 3064 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 3065 CID_BITS(&cce->cce_cid)); 3066 return -1; 3067 } 3068} 3069 3070 3071void 3072lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 3073 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now) 3074{ 3075 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 3076 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 3077 void *peer_ctx; 3078 3079 assert(cce_idx < conn->cn_n_cces); 3080 3081 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 3082 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 3083 3084 if (engine->purga) 3085 { 3086 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 3087 lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 3088 PUTY_CID_RETIRED, now); 3089 } 3090 conn->cn_cces_mask &= ~(1u << cce_idx); 3091 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 3092} 3093 3094 3095