lsquic_engine.c revision cca25415
1/* Copyright (c) 2017 - 2019 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#ifndef WIN32 17#include <sys/time.h> 18#include <netinet/in.h> 19#include <sys/types.h> 20#include <sys/stat.h> 21#include <fcntl.h> 22#include <unistd.h> 23#include <netdb.h> 24#endif 25 26#ifndef NDEBUG 27#include <sys/types.h> 28#include <regex.h> /* For code that loses packets */ 29#endif 30 31#if LOG_PACKET_CHECKSUM 32#include <zlib.h> 33#endif 34 35#include <openssl/aead.h> 36 37#include "lsquic.h" 38#include "lsquic_types.h" 39#include "lsquic_int_types.h" 40#include "lsquic_sizes.h" 41#include "lsquic_parse_common.h" 42#include "lsquic_parse.h" 43#include "lsquic_packet_in.h" 44#include "lsquic_packet_out.h" 45#include "lsquic_senhist.h" 46#include "lsquic_rtt.h" 47#include "lsquic_cubic.h" 48#include "lsquic_pacer.h" 49#include "lsquic_bw_sampler.h" 50#include "lsquic_minmax.h" 51#include "lsquic_bbr.h" 52#include "lsquic_send_ctl.h" 53#include "lsquic_set.h" 54#include "lsquic_conn_flow.h" 55#include "lsquic_sfcw.h" 56#include "lsquic_hash.h" 57#include "lsquic_conn.h" 58#include "lsquic_full_conn.h" 59#include "lsquic_util.h" 60#include "lsquic_qtags.h" 61#include "lsquic_enc_sess.h" 62#include "lsquic_mm.h" 63#include "lsquic_engine_public.h" 64#include "lsquic_eng_hist.h" 65#include "lsquic_ev_log.h" 66#include "lsquic_version.h" 67#include "lsquic_pr_queue.h" 68#include "lsquic_mini_conn.h" 69#include "lsquic_mini_conn_ietf.h" 70#include "lsquic_stock_shi.h" 71#include "lsquic_purga.h" 72#include "lsquic_tokgen.h" 73#include "lsquic_attq.h" 74#include "lsquic_min_heap.h" 75#include "lsquic_http1x_if.h" 76#include "lsquic_parse_common.h" 77#include "lsquic_h3_prio.h" 78#include "lsquic_handshake.h" 79 80#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 81#include "lsquic_logger.h" 82 83#define MIN(a, b) ((a) < (b) ? (a) : (b)) 84 85/* The batch of outgoing packets grows and shrinks dynamically */ 86#define MAX_OUT_BATCH_SIZE 1024 87#define MIN_OUT_BATCH_SIZE 4 88#define INITIAL_OUT_BATCH_SIZE 32 89 90struct out_batch 91{ 92 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 93 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 94 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 95 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 96 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 97}; 98 99typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 100 101static void 102process_connections (struct lsquic_engine *engine, conn_iter_f iter, 103 lsquic_time_t now); 104 105static void 106engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 107 108static lsquic_conn_t * 109engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 110 enum lsquic_conn_flags flag); 111 112static void 113force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 114 115#if LSQUIC_COUNT_ENGINE_CALLS 116#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 117#else 118#define ENGINE_CALLS_INCR(e) 119#endif 120 121/* Nested calls to LSQUIC are not supported */ 122#define ENGINE_IN(e) do { \ 123 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 124 (e)->pub.enp_flags |= ENPUB_PROC; \ 125 ENGINE_CALLS_INCR(e); \ 126} while (0) 127 128#define ENGINE_OUT(e) do { \ 129 assert((e)->pub.enp_flags & ENPUB_PROC); \ 130 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 131} while (0) 132 133/* A connection can be referenced from one of six places: 134 * 135 * 1. A hash. The engine maintains two hash tables -- one for full, and 136 * one for mini connections. A connection starts its life in one of 137 * those. 138 * 139 * 2. Outgoing queue. 140 * 141 * 3. Tickable queue 142 * 143 * 4. Advisory Tick Time queue. 144 * 145 * 5. Closing connections queue. This is a transient queue -- it only 146 * exists for the duration of process_connections() function call. 147 * 148 * 6. Ticked connections queue. Another transient queue, similar to (5). 149 * 150 * The idea is to destroy the connection when it is no longer referenced. 151 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 152 * that case, the connection is referenced from two places: (2) and (5). 153 * After its packets are sent, it is only referenced in (5), and at the 154 * end of the function call, when it is removed from (5), reference count 155 * goes to zero and the connection is destroyed. If not all packets can 156 * be sent, at the end of the function call, the connection is referenced 157 * by (2) and will only be removed once all outgoing packets have been 158 * sent. 159 */ 160#define CONN_REF_FLAGS (LSCONN_HASHED \ 161 |LSCONN_HAS_OUTGOING \ 162 |LSCONN_TICKABLE \ 163 |LSCONN_TICKED \ 164 |LSCONN_CLOSING \ 165 |LSCONN_ATTQ) 166 167 168 169 170struct cid_update_batch 171{ 172 lsquic_cids_update_f cub_update_cids; 173 void *cub_update_ctx; 174 unsigned cub_count; 175 lsquic_cid_t cub_cids[20]; 176 void *cub_peer_ctxs[20]; 177}; 178 179static void 180cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 181 182 183struct lsquic_engine 184{ 185 struct lsquic_engine_public pub; 186 enum { 187 ENG_SERVER = LSENG_SERVER, 188 ENG_HTTP = LSENG_HTTP, 189 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 190 ENG_PAST_DEADLINE 191 = (1 << 8), /* Previous call to a processing 192 * function went past time threshold. 193 */ 194 ENG_CONNS_BY_ADDR 195 = (1 << 9), /* Connections are hashed by address */ 196#ifndef NDEBUG 197 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 198 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 199 ENG_DTOR = (1 << 26), /* Engine destructor */ 200#endif 201 } flags; 202 lsquic_packets_out_f packets_out; 203 void *packets_out_ctx; 204 lsquic_cids_update_f report_new_scids; 205 lsquic_cids_update_f report_live_scids; 206 lsquic_cids_update_f report_old_scids; 207 void *scids_ctx; 208 struct lsquic_hash *conns_hash; 209 struct min_heap conns_tickable; 210 struct min_heap conns_out; 211 /* Use a union because only one iterator is being used at any one time */ 212 union { 213 struct { 214 struct cert_susp_head *head; 215 } resumed; 216 struct lsquic_conn *one_conn; 217 } iter_state; 218 struct eng_hist history; 219 unsigned batch_size; 220 struct pr_queue *pr_queue; 221 struct attq *attq; 222 /* Track time last time a packet was sent to give new connections 223 * priority lower than that of existing connections. 224 */ 225 lsquic_time_t last_sent; 226#ifndef NDEBUG 227 regex_t lose_packets_re; 228 const char *lose_packets_str; 229#endif 230 unsigned n_conns; 231 lsquic_time_t deadline; 232 lsquic_time_t resume_sending_at; 233 unsigned mini_conns_count; 234 struct lsquic_purga *purga; 235#if LSQUIC_CONN_STATS 236 struct { 237 unsigned conns; 238 } stats; 239 struct conn_stats conn_stats_sum; 240 FILE *stats_fh; 241#endif 242 struct cid_update_batch new_scids; 243 struct out_batch out_batch; 244#if LSQUIC_COUNT_ENGINE_CALLS 245 unsigned long n_engine_calls; 246#endif 247}; 248 249 250void 251lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 252 unsigned flags) 253{ 254 memset(settings, 0, sizeof(*settings)); 255 settings->es_versions = LSQUIC_DF_VERSIONS; 256 if (flags & ENG_SERVER) 257 { 258 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 259 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 260 settings->es_support_srej= LSQUIC_DF_SUPPORT_SREJ_SERVER; 261 settings->es_init_max_data 262 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 263 settings->es_init_max_stream_data_bidi_remote 264 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 265 settings->es_init_max_stream_data_bidi_local 266 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 267 settings->es_init_max_stream_data_uni 268 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 269 settings->es_init_max_streams_uni 270 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 271 settings->es_ping_period = 0; 272 } 273 else 274 { 275 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 276 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 277 settings->es_support_srej= LSQUIC_DF_SUPPORT_SREJ_CLIENT; 278 settings->es_init_max_data 279 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 280 settings->es_init_max_stream_data_bidi_remote 281 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 282 settings->es_init_max_stream_data_bidi_local 283 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 284 settings->es_init_max_stream_data_uni 285 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 286 settings->es_init_max_streams_uni 287 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 288 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 289 } 290 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 291 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 292 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 293 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 294 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 295 settings->es_max_header_list_size 296 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 297 settings->es_ua = LSQUIC_DF_UA; 298 settings->es_ecn = LSQUIC_DF_ECN; 299 300 settings->es_pdmd = QTAG_X509; 301 settings->es_aead = QTAG_AESG; 302 settings->es_kexs = QTAG_C255; 303 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 304 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 305 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 306 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 307 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 308 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 309 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 310 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 311 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 312 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 313 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 314 settings->es_sttl = LSQUIC_DF_STTL; 315 settings->es_init_max_streams_bidi 316 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 317 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 318 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 319 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 320 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 321 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 322 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 323 settings->es_h3_placeholders = LSQUIC_DF_H3_PLACEHOLDERS; 324 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 325} 326 327 328/* Note: if returning an error, err_buf must be valid if non-NULL */ 329int 330lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 331 unsigned flags, 332 char *err_buf, size_t err_buf_sz) 333{ 334 unsigned sum; 335 336 if (settings->es_cfcw < LSQUIC_MIN_FCW || 337 settings->es_sfcw < LSQUIC_MIN_FCW) 338 { 339 if (err_buf) 340 snprintf(err_buf, err_buf_sz, "%s", 341 "flow control window set too low"); 342 return -1; 343 } 344 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 345 { 346 if (err_buf) 347 snprintf(err_buf, err_buf_sz, "%s", 348 "No supported QUIC versions specified"); 349 return -1; 350 } 351 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 352 { 353 if (err_buf) 354 snprintf(err_buf, err_buf_sz, "%s", 355 "one or more unsupported QUIC version is specified"); 356 return -1; 357 } 358 if (flags & ENG_SERVER) 359 { 360 if (settings->es_handshake_to > 361 MAX_MINI_CONN_LIFESPAN_IN_USEC) 362 { 363 if (err_buf) 364 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 365 " usec is too large. The maximum for server is %u usec", 366 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 367 return -1; 368 } 369 } 370 if (settings->es_idle_timeout > 600) 371 { 372 if (err_buf) 373 snprintf(err_buf, err_buf_sz, "%s", 374 "The maximum value of idle timeout is 600 seconds"); 375 return -1; 376 } 377 if (!(!(flags & ENG_SERVER) && settings->es_scid_len == 0) 378 && (settings->es_scid_len < 4 || settings->es_scid_len > 18)) 379 { 380 if (err_buf) 381 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 382 "bytes long; it must be between 4 and 18.", 383 settings->es_scid_len); 384 return -1; 385 } 386 387 sum = settings->es_init_max_streams_bidi 388 + settings->es_init_max_streams_uni 389 + settings->es_h3_placeholders; 390 if (sum > H3_PRIO_MAX_ELEMS) 391 { 392 if (err_buf) 393 snprintf(err_buf, err_buf_sz, "Combined number of streams and " 394 "placeholders (%u) is greater than the maximum supported " 395 "number of elements in the HTTP/3 priority tree (%u)", 396 sum, H3_PRIO_MAX_ELEMS); 397 return -1; 398 } 399 400 if (settings->es_cc_algo > 2) 401 { 402 if (err_buf) 403 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 404 "algorithm value %u", settings->es_cc_algo); 405 return -1; 406 } 407 return 0; 408} 409 410 411static void 412free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 413{ 414 free(packet_data); 415} 416 417 418static void * 419malloc_buf (void *ctx, void *conn_ctx, unsigned short size, char is_ipv6) 420{ 421 return malloc(size); 422} 423 424 425static const struct lsquic_packout_mem_if stock_pmi = 426{ 427 malloc_buf, free_packet, free_packet, 428}; 429 430 431static int 432hash_conns_by_addr (const struct lsquic_engine *engine) 433{ 434 if (engine->flags & ENG_SERVER) 435 return 0; 436 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 437 return 1; 438 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 439 && engine->pub.enp_settings.es_support_tcid0) 440 return 1; 441 if (engine->pub.enp_settings.es_scid_len == 0) 442 return 1; 443 return 0; 444} 445 446 447lsquic_engine_t * 448lsquic_engine_new (unsigned flags, 449 const struct lsquic_engine_api *api) 450{ 451 lsquic_engine_t *engine; 452 char err_buf[100]; 453 454 if (!api->ea_packets_out) 455 { 456 LSQ_ERROR("packets_out callback is not specified"); 457 return NULL; 458 } 459 460 if (api->ea_settings && 461 0 != lsquic_engine_check_settings(api->ea_settings, flags, 462 err_buf, sizeof(err_buf))) 463 { 464 LSQ_ERROR("cannot create engine: %s", err_buf); 465 return NULL; 466 } 467 468 engine = calloc(1, sizeof(*engine)); 469 if (!engine) 470 return NULL; 471 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 472 { 473 free(engine); 474 return NULL; 475 } 476 if (api->ea_settings) 477 engine->pub.enp_settings = *api->ea_settings; 478 else 479 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 480 int tag_buf_len; 481 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 482 sizeof(engine->pub.enp_ver_tags_buf), 483 engine->pub.enp_settings.es_versions); 484 if (tag_buf_len <= 0) 485 { 486 LSQ_ERROR("cannot generate version tags buffer"); 487 free(engine); 488 return NULL; 489 } 490 engine->pub.enp_ver_tags_len = tag_buf_len; 491 engine->pub.enp_flags = ENPUB_CAN_SEND; 492 engine->pub.enp_stream_if = api->ea_stream_if; 493 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 494 495 engine->flags = flags; 496#ifndef NDEBUG 497 engine->flags |= ENG_COALESCE; 498#endif 499 engine->packets_out = api->ea_packets_out; 500 engine->packets_out_ctx = api->ea_packets_out_ctx; 501 engine->report_new_scids = api->ea_new_scids; 502 engine->report_live_scids = api->ea_live_scids; 503 engine->report_old_scids = api->ea_old_scids; 504 engine->scids_ctx = api->ea_cids_update_ctx; 505 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 506 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 507 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 508 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 509 if (api->ea_shi) 510 { 511 engine->pub.enp_shi = api->ea_shi; 512 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 513 } 514 else 515 { 516 engine->pub.enp_shi = &stock_shi; 517 engine->pub.enp_shi_ctx = stock_shared_hash_new(); 518 if (!engine->pub.enp_shi_ctx) 519 { 520 free(engine); 521 return NULL; 522 } 523 } 524 if (api->ea_hsi_if) 525 { 526 engine->pub.enp_hsi_if = api->ea_hsi_if; 527 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 528 } 529 else 530 { 531 engine->pub.enp_hsi_if = lsquic_http1x_if; 532 engine->pub.enp_hsi_ctx = NULL; 533 } 534 if (api->ea_pmi) 535 { 536 engine->pub.enp_pmi = api->ea_pmi; 537 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 538 } 539 else 540 { 541 engine->pub.enp_pmi = &stock_pmi; 542 engine->pub.enp_pmi_ctx = NULL; 543 } 544 engine->pub.enp_verify_cert = api->ea_verify_cert; 545 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 546 engine->pub.enp_kli = api->ea_keylog_if; 547 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 548 engine->pub.enp_engine = engine; 549 if (hash_conns_by_addr(engine)) 550 engine->flags |= ENG_CONNS_BY_ADDR; 551 engine->conns_hash = lsquic_hash_create(); 552 if (flags & ENG_SERVER) 553 { 554 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 555 if (!engine->pub.enp_tokgen) 556 return NULL; 557 engine->pr_queue = prq_create( 558 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 559 &engine->pub); 560 if (!engine->pr_queue) 561 { 562 lsquic_tg_destroy(engine->pub.enp_tokgen); 563 return NULL; 564 } 565 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 566 engine->report_old_scids, engine->scids_ctx); 567 if (!engine->purga) 568 { 569 lsquic_tg_destroy(engine->pub.enp_tokgen); 570 prq_destroy(engine->pr_queue); 571 return NULL; 572 } 573 } 574 engine->attq = attq_create(); 575 eng_hist_init(&engine->history); 576 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 577 if (engine->pub.enp_settings.es_honor_prst) 578 { 579 engine->pub.enp_srst_hash = lsquic_hash_create(); 580 if (!engine->pub.enp_srst_hash) 581 { 582 lsquic_engine_destroy(engine); 583 return NULL; 584 } 585 } 586 587#ifndef NDEBUG 588 { 589 const char *env; 590 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 591 if (env) 592 { 593 if (0 != regcomp(&engine->lose_packets_re, env, 594 REG_EXTENDED|REG_NOSUB)) 595 { 596 LSQ_ERROR("could not compile lost packet regex `%s'", env); 597 return NULL; 598 } 599 engine->flags |= ENG_LOSE_PACKETS; 600 engine->lose_packets_str = env; 601 LSQ_WARN("will lose packets that match the following regex: %s", 602 env); 603 } 604 env = getenv("LSQUIC_COALESCE"); 605 if (env) 606 { 607 engine->flags &= ~ENG_COALESCE; 608 if (atoi(env)) 609 { 610 engine->flags |= ENG_COALESCE; 611 LSQ_NOTICE("will coalesce packets"); 612 } 613 else 614 LSQ_NOTICE("will not coalesce packets"); 615 } 616 } 617#endif 618#if LSQUIC_CONN_STATS 619 engine->stats_fh = api->ea_stats_fh; 620#endif 621 622 LSQ_INFO("instantiated engine"); 623 return engine; 624} 625 626 627#if LOG_PACKET_CHECKSUM 628static void 629log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 630 const unsigned char *buf, size_t bufsz) 631{ 632 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 633 (uint32_t) crc32(0, buf, bufsz)); 634} 635 636 637#endif 638 639 640static void 641grow_batch_size (struct lsquic_engine *engine) 642{ 643 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 644} 645 646 647static void 648shrink_batch_size (struct lsquic_engine *engine) 649{ 650 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 651} 652 653 654struct cce_cid_iter 655{ 656 const struct lsquic_conn *conn; 657 unsigned todo, n; 658}; 659 660 661static struct conn_cid_elem * 662cce_iter_next (struct cce_cid_iter *citer) 663{ 664 struct conn_cid_elem *cce; 665 666 while (citer->todo) 667 if (citer->todo & (1 << citer->n)) 668 { 669 citer->todo &= ~(1 << citer->n); 670 cce = &citer->conn->cn_cces[ citer->n++ ]; 671 if (!(cce->cce_flags & CCE_PORT)) 672 return cce; 673 } 674 else 675 ++citer->n; 676 677 return NULL; 678} 679 680 681static struct conn_cid_elem * 682cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 683{ 684 citer->conn = conn; 685 citer->todo = conn->cn_cces_mask; 686 citer->n = 0; 687 return cce_iter_next(citer); 688} 689 690 691#if LSQUIC_CONN_STATS 692void 693update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 694{ 695 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 696 const unsigned long *src; 697 const struct conn_stats *stats; 698 unsigned i; 699 700 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 701 { 702 ++engine->stats.conns; 703 src = (unsigned long *) stats; 704 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 705 dst[i] += src[i]; 706 } 707} 708 709 710#endif 711 712 713/* Wrapper to make sure important things occur before the connection is 714 * really destroyed. 715 */ 716static void 717destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 718 lsquic_time_t now) 719{ 720 struct cce_cid_iter citer; 721 const struct conn_cid_elem *cce; 722 lsquic_time_t drain_time; 723 struct purga_el *puel; 724 725 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 726 if (engine->purga 727 /* Blacklist all CIDs except for promoted mini connections */ 728 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 729 != (LSCONN_MINI|LSCONN_PROMOTED)) 730 { 731 if (conn->cn_if->ci_drain_time && 732 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 733 { 734 for (cce = cce_iter_first(&citer, conn); cce; 735 cce = cce_iter_next(&citer)) 736 { 737 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 738 lsquic_conn_get_peer_ctx(conn, NULL), 739 PUTY_CONN_DRAIN, now); 740 if (puel) 741 puel->puel_time = now + drain_time; 742 } 743 } 744 else 745 { 746 for (cce = cce_iter_first(&citer, conn); cce; 747 cce = cce_iter_next(&citer)) 748 { 749 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 750 lsquic_conn_get_peer_ctx(conn, NULL), 751 PUTY_CONN_DELETED, now); 752 if (puel) 753 { 754 puel->puel_time = now; 755 puel->puel_count = 0; 756 } 757 } 758 } 759 } 760#if LSQUIC_CONN_STATS 761 update_stats_sum(engine, conn); 762#endif 763 --engine->n_conns; 764 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 765 conn->cn_if->ci_destroy(conn); 766} 767 768 769static int 770maybe_grow_conn_heaps (struct lsquic_engine *engine) 771{ 772 struct min_heap_elem *els; 773 unsigned count; 774 775 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 776 return 0; /* Nothing to do */ 777 778 if (lsquic_mh_nalloc(&engine->conns_tickable)) 779 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 780 else 781 count = 8; 782 783 els = malloc(sizeof(els[0]) * count); 784 if (!els) 785 { 786 LSQ_ERROR("%s: malloc failed", __func__); 787 return -1; 788 } 789 790 LSQ_DEBUG("grew heaps to %u elements", count / 2); 791 memcpy(&els[0], engine->conns_tickable.mh_elems, 792 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 793 memcpy(&els[count / 2], engine->conns_out.mh_elems, 794 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 795 free(engine->conns_tickable.mh_elems); 796 engine->conns_tickable.mh_elems = els; 797 engine->conns_out.mh_elems = &els[count / 2]; 798 engine->conns_tickable.mh_nalloc = count / 2; 799 engine->conns_out.mh_nalloc = count / 2; 800 return 0; 801} 802 803 804static void 805remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 806 unsigned todo) 807{ 808 unsigned n; 809 810 for (n = 0; todo; todo &= ~(1 << n++)) 811 if ((todo & (1 << n)) && 812 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 813 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 814} 815 816 817static void 818remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 819{ 820 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 821} 822 823 824static void 825cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 826 827 828static int 829insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 830 void *peer_ctx) 831{ 832 struct conn_cid_elem *cce; 833 unsigned todo, done, n; 834 835 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 836 if (todo & (1 << n)) 837 { 838 cce = &conn->cn_cces[n]; 839 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 840 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 841 cce->cce_cid.len, conn, &cce->cce_hash_el)) 842 done |= 1 << n; 843 else 844 goto err; 845 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 846 { 847 cce->cce_flags |= CCE_REG; 848 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 849 } 850 } 851 852 return 0; 853 854 err: 855 remove_cces_from_hash(engine->conns_hash, conn, done); 856 return -1; 857} 858 859 860static lsquic_conn_t * 861new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 862 lsquic_time_t now) 863{ 864 const lsquic_cid_t *cid; 865 server_conn_ctor_f ctor; 866 lsquic_conn_t *conn; 867 unsigned flags; 868 if (0 != maybe_grow_conn_heaps(engine)) 869 return NULL; 870 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 871 872 if (mini_conn->cn_flags & LSCONN_IETF) 873 ctor = lsquic_ietf_full_conn_server_new; 874 else 875 ctor = lsquic_gquic_full_conn_server_new; 876 877 conn = ctor(&engine->pub, flags, mini_conn); 878 if (!conn) 879 { 880 /* Otherwise, full_conn_server_new prints its own warnings */ 881 if (ENOMEM == errno) 882 { 883 cid = lsquic_conn_log_cid(mini_conn); 884 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 885 CID_BITS(cid), strerror(errno)); 886 } 887 return NULL; 888 } 889 ++engine->n_conns; 890 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 891 { 892 cid = lsquic_conn_log_cid(conn); 893 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 894 CID_BITS(cid)); 895 destroy_conn(engine, conn, now); 896 return NULL; 897 } 898 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 899 conn->cn_flags |= LSCONN_HASHED; 900 return conn; 901} 902 903 904static enum 905{ 906 VER_NOT_SPECIFIED, 907 VER_SUPPORTED, 908 VER_UNSUPPORTED, 909} 910 911 912version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 913 enum lsquic_version *pversion) 914{ 915 lsquic_ver_tag_t ver_tag; 916 enum lsquic_version version; 917 918 if (!packet_in->pi_quic_ver) 919 { 920 LSQ_DEBUG("packet does not specify version"); 921 return VER_NOT_SPECIFIED; 922 } 923 924 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 925 version = lsquic_tag2ver(ver_tag); 926 if (version < N_LSQVER) 927 { 928 if (engine->pub.enp_settings.es_versions & (1 << version)) 929 { 930 LSQ_DEBUG("client-supplied version %s is supported", 931 lsquic_ver2str[version]); 932 *pversion = version; 933 return VER_SUPPORTED; 934 } 935 else 936 LSQ_DEBUG("client-supplied version %s is not supported", 937 lsquic_ver2str[version]); 938 } 939 else 940 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 941 ver_tag); 942 943 return VER_UNSUPPORTED; 944} 945 946 947static void 948schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 949 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 950 const struct sockaddr *sa_peer, void *peer_ctx) 951{ 952 assert(engine->pr_queue); 953 if (0 == prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 954 sa_local, sa_peer)) 955 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 956 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 957 else 958 LSQ_DEBUG("cannot schedule %s packet: out of packet request objects", 959 lsquic_preqt2str[type]); 960} 961 962 963static unsigned short 964sa2port (const struct sockaddr *sa) 965{ 966 if (sa->sa_family == AF_INET) 967 { 968 struct sockaddr_in *const sa4 = (void *) sa; 969 return sa4->sin_port; 970 } 971 else 972 { 973 struct sockaddr_in6 *const sa6 = (void *) sa; 974 return sa6->sin6_port; 975 } 976} 977 978 979static struct lsquic_hash_elem * 980find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 981{ 982 unsigned short port; 983 984 port = sa2port(sa); 985 return lsquic_hash_find(hash, &port, sizeof(port)); 986} 987 988 989static lsquic_conn_t * 990find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 991 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 992{ 993 struct lsquic_hash_elem *el; 994 lsquic_conn_t *conn; 995 996 if (engine->flags & ENG_CONNS_BY_ADDR) 997 el = find_conn_by_addr(engine->conns_hash, sa_local); 998 else if (packet_in->pi_flags & PI_CONN_ID) 999 el = lsquic_hash_find(engine->conns_hash, 1000 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1001 else 1002 { 1003 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1004 return NULL; 1005 } 1006 1007 if (!el) 1008 return NULL; 1009 1010 conn = lsquic_hashelem_getdata(el); 1011 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1012 if ((engine->flags & ENG_CONNS_BY_ADDR) 1013 && !(conn->cn_flags & LSCONN_IETF) 1014 && (packet_in->pi_flags & PI_CONN_ID) 1015 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1016 { 1017 LSQ_DEBUG("connection IDs do not match"); 1018 return NULL; 1019 } 1020 1021 return conn; 1022} 1023 1024 1025static lsquic_conn_t * 1026find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1027 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1028 const struct sockaddr *sa_peer, void *peer_ctx) 1029{ 1030 struct lsquic_hash_elem *el; 1031 lsquic_cid_t odcid; 1032 struct purga_el *puel; 1033 lsquic_conn_t *conn; 1034 1035 if (!(packet_in->pi_flags & PI_CONN_ID)) 1036 { 1037 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1038 return NULL; 1039 } 1040 el = lsquic_hash_find(engine->conns_hash, 1041 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1042 1043 if (el) 1044 { 1045 conn = lsquic_hashelem_getdata(el); 1046 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1047 return conn; 1048 } 1049 1050 if (engine->flags & ENG_COOLDOWN) 1051 { /* Do not create incoming connections during cooldown */ 1052 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1053 return NULL; 1054 } 1055 1056 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1057 { 1058 LSQ_DEBUG("reached limit of %u inchoate connections", 1059 engine->pub.enp_settings.es_max_inchoate); 1060 return NULL; 1061 } 1062 1063 1064 if (engine->purga 1065 && (puel = lsquic_purga_contains(engine->purga, 1066 &packet_in->pi_conn_id), puel)) 1067 { 1068 switch (puel->puel_type) 1069 { 1070 case PUTY_CID_RETIRED: 1071 LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet", 1072 CID_BITS(&packet_in->pi_conn_id)); 1073 return NULL; 1074 case PUTY_CONN_DRAIN: 1075 LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64, 1076 puel->puel_time, packet_in->pi_received); 1077 if (puel->puel_time > packet_in->pi_received) 1078 { 1079 LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet", 1080 CID_BITS(&packet_in->pi_conn_id)); 1081 return NULL; 1082 } 1083 LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted", 1084 CID_BITS(&packet_in->pi_conn_id)); 1085 puel->puel_type = PUTY_CONN_DELETED; 1086 puel->puel_count = 0; 1087 puel->puel_time = 0; 1088 /* fall-through */ 1089 case PUTY_CONN_DELETED: 1090 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1091 CID_BITS(&packet_in->pi_conn_id)); 1092 if (puel->puel_time < packet_in->pi_received) 1093 { 1094 puel->puel_time = packet_in->pi_received 1095 /* Exponential back-off */ 1096 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1097 ++puel->puel_count; 1098 goto maybe_send_prst; 1099 } 1100 return NULL; 1101 default: 1102 assert(0); 1103 return NULL; 1104 } 1105 } 1106 1107 if (engine->pub.enp_settings.es_send_prst 1108 && !(packet_in->pi_flags & PI_GQUIC) 1109 && HETY_NOT_SET == packet_in->pi_header_type) 1110 goto maybe_send_prst; 1111 1112 if (0 != maybe_grow_conn_heaps(engine)) 1113 return NULL; 1114 1115 const struct parse_funcs *pf; 1116 enum lsquic_version version; 1117 switch (version_matches(engine, packet_in, &version)) 1118 { 1119 case VER_UNSUPPORTED: 1120 if (engine->flags & ENG_SERVER) 1121 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1122 sa_local, sa_peer, peer_ctx); 1123 return NULL; 1124 case VER_NOT_SPECIFIED: 1125 maybe_send_prst: 1126 if ((engine->flags & ENG_SERVER) && 1127 engine->pub.enp_settings.es_send_prst) 1128 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1129 sa_local, sa_peer, peer_ctx); 1130 return NULL; 1131 case VER_SUPPORTED: 1132 pf = select_pf_by_ver(version); 1133 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1134 break; 1135 } 1136 1137 1138 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1139 { 1140 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1141 sa_peer->sa_family == AF_INET, odcid.len ? &odcid : NULL); 1142 } 1143 else 1144 { 1145 conn = mini_conn_new(&engine->pub, packet_in, version); 1146 } 1147 if (!conn) 1148 return NULL; 1149 ++engine->mini_conns_count; 1150 ++engine->n_conns; 1151 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1152 { 1153 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1154 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1155 CID_BITS(cid)); 1156 destroy_conn(engine, conn, packet_in->pi_received); 1157 return NULL; 1158 } 1159 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1160 conn->cn_flags |= LSCONN_HASHED; 1161 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1162 conn->cn_last_sent = engine->last_sent; 1163 return conn; 1164} 1165 1166 1167lsquic_conn_t * 1168lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1169 const lsquic_cid_t *cid) 1170{ 1171 struct lsquic_hash_elem *el; 1172 lsquic_conn_t *conn = NULL; 1173 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1174 1175 if (el) 1176 conn = lsquic_hashelem_getdata(el); 1177 return conn; 1178} 1179 1180 1181#if !defined(NDEBUG) && __GNUC__ 1182__attribute__((weak)) 1183#endif 1184void 1185lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1186 lsquic_conn_t *conn) 1187{ 1188 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1189 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1190 { 1191 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1192 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1193 engine_incref_conn(conn, LSCONN_TICKABLE); 1194 } 1195} 1196 1197 1198void 1199lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1200 lsquic_conn_t *conn, lsquic_time_t tick_time) 1201{ 1202 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1203 if (conn->cn_flags & LSCONN_TICKABLE) 1204 { 1205 /* Optimization: no need to add the connection to the Advisory Tick 1206 * Time Queue: it is about to be ticked, after which it its next tick 1207 * time may be queried again. 1208 */; 1209 } 1210 else if (conn->cn_flags & LSCONN_ATTQ) 1211 { 1212 if (lsquic_conn_adv_time(conn) != tick_time) 1213 { 1214 attq_remove(engine->attq, conn); 1215 if (0 != attq_add(engine->attq, conn, tick_time)) 1216 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1217 } 1218 } 1219 else if (0 == attq_add(engine->attq, conn, tick_time)) 1220 engine_incref_conn(conn, LSCONN_ATTQ); 1221} 1222 1223 1224static struct lsquic_conn * 1225find_conn_by_srst (struct lsquic_engine *engine, 1226 const struct lsquic_packet_in *packet_in) 1227{ 1228 struct lsquic_hash_elem *el; 1229 struct lsquic_conn *conn; 1230 1231 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1232 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1233 return NULL; 1234 1235 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1236 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1237 IQUIC_SRESET_TOKEN_SZ); 1238 if (!el) 1239 return NULL; 1240 1241 conn = lsquic_hashelem_getdata(el); 1242 return conn; 1243} 1244 1245 1246/* Return 0 if packet is being processed by a real connection (mini or full), 1247 * otherwise return 1. 1248 */ 1249static int 1250process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1251 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1252 const struct sockaddr *sa_peer, void *peer_ctx) 1253{ 1254 lsquic_conn_t *conn; 1255 const unsigned char *packet_in_data; 1256 size_t packet_in_size; 1257 1258 if (lsquic_packet_in_is_gquic_prst(packet_in) 1259 && !engine->pub.enp_settings.es_honor_prst) 1260 { 1261 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1262 LSQ_DEBUG("public reset packet: discarding"); 1263 return 1; 1264 } 1265 1266 if (engine->flags & ENG_SERVER) 1267 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1268 sa_peer, peer_ctx); 1269 else 1270 conn = find_conn(engine, packet_in, ppstate, sa_local); 1271 1272 if (!conn) 1273 { 1274 if (engine->pub.enp_settings.es_honor_prst 1275 && !(packet_in->pi_flags & PI_GQUIC) 1276 && engine->pub.enp_srst_hash 1277 && (conn = find_conn_by_srst(engine, packet_in))) 1278 { 1279 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1280 CID_BITS(lsquic_conn_log_cid(conn))); 1281 conn->cn_if->ci_stateless_reset(conn); 1282 if (!(conn->cn_flags & LSCONN_TICKABLE) 1283 && conn->cn_if->ci_is_tickable(conn)) 1284 { 1285 lsquic_mh_insert(&engine->conns_tickable, conn, 1286 conn->cn_last_ticked); 1287 engine_incref_conn(conn, LSCONN_TICKABLE); 1288 } 1289 /* Even though the connection processes this packet, we return 1290 * 1 so that the caller does not add reset packet's random 1291 * bytes to the list of valid CIDs. 1292 */ 1293 } 1294 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1295 return 1; 1296 } 1297 1298 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1299 { 1300 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1301 engine_incref_conn(conn, LSCONN_TICKABLE); 1302 } 1303 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1304 sa_local, sa_peer); 1305 lsquic_packet_in_upref(packet_in); 1306#if LOG_PACKET_CHECKSUM 1307 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1308 packet_in->pi_data_sz); 1309#endif 1310 /* Note on QLog: 1311 * For the PACKET_RX QLog event, we are interested in logging these things: 1312 * - raw packet (however it comes in, encrypted or not) 1313 * - frames (list of frame names) 1314 * - packet type and number 1315 * - packet rx timestamp 1316 * 1317 * Since only some of these items are available at this code 1318 * juncture, we will wait until after the packet has been 1319 * decrypted (if necessary) and parsed to call the log functions. 1320 * 1321 * Once the PACKET_RX event is finally logged, the timestamp 1322 * will come from packet_in->pi_received. For correct sequential 1323 * ordering of QLog events, be sure to process the QLogs downstream. 1324 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1325 */ 1326 packet_in_data = packet_in->pi_data; 1327 packet_in_size = packet_in->pi_data_sz; 1328 conn->cn_if->ci_packet_in(conn, packet_in); 1329 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1330 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1331 return 0; 1332} 1333 1334 1335void 1336lsquic_engine_destroy (lsquic_engine_t *engine) 1337{ 1338 struct lsquic_hash_elem *el; 1339 lsquic_conn_t *conn; 1340 1341 LSQ_DEBUG("destroying engine"); 1342#ifndef NDEBUG 1343 engine->flags |= ENG_DTOR; 1344#endif 1345 1346 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1347 { 1348 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1349 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1350 } 1351 1352 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1353 { 1354 assert(conn->cn_flags & LSCONN_TICKABLE); 1355 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1356 } 1357 1358 for (el = lsquic_hash_first(engine->conns_hash); el; 1359 el = lsquic_hash_next(engine->conns_hash)) 1360 { 1361 conn = lsquic_hashelem_getdata(el); 1362 force_close_conn(engine, conn); 1363 } 1364 lsquic_hash_destroy(engine->conns_hash); 1365 1366 assert(0 == engine->n_conns); 1367 assert(0 == engine->mini_conns_count); 1368 if (engine->pr_queue) 1369 prq_destroy(engine->pr_queue); 1370 if (engine->purga) 1371 lsquic_purga_destroy(engine->purga); 1372 attq_destroy(engine->attq); 1373 1374 assert(0 == lsquic_mh_count(&engine->conns_out)); 1375 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1376 if (engine->pub.enp_shi == &stock_shi) 1377 stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1378 lsquic_mm_cleanup(&engine->pub.enp_mm); 1379 free(engine->conns_tickable.mh_elems); 1380#ifndef NDEBUG 1381 if (engine->flags & ENG_LOSE_PACKETS) 1382 regfree(&engine->lose_packets_re); 1383#endif 1384 if (engine->pub.enp_tokgen) 1385 lsquic_tg_destroy(engine->pub.enp_tokgen); 1386#if LSQUIC_CONN_STATS 1387 if (engine->stats_fh) 1388 { 1389 const struct conn_stats *const stats = &engine->conn_stats_sum; 1390 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1391 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1392 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1393 fprintf(engine->stats_fh, "In:\n"); 1394 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1395 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1396 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1397 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1398 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1399 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1400 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1401 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1402 stats->in.headers_comp, stats->in.headers_uncomp, 1403 stats->in.headers_uncomp ? 1404 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1405 : 0); 1406 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1407 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1408 fprintf(engine->stats_fh, " ACK frames merged to new: %lu\n", stats->in.n_acks_merged[0]); 1409 fprintf(engine->stats_fh, " ACK frames merged to old: %lu\n", stats->in.n_acks_merged[1]); 1410 fprintf(engine->stats_fh, "Out:\n"); 1411 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1412 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1413 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1414 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1415 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1416 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1417 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1418 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1419 stats->out.headers_comp, stats->out.headers_uncomp, 1420 stats->out.headers_uncomp ? 1421 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1422 : 0); 1423 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1424 } 1425#endif 1426 if (engine->pub.enp_srst_hash) 1427 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1428#if LSQUIC_COUNT_ENGINE_CALLS 1429 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1430#endif 1431 free(engine); 1432} 1433 1434 1435static struct conn_cid_elem * 1436find_free_cce (struct lsquic_conn *conn) 1437{ 1438 struct conn_cid_elem *cce; 1439 1440 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1441 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1442 return cce; 1443 1444 return NULL; 1445} 1446 1447 1448static int 1449add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1450 const struct sockaddr *local_sa, void *peer_ctx) 1451{ 1452 struct conn_cid_elem *cce; 1453 1454 if (engine->flags & ENG_CONNS_BY_ADDR) 1455 { 1456 cce = find_free_cce(conn); 1457 if (!cce) 1458 { 1459 LSQ_ERROR("cannot find free CCE"); 1460 return -1; 1461 } 1462 cce->cce_port = sa2port(local_sa); 1463 cce->cce_flags = CCE_PORT; 1464 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1465 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1466 { 1467 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1468 return 0; 1469 } 1470 else 1471 return -1; 1472 1473 } 1474 else 1475 return insert_conn_into_hash(engine, conn, peer_ctx); 1476} 1477 1478 1479lsquic_conn_t * 1480lsquic_engine_connect (lsquic_engine_t *engine, const struct sockaddr *local_sa, 1481 const struct sockaddr *peer_sa, 1482 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1483 const char *hostname, unsigned short max_packet_size, 1484 const unsigned char *zero_rtt, size_t zero_rtt_len, 1485 const unsigned char *token, size_t token_sz) 1486{ 1487 lsquic_conn_t *conn; 1488 unsigned flags; 1489 int is_ipv4; 1490 1491 ENGINE_IN(engine); 1492 1493 if (engine->flags & ENG_SERVER) 1494 { 1495 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1496 goto err; 1497 } 1498 1499 if (engine->flags & ENG_CONNS_BY_ADDR 1500 && find_conn_by_addr(engine->conns_hash, local_sa)) 1501 { 1502 LSQ_ERROR("cannot have more than one connection on the same port"); 1503 goto err; 1504 } 1505 1506 if (0 != maybe_grow_conn_heaps(engine)) 1507 return NULL; 1508 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1509 is_ipv4 = peer_sa->sa_family == AF_INET; 1510 if (engine->pub.enp_settings.es_versions & LSQUIC_IETF_VERSIONS) 1511 conn = lsquic_ietf_full_conn_client_new(&engine->pub, 1512 flags, hostname, max_packet_size, 1513 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1514 else 1515 conn = lsquic_gquic_full_conn_client_new(&engine->pub, flags, 1516 hostname, max_packet_size, is_ipv4, 1517 zero_rtt, zero_rtt_len); 1518 if (!conn) 1519 goto err; 1520 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1521 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1522 lsquic_ver2str[conn->cn_version]); 1523 ++engine->n_conns; 1524 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1525 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1526 { 1527 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1528 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1529 CID_BITS(cid)); 1530 destroy_conn(engine, conn, lsquic_time_now()); 1531 goto err; 1532 } 1533 assert(!(conn->cn_flags & 1534 (CONN_REF_FLAGS 1535 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1536 callbacks */ 1537 ))); 1538 conn->cn_flags |= LSCONN_HASHED; 1539 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1540 engine_incref_conn(conn, LSCONN_TICKABLE); 1541 lsquic_conn_set_ctx(conn, conn_ctx); 1542 conn->cn_if->ci_client_call_on_new(conn); 1543 end: 1544 ENGINE_OUT(engine); 1545 return conn; 1546 err: 1547 conn = NULL; 1548 goto end; 1549} 1550 1551 1552static void 1553remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1554{ 1555 remove_all_cces_from_hash(engine->conns_hash, conn); 1556 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1557} 1558 1559 1560static void 1561refflags2str (enum lsquic_conn_flags flags, char s[6]) 1562{ 1563 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1564 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1565 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1566 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1567 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1568 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1569 *s = '\0'; 1570} 1571 1572 1573static void 1574engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1575{ 1576 const lsquic_cid_t *cid; 1577 char str[2][7]; 1578 assert(flag & CONN_REF_FLAGS); 1579 assert(!(conn->cn_flags & flag)); 1580 conn->cn_flags |= flag; 1581 cid = lsquic_conn_log_cid(conn); 1582 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", CID_BITS(cid), 1583 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1584 (refflags2str(conn->cn_flags, str[1]), str[1])); 1585} 1586 1587 1588static lsquic_conn_t * 1589engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1590 enum lsquic_conn_flags flags) 1591{ 1592 const lsquic_cid_t *cid; 1593 char str[2][7]; 1594 lsquic_time_t now; 1595 assert(flags & CONN_REF_FLAGS); 1596 assert(conn->cn_flags & flags); 1597#ifndef NDEBUG 1598 if (flags & LSCONN_CLOSING) 1599 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1600#endif 1601 conn->cn_flags &= ~flags; 1602 cid = lsquic_conn_log_cid(conn); 1603 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", CID_BITS(cid), 1604 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1605 (refflags2str(conn->cn_flags, str[1]), str[1])); 1606 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1607 { 1608 now = lsquic_time_now(); 1609 if (conn->cn_flags & LSCONN_MINI) 1610 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1611 else 1612 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1613 destroy_conn(engine, conn, now); 1614 return NULL; 1615 } 1616 else 1617 return conn; 1618} 1619 1620 1621/* This is not a general-purpose function. Only call from engine dtor. */ 1622static void 1623force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1624{ 1625 assert(engine->flags & ENG_DTOR); 1626 const enum lsquic_conn_flags flags = conn->cn_flags; 1627 assert(conn->cn_flags & CONN_REF_FLAGS); 1628 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1629 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1630 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1631 if (flags & LSCONN_ATTQ) 1632 { 1633 attq_remove(engine->attq, conn); 1634 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1635 } 1636 if (flags & LSCONN_HASHED) 1637 remove_conn_from_hash(engine, conn); 1638} 1639 1640 1641/* Iterator for tickable connections (those on the Tickable Queue). Before 1642 * a connection is returned, it is removed from the Advisory Tick Time queue 1643 * if necessary. 1644 */ 1645static lsquic_conn_t * 1646conn_iter_next_tickable (struct lsquic_engine *engine) 1647{ 1648 lsquic_conn_t *conn; 1649 1650 if (engine->flags & ENG_SERVER) 1651 while (1) 1652 { 1653 conn = lsquic_mh_pop(&engine->conns_tickable); 1654 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1655 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1656 else 1657 break; 1658 } 1659 else 1660 conn = lsquic_mh_pop(&engine->conns_tickable); 1661 1662 if (conn) 1663 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1664 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1665 { 1666 attq_remove(engine->attq, conn); 1667 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1668 } 1669 1670 return conn; 1671} 1672 1673 1674static void 1675cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1676 void *update_ctx) 1677{ 1678 cub->cub_update_cids = update; 1679 cub->cub_update_ctx = update_ctx; 1680 cub->cub_count = 0; 1681} 1682 1683 1684static void 1685cub_flush (struct cid_update_batch *cub) 1686{ 1687 if (cub->cub_count > 0 && cub->cub_update_cids) 1688 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1689 cub->cub_cids, cub->cub_count); 1690 cub->cub_count = 0; 1691} 1692 1693 1694static void 1695cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1696{ 1697 cub->cub_cids [ cub->cub_count ] = *cid; 1698 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1699 ++cub->cub_count; 1700 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1701 cub_flush(cub); 1702} 1703 1704 1705/* Process registered CIDs */ 1706static void 1707cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1708{ 1709 struct cce_cid_iter citer; 1710 struct conn_cid_elem *cce; 1711 void *peer_ctx; 1712 1713 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1714 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1715 if (cce->cce_flags & CCE_REG) 1716 cub_add(cub, &cce->cce_cid, peer_ctx); 1717} 1718 1719 1720static void 1721drop_all_mini_conns (lsquic_engine_t *engine) 1722{ 1723 struct lsquic_hash_elem *el; 1724 lsquic_conn_t *conn; 1725 struct cid_update_batch cub; 1726 1727 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1728 1729 for (el = lsquic_hash_first(engine->conns_hash); el; 1730 el = lsquic_hash_next(engine->conns_hash)) 1731 { 1732 conn = lsquic_hashelem_getdata(el); 1733 if (conn->cn_flags & LSCONN_MINI) 1734 { 1735 /* If promoted, why is it still in this hash? */ 1736 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1737 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1738 cub_add_cids_from_cces(&cub, conn); 1739 remove_conn_from_hash(engine, conn); 1740 } 1741 } 1742 1743 cub_flush(&cub); 1744} 1745 1746 1747void 1748lsquic_engine_process_conns (lsquic_engine_t *engine) 1749{ 1750 lsquic_conn_t *conn; 1751 lsquic_time_t now; 1752 1753 ENGINE_IN(engine); 1754 1755 now = lsquic_time_now(); 1756 while ((conn = attq_pop(engine->attq, now))) 1757 { 1758 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1759 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 1760 { 1761 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1762 engine_incref_conn(conn, LSCONN_TICKABLE); 1763 } 1764 } 1765 1766 process_connections(engine, conn_iter_next_tickable, now); 1767 ENGINE_OUT(engine); 1768} 1769 1770 1771static void 1772release_or_return_enc_data (struct lsquic_engine *engine, 1773 void (*pmi_rel_or_ret) (void *, void *, void *, char), 1774 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 1775{ 1776 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1777 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 1778 packet_out->po_flags &= ~PO_ENCRYPTED; 1779 packet_out->po_enc_data = NULL; 1780} 1781 1782 1783static void 1784release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1785 struct lsquic_packet_out *packet_out) 1786{ 1787 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 1788 conn, packet_out); 1789} 1790 1791 1792static void 1793return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1794 struct lsquic_packet_out *packet_out) 1795{ 1796 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 1797 conn, packet_out); 1798} 1799 1800 1801static int 1802copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 1803 struct lsquic_packet_out *packet_out) 1804{ 1805 int ipv6; 1806 1807 ipv6 = NP_IS_IPv6(packet_out->po_path); 1808 if (packet_out->po_flags & PO_ENCRYPTED) 1809 { 1810 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1811 && packet_out->po_data_sz == packet_out->po_enc_data_sz 1812 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 1813 packet_out->po_data_sz)) 1814 return 0; 1815 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1816 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 1817 goto copy; 1818 return_enc_data(engine, conn, packet_out); 1819 } 1820 1821 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 1822 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1823 packet_out->po_data_sz, ipv6); 1824 if (!packet_out->po_enc_data) 1825 { 1826 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 1827 "of size %hu", packet_out->po_data_sz); 1828 return -1; 1829 } 1830 1831 copy: 1832 memcpy(packet_out->po_enc_data, packet_out->po_data, 1833 packet_out->po_data_sz); 1834 packet_out->po_enc_data_sz = packet_out->po_data_sz; 1835 packet_out->po_sent_sz = packet_out->po_data_sz; 1836 packet_out->po_flags &= ~PO_IPv6; 1837 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 1838 1839 return 0; 1840} 1841 1842 1843STAILQ_HEAD(conns_stailq, lsquic_conn); 1844TAILQ_HEAD(conns_tailq, lsquic_conn); 1845 1846 1847struct conns_out_iter 1848{ 1849 struct min_heap *coi_heap; 1850 struct pr_queue *coi_prq; 1851 TAILQ_HEAD(, lsquic_conn) coi_active_list, 1852 coi_inactive_list; 1853 lsquic_conn_t *coi_next; 1854#ifndef NDEBUG 1855 lsquic_time_t coi_last_sent; 1856#endif 1857}; 1858 1859 1860static void 1861coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 1862{ 1863 iter->coi_heap = &engine->conns_out; 1864 iter->coi_prq = engine->pr_queue; 1865 iter->coi_next = NULL; 1866 TAILQ_INIT(&iter->coi_active_list); 1867 TAILQ_INIT(&iter->coi_inactive_list); 1868#ifndef NDEBUG 1869 iter->coi_last_sent = 0; 1870#endif 1871} 1872 1873 1874static lsquic_conn_t * 1875coi_next (struct conns_out_iter *iter) 1876{ 1877 lsquic_conn_t *conn; 1878 1879 if (lsquic_mh_count(iter->coi_heap) > 0) 1880 { 1881 conn = lsquic_mh_pop(iter->coi_heap); 1882 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1883 conn->cn_flags |= LSCONN_COI_ACTIVE; 1884#ifndef NDEBUG 1885 if (iter->coi_last_sent) 1886 assert(iter->coi_last_sent <= conn->cn_last_sent); 1887 iter->coi_last_sent = conn->cn_last_sent; 1888#endif 1889 return conn; 1890 } 1891 else if (iter->coi_prq && (conn = prq_next_conn(iter->coi_prq))) 1892 { 1893 return conn; 1894 } 1895 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 1896 { 1897 iter->coi_prq = NULL; /* Save function call in previous conditional */ 1898 conn = iter->coi_next; 1899 if (!conn) 1900 conn = TAILQ_FIRST(&iter->coi_active_list); 1901 if (conn) 1902 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 1903 return conn; 1904 } 1905 else 1906 return NULL; 1907} 1908 1909 1910static void 1911coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1912{ 1913 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 1914 { 1915 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 1916 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1917 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1918 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 1919 conn->cn_flags |= LSCONN_COI_INACTIVE; 1920 } 1921} 1922 1923 1924static void 1925coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1926{ 1927 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 1928 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 1929 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 1930 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1931 conn->cn_flags |= LSCONN_COI_ACTIVE; 1932} 1933 1934 1935static void 1936coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 1937{ 1938 lsquic_conn_t *conn; 1939 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 1940 { 1941 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1942 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1943 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 1944 } 1945 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 1946 { 1947 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 1948 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 1949 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1950 } 1951} 1952 1953 1954#ifndef NDEBUG 1955static void 1956lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 1957 unsigned n) 1958{ 1959 const lsquic_cid_t *cid; 1960 struct iovec *iov; 1961 unsigned i; 1962 char packno_str[22]; 1963 1964 for (i = 0; i < n; ++i) 1965 { 1966 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 1967 batch->packets[i]->po_packno); 1968 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 1969 { 1970 for (iov = batch->outs[i].iov; iov < 1971 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 1972 batch->outs[i].iov->iov_len -= 1; 1973 cid = lsquic_conn_log_cid(batch->conns[i]); 1974 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 1975 CID_BITS(cid)); 1976 } 1977 } 1978} 1979 1980 1981#endif 1982 1983 1984#ifdef NDEBUG 1985#define CONST_BATCH const 1986#else 1987#define CONST_BATCH 1988#endif 1989 1990static unsigned 1991send_batch (lsquic_engine_t *engine, struct conns_out_iter *conns_iter, 1992 CONST_BATCH struct out_batch *batch, unsigned n_to_send) 1993{ 1994 int n_sent, i; 1995 lsquic_time_t now; 1996 unsigned off; 1997 size_t count; 1998 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 1999 2000#ifndef NDEBUG 2001 if (engine->flags & ENG_LOSE_PACKETS) 2002 lose_matching_packets(engine, batch, n_to_send); 2003#endif 2004 /* Set sent time before the write to avoid underestimating RTT */ 2005 now = lsquic_time_now(); 2006 for (i = 0; i < (int) n_to_send; ++i) 2007 { 2008 off = batch->pack_off[i]; 2009 count = batch->outs[i].iovlen; 2010 assert(count > 0); 2011 packet_out = &batch->packets[off]; 2012 end = packet_out + count; 2013 do 2014 (*packet_out)->po_sent = now; 2015 while (++packet_out < end); 2016 } 2017 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs, 2018 n_to_send); 2019 if (n_sent < (int) n_to_send) 2020 { 2021 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2022 engine->resume_sending_at = now + 1000000; 2023 LSQ_DEBUG("cannot send packets"); 2024 EV_LOG_GENERIC_EVENT("cannot send packets"); 2025 } 2026 if (n_sent >= 0) 2027 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, n_to_send); 2028 else 2029 { 2030 LSQ_DEBUG("packets out returned an error: %s", strerror(errno)); 2031 n_sent = 0; 2032 } 2033 if (n_sent > 0) 2034 engine->last_sent = now + n_sent; 2035 for (i = 0; i < n_sent; ++i) 2036 { 2037 eng_hist_inc(&engine->history, now, sl_packets_out); 2038 /* `i' is added to maintain relative order */ 2039 batch->conns[i]->cn_last_sent = now + i; 2040 2041 off = batch->pack_off[i]; 2042 count = batch->outs[i].iovlen; 2043 assert(count > 0); 2044 packet_out = &batch->packets[off]; 2045 end = packet_out + count; 2046 do 2047 { 2048#if LOG_PACKET_CHECKSUM 2049 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2050 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2051 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2052#endif 2053 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2054 *packet_out); 2055 /* Release packet out buffer as soon as the packet is sent 2056 * successfully. If not successfully sent, we hold on to 2057 * this buffer until the packet sending is attempted again 2058 * or until it times out and regenerated. 2059 */ 2060 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2061 release_enc_data(engine, batch->conns[i], *packet_out); 2062 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2063 *packet_out); 2064 } 2065 while (++packet_out < end); 2066 } 2067 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2068 for ( ; i < (int) n_to_send; ++i) 2069 { 2070 off = batch->pack_off[i]; 2071 count = batch->outs[i].iovlen; 2072 assert(count > 0); 2073 packet_out = &batch->packets[off]; 2074 end = packet_out + count; 2075 do 2076 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2077 *packet_out); 2078 while (++packet_out < end); 2079 } 2080 /* Return packets to the connection in reverse order so that the packet 2081 * ordering is maintained. 2082 */ 2083 for (i = (int) n_to_send - 1; i >= n_sent; --i) 2084 { 2085 off = batch->pack_off[i]; 2086 count = batch->outs[i].iovlen; 2087 assert(count > 0); 2088 packet_out = &batch->packets[off + count - 1]; 2089 end = &batch->packets[off - 1]; 2090 do 2091 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2092 *packet_out); 2093 while (--packet_out > end); 2094 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2095 coi_reactivate(conns_iter, batch->conns[i]); 2096 } 2097 return n_sent; 2098} 2099 2100 2101/* Return 1 if went past deadline, 0 otherwise */ 2102static int 2103check_deadline (lsquic_engine_t *engine) 2104{ 2105 if (engine->pub.enp_settings.es_proc_time_thresh && 2106 lsquic_time_now() > engine->deadline) 2107 { 2108 LSQ_INFO("went past threshold of %u usec, stop sending", 2109 engine->pub.enp_settings.es_proc_time_thresh); 2110 engine->flags |= ENG_PAST_DEADLINE; 2111 return 1; 2112 } 2113 else 2114 return 0; 2115} 2116 2117 2118static size_t 2119iov_size (const struct iovec *iov, const struct iovec *const end) 2120{ 2121 size_t size; 2122 2123 assert(iov < end); 2124 2125 size = 0; 2126 do 2127 size += iov->iov_len; 2128 while (++iov < end); 2129 2130 return size; 2131} 2132 2133 2134/* XXX A lot of extra setup -- two extra arguments to this function, two extra 2135 * connection ref flags and queues -- is just to handle the ENCPA_BADCRYPT case, 2136 * which never really happens. 2137 */ 2138static void 2139send_packets_out (struct lsquic_engine *engine, 2140 struct conns_tailq *ticked_conns, 2141 struct conns_stailq *closed_conns) 2142{ 2143 const lsquic_cid_t *cid; 2144 unsigned n, w, n_sent, n_batches_sent; 2145 lsquic_packet_out_t *packet_out; 2146 struct lsquic_packet_out **packet; 2147 lsquic_conn_t *conn; 2148 struct out_batch *const batch = &engine->out_batch; 2149 struct iovec *iov, *packet_iov; 2150 struct conns_out_iter conns_iter; 2151 int shrink, deadline_exceeded; 2152 2153 coi_init(&conns_iter, engine); 2154 n_batches_sent = 0; 2155 n_sent = 0, n = 0; 2156 shrink = 0; 2157 deadline_exceeded = 0; 2158 iov = batch->iov; 2159 packet = batch->packets; 2160 2161 while ((conn = coi_next(&conns_iter))) 2162 { 2163 cid = lsquic_conn_log_cid(conn); 2164 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2165 if (!packet_out) { 2166 /* Evanescent connection always has a packet to send: */ 2167 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2168 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2169 (conn->cn_flags & LSCONN_MINI ? "mini" : 2170 "full"), CID_BITS(cid)); 2171 coi_deactivate(&conns_iter, conn); 2172 continue; 2173 } 2174 batch->outs[n].iov = packet_iov = iov; 2175 next_coa: 2176 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2177 { 2178 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2179 &engine->pub, conn, packet_out)) 2180 { 2181 case ENCPA_NOMEM: 2182 /* Send what we have and wait for a more opportune moment */ 2183 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2184 goto end_for; 2185 case ENCPA_BADCRYPT: 2186 /* This is pretty bad: close connection immediately */ 2187 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2188 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2189 CID_BITS(cid)); 2190 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2191 { 2192 if (!(conn->cn_flags & LSCONN_CLOSING)) 2193 { 2194 STAILQ_INSERT_TAIL(closed_conns, conn, cn_next_closed_conn); 2195 engine_incref_conn(conn, LSCONN_CLOSING); 2196 if (conn->cn_flags & LSCONN_HASHED) 2197 remove_conn_from_hash(engine, conn); 2198 } 2199 coi_deactivate(&conns_iter, conn); 2200 if (conn->cn_flags & LSCONN_TICKED) 2201 { 2202 TAILQ_REMOVE(ticked_conns, conn, cn_next_ticked); 2203 engine_decref_conn(engine, conn, LSCONN_TICKED); 2204 } 2205 } 2206 continue; 2207 case ENCPA_OK: 2208 break; 2209 } 2210 } 2211 else if ((packet_out->po_flags & PO_NOENCRYPT) 2212 && engine->pub.enp_pmi != &stock_pmi) 2213 { 2214 if (0 != copy_packet(engine, conn, packet_out)) 2215 { 2216 /* Copy can only fail if packet could not be allocated */ 2217 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2218 goto end_for; 2219 } 2220 } 2221 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2222 packet_out->po_packno, CID_BITS(cid)); 2223 if (packet_out->po_flags & PO_ENCRYPTED) 2224 { 2225 iov->iov_base = packet_out->po_enc_data; 2226 iov->iov_len = packet_out->po_enc_data_sz; 2227 } 2228 else 2229 { 2230 iov->iov_base = packet_out->po_data; 2231 iov->iov_len = packet_out->po_data_sz; 2232 } 2233 if (packet_iov == iov) 2234 { 2235 batch->pack_off[n] = packet - batch->packets; 2236 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2237 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2238 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2239 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2240 batch->conns [n] = conn; 2241 } 2242 *packet = packet_out; 2243 ++packet; 2244 ++iov; 2245 if ((conn->cn_flags & LSCONN_IETF) 2246 && ((1 << packet_out->po_header_type) 2247 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2248#ifndef NDEBUG 2249 && (engine->flags & ENG_COALESCE) 2250#endif 2251 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2252 { 2253 const size_t size = iov_size(packet_iov, iov); 2254 packet_out = conn->cn_if->ci_next_packet_to_send(conn, size); 2255 if (packet_out) 2256 goto next_coa; 2257 } 2258 batch->outs [n].iovlen = iov - packet_iov; 2259 ++n; 2260 if (n == engine->batch_size 2261 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2262 { 2263 w = send_batch(engine, &conns_iter, batch, n); 2264 n = 0; 2265 iov = batch->iov; 2266 packet = batch->packets; 2267 ++n_batches_sent; 2268 n_sent += w; 2269 if (w < engine->batch_size) 2270 { 2271 shrink = 1; 2272 break; 2273 } 2274 deadline_exceeded = check_deadline(engine); 2275 if (deadline_exceeded) 2276 break; 2277 grow_batch_size(engine); 2278 } 2279 } 2280 end_for: 2281 2282 if (n > 0) { 2283 w = send_batch(engine, &conns_iter, batch, n); 2284 n_sent += w; 2285 shrink = w < n; 2286 ++n_batches_sent; 2287 } 2288 2289 if (shrink) 2290 shrink_batch_size(engine); 2291 else if (n_batches_sent > 1) 2292 { 2293 deadline_exceeded = check_deadline(engine); 2294 if (!deadline_exceeded) 2295 grow_batch_size(engine); 2296 } 2297 2298 coi_reheap(&conns_iter, engine); 2299 2300 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2301} 2302 2303 2304int 2305lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2306{ 2307 return lsquic_mh_count(&engine->conns_out) > 0 2308 || (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2309 ; 2310} 2311 2312 2313static void 2314reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2315{ 2316 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2317 engine->flags &= ~ENG_PAST_DEADLINE; 2318} 2319 2320 2321/* TODO: this is a user-facing function, account for load */ 2322void 2323lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2324{ 2325 lsquic_conn_t *conn; 2326 struct conns_stailq closed_conns; 2327 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2328 struct cid_update_batch cub; 2329 2330 ENGINE_IN(engine); 2331 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2332 STAILQ_INIT(&closed_conns); 2333 reset_deadline(engine, lsquic_time_now()); 2334 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2335 { 2336 LSQ_DEBUG("can send again"); 2337 EV_LOG_GENERIC_EVENT("can send again"); 2338 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2339 } 2340 2341 send_packets_out(engine, &ticked_conns, &closed_conns); 2342 2343 while ((conn = STAILQ_FIRST(&closed_conns))) { 2344 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2345 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2346 cub_add_cids_from_cces(&cub, conn); 2347 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2348 } 2349 2350 cub_flush(&cub); 2351 ENGINE_OUT(engine); 2352} 2353 2354 2355static lsquic_conn_t * 2356next_new_full_conn (struct conns_stailq *new_full_conns) 2357{ 2358 lsquic_conn_t *conn; 2359 2360 conn = STAILQ_FIRST(new_full_conns); 2361 if (conn) 2362 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2363 return conn; 2364} 2365 2366 2367static void 2368process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2369 lsquic_time_t now) 2370{ 2371 lsquic_conn_t *conn; 2372 enum tick_st tick_st; 2373 unsigned i; 2374 lsquic_time_t next_tick_time; 2375 struct conns_stailq closed_conns; 2376 struct conns_tailq ticked_conns; 2377 struct conns_stailq new_full_conns; 2378 struct cid_update_batch cub_old, cub_live; 2379 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2380 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2381 2382 eng_hist_tick(&engine->history, now); 2383 2384 STAILQ_INIT(&closed_conns); 2385 TAILQ_INIT(&ticked_conns); 2386 reset_deadline(engine, now); 2387 STAILQ_INIT(&new_full_conns); 2388 2389 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2390 && now > engine->resume_sending_at) 2391 { 2392 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2393 "timeout"); 2394 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2395 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2396 } 2397 2398 i = 0; 2399 while ((conn = next_conn(engine)) 2400 || (conn = next_new_full_conn(&new_full_conns))) 2401 { 2402 tick_st = conn->cn_if->ci_tick(conn, now); 2403 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2404 if (tick_st & TICK_PROMOTE) 2405 { 2406 lsquic_conn_t *new_conn; 2407 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2408 "scheduled for promotion"); 2409 assert(conn->cn_flags & LSCONN_MINI); 2410 new_conn = new_full_conn_server(engine, conn, now); 2411 if (new_conn) 2412 { 2413 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2414 new_conn->cn_last_sent = engine->last_sent; 2415 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2416 } 2417 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2418 conn->cn_flags |= LSCONN_PROMOTED; 2419 } 2420 if (tick_st & TICK_SEND) 2421 { 2422 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2423 { 2424 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2425 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2426 } 2427 } 2428 if (tick_st & TICK_CLOSE) 2429 { 2430 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2431 engine_incref_conn(conn, LSCONN_CLOSING); 2432 if (conn->cn_flags & LSCONN_HASHED) 2433 remove_conn_from_hash(engine, conn); 2434 } 2435 else 2436 { 2437 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2438 engine_incref_conn(conn, LSCONN_TICKED); 2439 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2440 && conn->cn_if->ci_report_live(conn, now)) 2441 cub_add_cids_from_cces(&cub_live, conn); 2442 } 2443 } 2444 2445 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2446 && lsquic_engine_has_unsent_packets(engine)) 2447 send_packets_out(engine, &ticked_conns, &closed_conns); 2448 2449 while ((conn = STAILQ_FIRST(&closed_conns))) { 2450 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2451 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2452 cub_add_cids_from_cces(&cub_old, conn); 2453 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2454 } 2455 2456 /* TODO Heapification can be optimized by switching to the Floyd method: 2457 * https://en.wikipedia.org/wiki/Binary_heap#Building_a_heap 2458 */ 2459 while ((conn = TAILQ_FIRST(&ticked_conns))) 2460 { 2461 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2462 engine_decref_conn(engine, conn, LSCONN_TICKED); 2463 if (!(conn->cn_flags & LSCONN_TICKABLE) 2464 && conn->cn_if->ci_is_tickable(conn)) 2465 { 2466 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2467 engine_incref_conn(conn, LSCONN_TICKABLE); 2468 } 2469 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2470 { 2471 next_tick_time = conn->cn_if->ci_next_tick_time(conn); 2472 if (next_tick_time) 2473 { 2474 if (0 == attq_add(engine->attq, conn, next_tick_time)) 2475 engine_incref_conn(conn, LSCONN_ATTQ); 2476 } 2477 else 2478 assert(0); 2479 } 2480 } 2481 2482 cub_flush(&engine->new_scids); 2483 cub_flush(&cub_live); 2484 cub_flush(&cub_old); 2485} 2486 2487 2488/* Return 0 if packet is being processed by a real connection, 1 if the 2489 * packet was processed, but not by a connection, and -1 on error. 2490 */ 2491int 2492lsquic_engine_packet_in (lsquic_engine_t *engine, 2493 const unsigned char *packet_in_data, size_t packet_in_size, 2494 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2495 void *peer_ctx, int ecn) 2496{ 2497 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2498 struct packin_parse_state ppstate; 2499 lsquic_packet_in_t *packet_in; 2500 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2501 int is_server, unsigned cid_len, struct packin_parse_state *); 2502 unsigned n_zeroes; 2503 int s; 2504 2505 ENGINE_CALLS_INCR(engine); 2506 2507 if (engine->flags & ENG_SERVER) 2508 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2509 else 2510 if (engine->flags & ENG_CONNS_BY_ADDR) 2511 { 2512 struct lsquic_hash_elem *el; 2513 const struct lsquic_conn *conn; 2514 el = find_conn_by_addr(engine->conns_hash, sa_local); 2515 if (!el) 2516 return -1; 2517 conn = lsquic_hashelem_getdata(el); 2518 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2519 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2520 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2521 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2522 else 2523 { 2524 assert(conn->cn_version == LSQVER_046 2525#if LSQUIC_USE_Q098 2526 || conn->cn_version == LSQVER_098 2527#endif 2528 2529 ); 2530 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2531 } 2532 } 2533 else 2534 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2535 2536 n_zeroes = 0; 2537 do 2538 { 2539 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 2540 if (!packet_in) 2541 return -1; 2542 /* Library does not modify packet_in_data, it is not referenced after 2543 * this function returns and subsequent release of pi_data is guarded 2544 * by PI_OWN_DATA flag. 2545 */ 2546 packet_in->pi_data = (unsigned char *) packet_in_data; 2547 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 2548 engine->flags & ENG_SERVER, 2549 engine->pub.enp_settings.es_scid_len, &ppstate)) 2550 { 2551 LSQ_DEBUG("Cannot parse incoming packet's header"); 2552 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 2553 errno = EINVAL; 2554 return -1; 2555 } 2556 2557 packet_in_data += packet_in->pi_data_sz; 2558 packet_in->pi_received = lsquic_time_now(); 2559 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 2560 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 2561 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 2562 peer_ctx); 2563 n_zeroes += s == 0; 2564 } 2565 while (0 == s && packet_in_data < packet_end); 2566 2567 return n_zeroes > 0 ? 0 : s; 2568} 2569 2570 2571#if __GNUC__ && !defined(NDEBUG) 2572__attribute__((weak)) 2573#endif 2574unsigned 2575lsquic_engine_quic_versions (const lsquic_engine_t *engine) 2576{ 2577 return engine->pub.enp_settings.es_versions; 2578} 2579 2580 2581void 2582lsquic_engine_cooldown (lsquic_engine_t *engine) 2583{ 2584 struct lsquic_hash_elem *el; 2585 lsquic_conn_t *conn; 2586 2587 if (engine->flags & ENG_COOLDOWN) 2588 /* AFAICT, there is no harm in calling this function more than once, 2589 * but log it just in case, as it may indicate an error in the caller. 2590 */ 2591 LSQ_INFO("cooldown called again"); 2592 engine->flags |= ENG_COOLDOWN; 2593 LSQ_INFO("entering cooldown mode"); 2594 if (engine->flags & ENG_SERVER) 2595 drop_all_mini_conns(engine); 2596 for (el = lsquic_hash_first(engine->conns_hash); el; 2597 el = lsquic_hash_next(engine->conns_hash)) 2598 { 2599 conn = lsquic_hashelem_getdata(el); 2600 lsquic_conn_going_away(conn); 2601 } 2602} 2603 2604 2605int 2606lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 2607{ 2608 const lsquic_time_t *next_attq_time; 2609 lsquic_time_t now, next_time; 2610 2611 ENGINE_CALLS_INCR(engine); 2612 2613 if (((engine->flags & ENG_PAST_DEADLINE) 2614 && lsquic_mh_count(&engine->conns_out)) 2615 || (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2616 || lsquic_mh_count(&engine->conns_tickable)) 2617 { 2618 *diff = 0; 2619 return 1; 2620 } 2621 2622 next_attq_time = attq_next_time(engine->attq); 2623 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 2624 { 2625 if (next_attq_time) 2626 next_time = *next_attq_time; 2627 else 2628 return 0; 2629 } 2630 else 2631 { 2632 if (next_attq_time) 2633 next_time = MIN(*next_attq_time, engine->resume_sending_at); 2634 else 2635 next_time = engine->resume_sending_at; 2636 } 2637 2638 now = lsquic_time_now(); 2639 *diff = (int) ((int64_t) next_time - (int64_t) now); 2640 return 1; 2641} 2642 2643 2644unsigned 2645lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 2646{ 2647 lsquic_time_t now; 2648 ENGINE_CALLS_INCR(engine); 2649 now = lsquic_time_now(); 2650 if (from_now < 0) 2651 now -= from_now; 2652 else 2653 now += from_now; 2654 return attq_count_before(engine->attq, now); 2655} 2656 2657 2658int 2659lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 2660 struct lsquic_conn *conn, unsigned cce_idx) 2661{ 2662 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2663 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2664 void *peer_ctx; 2665 2666 assert(cce_idx < conn->cn_n_cces); 2667 assert(conn->cn_cces_mask & (1 << cce_idx)); 2668 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 2669 2670 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 2671 cce->cce_cid.len, conn, &cce->cce_hash_el)) 2672 { 2673 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 2674 CID_BITS(&cce->cce_cid)); 2675 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2676 cce->cce_flags |= CCE_REG; 2677 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 2678 return 0; 2679 } 2680 else 2681 { 2682 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 2683 CID_BITS(&cce->cce_cid)); 2684 return -1; 2685 } 2686} 2687 2688 2689void 2690lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 2691 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now) 2692{ 2693 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2694 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2695 void *peer_ctx; 2696 2697 assert(cce_idx < conn->cn_n_cces); 2698 2699 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 2700 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 2701 2702 if (engine->purga) 2703 { 2704 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2705 lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 2706 PUTY_CID_RETIRED, now); 2707 } 2708 conn->cn_cces_mask &= ~(1u << cce_idx); 2709 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 2710} 2711 2712 2713