lsquic_engine.c revision f2a7fa84
1/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#ifndef WIN32 17#include <sys/time.h> 18#include <netinet/in.h> 19#include <sys/types.h> 20#include <sys/stat.h> 21#include <fcntl.h> 22#include <unistd.h> 23#include <netdb.h> 24#endif 25 26#ifndef NDEBUG 27#include <sys/types.h> 28#include <regex.h> /* For code that loses packets */ 29#endif 30 31#if LOG_PACKET_CHECKSUM 32#include <zlib.h> 33#endif 34 35#include <openssl/aead.h> 36 37#include "lsquic.h" 38#include "lsquic_types.h" 39#include "lsquic_int_types.h" 40#include "lsquic_sizes.h" 41#include "lsquic_parse_common.h" 42#include "lsquic_parse.h" 43#include "lsquic_packet_in.h" 44#include "lsquic_packet_out.h" 45#include "lsquic_senhist.h" 46#include "lsquic_rtt.h" 47#include "lsquic_cubic.h" 48#include "lsquic_pacer.h" 49#include "lsquic_bw_sampler.h" 50#include "lsquic_minmax.h" 51#include "lsquic_bbr.h" 52#include "lsquic_send_ctl.h" 53#include "lsquic_set.h" 54#include "lsquic_conn_flow.h" 55#include "lsquic_sfcw.h" 56#include "lsquic_hash.h" 57#include "lsquic_conn.h" 58#include "lsquic_full_conn.h" 59#include "lsquic_util.h" 60#include "lsquic_qtags.h" 61#include "lsquic_enc_sess.h" 62#include "lsquic_mm.h" 63#include "lsquic_engine_public.h" 64#include "lsquic_eng_hist.h" 65#include "lsquic_ev_log.h" 66#include "lsquic_version.h" 67#include "lsquic_pr_queue.h" 68#include "lsquic_mini_conn.h" 69#include "lsquic_mini_conn_ietf.h" 70#include "lsquic_stock_shi.h" 71#include "lsquic_purga.h" 72#include "lsquic_tokgen.h" 73#include "lsquic_attq.h" 74#include "lsquic_min_heap.h" 75#include "lsquic_http1x_if.h" 76#include "lsquic_parse_common.h" 77#include "lsquic_handshake.h" 78#include "lsquic_crand.h" 79#include "lsquic_ietf.h" 80 81#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 82#include "lsquic_logger.h" 83 84#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 85#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 86#endif 87 88#if LSQUIC_DEBUG_NEXT_ADV_TICK 89#include "lsquic_alarmset.h" 90#endif 91 92#define MIN(a, b) ((a) < (b) ? (a) : (b)) 93 94/* The batch of outgoing packets grows and shrinks dynamically */ 95#define MAX_OUT_BATCH_SIZE 1024 96#define MIN_OUT_BATCH_SIZE 4 97#define INITIAL_OUT_BATCH_SIZE 32 98 99struct out_batch 100{ 101 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 102 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 103 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 104 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 105 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 106}; 107 108typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 109 110static void 111process_connections (struct lsquic_engine *engine, conn_iter_f iter, 112 lsquic_time_t now); 113 114static void 115engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 116 117static lsquic_conn_t * 118engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 119 enum lsquic_conn_flags flag); 120 121static void 122force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 123 124#if LSQUIC_COUNT_ENGINE_CALLS 125#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 126#else 127#define ENGINE_CALLS_INCR(e) 128#endif 129 130/* Nested calls to LSQUIC are not supported */ 131#define ENGINE_IN(e) do { \ 132 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 133 (e)->pub.enp_flags |= ENPUB_PROC; \ 134 ENGINE_CALLS_INCR(e); \ 135} while (0) 136 137#define ENGINE_OUT(e) do { \ 138 assert((e)->pub.enp_flags & ENPUB_PROC); \ 139 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 140} while (0) 141 142/* A connection can be referenced from one of six places: 143 * 144 * 1. A hash. The engine maintains two hash tables -- one for full, and 145 * one for mini connections. A connection starts its life in one of 146 * those. 147 * 148 * 2. Outgoing queue. 149 * 150 * 3. Tickable queue 151 * 152 * 4. Advisory Tick Time queue. 153 * 154 * 5. Closing connections queue. This is a transient queue -- it only 155 * exists for the duration of process_connections() function call. 156 * 157 * 6. Ticked connections queue. Another transient queue, similar to (5). 158 * 159 * The idea is to destroy the connection when it is no longer referenced. 160 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 161 * that case, the connection is referenced from two places: (2) and (5). 162 * After its packets are sent, it is only referenced in (5), and at the 163 * end of the function call, when it is removed from (5), reference count 164 * goes to zero and the connection is destroyed. If not all packets can 165 * be sent, at the end of the function call, the connection is referenced 166 * by (2) and will only be removed once all outgoing packets have been 167 * sent. 168 */ 169#define CONN_REF_FLAGS (LSCONN_HASHED \ 170 |LSCONN_HAS_OUTGOING \ 171 |LSCONN_TICKABLE \ 172 |LSCONN_TICKED \ 173 |LSCONN_CLOSING \ 174 |LSCONN_ATTQ) 175 176 177 178 179struct cid_update_batch 180{ 181 lsquic_cids_update_f cub_update_cids; 182 void *cub_update_ctx; 183 unsigned cub_count; 184 lsquic_cid_t cub_cids[20]; 185 void *cub_peer_ctxs[20]; 186}; 187 188static void 189cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 190 191 192struct lsquic_engine 193{ 194 struct lsquic_engine_public pub; 195 enum { 196 ENG_SERVER = LSENG_SERVER, 197 ENG_HTTP = LSENG_HTTP, 198 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 199 ENG_PAST_DEADLINE 200 = (1 << 8), /* Previous call to a processing 201 * function went past time threshold. 202 */ 203 ENG_CONNS_BY_ADDR 204 = (1 << 9), /* Connections are hashed by address */ 205#ifndef NDEBUG 206 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 207 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 208 ENG_DTOR = (1 << 26), /* Engine destructor */ 209#endif 210 } flags; 211 lsquic_packets_out_f packets_out; 212 void *packets_out_ctx; 213 lsquic_cids_update_f report_new_scids; 214 lsquic_cids_update_f report_live_scids; 215 lsquic_cids_update_f report_old_scids; 216 void *scids_ctx; 217 struct lsquic_hash *conns_hash; 218 struct min_heap conns_tickable; 219 struct min_heap conns_out; 220 /* Use a union because only one iterator is being used at any one time */ 221 union { 222 struct { 223 struct cert_susp_head *head; 224 } resumed; 225 struct lsquic_conn *one_conn; 226 } iter_state; 227 struct eng_hist history; 228 unsigned batch_size; 229 struct pr_queue *pr_queue; 230 struct attq *attq; 231 /* Track time last time a packet was sent to give new connections 232 * priority lower than that of existing connections. 233 */ 234 lsquic_time_t last_sent; 235#ifndef NDEBUG 236 regex_t lose_packets_re; 237 const char *lose_packets_str; 238#endif 239 unsigned n_conns; 240 lsquic_time_t deadline; 241 lsquic_time_t resume_sending_at; 242 unsigned mini_conns_count; 243 struct lsquic_purga *purga; 244#if LSQUIC_CONN_STATS 245 struct { 246 unsigned conns; 247 } stats; 248 struct conn_stats conn_stats_sum; 249 FILE *stats_fh; 250#endif 251 struct cid_update_batch new_scids; 252 struct out_batch out_batch; 253#if LSQUIC_COUNT_ENGINE_CALLS 254 unsigned long n_engine_calls; 255#endif 256#if LSQUIC_DEBUG_NEXT_ADV_TICK 257 uintptr_t last_logged_conn; 258 unsigned last_logged_ae_why; 259 int last_tick_diff; 260#endif 261 struct crand crand; 262 EVP_AEAD_CTX retry_aead_ctx; 263}; 264 265 266void 267lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 268 unsigned flags) 269{ 270 memset(settings, 0, sizeof(*settings)); 271 settings->es_versions = LSQUIC_DF_VERSIONS; 272 if (flags & ENG_SERVER) 273 { 274 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 275 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 276 settings->es_init_max_data 277 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 278 settings->es_init_max_stream_data_bidi_remote 279 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 280 settings->es_init_max_stream_data_bidi_local 281 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 282 settings->es_init_max_stream_data_uni 283 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 284 settings->es_init_max_streams_uni 285 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 286 settings->es_ping_period = 0; 287 } 288 else 289 { 290 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 291 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 292 settings->es_init_max_data 293 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 294 settings->es_init_max_stream_data_bidi_remote 295 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 296 settings->es_init_max_stream_data_bidi_local 297 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 298 settings->es_init_max_stream_data_uni 299 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 300 settings->es_init_max_streams_uni 301 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 302 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 303 } 304 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 305 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 306 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 307 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 308 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 309 settings->es_max_header_list_size 310 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 311 settings->es_ua = LSQUIC_DF_UA; 312 settings->es_ecn = LSQUIC_DF_ECN; 313 314 settings->es_pdmd = QTAG_X509; 315 settings->es_aead = QTAG_AESG; 316 settings->es_kexs = QTAG_C255; 317 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 318 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 319 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 320 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 321 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 322 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 323 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 324 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 325 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 326 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 327 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 328 settings->es_sttl = LSQUIC_DF_STTL; 329 settings->es_init_max_streams_bidi 330 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 331 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 332 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 333 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 334 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 335 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 336 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 337 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 338 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 339 settings->es_spin = LSQUIC_DF_SPIN; 340} 341 342 343/* Note: if returning an error, err_buf must be valid if non-NULL */ 344int 345lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 346 unsigned flags, 347 char *err_buf, size_t err_buf_sz) 348{ 349 if (settings->es_cfcw < LSQUIC_MIN_FCW || 350 settings->es_sfcw < LSQUIC_MIN_FCW) 351 { 352 if (err_buf) 353 snprintf(err_buf, err_buf_sz, "%s", 354 "flow control window set too low"); 355 return -1; 356 } 357 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 358 { 359 if (err_buf) 360 snprintf(err_buf, err_buf_sz, "%s", 361 "No supported QUIC versions specified"); 362 return -1; 363 } 364 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 365 { 366 if (err_buf) 367 snprintf(err_buf, err_buf_sz, "%s", 368 "one or more unsupported QUIC version is specified"); 369 return -1; 370 } 371 if (flags & ENG_SERVER) 372 { 373 if (settings->es_handshake_to > 374 MAX_MINI_CONN_LIFESPAN_IN_USEC) 375 { 376 if (err_buf) 377 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 378 " usec is too large. The maximum for server is %u usec", 379 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 380 return -1; 381 } 382 } 383 if (settings->es_idle_timeout > 600) 384 { 385 if (err_buf) 386 snprintf(err_buf, err_buf_sz, "%s", 387 "The maximum value of idle timeout is 600 seconds"); 388 return -1; 389 } 390 if (settings->es_scid_len > MAX_CID_LEN) 391 { 392 if (err_buf) 393 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 394 "bytes long; it must be between 0 and %u.", 395 settings->es_scid_len, MAX_CID_LEN); 396 return -1; 397 } 398 399 if (settings->es_cc_algo > 2) 400 { 401 if (err_buf) 402 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 403 "algorithm value %u", settings->es_cc_algo); 404 return -1; 405 } 406 407 if (!(settings->es_ql_bits >= -1 && settings->es_ql_bits <= 2)) 408 { 409 if (err_buf) 410 snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ", 411 settings->es_ql_bits); 412 return -1; 413 } 414 415 if (!(settings->es_spin == 0 || settings->es_spin == 1)) 416 { 417 if (err_buf) 418 snprintf(err_buf, err_buf_sz, "Invalid spin value %d", 419 settings->es_spin); 420 return -1; 421 } 422 423 return 0; 424} 425 426 427static void 428free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 429{ 430 free(packet_data); 431} 432 433 434static void * 435malloc_buf (void *ctx, void *conn_ctx, unsigned short size, char is_ipv6) 436{ 437 return malloc(size); 438} 439 440 441static const struct lsquic_packout_mem_if stock_pmi = 442{ 443 malloc_buf, free_packet, free_packet, 444}; 445 446 447static int 448hash_conns_by_addr (const struct lsquic_engine *engine) 449{ 450 if (engine->flags & ENG_SERVER) 451 return 0; 452 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 453 return 1; 454 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 455 && engine->pub.enp_settings.es_support_tcid0) 456 return 1; 457 if (engine->pub.enp_settings.es_scid_len == 0) 458 return 1; 459 return 0; 460} 461 462 463lsquic_engine_t * 464lsquic_engine_new (unsigned flags, 465 const struct lsquic_engine_api *api) 466{ 467 lsquic_engine_t *engine; 468 char err_buf[100]; 469 470 if (!api->ea_packets_out) 471 { 472 LSQ_ERROR("packets_out callback is not specified"); 473 return NULL; 474 } 475 476 if (api->ea_settings && 477 0 != lsquic_engine_check_settings(api->ea_settings, flags, 478 err_buf, sizeof(err_buf))) 479 { 480 LSQ_ERROR("cannot create engine: %s", err_buf); 481 return NULL; 482 } 483 484 engine = calloc(1, sizeof(*engine)); 485 if (!engine) 486 return NULL; 487 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 488 { 489 free(engine); 490 return NULL; 491 } 492 if (api->ea_settings) 493 engine->pub.enp_settings = *api->ea_settings; 494 else 495 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 496 int tag_buf_len; 497 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 498 sizeof(engine->pub.enp_ver_tags_buf), 499 engine->pub.enp_settings.es_versions); 500 if (tag_buf_len <= 0) 501 { 502 LSQ_ERROR("cannot generate version tags buffer"); 503 free(engine); 504 return NULL; 505 } 506 engine->pub.enp_ver_tags_len = tag_buf_len; 507 engine->pub.enp_flags = ENPUB_CAN_SEND; 508 engine->pub.enp_stream_if = api->ea_stream_if; 509 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 510 511 engine->flags = flags; 512#ifndef NDEBUG 513 engine->flags |= ENG_COALESCE; 514#endif 515 engine->packets_out = api->ea_packets_out; 516 engine->packets_out_ctx = api->ea_packets_out_ctx; 517 engine->report_new_scids = api->ea_new_scids; 518 engine->report_live_scids = api->ea_live_scids; 519 engine->report_old_scids = api->ea_old_scids; 520 engine->scids_ctx = api->ea_cids_update_ctx; 521 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 522 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 523 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 524 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 525 if (api->ea_shi) 526 { 527 engine->pub.enp_shi = api->ea_shi; 528 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 529 } 530 else 531 { 532 engine->pub.enp_shi = &stock_shi; 533 engine->pub.enp_shi_ctx = stock_shared_hash_new(); 534 if (!engine->pub.enp_shi_ctx) 535 { 536 free(engine); 537 return NULL; 538 } 539 } 540 if (api->ea_hsi_if) 541 { 542 engine->pub.enp_hsi_if = api->ea_hsi_if; 543 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 544 } 545 else 546 { 547 engine->pub.enp_hsi_if = lsquic_http1x_if; 548 engine->pub.enp_hsi_ctx = NULL; 549 } 550 if (api->ea_pmi) 551 { 552 engine->pub.enp_pmi = api->ea_pmi; 553 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 554 } 555 else 556 { 557 engine->pub.enp_pmi = &stock_pmi; 558 engine->pub.enp_pmi_ctx = NULL; 559 } 560 engine->pub.enp_verify_cert = api->ea_verify_cert; 561 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 562 engine->pub.enp_kli = api->ea_keylog_if; 563 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 564 engine->pub.enp_engine = engine; 565 if (hash_conns_by_addr(engine)) 566 engine->flags |= ENG_CONNS_BY_ADDR; 567 engine->conns_hash = lsquic_hash_create(); 568 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 569 if (!engine->pub.enp_tokgen) 570 return NULL; 571 engine->pub.enp_crand = &engine->crand; 572 if (flags & ENG_SERVER) 573 { 574 engine->pr_queue = prq_create( 575 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 576 &engine->pub); 577 if (!engine->pr_queue) 578 { 579 lsquic_tg_destroy(engine->pub.enp_tokgen); 580 return NULL; 581 } 582 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 583 engine->report_old_scids, engine->scids_ctx); 584 if (!engine->purga) 585 { 586 lsquic_tg_destroy(engine->pub.enp_tokgen); 587 prq_destroy(engine->pr_queue); 588 return NULL; 589 } 590 } 591 engine->attq = attq_create(); 592 eng_hist_init(&engine->history); 593 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 594 if (engine->pub.enp_settings.es_honor_prst) 595 { 596 engine->pub.enp_srst_hash = lsquic_hash_create(); 597 if (!engine->pub.enp_srst_hash) 598 { 599 lsquic_engine_destroy(engine); 600 return NULL; 601 } 602 } 603 604#ifndef NDEBUG 605 { 606 const char *env; 607 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 608 if (env) 609 { 610 if (0 != regcomp(&engine->lose_packets_re, env, 611 REG_EXTENDED|REG_NOSUB)) 612 { 613 LSQ_ERROR("could not compile lost packet regex `%s'", env); 614 return NULL; 615 } 616 engine->flags |= ENG_LOSE_PACKETS; 617 engine->lose_packets_str = env; 618 LSQ_WARN("will lose packets that match the following regex: %s", 619 env); 620 } 621 env = getenv("LSQUIC_COALESCE"); 622 if (env) 623 { 624 engine->flags &= ~ENG_COALESCE; 625 if (atoi(env)) 626 { 627 engine->flags |= ENG_COALESCE; 628 LSQ_NOTICE("will coalesce packets"); 629 } 630 else 631 LSQ_NOTICE("will not coalesce packets"); 632 } 633 } 634#endif 635#if LSQUIC_CONN_STATS 636 engine->stats_fh = api->ea_stats_fh; 637#endif 638 if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx, EVP_aead_aes_128_gcm(), 639 IETF_RETRY_KEY_BUF, IETF_RETRY_KEY_SZ, 16, NULL)) 640 { 641 LSQ_ERROR("could not initialize retry AEAD ctx"); 642 lsquic_engine_destroy(engine); 643 return NULL; 644 } 645 engine->pub.enp_retry_aead_ctx = &engine->retry_aead_ctx; 646 647 LSQ_INFO("instantiated engine"); 648 return engine; 649} 650 651 652#if LOG_PACKET_CHECKSUM 653static void 654log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 655 const unsigned char *buf, size_t bufsz) 656{ 657 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 658 (uint32_t) crc32(0, buf, bufsz)); 659} 660 661 662#endif 663 664 665static void 666grow_batch_size (struct lsquic_engine *engine) 667{ 668 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 669} 670 671 672static void 673shrink_batch_size (struct lsquic_engine *engine) 674{ 675 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 676} 677 678 679struct cce_cid_iter 680{ 681 const struct lsquic_conn *conn; 682 unsigned todo, n; 683}; 684 685 686static struct conn_cid_elem * 687cce_iter_next (struct cce_cid_iter *citer) 688{ 689 struct conn_cid_elem *cce; 690 691 while (citer->todo) 692 if (citer->todo & (1 << citer->n)) 693 { 694 citer->todo &= ~(1 << citer->n); 695 cce = &citer->conn->cn_cces[ citer->n++ ]; 696 if (!(cce->cce_flags & CCE_PORT)) 697 return cce; 698 } 699 else 700 ++citer->n; 701 702 return NULL; 703} 704 705 706static struct conn_cid_elem * 707cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 708{ 709 citer->conn = conn; 710 citer->todo = conn->cn_cces_mask; 711 citer->n = 0; 712 return cce_iter_next(citer); 713} 714 715 716#if LSQUIC_CONN_STATS 717void 718update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 719{ 720 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 721 const unsigned long *src; 722 const struct conn_stats *stats; 723 unsigned i; 724 725 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 726 { 727 ++engine->stats.conns; 728 src = (unsigned long *) stats; 729 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 730 dst[i] += src[i]; 731 } 732} 733 734 735#endif 736 737 738/* Wrapper to make sure important things occur before the connection is 739 * really destroyed. 740 */ 741static void 742destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 743 lsquic_time_t now) 744{ 745 struct cce_cid_iter citer; 746 const struct conn_cid_elem *cce; 747 lsquic_time_t drain_time; 748 struct purga_el *puel; 749 750 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 751 if (engine->purga 752 /* Blacklist all CIDs except for promoted mini connections */ 753 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 754 != (LSCONN_MINI|LSCONN_PROMOTED)) 755 { 756 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 757 && conn->cn_if->ci_drain_time && 758 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 759 { 760 for (cce = cce_iter_first(&citer, conn); cce; 761 cce = cce_iter_next(&citer)) 762 { 763 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 764 lsquic_conn_get_peer_ctx(conn, NULL), 765 PUTY_CONN_DRAIN, now); 766 if (puel) 767 puel->puel_time = now + drain_time; 768 } 769 } 770 else 771 { 772 for (cce = cce_iter_first(&citer, conn); cce; 773 cce = cce_iter_next(&citer)) 774 { 775 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 776 lsquic_conn_get_peer_ctx(conn, NULL), 777 PUTY_CONN_DELETED, now); 778 if (puel) 779 { 780 puel->puel_time = now; 781 puel->puel_count = 0; 782 } 783 } 784 } 785 } 786#if LSQUIC_CONN_STATS 787 update_stats_sum(engine, conn); 788#endif 789 --engine->n_conns; 790 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 791 conn->cn_if->ci_destroy(conn); 792} 793 794 795static int 796maybe_grow_conn_heaps (struct lsquic_engine *engine) 797{ 798 struct min_heap_elem *els; 799 unsigned count; 800 801 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 802 return 0; /* Nothing to do */ 803 804 if (lsquic_mh_nalloc(&engine->conns_tickable)) 805 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 806 else 807 count = 8; 808 809 els = malloc(sizeof(els[0]) * count); 810 if (!els) 811 { 812 LSQ_ERROR("%s: malloc failed", __func__); 813 return -1; 814 } 815 816 LSQ_DEBUG("grew heaps to %u elements", count / 2); 817 memcpy(&els[0], engine->conns_tickable.mh_elems, 818 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 819 memcpy(&els[count / 2], engine->conns_out.mh_elems, 820 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 821 free(engine->conns_tickable.mh_elems); 822 engine->conns_tickable.mh_elems = els; 823 engine->conns_out.mh_elems = &els[count / 2]; 824 engine->conns_tickable.mh_nalloc = count / 2; 825 engine->conns_out.mh_nalloc = count / 2; 826 return 0; 827} 828 829 830static void 831remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 832 unsigned todo) 833{ 834 unsigned n; 835 836 for (n = 0; todo; todo &= ~(1 << n++)) 837 if ((todo & (1 << n)) && 838 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 839 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 840} 841 842 843static void 844remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 845{ 846 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 847} 848 849 850static void 851cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 852 853 854static int 855insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 856 void *peer_ctx) 857{ 858 struct conn_cid_elem *cce; 859 unsigned todo, done, n; 860 861 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 862 if (todo & (1 << n)) 863 { 864 cce = &conn->cn_cces[n]; 865 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 866 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 867 cce->cce_cid.len, conn, &cce->cce_hash_el)) 868 done |= 1 << n; 869 else 870 goto err; 871 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 872 { 873 cce->cce_flags |= CCE_REG; 874 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 875 } 876 } 877 878 return 0; 879 880 err: 881 remove_cces_from_hash(engine->conns_hash, conn, done); 882 return -1; 883} 884 885 886static lsquic_conn_t * 887new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 888 lsquic_time_t now) 889{ 890 const lsquic_cid_t *cid; 891 server_conn_ctor_f ctor; 892 lsquic_conn_t *conn; 893 unsigned flags; 894 if (0 != maybe_grow_conn_heaps(engine)) 895 return NULL; 896 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 897 898 if (mini_conn->cn_flags & LSCONN_IETF) 899 { 900 if (mini_conn->cn_version == LSQVER_ID24) 901 ctor = lsquic_id24_full_conn_server_new; 902 else 903 ctor = lsquic_ietf_full_conn_server_new; 904 } 905 else 906 ctor = lsquic_gquic_full_conn_server_new; 907 908 conn = ctor(&engine->pub, flags, mini_conn); 909 if (!conn) 910 { 911 /* Otherwise, full_conn_server_new prints its own warnings */ 912 if (ENOMEM == errno) 913 { 914 cid = lsquic_conn_log_cid(mini_conn); 915 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 916 CID_BITS(cid), strerror(errno)); 917 } 918 return NULL; 919 } 920 ++engine->n_conns; 921 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 922 { 923 cid = lsquic_conn_log_cid(conn); 924 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 925 CID_BITS(cid)); 926 destroy_conn(engine, conn, now); 927 return NULL; 928 } 929 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 930 conn->cn_flags |= LSCONN_HASHED; 931 return conn; 932} 933 934 935static enum 936{ 937 VER_NOT_SPECIFIED, 938 VER_SUPPORTED, 939 VER_UNSUPPORTED, 940} 941 942 943version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 944 enum lsquic_version *pversion) 945{ 946 lsquic_ver_tag_t ver_tag; 947 enum lsquic_version version; 948 949 if (!packet_in->pi_quic_ver) 950 { 951 LSQ_DEBUG("packet does not specify version"); 952 return VER_NOT_SPECIFIED; 953 } 954 955 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 956 version = lsquic_tag2ver(ver_tag); 957 if (version < N_LSQVER) 958 { 959 if (engine->pub.enp_settings.es_versions & (1 << version)) 960 { 961 LSQ_DEBUG("client-supplied version %s is supported", 962 lsquic_ver2str[version]); 963 *pversion = version; 964 return VER_SUPPORTED; 965 } 966 else 967 LSQ_DEBUG("client-supplied version %s is not supported", 968 lsquic_ver2str[version]); 969 } 970 else 971 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 972 ver_tag); 973 974 return VER_UNSUPPORTED; 975} 976 977 978static void 979schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 980 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 981 const struct sockaddr *sa_peer, void *peer_ctx) 982{ 983 assert(engine->pr_queue); 984 if (0 == prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 985 sa_local, sa_peer)) 986 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 987 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 988 else 989 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 990} 991 992 993static unsigned short 994sa2port (const struct sockaddr *sa) 995{ 996 if (sa->sa_family == AF_INET) 997 { 998 struct sockaddr_in *const sa4 = (void *) sa; 999 return sa4->sin_port; 1000 } 1001 else 1002 { 1003 struct sockaddr_in6 *const sa6 = (void *) sa; 1004 return sa6->sin6_port; 1005 } 1006} 1007 1008 1009static struct lsquic_hash_elem * 1010find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 1011{ 1012 unsigned short port; 1013 1014 port = sa2port(sa); 1015 return lsquic_hash_find(hash, &port, sizeof(port)); 1016} 1017 1018 1019static lsquic_conn_t * 1020find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1021 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 1022{ 1023 struct lsquic_hash_elem *el; 1024 lsquic_conn_t *conn; 1025 1026 if (engine->flags & ENG_CONNS_BY_ADDR) 1027 el = find_conn_by_addr(engine->conns_hash, sa_local); 1028 else if (packet_in->pi_flags & PI_CONN_ID) 1029 el = lsquic_hash_find(engine->conns_hash, 1030 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1031 else 1032 { 1033 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1034 return NULL; 1035 } 1036 1037 if (!el) 1038 return NULL; 1039 1040 conn = lsquic_hashelem_getdata(el); 1041 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1042 if ((engine->flags & ENG_CONNS_BY_ADDR) 1043 && !(conn->cn_flags & LSCONN_IETF) 1044 && (packet_in->pi_flags & PI_CONN_ID) 1045 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1046 { 1047 LSQ_DEBUG("connection IDs do not match"); 1048 return NULL; 1049 } 1050 1051 return conn; 1052} 1053 1054 1055static lsquic_conn_t * 1056find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1057 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1058 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1059{ 1060 struct lsquic_hash_elem *el; 1061 struct purga_el *puel; 1062 lsquic_conn_t *conn; 1063 1064 if (!(packet_in->pi_flags & PI_CONN_ID)) 1065 { 1066 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1067 return NULL; 1068 } 1069 el = lsquic_hash_find(engine->conns_hash, 1070 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1071 1072 if (el) 1073 { 1074 conn = lsquic_hashelem_getdata(el); 1075 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1076 return conn; 1077 } 1078 1079 if (engine->flags & ENG_COOLDOWN) 1080 { /* Do not create incoming connections during cooldown */ 1081 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1082 return NULL; 1083 } 1084 1085 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1086 { 1087 LSQ_DEBUG("reached limit of %u inchoate connections", 1088 engine->pub.enp_settings.es_max_inchoate); 1089 return NULL; 1090 } 1091 1092 1093 if (engine->purga 1094 && (puel = lsquic_purga_contains(engine->purga, 1095 &packet_in->pi_conn_id), puel)) 1096 { 1097 switch (puel->puel_type) 1098 { 1099 case PUTY_CID_RETIRED: 1100 LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet", 1101 CID_BITS(&packet_in->pi_conn_id)); 1102 return NULL; 1103 case PUTY_CONN_DRAIN: 1104 LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64, 1105 puel->puel_time, packet_in->pi_received); 1106 if (puel->puel_time > packet_in->pi_received) 1107 { 1108 LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet", 1109 CID_BITS(&packet_in->pi_conn_id)); 1110 return NULL; 1111 } 1112 LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted", 1113 CID_BITS(&packet_in->pi_conn_id)); 1114 puel->puel_type = PUTY_CONN_DELETED; 1115 puel->puel_count = 0; 1116 puel->puel_time = 0; 1117 /* fall-through */ 1118 case PUTY_CONN_DELETED: 1119 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1120 CID_BITS(&packet_in->pi_conn_id)); 1121 if (puel->puel_time < packet_in->pi_received) 1122 { 1123 puel->puel_time = packet_in->pi_received 1124 /* Exponential back-off */ 1125 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1126 ++puel->puel_count; 1127 goto maybe_send_prst; 1128 } 1129 return NULL; 1130 default: 1131 assert(0); 1132 return NULL; 1133 } 1134 } 1135 1136 if (engine->pub.enp_settings.es_send_prst 1137 && !(packet_in->pi_flags & PI_GQUIC) 1138 && HETY_NOT_SET == packet_in->pi_header_type) 1139 goto maybe_send_prst; 1140 1141 if (0 != maybe_grow_conn_heaps(engine)) 1142 return NULL; 1143 1144 const struct parse_funcs *pf; 1145 enum lsquic_version version; 1146 switch (version_matches(engine, packet_in, &version)) 1147 { 1148 case VER_UNSUPPORTED: 1149 if (engine->flags & ENG_SERVER) 1150 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1151 sa_local, sa_peer, peer_ctx); 1152 return NULL; 1153 case VER_NOT_SPECIFIED: 1154 maybe_send_prst: 1155 if ((engine->flags & ENG_SERVER) && 1156 engine->pub.enp_settings.es_send_prst) 1157 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1158 sa_local, sa_peer, peer_ctx); 1159 return NULL; 1160 case VER_SUPPORTED: 1161 pf = select_pf_by_ver(version); 1162 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1163 break; 1164 } 1165 1166 1167 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1168 { 1169 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1170 sa_peer->sa_family == AF_INET, NULL, packet_in_size); 1171 } 1172 else 1173 { 1174 conn = mini_conn_new(&engine->pub, packet_in, version); 1175 } 1176 if (!conn) 1177 return NULL; 1178 ++engine->mini_conns_count; 1179 ++engine->n_conns; 1180 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1181 { 1182 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1183 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1184 CID_BITS(cid)); 1185 destroy_conn(engine, conn, packet_in->pi_received); 1186 return NULL; 1187 } 1188 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1189 conn->cn_flags |= LSCONN_HASHED; 1190 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1191 conn->cn_last_sent = engine->last_sent; 1192 return conn; 1193} 1194 1195 1196lsquic_conn_t * 1197lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1198 const lsquic_cid_t *cid) 1199{ 1200 struct lsquic_hash_elem *el; 1201 lsquic_conn_t *conn = NULL; 1202 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1203 1204 if (el) 1205 conn = lsquic_hashelem_getdata(el); 1206 return conn; 1207} 1208 1209 1210#if !defined(NDEBUG) && __GNUC__ 1211__attribute__((weak)) 1212#endif 1213void 1214lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1215 lsquic_conn_t *conn) 1216{ 1217 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1218 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1219 { 1220 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1221 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1222 engine_incref_conn(conn, LSCONN_TICKABLE); 1223 } 1224} 1225 1226 1227void 1228lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1229 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1230{ 1231 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1232 if (conn->cn_flags & LSCONN_TICKABLE) 1233 { 1234 /* Optimization: no need to add the connection to the Advisory Tick 1235 * Time Queue: it is about to be ticked, after which it its next tick 1236 * time may be queried again. 1237 */; 1238 } 1239 else if (conn->cn_flags & LSCONN_ATTQ) 1240 { 1241 if (lsquic_conn_adv_time(conn) != tick_time) 1242 { 1243 attq_remove(engine->attq, conn); 1244 if (0 != attq_add(engine->attq, conn, tick_time, why)) 1245 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1246 } 1247 } 1248 else if (0 == attq_add(engine->attq, conn, tick_time, why)) 1249 engine_incref_conn(conn, LSCONN_ATTQ); 1250} 1251 1252 1253static struct lsquic_conn * 1254find_conn_by_srst (struct lsquic_engine *engine, 1255 const struct lsquic_packet_in *packet_in) 1256{ 1257 struct lsquic_hash_elem *el; 1258 struct lsquic_conn *conn; 1259 1260 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1261 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1262 return NULL; 1263 1264 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1265 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1266 IQUIC_SRESET_TOKEN_SZ); 1267 if (!el) 1268 return NULL; 1269 1270 conn = lsquic_hashelem_getdata(el); 1271 return conn; 1272} 1273 1274 1275/* Return 0 if packet is being processed by a real connection (mini or full), 1276 * otherwise return 1. 1277 */ 1278static int 1279process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1280 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1281 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1282{ 1283 lsquic_conn_t *conn; 1284 const unsigned char *packet_in_data; 1285 1286 if (lsquic_packet_in_is_gquic_prst(packet_in) 1287 && !engine->pub.enp_settings.es_honor_prst) 1288 { 1289 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1290 LSQ_DEBUG("public reset packet: discarding"); 1291 return 1; 1292 } 1293 1294 if (engine->flags & ENG_SERVER) 1295 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1296 sa_peer, peer_ctx, packet_in_size); 1297 else 1298 conn = find_conn(engine, packet_in, ppstate, sa_local); 1299 1300 if (!conn) 1301 { 1302 if (engine->pub.enp_settings.es_honor_prst 1303 && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */ 1304 && !(packet_in->pi_flags & PI_GQUIC) 1305 && engine->pub.enp_srst_hash 1306 && (conn = find_conn_by_srst(engine, packet_in))) 1307 { 1308 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1309 CID_BITS(lsquic_conn_log_cid(conn))); 1310 conn->cn_if->ci_stateless_reset(conn); 1311 if (!(conn->cn_flags & LSCONN_TICKABLE) 1312 && conn->cn_if->ci_is_tickable(conn)) 1313 { 1314 lsquic_mh_insert(&engine->conns_tickable, conn, 1315 conn->cn_last_ticked); 1316 engine_incref_conn(conn, LSCONN_TICKABLE); 1317 } 1318 /* Even though the connection processes this packet, we return 1319 * 1 so that the caller does not add reset packet's random 1320 * bytes to the list of valid CIDs. 1321 */ 1322 } 1323 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1324 return 1; 1325 } 1326 1327 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1328 { 1329 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1330 engine_incref_conn(conn, LSCONN_TICKABLE); 1331 } 1332 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1333 sa_local, sa_peer); 1334 lsquic_packet_in_upref(packet_in); 1335#if LOG_PACKET_CHECKSUM 1336 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1337 packet_in->pi_data_sz); 1338#endif 1339 /* Note on QLog: 1340 * For the PACKET_RX QLog event, we are interested in logging these things: 1341 * - raw packet (however it comes in, encrypted or not) 1342 * - frames (list of frame names) 1343 * - packet type and number 1344 * - packet rx timestamp 1345 * 1346 * Since only some of these items are available at this code 1347 * juncture, we will wait until after the packet has been 1348 * decrypted (if necessary) and parsed to call the log functions. 1349 * 1350 * Once the PACKET_RX event is finally logged, the timestamp 1351 * will come from packet_in->pi_received. For correct sequential 1352 * ordering of QLog events, be sure to process the QLogs downstream. 1353 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1354 */ 1355 packet_in_data = packet_in->pi_data; 1356 packet_in_size = packet_in->pi_data_sz; 1357 conn->cn_if->ci_packet_in(conn, packet_in); 1358 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1359 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1360 return 0; 1361} 1362 1363 1364void 1365lsquic_engine_destroy (lsquic_engine_t *engine) 1366{ 1367 struct lsquic_hash_elem *el; 1368 lsquic_conn_t *conn; 1369 1370 LSQ_DEBUG("destroying engine"); 1371#ifndef NDEBUG 1372 engine->flags |= ENG_DTOR; 1373#endif 1374 1375 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1376 { 1377 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1378 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1379 } 1380 1381 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1382 { 1383 assert(conn->cn_flags & LSCONN_TICKABLE); 1384 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1385 } 1386 1387 for (el = lsquic_hash_first(engine->conns_hash); el; 1388 el = lsquic_hash_next(engine->conns_hash)) 1389 { 1390 conn = lsquic_hashelem_getdata(el); 1391 force_close_conn(engine, conn); 1392 } 1393 lsquic_hash_destroy(engine->conns_hash); 1394 1395 assert(0 == engine->n_conns); 1396 assert(0 == engine->mini_conns_count); 1397 if (engine->pr_queue) 1398 prq_destroy(engine->pr_queue); 1399 if (engine->purga) 1400 lsquic_purga_destroy(engine->purga); 1401 attq_destroy(engine->attq); 1402 1403 assert(0 == lsquic_mh_count(&engine->conns_out)); 1404 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1405 if (engine->pub.enp_shi == &stock_shi) 1406 stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1407 lsquic_mm_cleanup(&engine->pub.enp_mm); 1408 free(engine->conns_tickable.mh_elems); 1409#ifndef NDEBUG 1410 if (engine->flags & ENG_LOSE_PACKETS) 1411 regfree(&engine->lose_packets_re); 1412#endif 1413 if (engine->pub.enp_tokgen) 1414 lsquic_tg_destroy(engine->pub.enp_tokgen); 1415#if LSQUIC_CONN_STATS 1416 if (engine->stats_fh) 1417 { 1418 const struct conn_stats *const stats = &engine->conn_stats_sum; 1419 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1420 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1421 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1422 fprintf(engine->stats_fh, "In:\n"); 1423 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1424 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1425 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1426 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1427 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1428 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1429 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1430 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1431 stats->in.headers_comp, stats->in.headers_uncomp, 1432 stats->in.headers_uncomp ? 1433 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1434 : 0); 1435 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1436 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1437 fprintf(engine->stats_fh, " ACK frames merged to new: %lu\n", stats->in.n_acks_merged[0]); 1438 fprintf(engine->stats_fh, " ACK frames merged to old: %lu\n", stats->in.n_acks_merged[1]); 1439 fprintf(engine->stats_fh, "Out:\n"); 1440 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1441 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1442 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1443 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1444 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1445 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1446 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1447 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1448 stats->out.headers_comp, stats->out.headers_uncomp, 1449 stats->out.headers_uncomp ? 1450 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1451 : 0); 1452 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1453 } 1454#endif 1455 if (engine->pub.enp_srst_hash) 1456 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1457#if LSQUIC_COUNT_ENGINE_CALLS 1458 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1459#endif 1460 if (engine->pub.enp_retry_aead_ctx) 1461 EVP_AEAD_CTX_cleanup(engine->pub.enp_retry_aead_ctx); 1462 free(engine); 1463} 1464 1465 1466static struct conn_cid_elem * 1467find_free_cce (struct lsquic_conn *conn) 1468{ 1469 struct conn_cid_elem *cce; 1470 1471 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1472 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1473 return cce; 1474 1475 return NULL; 1476} 1477 1478 1479static int 1480add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1481 const struct sockaddr *local_sa, void *peer_ctx) 1482{ 1483 struct conn_cid_elem *cce; 1484 1485 if (engine->flags & ENG_CONNS_BY_ADDR) 1486 { 1487 cce = find_free_cce(conn); 1488 if (!cce) 1489 { 1490 LSQ_ERROR("cannot find free CCE"); 1491 return -1; 1492 } 1493 cce->cce_port = sa2port(local_sa); 1494 cce->cce_flags = CCE_PORT; 1495 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1496 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1497 { 1498 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1499 return 0; 1500 } 1501 else 1502 return -1; 1503 1504 } 1505 else 1506 return insert_conn_into_hash(engine, conn, peer_ctx); 1507} 1508 1509 1510lsquic_conn_t * 1511lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1512 const struct sockaddr *local_sa, 1513 const struct sockaddr *peer_sa, 1514 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1515 const char *hostname, unsigned short max_packet_size, 1516 const unsigned char *zero_rtt, size_t zero_rtt_len, 1517 const unsigned char *token, size_t token_sz) 1518{ 1519 lsquic_conn_t *conn; 1520 unsigned flags, versions; 1521 int is_ipv4; 1522 1523 ENGINE_IN(engine); 1524 1525 if (engine->flags & ENG_SERVER) 1526 { 1527 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1528 goto err; 1529 } 1530 1531 if (engine->flags & ENG_CONNS_BY_ADDR 1532 && find_conn_by_addr(engine->conns_hash, local_sa)) 1533 { 1534 LSQ_ERROR("cannot have more than one connection on the same port"); 1535 goto err; 1536 } 1537 1538 if (0 != maybe_grow_conn_heaps(engine)) 1539 return NULL; 1540 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1541 is_ipv4 = peer_sa->sa_family == AF_INET; 1542 if (zero_rtt && zero_rtt_len) 1543 { 1544 version = lsquic_zero_rtt_version(zero_rtt, zero_rtt_len); 1545 if (version >= N_LSQVER) 1546 { 1547 LSQ_INFO("zero-rtt version is bad, won't use"); 1548 zero_rtt = NULL; 1549 zero_rtt_len = 0; 1550 } 1551 } 1552 if (version >= N_LSQVER) 1553 { 1554 if (version > N_LSQVER) 1555 LSQ_WARN("invalid version specified, engine will pick"); 1556 versions = engine->pub.enp_settings.es_versions; 1557 } 1558 else 1559 versions = 1u << version; 1560 if (versions & LSQUIC_IETF_VERSIONS) 1561 { 1562 if (version == LSQVER_ID24) 1563 conn = lsquic_id24_full_conn_client_new(&engine->pub, versions, 1564 flags, hostname, max_packet_size, 1565 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1566 else 1567 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1568 flags, hostname, max_packet_size, 1569 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1570 } 1571 else 1572 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1573 flags, hostname, max_packet_size, is_ipv4, 1574 zero_rtt, zero_rtt_len); 1575 if (!conn) 1576 goto err; 1577 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1578 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1579 lsquic_ver2str[conn->cn_version]); 1580 ++engine->n_conns; 1581 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1582 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1583 { 1584 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1585 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1586 CID_BITS(cid)); 1587 destroy_conn(engine, conn, lsquic_time_now()); 1588 goto err; 1589 } 1590 assert(!(conn->cn_flags & 1591 (CONN_REF_FLAGS 1592 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1593 callbacks */ 1594 ))); 1595 conn->cn_flags |= LSCONN_HASHED; 1596 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1597 engine_incref_conn(conn, LSCONN_TICKABLE); 1598 lsquic_conn_set_ctx(conn, conn_ctx); 1599 conn->cn_if->ci_client_call_on_new(conn); 1600 end: 1601 ENGINE_OUT(engine); 1602 return conn; 1603 err: 1604 conn = NULL; 1605 goto end; 1606} 1607 1608 1609static void 1610remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1611{ 1612 remove_all_cces_from_hash(engine->conns_hash, conn); 1613 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1614} 1615 1616 1617static void 1618refflags2str (enum lsquic_conn_flags flags, char s[6]) 1619{ 1620 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1621 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1622 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1623 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1624 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1625 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1626 *s = '\0'; 1627} 1628 1629 1630static void 1631engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1632{ 1633 char str[2][7]; 1634 assert(flag & CONN_REF_FLAGS); 1635 assert(!(conn->cn_flags & flag)); 1636 conn->cn_flags |= flag; 1637 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1638 CID_BITS(lsquic_conn_log_cid(conn)), 1639 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1640 (refflags2str(conn->cn_flags, str[1]), str[1])); 1641} 1642 1643 1644static lsquic_conn_t * 1645engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1646 enum lsquic_conn_flags flags) 1647{ 1648 char str[2][7]; 1649 lsquic_time_t now; 1650 assert(flags & CONN_REF_FLAGS); 1651 assert(conn->cn_flags & flags); 1652#ifndef NDEBUG 1653 if (flags & LSCONN_CLOSING) 1654 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1655#endif 1656 conn->cn_flags &= ~flags; 1657 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1658 CID_BITS(lsquic_conn_log_cid(conn)), 1659 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1660 (refflags2str(conn->cn_flags, str[1]), str[1])); 1661 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1662 { 1663 now = lsquic_time_now(); 1664 if (conn->cn_flags & LSCONN_MINI) 1665 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1666 else 1667 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1668 destroy_conn(engine, conn, now); 1669 return NULL; 1670 } 1671 else 1672 return conn; 1673} 1674 1675 1676/* This is not a general-purpose function. Only call from engine dtor. */ 1677static void 1678force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1679{ 1680 assert(engine->flags & ENG_DTOR); 1681 const enum lsquic_conn_flags flags = conn->cn_flags; 1682 assert(conn->cn_flags & CONN_REF_FLAGS); 1683 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1684 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1685 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1686 if (flags & LSCONN_ATTQ) 1687 { 1688 attq_remove(engine->attq, conn); 1689 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1690 } 1691 if (flags & LSCONN_HASHED) 1692 remove_conn_from_hash(engine, conn); 1693} 1694 1695 1696/* Iterator for tickable connections (those on the Tickable Queue). Before 1697 * a connection is returned, it is removed from the Advisory Tick Time queue 1698 * if necessary. 1699 */ 1700static lsquic_conn_t * 1701conn_iter_next_tickable (struct lsquic_engine *engine) 1702{ 1703 lsquic_conn_t *conn; 1704 1705 if (engine->flags & ENG_SERVER) 1706 while (1) 1707 { 1708 conn = lsquic_mh_pop(&engine->conns_tickable); 1709 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1710 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1711 else 1712 break; 1713 } 1714 else 1715 conn = lsquic_mh_pop(&engine->conns_tickable); 1716 1717 if (conn) 1718 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1719 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1720 { 1721 attq_remove(engine->attq, conn); 1722 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1723 } 1724 1725 return conn; 1726} 1727 1728 1729static void 1730cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1731 void *update_ctx) 1732{ 1733 cub->cub_update_cids = update; 1734 cub->cub_update_ctx = update_ctx; 1735 cub->cub_count = 0; 1736} 1737 1738 1739static void 1740cub_flush (struct cid_update_batch *cub) 1741{ 1742 if (cub->cub_count > 0 && cub->cub_update_cids) 1743 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1744 cub->cub_cids, cub->cub_count); 1745 cub->cub_count = 0; 1746} 1747 1748 1749static void 1750cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1751{ 1752 cub->cub_cids [ cub->cub_count ] = *cid; 1753 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1754 ++cub->cub_count; 1755 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1756 cub_flush(cub); 1757} 1758 1759 1760/* Process registered CIDs */ 1761static void 1762cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1763{ 1764 struct cce_cid_iter citer; 1765 struct conn_cid_elem *cce; 1766 void *peer_ctx; 1767 1768 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1769 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1770 if (cce->cce_flags & CCE_REG) 1771 cub_add(cub, &cce->cce_cid, peer_ctx); 1772} 1773 1774 1775static void 1776drop_all_mini_conns (lsquic_engine_t *engine) 1777{ 1778 struct lsquic_hash_elem *el; 1779 lsquic_conn_t *conn; 1780 struct cid_update_batch cub; 1781 1782 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1783 1784 for (el = lsquic_hash_first(engine->conns_hash); el; 1785 el = lsquic_hash_next(engine->conns_hash)) 1786 { 1787 conn = lsquic_hashelem_getdata(el); 1788 if (conn->cn_flags & LSCONN_MINI) 1789 { 1790 /* If promoted, why is it still in this hash? */ 1791 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1792 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1793 cub_add_cids_from_cces(&cub, conn); 1794 remove_conn_from_hash(engine, conn); 1795 } 1796 } 1797 1798 cub_flush(&cub); 1799} 1800 1801 1802void 1803lsquic_engine_process_conns (lsquic_engine_t *engine) 1804{ 1805 lsquic_conn_t *conn; 1806 lsquic_time_t now; 1807 1808 ENGINE_IN(engine); 1809 1810 now = lsquic_time_now(); 1811 while ((conn = attq_pop(engine->attq, now))) 1812 { 1813 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1814 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 1815 { 1816 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1817 engine_incref_conn(conn, LSCONN_TICKABLE); 1818 } 1819 } 1820 1821 process_connections(engine, conn_iter_next_tickable, now); 1822 ENGINE_OUT(engine); 1823} 1824 1825 1826static void 1827release_or_return_enc_data (struct lsquic_engine *engine, 1828 void (*pmi_rel_or_ret) (void *, void *, void *, char), 1829 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 1830{ 1831 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1832 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 1833 packet_out->po_flags &= ~PO_ENCRYPTED; 1834 packet_out->po_enc_data = NULL; 1835} 1836 1837 1838static void 1839release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1840 struct lsquic_packet_out *packet_out) 1841{ 1842 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 1843 conn, packet_out); 1844} 1845 1846 1847static void 1848return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1849 struct lsquic_packet_out *packet_out) 1850{ 1851 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 1852 conn, packet_out); 1853} 1854 1855 1856static int 1857copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 1858 struct lsquic_packet_out *packet_out) 1859{ 1860 int ipv6; 1861 1862 ipv6 = NP_IS_IPv6(packet_out->po_path); 1863 if (packet_out->po_flags & PO_ENCRYPTED) 1864 { 1865 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1866 && packet_out->po_data_sz == packet_out->po_enc_data_sz 1867 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 1868 packet_out->po_data_sz)) 1869 return 0; 1870 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1871 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 1872 goto copy; 1873 return_enc_data(engine, conn, packet_out); 1874 } 1875 1876 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 1877 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1878 packet_out->po_data_sz, ipv6); 1879 if (!packet_out->po_enc_data) 1880 { 1881 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 1882 "of size %hu", packet_out->po_data_sz); 1883 return -1; 1884 } 1885 1886 copy: 1887 memcpy(packet_out->po_enc_data, packet_out->po_data, 1888 packet_out->po_data_sz); 1889 packet_out->po_enc_data_sz = packet_out->po_data_sz; 1890 packet_out->po_sent_sz = packet_out->po_data_sz; 1891 packet_out->po_flags &= ~PO_IPv6; 1892 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 1893 1894 return 0; 1895} 1896 1897 1898STAILQ_HEAD(conns_stailq, lsquic_conn); 1899TAILQ_HEAD(conns_tailq, lsquic_conn); 1900 1901 1902struct conns_out_iter 1903{ 1904 struct min_heap *coi_heap; 1905 struct pr_queue *coi_prq; 1906 TAILQ_HEAD(, lsquic_conn) coi_active_list, 1907 coi_inactive_list; 1908 lsquic_conn_t *coi_next; 1909#ifndef NDEBUG 1910 lsquic_time_t coi_last_sent; 1911#endif 1912}; 1913 1914 1915static void 1916coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 1917{ 1918 iter->coi_heap = &engine->conns_out; 1919 iter->coi_prq = engine->pr_queue; 1920 iter->coi_next = NULL; 1921 TAILQ_INIT(&iter->coi_active_list); 1922 TAILQ_INIT(&iter->coi_inactive_list); 1923#ifndef NDEBUG 1924 iter->coi_last_sent = 0; 1925#endif 1926} 1927 1928 1929static lsquic_conn_t * 1930coi_next (struct conns_out_iter *iter) 1931{ 1932 lsquic_conn_t *conn; 1933 1934 if (lsquic_mh_count(iter->coi_heap) > 0) 1935 { 1936 conn = lsquic_mh_pop(iter->coi_heap); 1937 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1938 conn->cn_flags |= LSCONN_COI_ACTIVE; 1939#ifndef NDEBUG 1940 if (iter->coi_last_sent) 1941 assert(iter->coi_last_sent <= conn->cn_last_sent); 1942 iter->coi_last_sent = conn->cn_last_sent; 1943#endif 1944 return conn; 1945 } 1946 else if (iter->coi_prq && (conn = prq_next_conn(iter->coi_prq))) 1947 { 1948 return conn; 1949 } 1950 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 1951 { 1952 iter->coi_prq = NULL; /* Save function call in previous conditional */ 1953 conn = iter->coi_next; 1954 if (!conn) 1955 conn = TAILQ_FIRST(&iter->coi_active_list); 1956 if (conn) 1957 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 1958 return conn; 1959 } 1960 else 1961 return NULL; 1962} 1963 1964 1965static void 1966coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1967{ 1968 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 1969 { 1970 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 1971 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1972 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1973 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 1974 conn->cn_flags |= LSCONN_COI_INACTIVE; 1975 } 1976} 1977 1978 1979static void 1980coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1981{ 1982 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 1983 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 1984 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 1985 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1986 conn->cn_flags |= LSCONN_COI_ACTIVE; 1987} 1988 1989 1990static void 1991coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 1992{ 1993 lsquic_conn_t *conn; 1994 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 1995 { 1996 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1997 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1998 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 1999 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 2000 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 2001 else /* Closed connection gets one shot at sending packets */ 2002 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2003 } 2004 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 2005 { 2006 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2007 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2008 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2009 } 2010} 2011 2012 2013#ifndef NDEBUG 2014static void 2015lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 2016 unsigned n) 2017{ 2018 const lsquic_cid_t *cid; 2019 struct iovec *iov; 2020 unsigned i; 2021 char packno_str[22]; 2022 2023 for (i = 0; i < n; ++i) 2024 { 2025 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 2026 batch->packets[i]->po_packno); 2027 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 2028 { 2029 for (iov = batch->outs[i].iov; iov < 2030 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 2031 batch->outs[i].iov->iov_len -= 1; 2032 cid = lsquic_conn_log_cid(batch->conns[i]); 2033 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 2034 CID_BITS(cid)); 2035 } 2036 } 2037} 2038 2039 2040#endif 2041 2042 2043#ifdef NDEBUG 2044#define CONST_BATCH const 2045#else 2046#define CONST_BATCH 2047#endif 2048 2049 2050struct send_batch_ctx { 2051 struct conns_stailq *closed_conns; 2052 struct conns_tailq *ticked_conns; 2053 struct conns_out_iter *conns_iter; 2054 CONST_BATCH struct out_batch *batch; 2055}; 2056 2057 2058static void 2059close_conn_immediately (struct lsquic_engine *engine, 2060 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2061{ 2062 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2063 if (!(conn->cn_flags & LSCONN_CLOSING)) 2064 { 2065 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2066 engine_incref_conn(conn, LSCONN_CLOSING); 2067 if (conn->cn_flags & LSCONN_HASHED) 2068 remove_conn_from_hash(engine, conn); 2069 } 2070 if (conn->cn_flags & LSCONN_TICKED) 2071 { 2072 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2073 engine_decref_conn(engine, conn, LSCONN_TICKED); 2074 } 2075} 2076 2077 2078static void 2079close_conn_on_send_error (struct lsquic_engine *engine, 2080 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2081{ 2082 const struct out_batch *batch = sb_ctx->batch; 2083 struct lsquic_conn *const conn = batch->conns[n]; 2084 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2085 2086 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2087 "src: %s; dst: %s; errno: %d", 2088 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2089 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2090 CID_BITS(lsquic_conn_log_cid(conn)), 2091 SA2STR(batch->outs[n].local_sa, buf[0]), 2092 SA2STR(batch->outs[n].dest_sa, buf[1]), 2093 e_val); 2094 if (conn->cn_flags & LSCONN_EVANESCENT) 2095 lsquic_prq_drop(conn); 2096 else 2097 close_conn_immediately(engine, sb_ctx, conn); 2098} 2099 2100 2101static unsigned 2102send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2103 unsigned n_to_send) 2104{ 2105 int n_sent, i, e_val; 2106 lsquic_time_t now; 2107 unsigned off; 2108 size_t count; 2109 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2110 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2111 2112#ifndef NDEBUG 2113 if (engine->flags & ENG_LOSE_PACKETS) 2114 lose_matching_packets(engine, batch, n_to_send); 2115#endif 2116 /* Set sent time before the write to avoid underestimating RTT */ 2117 now = lsquic_time_now(); 2118 for (i = 0; i < (int) n_to_send; ++i) 2119 { 2120 off = batch->pack_off[i]; 2121 count = batch->outs[i].iovlen; 2122 assert(count > 0); 2123 packet_out = &batch->packets[off]; 2124 end = packet_out + count; 2125 do 2126 (*packet_out)->po_sent = now; 2127 while (++packet_out < end); 2128 } 2129 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs, 2130 n_to_send); 2131 e_val = errno; 2132 if (n_sent < (int) n_to_send) 2133 { 2134 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2135 engine->resume_sending_at = now + 1000000; 2136 LSQ_DEBUG("cannot send packets"); 2137 EV_LOG_GENERIC_EVENT("cannot send packets"); 2138 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2139 close_conn_on_send_error(engine, sb_ctx, 2140 n_sent < 0 ? 0 : n_sent, e_val); 2141 } 2142 if (n_sent >= 0) 2143 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, n_to_send); 2144 else 2145 { 2146 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2147 n_sent = 0; 2148 } 2149 if (n_sent > 0) 2150 engine->last_sent = now + n_sent; 2151 for (i = 0; i < n_sent; ++i) 2152 { 2153 eng_hist_inc(&engine->history, now, sl_packets_out); 2154 /* `i' is added to maintain relative order */ 2155 batch->conns[i]->cn_last_sent = now + i; 2156 2157 off = batch->pack_off[i]; 2158 count = batch->outs[i].iovlen; 2159 assert(count > 0); 2160 packet_out = &batch->packets[off]; 2161 end = packet_out + count; 2162 do 2163 { 2164#if LOG_PACKET_CHECKSUM 2165 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2166 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2167 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2168#endif 2169 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2170 *packet_out); 2171 /* Release packet out buffer as soon as the packet is sent 2172 * successfully. If not successfully sent, we hold on to 2173 * this buffer until the packet sending is attempted again 2174 * or until it times out and regenerated. 2175 */ 2176 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2177 release_enc_data(engine, batch->conns[i], *packet_out); 2178 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2179 *packet_out); 2180 } 2181 while (++packet_out < end); 2182 } 2183 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2184 for ( ; i < (int) n_to_send; ++i) 2185 { 2186 off = batch->pack_off[i]; 2187 count = batch->outs[i].iovlen; 2188 assert(count > 0); 2189 packet_out = &batch->packets[off]; 2190 end = packet_out + count; 2191 do 2192 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2193 *packet_out); 2194 while (++packet_out < end); 2195 } 2196 /* Return packets to the connection in reverse order so that the packet 2197 * ordering is maintained. 2198 */ 2199 for (i = (int) n_to_send - 1; i >= n_sent; --i) 2200 { 2201 off = batch->pack_off[i]; 2202 count = batch->outs[i].iovlen; 2203 assert(count > 0); 2204 packet_out = &batch->packets[off + count - 1]; 2205 end = &batch->packets[off - 1]; 2206 do 2207 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2208 *packet_out); 2209 while (--packet_out > end); 2210 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2211 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2212 } 2213 return n_sent; 2214} 2215 2216 2217/* Return 1 if went past deadline, 0 otherwise */ 2218static int 2219check_deadline (lsquic_engine_t *engine) 2220{ 2221 if (engine->pub.enp_settings.es_proc_time_thresh && 2222 lsquic_time_now() > engine->deadline) 2223 { 2224 LSQ_INFO("went past threshold of %u usec, stop sending", 2225 engine->pub.enp_settings.es_proc_time_thresh); 2226 engine->flags |= ENG_PAST_DEADLINE; 2227 return 1; 2228 } 2229 else 2230 return 0; 2231} 2232 2233 2234static size_t 2235iov_size (const struct iovec *iov, const struct iovec *const end) 2236{ 2237 size_t size; 2238 2239 assert(iov < end); 2240 2241 size = 0; 2242 do 2243 size += iov->iov_len; 2244 while (++iov < end); 2245 2246 return size; 2247} 2248 2249 2250static void 2251send_packets_out (struct lsquic_engine *engine, 2252 struct conns_tailq *ticked_conns, 2253 struct conns_stailq *closed_conns) 2254{ 2255 unsigned n, w, n_sent, n_batches_sent; 2256 lsquic_packet_out_t *packet_out; 2257 struct lsquic_packet_out **packet; 2258 lsquic_conn_t *conn; 2259 struct out_batch *const batch = &engine->out_batch; 2260 struct iovec *iov, *packet_iov; 2261 struct conns_out_iter conns_iter; 2262 int shrink, deadline_exceeded; 2263 const struct send_batch_ctx sb_ctx = { 2264 closed_conns, 2265 ticked_conns, 2266 &conns_iter, 2267 &engine->out_batch, 2268 }; 2269 2270 coi_init(&conns_iter, engine); 2271 n_batches_sent = 0; 2272 n_sent = 0, n = 0; 2273 shrink = 0; 2274 deadline_exceeded = 0; 2275 iov = batch->iov; 2276 packet = batch->packets; 2277 2278 while ((conn = coi_next(&conns_iter))) 2279 { 2280 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2281 if (!packet_out) { 2282 /* Evanescent connection always has a packet to send: */ 2283 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2284 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2285 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2286 CID_BITS(lsquic_conn_log_cid(conn))); 2287 coi_deactivate(&conns_iter, conn); 2288 continue; 2289 } 2290 batch->outs[n].iov = packet_iov = iov; 2291 next_coa: 2292 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2293 { 2294 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2295 &engine->pub, conn, packet_out)) 2296 { 2297 case ENCPA_NOMEM: 2298 /* Send what we have and wait for a more opportune moment */ 2299 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2300 goto end_for; 2301 case ENCPA_BADCRYPT: 2302 /* This is pretty bad: close connection immediately */ 2303 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2304 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2305 CID_BITS(lsquic_conn_log_cid(conn))); 2306 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2307 { 2308 close_conn_immediately(engine, &sb_ctx, conn); 2309 coi_deactivate(&conns_iter, conn); 2310 } 2311 continue; 2312 case ENCPA_OK: 2313 break; 2314 } 2315 } 2316 else if ((packet_out->po_flags & PO_NOENCRYPT) 2317 && engine->pub.enp_pmi != &stock_pmi) 2318 { 2319 if (0 != copy_packet(engine, conn, packet_out)) 2320 { 2321 /* Copy can only fail if packet could not be allocated */ 2322 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2323 goto end_for; 2324 } 2325 } 2326 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2327 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2328 if (packet_out->po_flags & PO_ENCRYPTED) 2329 { 2330 iov->iov_base = packet_out->po_enc_data; 2331 iov->iov_len = packet_out->po_enc_data_sz; 2332 } 2333 else 2334 { 2335 iov->iov_base = packet_out->po_data; 2336 iov->iov_len = packet_out->po_data_sz; 2337 } 2338 if (packet_iov == iov) 2339 { 2340 batch->pack_off[n] = packet - batch->packets; 2341 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2342 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2343 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2344 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2345 batch->conns [n] = conn; 2346 } 2347 *packet = packet_out; 2348 ++packet; 2349 ++iov; 2350 if ((conn->cn_flags & LSCONN_IETF) 2351 && ((1 << packet_out->po_header_type) 2352 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2353#ifndef NDEBUG 2354 && (engine->flags & ENG_COALESCE) 2355#endif 2356 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2357 { 2358 const size_t size = iov_size(packet_iov, iov); 2359 packet_out = conn->cn_if->ci_next_packet_to_send(conn, size); 2360 if (packet_out) 2361 goto next_coa; 2362 } 2363 batch->outs [n].iovlen = iov - packet_iov; 2364 ++n; 2365 if (n == engine->batch_size 2366 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2367 { 2368 w = send_batch(engine, &sb_ctx, n); 2369 n = 0; 2370 iov = batch->iov; 2371 packet = batch->packets; 2372 ++n_batches_sent; 2373 n_sent += w; 2374 if (w < engine->batch_size) 2375 { 2376 shrink = 1; 2377 break; 2378 } 2379 deadline_exceeded = check_deadline(engine); 2380 if (deadline_exceeded) 2381 break; 2382 grow_batch_size(engine); 2383 } 2384 } 2385 end_for: 2386 2387 if (n > 0) { 2388 w = send_batch(engine, &sb_ctx, n); 2389 n_sent += w; 2390 shrink = w < n; 2391 ++n_batches_sent; 2392 } 2393 2394 if (shrink) 2395 shrink_batch_size(engine); 2396 else if (n_batches_sent > 1) 2397 { 2398 deadline_exceeded = check_deadline(engine); 2399 if (!deadline_exceeded) 2400 grow_batch_size(engine); 2401 } 2402 2403 coi_reheap(&conns_iter, engine); 2404 2405 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2406} 2407 2408 2409int 2410lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2411{ 2412 return lsquic_mh_count(&engine->conns_out) > 0 2413 || (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2414 ; 2415} 2416 2417 2418static void 2419reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2420{ 2421 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2422 engine->flags &= ~ENG_PAST_DEADLINE; 2423} 2424 2425 2426void 2427lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2428{ 2429 lsquic_conn_t *conn; 2430 struct conns_stailq closed_conns; 2431 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2432 struct cid_update_batch cub; 2433 2434 ENGINE_IN(engine); 2435 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2436 STAILQ_INIT(&closed_conns); 2437 reset_deadline(engine, lsquic_time_now()); 2438 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2439 { 2440 LSQ_DEBUG("can send again"); 2441 EV_LOG_GENERIC_EVENT("can send again"); 2442 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2443 } 2444 2445 send_packets_out(engine, &ticked_conns, &closed_conns); 2446 2447 while ((conn = STAILQ_FIRST(&closed_conns))) { 2448 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2449 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2450 cub_add_cids_from_cces(&cub, conn); 2451 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2452 } 2453 2454 cub_flush(&cub); 2455 ENGINE_OUT(engine); 2456} 2457 2458 2459static lsquic_conn_t * 2460next_new_full_conn (struct conns_stailq *new_full_conns) 2461{ 2462 lsquic_conn_t *conn; 2463 2464 conn = STAILQ_FIRST(new_full_conns); 2465 if (conn) 2466 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2467 return conn; 2468} 2469 2470 2471static void 2472process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2473 lsquic_time_t now) 2474{ 2475 lsquic_conn_t *conn; 2476 enum tick_st tick_st; 2477 unsigned i, why; 2478 lsquic_time_t next_tick_time; 2479 struct conns_stailq closed_conns; 2480 struct conns_tailq ticked_conns; 2481 struct conns_stailq new_full_conns; 2482 struct cid_update_batch cub_old, cub_live; 2483 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2484 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2485 2486 eng_hist_tick(&engine->history, now); 2487 2488 STAILQ_INIT(&closed_conns); 2489 TAILQ_INIT(&ticked_conns); 2490 reset_deadline(engine, now); 2491 STAILQ_INIT(&new_full_conns); 2492 2493 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2494 && now > engine->resume_sending_at) 2495 { 2496 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2497 "timeout"); 2498 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2499 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2500 } 2501 2502 i = 0; 2503 while ((conn = next_conn(engine)) 2504 || (conn = next_new_full_conn(&new_full_conns))) 2505 { 2506 tick_st = conn->cn_if->ci_tick(conn, now); 2507 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2508 if (tick_st & TICK_PROMOTE) 2509 { 2510 lsquic_conn_t *new_conn; 2511 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2512 "scheduled for promotion"); 2513 assert(conn->cn_flags & LSCONN_MINI); 2514 new_conn = new_full_conn_server(engine, conn, now); 2515 if (new_conn) 2516 { 2517 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2518 new_conn->cn_last_sent = engine->last_sent; 2519 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2520 conn->cn_flags |= LSCONN_PROMOTED; 2521 } 2522 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2523 } 2524 if (tick_st & TICK_SEND) 2525 { 2526 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2527 { 2528 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2529 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2530 } 2531 } 2532 if (tick_st & TICK_CLOSE) 2533 { 2534 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2535 engine_incref_conn(conn, LSCONN_CLOSING); 2536 if (conn->cn_flags & LSCONN_HASHED) 2537 remove_conn_from_hash(engine, conn); 2538 } 2539 else 2540 { 2541 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2542 engine_incref_conn(conn, LSCONN_TICKED); 2543 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2544 && conn->cn_if->ci_report_live(conn, now)) 2545 cub_add_cids_from_cces(&cub_live, conn); 2546 } 2547 } 2548 2549 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2550 && lsquic_engine_has_unsent_packets(engine)) 2551 send_packets_out(engine, &ticked_conns, &closed_conns); 2552 2553 while ((conn = STAILQ_FIRST(&closed_conns))) { 2554 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2555 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2556 cub_add_cids_from_cces(&cub_old, conn); 2557 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2558 } 2559 2560 while ((conn = TAILQ_FIRST(&ticked_conns))) 2561 { 2562 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2563 engine_decref_conn(engine, conn, LSCONN_TICKED); 2564 if (!(conn->cn_flags & LSCONN_TICKABLE) 2565 && conn->cn_if->ci_is_tickable(conn)) 2566 { 2567 /* Floyd heapification is not faster, don't bother. */ 2568 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2569 engine_incref_conn(conn, LSCONN_TICKABLE); 2570 } 2571 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2572 { 2573 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2574 if (next_tick_time) 2575 { 2576 if (0 == attq_add(engine->attq, conn, next_tick_time, why)) 2577 engine_incref_conn(conn, LSCONN_ATTQ); 2578 } 2579 else 2580 /* In all other cases, the idle timeout would make the next 2581 * tick time non-zero: 2582 */ 2583 assert((conn->cn_flags & LSCONN_IETF) 2584 && engine->pub.enp_settings.es_idle_timeout == 0); 2585 } 2586 } 2587 2588 cub_flush(&engine->new_scids); 2589 cub_flush(&cub_live); 2590 cub_flush(&cub_old); 2591} 2592 2593 2594/* Return 0 if packet is being processed by a real connection, 1 if the 2595 * packet was processed, but not by a connection, and -1 on error. 2596 */ 2597int 2598lsquic_engine_packet_in (lsquic_engine_t *engine, 2599 const unsigned char *packet_in_data, size_t packet_in_size, 2600 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2601 void *peer_ctx, int ecn) 2602{ 2603 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2604 struct packin_parse_state ppstate; 2605 lsquic_packet_in_t *packet_in; 2606 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2607 int is_server, unsigned cid_len, struct packin_parse_state *); 2608 unsigned n_zeroes; 2609 int s; 2610 2611 ENGINE_CALLS_INCR(engine); 2612 2613 if (engine->flags & ENG_SERVER) 2614 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2615 else 2616 if (engine->flags & ENG_CONNS_BY_ADDR) 2617 { 2618 struct lsquic_hash_elem *el; 2619 const struct lsquic_conn *conn; 2620 el = find_conn_by_addr(engine->conns_hash, sa_local); 2621 if (!el) 2622 return -1; 2623 conn = lsquic_hashelem_getdata(el); 2624 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2625 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2626 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2627 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2628 else if (conn->cn_version == LSQVER_050) 2629 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 2630 else 2631 { 2632 assert(conn->cn_version == LSQVER_046 2633#if LSQUIC_USE_Q098 2634 || conn->cn_version == LSQVER_098 2635#endif 2636 2637 ); 2638 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2639 } 2640 } 2641 else 2642 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2643 2644 n_zeroes = 0; 2645 do 2646 { 2647 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 2648 if (!packet_in) 2649 return -1; 2650 /* Library does not modify packet_in_data, it is not referenced after 2651 * this function returns and subsequent release of pi_data is guarded 2652 * by PI_OWN_DATA flag. 2653 */ 2654 packet_in->pi_data = (unsigned char *) packet_in_data; 2655 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 2656 engine->flags & ENG_SERVER, 2657 engine->pub.enp_settings.es_scid_len, &ppstate)) 2658 { 2659 LSQ_DEBUG("Cannot parse incoming packet's header"); 2660 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 2661 errno = EINVAL; 2662 return -1; 2663 } 2664 2665 packet_in_data += packet_in->pi_data_sz; 2666 packet_in->pi_received = lsquic_time_now(); 2667 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 2668 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 2669 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 2670 peer_ctx, packet_in_size); 2671 n_zeroes += s == 0; 2672 } 2673 while (0 == s && packet_in_data < packet_end); 2674 2675 return n_zeroes > 0 ? 0 : s; 2676} 2677 2678 2679#if __GNUC__ && !defined(NDEBUG) 2680__attribute__((weak)) 2681#endif 2682unsigned 2683lsquic_engine_quic_versions (const lsquic_engine_t *engine) 2684{ 2685 return engine->pub.enp_settings.es_versions; 2686} 2687 2688 2689void 2690lsquic_engine_cooldown (lsquic_engine_t *engine) 2691{ 2692 struct lsquic_hash_elem *el; 2693 lsquic_conn_t *conn; 2694 2695 if (engine->flags & ENG_COOLDOWN) 2696 /* AFAICT, there is no harm in calling this function more than once, 2697 * but log it just in case, as it may indicate an error in the caller. 2698 */ 2699 LSQ_INFO("cooldown called again"); 2700 engine->flags |= ENG_COOLDOWN; 2701 LSQ_INFO("entering cooldown mode"); 2702 if (engine->flags & ENG_SERVER) 2703 drop_all_mini_conns(engine); 2704 for (el = lsquic_hash_first(engine->conns_hash); el; 2705 el = lsquic_hash_next(engine->conns_hash)) 2706 { 2707 conn = lsquic_hashelem_getdata(el); 2708 lsquic_conn_going_away(conn); 2709 } 2710} 2711 2712 2713int 2714lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 2715{ 2716 const struct attq_elem *next_attq; 2717 lsquic_time_t now, next_time; 2718#if LSQUIC_DEBUG_NEXT_ADV_TICK 2719 const struct lsquic_conn *conn; 2720 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 2721#endif 2722 2723 ENGINE_CALLS_INCR(engine); 2724 2725 if ((engine->flags & ENG_PAST_DEADLINE) 2726 && lsquic_mh_count(&engine->conns_out)) 2727 { 2728#if LSQUIC_DEBUG_NEXT_ADV_TICK 2729 conn = lsquic_mh_peek(&engine->conns_out); 2730 engine->last_logged_conn = 0; 2731 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 2732 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 2733 lsquic_mh_count(&engine->conns_out), 2734 lsquic_mh_count(&engine->conns_out) != 1, "s", 2735 CID_BITS(lsquic_conn_log_cid(conn))); 2736#endif 2737 *diff = 0; 2738 return 1; 2739 } 2740 2741 if (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2742 { 2743#if LSQUIC_DEBUG_NEXT_ADV_TICK 2744 engine->last_logged_conn = 0; 2745 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 2746#endif 2747 *diff = 0; 2748 return 1; 2749 } 2750 2751 if (lsquic_mh_count(&engine->conns_tickable)) 2752 { 2753#if LSQUIC_DEBUG_NEXT_ADV_TICK 2754 conn = lsquic_mh_peek(&engine->conns_tickable); 2755 engine->last_logged_conn = 0; 2756 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 2757 "connection%.*s (%"CID_FMT" first)", 2758 lsquic_mh_count(&engine->conns_tickable), 2759 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 2760 CID_BITS(lsquic_conn_log_cid(conn))); 2761#endif 2762 *diff = 0; 2763 return 1; 2764 } 2765 2766 next_attq = attq_next(engine->attq); 2767 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 2768 { 2769 if (next_attq) 2770 next_time = next_attq->ae_adv_time; 2771 else 2772 return 0; 2773 } 2774 else 2775 { 2776 if (next_attq) 2777 { 2778 next_time = next_attq->ae_adv_time; 2779 if (engine->resume_sending_at < next_time) 2780 { 2781 next_time = engine->resume_sending_at; 2782 next_attq = NULL; 2783 } 2784 } 2785 else 2786 next_time = engine->resume_sending_at; 2787 } 2788 2789 now = lsquic_time_now(); 2790 *diff = (int) ((int64_t) next_time - (int64_t) now); 2791#if LSQUIC_DEBUG_NEXT_ADV_TICK 2792 if (next_attq) 2793 { 2794 /* Deduplicate consecutive log messages about the same reason for the 2795 * same connection. 2796 * If diff is always zero or diff reset to a higher value, event is 2797 * still logged. 2798 */ 2799 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 2800 && (uintptr_t) next_attq->ae_conn 2801 == engine->last_logged_conn 2802 && *diff < engine->last_tick_diff)) 2803 { 2804 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 2805 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 2806 engine->last_tick_diff = *diff; 2807 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 2808 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 2809 lsquic_attq_why2str(next_attq->ae_why)); 2810 } 2811 } 2812 else 2813 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 2814#endif 2815 return 1; 2816} 2817 2818 2819unsigned 2820lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 2821{ 2822 lsquic_time_t now; 2823 ENGINE_CALLS_INCR(engine); 2824 now = lsquic_time_now(); 2825 if (from_now < 0) 2826 now -= from_now; 2827 else 2828 now += from_now; 2829 return attq_count_before(engine->attq, now); 2830} 2831 2832 2833int 2834lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 2835 struct lsquic_conn *conn, unsigned cce_idx) 2836{ 2837 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2838 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2839 void *peer_ctx; 2840 2841 assert(cce_idx < conn->cn_n_cces); 2842 assert(conn->cn_cces_mask & (1 << cce_idx)); 2843 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 2844 2845 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 2846 cce->cce_cid.len, conn, &cce->cce_hash_el)) 2847 { 2848 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 2849 CID_BITS(&cce->cce_cid)); 2850 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2851 cce->cce_flags |= CCE_REG; 2852 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 2853 return 0; 2854 } 2855 else 2856 { 2857 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 2858 CID_BITS(&cce->cce_cid)); 2859 return -1; 2860 } 2861} 2862 2863 2864void 2865lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 2866 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now) 2867{ 2868 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2869 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2870 void *peer_ctx; 2871 2872 assert(cce_idx < conn->cn_n_cces); 2873 2874 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 2875 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 2876 2877 if (engine->purga) 2878 { 2879 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2880 lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 2881 PUTY_CID_RETIRED, now); 2882 } 2883 conn->cn_cces_mask &= ~(1u << cce_idx); 2884 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 2885} 2886 2887 2888