lsquic_engine.c revision feca77f5
1/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#ifndef WIN32 17#include <sys/time.h> 18#include <netinet/in.h> 19#include <sys/types.h> 20#include <sys/stat.h> 21#include <fcntl.h> 22#include <unistd.h> 23#include <netdb.h> 24#endif 25 26#ifndef NDEBUG 27#include <sys/types.h> 28#include <regex.h> /* For code that loses packets */ 29#endif 30 31#if LOG_PACKET_CHECKSUM 32#include <zlib.h> 33#endif 34 35#include <openssl/aead.h> 36 37#include "lsquic.h" 38#include "lsquic_types.h" 39#include "lsquic_int_types.h" 40#include "lsquic_sizes.h" 41#include "lsquic_parse_common.h" 42#include "lsquic_parse.h" 43#include "lsquic_packet_in.h" 44#include "lsquic_packet_out.h" 45#include "lsquic_senhist.h" 46#include "lsquic_rtt.h" 47#include "lsquic_cubic.h" 48#include "lsquic_pacer.h" 49#include "lsquic_bw_sampler.h" 50#include "lsquic_minmax.h" 51#include "lsquic_bbr.h" 52#include "lsquic_send_ctl.h" 53#include "lsquic_set.h" 54#include "lsquic_conn_flow.h" 55#include "lsquic_sfcw.h" 56#include "lsquic_hash.h" 57#include "lsquic_conn.h" 58#include "lsquic_full_conn.h" 59#include "lsquic_util.h" 60#include "lsquic_qtags.h" 61#include "lsquic_enc_sess.h" 62#include "lsquic_mm.h" 63#include "lsquic_engine_public.h" 64#include "lsquic_eng_hist.h" 65#include "lsquic_ev_log.h" 66#include "lsquic_version.h" 67#include "lsquic_pr_queue.h" 68#include "lsquic_mini_conn.h" 69#include "lsquic_mini_conn_ietf.h" 70#include "lsquic_stock_shi.h" 71#include "lsquic_purga.h" 72#include "lsquic_tokgen.h" 73#include "lsquic_attq.h" 74#include "lsquic_min_heap.h" 75#include "lsquic_http1x_if.h" 76#include "lsquic_handshake.h" 77#include "lsquic_crand.h" 78#include "lsquic_ietf.h" 79 80#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 81#include "lsquic_logger.h" 82 83#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 84#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 85#endif 86 87#if LSQUIC_DEBUG_NEXT_ADV_TICK 88#include "lsquic_alarmset.h" 89#endif 90 91#define MIN(a, b) ((a) < (b) ? (a) : (b)) 92 93/* The batch of outgoing packets grows and shrinks dynamically */ 94#define MAX_OUT_BATCH_SIZE 1024 95#define MIN_OUT_BATCH_SIZE 4 96#define INITIAL_OUT_BATCH_SIZE 32 97 98struct out_batch 99{ 100 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 101 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 102 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 103 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 104 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 105}; 106 107typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 108 109static void 110process_connections (struct lsquic_engine *engine, conn_iter_f iter, 111 lsquic_time_t now); 112 113static void 114engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 115 116static lsquic_conn_t * 117engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 118 enum lsquic_conn_flags flag); 119 120static void 121force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 122 123#if LSQUIC_COUNT_ENGINE_CALLS 124#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 125#else 126#define ENGINE_CALLS_INCR(e) 127#endif 128 129/* Nested calls to LSQUIC are not supported */ 130#define ENGINE_IN(e) do { \ 131 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 132 (e)->pub.enp_flags |= ENPUB_PROC; \ 133 ENGINE_CALLS_INCR(e); \ 134} while (0) 135 136#define ENGINE_OUT(e) do { \ 137 assert((e)->pub.enp_flags & ENPUB_PROC); \ 138 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 139} while (0) 140 141/* A connection can be referenced from one of six places: 142 * 143 * 1. A hash is used to find connections in order to dispatch an incoming 144 * packet. Connections can be hashed by CIDs or by address. In the 145 * former case, each connection has one or more mappings in the hash 146 * table. IETF QUIC connections have up to eight (in our implementation) 147 * source CIDs and each of those would have a mapping. In client mode, 148 * depending on QUIC versions and options selected, it is may be 149 * necessary to hash connections by address, in which case incoming 150 * packets are delivered to connections based on the address. 151 * 152 * 2. Outgoing queue. 153 * 154 * 3. Tickable queue 155 * 156 * 4. Advisory Tick Time queue. 157 * 158 * 5. Closing connections queue. This is a transient queue -- it only 159 * exists for the duration of process_connections() function call. 160 * 161 * 6. Ticked connections queue. Another transient queue, similar to (5). 162 * 163 * The idea is to destroy the connection when it is no longer referenced. 164 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 165 * that case, the connection is referenced from two places: (2) and (5). 166 * After its packets are sent, it is only referenced in (5), and at the 167 * end of the function call, when it is removed from (5), reference count 168 * goes to zero and the connection is destroyed. If not all packets can 169 * be sent, at the end of the function call, the connection is referenced 170 * by (2) and will only be removed once all outgoing packets have been 171 * sent. 172 */ 173#define CONN_REF_FLAGS (LSCONN_HASHED \ 174 |LSCONN_HAS_OUTGOING \ 175 |LSCONN_TICKABLE \ 176 |LSCONN_TICKED \ 177 |LSCONN_CLOSING \ 178 |LSCONN_ATTQ) 179 180 181 182 183struct cid_update_batch 184{ 185 lsquic_cids_update_f cub_update_cids; 186 void *cub_update_ctx; 187 unsigned cub_count; 188 lsquic_cid_t cub_cids[20]; 189 void *cub_peer_ctxs[20]; 190}; 191 192static void 193cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 194 195 196struct lsquic_engine 197{ 198 struct lsquic_engine_public pub; 199 enum { 200 ENG_SERVER = LSENG_SERVER, 201 ENG_HTTP = LSENG_HTTP, 202 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 203 ENG_PAST_DEADLINE 204 = (1 << 8), /* Previous call to a processing 205 * function went past time threshold. 206 */ 207 ENG_CONNS_BY_ADDR 208 = (1 << 9), /* Connections are hashed by address */ 209#ifndef NDEBUG 210 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 211 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 212 ENG_DTOR = (1 << 26), /* Engine destructor */ 213#endif 214 } flags; 215 lsquic_packets_out_f packets_out; 216 void *packets_out_ctx; 217 lsquic_cids_update_f report_new_scids; 218 lsquic_cids_update_f report_live_scids; 219 lsquic_cids_update_f report_old_scids; 220 void *scids_ctx; 221 struct lsquic_hash *conns_hash; 222 struct min_heap conns_tickable; 223 struct min_heap conns_out; 224 struct eng_hist history; 225 unsigned batch_size; 226 struct pr_queue *pr_queue; 227 struct attq *attq; 228 /* Track time last time a packet was sent to give new connections 229 * priority lower than that of existing connections. 230 */ 231 lsquic_time_t last_sent; 232#ifndef NDEBUG 233 regex_t lose_packets_re; 234 const char *lose_packets_str; 235#endif 236 unsigned n_conns; 237 lsquic_time_t deadline; 238 lsquic_time_t resume_sending_at; 239 unsigned mini_conns_count; 240 struct lsquic_purga *purga; 241#if LSQUIC_CONN_STATS 242 struct { 243 unsigned conns; 244 } stats; 245 struct conn_stats conn_stats_sum; 246 FILE *stats_fh; 247#endif 248 struct cid_update_batch new_scids; 249 struct out_batch out_batch; 250#if LSQUIC_COUNT_ENGINE_CALLS 251 unsigned long n_engine_calls; 252#endif 253#if LSQUIC_DEBUG_NEXT_ADV_TICK 254 uintptr_t last_logged_conn; 255 unsigned last_logged_ae_why; 256 int last_tick_diff; 257#endif 258 struct crand crand; 259 EVP_AEAD_CTX retry_aead_ctx; 260}; 261 262 263void 264lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 265 unsigned flags) 266{ 267 memset(settings, 0, sizeof(*settings)); 268 settings->es_versions = LSQUIC_DF_VERSIONS; 269 if (flags & ENG_SERVER) 270 { 271 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 272 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 273 settings->es_init_max_data 274 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 275 settings->es_init_max_stream_data_bidi_remote 276 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 277 settings->es_init_max_stream_data_bidi_local 278 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 279 settings->es_init_max_stream_data_uni 280 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 281 settings->es_init_max_streams_uni 282 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 283 settings->es_ping_period = 0; 284 } 285 else 286 { 287 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 288 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 289 settings->es_init_max_data 290 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 291 settings->es_init_max_stream_data_bidi_remote 292 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 293 settings->es_init_max_stream_data_bidi_local 294 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 295 settings->es_init_max_stream_data_uni 296 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 297 settings->es_init_max_streams_uni 298 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 299 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 300 } 301 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 302 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 303 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 304 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 305 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 306 settings->es_max_header_list_size 307 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 308 settings->es_ua = LSQUIC_DF_UA; 309 settings->es_ecn = LSQUIC_DF_ECN; 310 311 settings->es_pdmd = QTAG_X509; 312 settings->es_aead = QTAG_AESG; 313 settings->es_kexs = QTAG_C255; 314 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 315 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 316 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 317 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 318 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 319 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 320 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 321 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 322 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 323 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 324 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 325 settings->es_sttl = LSQUIC_DF_STTL; 326 settings->es_init_max_streams_bidi 327 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 328 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 329 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 330 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 331 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 332 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 333 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 334 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 335 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 336 settings->es_spin = LSQUIC_DF_SPIN; 337 settings->es_delayed_acks = LSQUIC_DF_DELAYED_ACKS; 338} 339 340 341/* Note: if returning an error, err_buf must be valid if non-NULL */ 342int 343lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 344 unsigned flags, 345 char *err_buf, size_t err_buf_sz) 346{ 347 if (settings->es_cfcw < LSQUIC_MIN_FCW || 348 settings->es_sfcw < LSQUIC_MIN_FCW) 349 { 350 if (err_buf) 351 snprintf(err_buf, err_buf_sz, "%s", 352 "flow control window set too low"); 353 return -1; 354 } 355 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 356 { 357 if (err_buf) 358 snprintf(err_buf, err_buf_sz, "%s", 359 "No supported QUIC versions specified"); 360 return -1; 361 } 362 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 363 { 364 if (err_buf) 365 snprintf(err_buf, err_buf_sz, "%s", 366 "one or more unsupported QUIC version is specified"); 367 return -1; 368 } 369 if (flags & ENG_SERVER) 370 { 371 if (settings->es_handshake_to > 372 MAX_MINI_CONN_LIFESPAN_IN_USEC) 373 { 374 if (err_buf) 375 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 376 " usec is too large. The maximum for server is %u usec", 377 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 378 return -1; 379 } 380 } 381 if (settings->es_idle_timeout > 600) 382 { 383 if (err_buf) 384 snprintf(err_buf, err_buf_sz, "%s", 385 "The maximum value of idle timeout is 600 seconds"); 386 return -1; 387 } 388 if (settings->es_scid_len > MAX_CID_LEN) 389 { 390 if (err_buf) 391 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 392 "bytes long; it must be between 0 and %u.", 393 settings->es_scid_len, MAX_CID_LEN); 394 return -1; 395 } 396 397 if (settings->es_cc_algo > 2) 398 { 399 if (err_buf) 400 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 401 "algorithm value %u", settings->es_cc_algo); 402 return -1; 403 } 404 405 if (!(settings->es_ql_bits >= 0 && settings->es_ql_bits <= 2)) 406 { 407 if (err_buf) 408 snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ", 409 settings->es_ql_bits); 410 return -1; 411 } 412 413 if (!(settings->es_spin == 0 || settings->es_spin == 1)) 414 { 415 if (err_buf) 416 snprintf(err_buf, err_buf_sz, "Invalid spin value %d", 417 settings->es_spin); 418 return -1; 419 } 420 421 return 0; 422} 423 424 425static void 426free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 427{ 428 free(packet_data); 429} 430 431 432static void * 433malloc_buf (void *ctx, void *conn_ctx, unsigned short size, char is_ipv6) 434{ 435 return malloc(size); 436} 437 438 439static const struct lsquic_packout_mem_if stock_pmi = 440{ 441 malloc_buf, free_packet, free_packet, 442}; 443 444 445static int 446hash_conns_by_addr (const struct lsquic_engine *engine) 447{ 448 if (engine->flags & ENG_SERVER) 449 return 0; 450 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 451 return 1; 452 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 453 && engine->pub.enp_settings.es_support_tcid0) 454 return 1; 455 if (engine->pub.enp_settings.es_scid_len == 0) 456 return 1; 457 return 0; 458} 459 460 461lsquic_engine_t * 462lsquic_engine_new (unsigned flags, 463 const struct lsquic_engine_api *api) 464{ 465 lsquic_engine_t *engine; 466 char err_buf[100]; 467 468 if (!api->ea_packets_out) 469 { 470 LSQ_ERROR("packets_out callback is not specified"); 471 return NULL; 472 } 473 474 if (!api->ea_stream_if) 475 { 476 LSQ_ERROR("stream interface is not specified"); 477 return NULL; 478 } 479 480 if (api->ea_settings && 481 0 != lsquic_engine_check_settings(api->ea_settings, flags, 482 err_buf, sizeof(err_buf))) 483 { 484 LSQ_ERROR("cannot create engine: %s", err_buf); 485 return NULL; 486 } 487 488 engine = calloc(1, sizeof(*engine)); 489 if (!engine) 490 return NULL; 491 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 492 { 493 free(engine); 494 return NULL; 495 } 496 if (api->ea_settings) 497 engine->pub.enp_settings = *api->ea_settings; 498 else 499 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 500 int tag_buf_len; 501 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 502 sizeof(engine->pub.enp_ver_tags_buf), 503 engine->pub.enp_settings.es_versions); 504 if (tag_buf_len <= 0) 505 { 506 LSQ_ERROR("cannot generate version tags buffer"); 507 free(engine); 508 return NULL; 509 } 510 engine->pub.enp_ver_tags_len = tag_buf_len; 511 engine->pub.enp_flags = ENPUB_CAN_SEND; 512 engine->pub.enp_stream_if = api->ea_stream_if; 513 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 514 515 engine->flags = flags; 516#ifndef NDEBUG 517 engine->flags |= ENG_COALESCE; 518#endif 519 engine->packets_out = api->ea_packets_out; 520 engine->packets_out_ctx = api->ea_packets_out_ctx; 521 engine->report_new_scids = api->ea_new_scids; 522 engine->report_live_scids = api->ea_live_scids; 523 engine->report_old_scids = api->ea_old_scids; 524 engine->scids_ctx = api->ea_cids_update_ctx; 525 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 526 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 527 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 528 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 529 if (api->ea_shi) 530 { 531 engine->pub.enp_shi = api->ea_shi; 532 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 533 } 534 else 535 { 536 engine->pub.enp_shi = &stock_shi; 537 engine->pub.enp_shi_ctx = stock_shared_hash_new(); 538 if (!engine->pub.enp_shi_ctx) 539 { 540 free(engine); 541 return NULL; 542 } 543 } 544 if (api->ea_hsi_if) 545 { 546 engine->pub.enp_hsi_if = api->ea_hsi_if; 547 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 548 } 549 else 550 { 551 engine->pub.enp_hsi_if = lsquic_http1x_if; 552 engine->pub.enp_hsi_ctx = NULL; 553 } 554 if (api->ea_pmi) 555 { 556 engine->pub.enp_pmi = api->ea_pmi; 557 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 558 } 559 else 560 { 561 engine->pub.enp_pmi = &stock_pmi; 562 engine->pub.enp_pmi_ctx = NULL; 563 } 564 engine->pub.enp_verify_cert = api->ea_verify_cert; 565 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 566 engine->pub.enp_kli = api->ea_keylog_if; 567 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 568 engine->pub.enp_engine = engine; 569 if (hash_conns_by_addr(engine)) 570 engine->flags |= ENG_CONNS_BY_ADDR; 571 engine->conns_hash = lsquic_hash_create(); 572 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 573 if (!engine->pub.enp_tokgen) 574 return NULL; 575 engine->pub.enp_crand = &engine->crand; 576 if (flags & ENG_SERVER) 577 { 578 engine->pr_queue = prq_create( 579 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 580 &engine->pub); 581 if (!engine->pr_queue) 582 { 583 lsquic_tg_destroy(engine->pub.enp_tokgen); 584 return NULL; 585 } 586 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 587 engine->report_old_scids, engine->scids_ctx); 588 if (!engine->purga) 589 { 590 lsquic_tg_destroy(engine->pub.enp_tokgen); 591 prq_destroy(engine->pr_queue); 592 return NULL; 593 } 594 } 595 engine->attq = attq_create(); 596 eng_hist_init(&engine->history); 597 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 598 if (engine->pub.enp_settings.es_honor_prst) 599 { 600 engine->pub.enp_srst_hash = lsquic_hash_create(); 601 if (!engine->pub.enp_srst_hash) 602 { 603 lsquic_engine_destroy(engine); 604 return NULL; 605 } 606 } 607 608#ifndef NDEBUG 609 { 610 const char *env; 611 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 612 if (env) 613 { 614 if (0 != regcomp(&engine->lose_packets_re, env, 615 REG_EXTENDED|REG_NOSUB)) 616 { 617 LSQ_ERROR("could not compile lost packet regex `%s'", env); 618 return NULL; 619 } 620 engine->flags |= ENG_LOSE_PACKETS; 621 engine->lose_packets_str = env; 622 LSQ_WARN("will lose packets that match the following regex: %s", 623 env); 624 } 625 env = getenv("LSQUIC_COALESCE"); 626 if (env) 627 { 628 engine->flags &= ~ENG_COALESCE; 629 if (atoi(env)) 630 { 631 engine->flags |= ENG_COALESCE; 632 LSQ_NOTICE("will coalesce packets"); 633 } 634 else 635 LSQ_NOTICE("will not coalesce packets"); 636 } 637 } 638#endif 639#if LSQUIC_CONN_STATS 640 engine->stats_fh = api->ea_stats_fh; 641#endif 642 if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx, EVP_aead_aes_128_gcm(), 643 IETF_RETRY_KEY_BUF, IETF_RETRY_KEY_SZ, 16, NULL)) 644 { 645 LSQ_ERROR("could not initialize retry AEAD ctx"); 646 lsquic_engine_destroy(engine); 647 return NULL; 648 } 649 engine->pub.enp_retry_aead_ctx = &engine->retry_aead_ctx; 650 651 LSQ_INFO("instantiated engine"); 652 return engine; 653} 654 655 656#if LOG_PACKET_CHECKSUM 657static void 658log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 659 const unsigned char *buf, size_t bufsz) 660{ 661 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 662 (uint32_t) crc32(0, buf, bufsz)); 663} 664 665 666#endif 667 668 669static void 670grow_batch_size (struct lsquic_engine *engine) 671{ 672 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 673} 674 675 676static void 677shrink_batch_size (struct lsquic_engine *engine) 678{ 679 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 680} 681 682 683struct cce_cid_iter 684{ 685 const struct lsquic_conn *conn; 686 unsigned todo, n; 687}; 688 689 690static struct conn_cid_elem * 691cce_iter_next (struct cce_cid_iter *citer) 692{ 693 struct conn_cid_elem *cce; 694 695 while (citer->todo) 696 if (citer->todo & (1 << citer->n)) 697 { 698 citer->todo &= ~(1 << citer->n); 699 cce = &citer->conn->cn_cces[ citer->n++ ]; 700 if (!(cce->cce_flags & CCE_PORT)) 701 return cce; 702 } 703 else 704 ++citer->n; 705 706 return NULL; 707} 708 709 710static struct conn_cid_elem * 711cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 712{ 713 citer->conn = conn; 714 citer->todo = conn->cn_cces_mask; 715 citer->n = 0; 716 return cce_iter_next(citer); 717} 718 719 720#if LSQUIC_CONN_STATS 721void 722update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 723{ 724 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 725 const unsigned long *src; 726 const struct conn_stats *stats; 727 unsigned i; 728 729 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 730 { 731 ++engine->stats.conns; 732 src = (unsigned long *) stats; 733 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 734 dst[i] += src[i]; 735 } 736} 737 738 739#endif 740 741 742/* Wrapper to make sure important things occur before the connection is 743 * really destroyed. 744 */ 745static void 746destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 747 lsquic_time_t now) 748{ 749 struct cce_cid_iter citer; 750 const struct conn_cid_elem *cce; 751 lsquic_time_t drain_time; 752 struct purga_el *puel; 753 754 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 755 if (engine->purga 756 /* Blacklist all CIDs except for promoted mini connections */ 757 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 758 != (LSCONN_MINI|LSCONN_PROMOTED)) 759 { 760 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 761 && conn->cn_if->ci_drain_time && 762 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 763 { 764 for (cce = cce_iter_first(&citer, conn); cce; 765 cce = cce_iter_next(&citer)) 766 { 767 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 768 lsquic_conn_get_peer_ctx(conn, NULL), 769 PUTY_CONN_DRAIN, now); 770 if (puel) 771 puel->puel_time = now + drain_time; 772 } 773 } 774 else 775 { 776 for (cce = cce_iter_first(&citer, conn); cce; 777 cce = cce_iter_next(&citer)) 778 { 779 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 780 lsquic_conn_get_peer_ctx(conn, NULL), 781 PUTY_CONN_DELETED, now); 782 if (puel) 783 { 784 puel->puel_time = now; 785 puel->puel_count = 0; 786 } 787 } 788 } 789 } 790#if LSQUIC_CONN_STATS 791 update_stats_sum(engine, conn); 792#endif 793 --engine->n_conns; 794 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 795 conn->cn_if->ci_destroy(conn); 796} 797 798 799static int 800maybe_grow_conn_heaps (struct lsquic_engine *engine) 801{ 802 struct min_heap_elem *els; 803 unsigned count; 804 805 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 806 return 0; /* Nothing to do */ 807 808 if (lsquic_mh_nalloc(&engine->conns_tickable)) 809 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 810 else 811 count = 8; 812 813 els = malloc(sizeof(els[0]) * count); 814 if (!els) 815 { 816 LSQ_ERROR("%s: malloc failed", __func__); 817 return -1; 818 } 819 820 LSQ_DEBUG("grew heaps to %u elements", count / 2); 821 memcpy(&els[0], engine->conns_tickable.mh_elems, 822 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 823 memcpy(&els[count / 2], engine->conns_out.mh_elems, 824 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 825 free(engine->conns_tickable.mh_elems); 826 engine->conns_tickable.mh_elems = els; 827 engine->conns_out.mh_elems = &els[count / 2]; 828 engine->conns_tickable.mh_nalloc = count / 2; 829 engine->conns_out.mh_nalloc = count / 2; 830 return 0; 831} 832 833 834static void 835remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 836 unsigned todo) 837{ 838 unsigned n; 839 840 for (n = 0; todo; todo &= ~(1 << n++)) 841 if ((todo & (1 << n)) && 842 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 843 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 844} 845 846 847static void 848remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 849{ 850 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 851} 852 853 854static void 855cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 856 857 858static int 859insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 860 void *peer_ctx) 861{ 862 struct conn_cid_elem *cce; 863 unsigned todo, done, n; 864 865 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 866 if (todo & (1 << n)) 867 { 868 cce = &conn->cn_cces[n]; 869 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 870 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 871 cce->cce_cid.len, conn, &cce->cce_hash_el)) 872 done |= 1 << n; 873 else 874 goto err; 875 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 876 { 877 cce->cce_flags |= CCE_REG; 878 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 879 } 880 } 881 882 return 0; 883 884 err: 885 remove_cces_from_hash(engine->conns_hash, conn, done); 886 return -1; 887} 888 889 890static lsquic_conn_t * 891new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 892 lsquic_time_t now) 893{ 894 const lsquic_cid_t *cid; 895 server_conn_ctor_f ctor; 896 lsquic_conn_t *conn; 897 unsigned flags; 898 if (0 != maybe_grow_conn_heaps(engine)) 899 return NULL; 900 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 901 902 if (mini_conn->cn_flags & LSCONN_IETF) 903 { 904 if (mini_conn->cn_version == LSQVER_ID24) 905 ctor = lsquic_id24_full_conn_server_new; 906 else 907 ctor = lsquic_ietf_full_conn_server_new; 908 } 909 else 910 ctor = lsquic_gquic_full_conn_server_new; 911 912 conn = ctor(&engine->pub, flags, mini_conn); 913 if (!conn) 914 { 915 /* Otherwise, full_conn_server_new prints its own warnings */ 916 if (ENOMEM == errno) 917 { 918 cid = lsquic_conn_log_cid(mini_conn); 919 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 920 CID_BITS(cid), strerror(errno)); 921 } 922 return NULL; 923 } 924 ++engine->n_conns; 925 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 926 { 927 cid = lsquic_conn_log_cid(conn); 928 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 929 CID_BITS(cid)); 930 destroy_conn(engine, conn, now); 931 return NULL; 932 } 933 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 934 conn->cn_flags |= LSCONN_HASHED; 935 return conn; 936} 937 938 939static enum 940{ 941 VER_NOT_SPECIFIED, 942 VER_SUPPORTED, 943 VER_UNSUPPORTED, 944} 945 946 947version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 948 enum lsquic_version *pversion) 949{ 950 lsquic_ver_tag_t ver_tag; 951 enum lsquic_version version; 952 953 if (!packet_in->pi_quic_ver) 954 { 955 LSQ_DEBUG("packet does not specify version"); 956 return VER_NOT_SPECIFIED; 957 } 958 959 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 960 version = lsquic_tag2ver(ver_tag); 961 if (version < N_LSQVER) 962 { 963 if (engine->pub.enp_settings.es_versions & (1 << version)) 964 { 965 LSQ_DEBUG("client-supplied version %s is supported", 966 lsquic_ver2str[version]); 967 *pversion = version; 968 return VER_SUPPORTED; 969 } 970 else 971 LSQ_DEBUG("client-supplied version %s is not supported", 972 lsquic_ver2str[version]); 973 } 974 else 975 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 976 ver_tag); 977 978 return VER_UNSUPPORTED; 979} 980 981 982static void 983schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 984 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 985 const struct sockaddr *sa_peer, void *peer_ctx) 986{ 987 assert(engine->pr_queue); 988 if (0 == prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 989 sa_local, sa_peer)) 990 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 991 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 992 else 993 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 994} 995 996 997static unsigned short 998sa2port (const struct sockaddr *sa) 999{ 1000 if (sa->sa_family == AF_INET) 1001 { 1002 struct sockaddr_in *const sa4 = (void *) sa; 1003 return sa4->sin_port; 1004 } 1005 else 1006 { 1007 struct sockaddr_in6 *const sa6 = (void *) sa; 1008 return sa6->sin6_port; 1009 } 1010} 1011 1012 1013static struct lsquic_hash_elem * 1014find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 1015{ 1016 unsigned short port; 1017 1018 port = sa2port(sa); 1019 return lsquic_hash_find(hash, &port, sizeof(port)); 1020} 1021 1022 1023static lsquic_conn_t * 1024find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1025 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 1026{ 1027 struct lsquic_hash_elem *el; 1028 lsquic_conn_t *conn; 1029 1030 if (engine->flags & ENG_CONNS_BY_ADDR) 1031 el = find_conn_by_addr(engine->conns_hash, sa_local); 1032 else if (packet_in->pi_flags & PI_CONN_ID) 1033 el = lsquic_hash_find(engine->conns_hash, 1034 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1035 else 1036 { 1037 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1038 return NULL; 1039 } 1040 1041 if (!el) 1042 return NULL; 1043 1044 conn = lsquic_hashelem_getdata(el); 1045 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1046 if ((engine->flags & ENG_CONNS_BY_ADDR) 1047 && !(conn->cn_flags & LSCONN_IETF) 1048 && (packet_in->pi_flags & PI_CONN_ID) 1049 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1050 { 1051 LSQ_DEBUG("connection IDs do not match"); 1052 return NULL; 1053 } 1054 1055 return conn; 1056} 1057 1058 1059static lsquic_conn_t * 1060find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1061 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1062 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1063{ 1064 struct lsquic_hash_elem *el; 1065 struct purga_el *puel; 1066 lsquic_conn_t *conn; 1067 1068 if (!(packet_in->pi_flags & PI_CONN_ID)) 1069 { 1070 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1071 return NULL; 1072 } 1073 el = lsquic_hash_find(engine->conns_hash, 1074 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1075 1076 if (el) 1077 { 1078 conn = lsquic_hashelem_getdata(el); 1079 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1080 return conn; 1081 } 1082 1083 if (engine->flags & ENG_COOLDOWN) 1084 { /* Do not create incoming connections during cooldown */ 1085 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1086 return NULL; 1087 } 1088 1089 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1090 { 1091 LSQ_DEBUG("reached limit of %u inchoate connections", 1092 engine->pub.enp_settings.es_max_inchoate); 1093 return NULL; 1094 } 1095 1096 1097 if (engine->purga 1098 && (puel = lsquic_purga_contains(engine->purga, 1099 &packet_in->pi_conn_id), puel)) 1100 { 1101 switch (puel->puel_type) 1102 { 1103 case PUTY_CID_RETIRED: 1104 LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet", 1105 CID_BITS(&packet_in->pi_conn_id)); 1106 return NULL; 1107 case PUTY_CONN_DRAIN: 1108 LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64, 1109 puel->puel_time, packet_in->pi_received); 1110 if (puel->puel_time > packet_in->pi_received) 1111 { 1112 LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet", 1113 CID_BITS(&packet_in->pi_conn_id)); 1114 return NULL; 1115 } 1116 LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted", 1117 CID_BITS(&packet_in->pi_conn_id)); 1118 puel->puel_type = PUTY_CONN_DELETED; 1119 puel->puel_count = 0; 1120 puel->puel_time = 0; 1121 /* fall-through */ 1122 case PUTY_CONN_DELETED: 1123 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1124 CID_BITS(&packet_in->pi_conn_id)); 1125 if (puel->puel_time < packet_in->pi_received) 1126 { 1127 puel->puel_time = packet_in->pi_received 1128 /* Exponential back-off */ 1129 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1130 ++puel->puel_count; 1131 goto maybe_send_prst; 1132 } 1133 return NULL; 1134 default: 1135 assert(0); 1136 return NULL; 1137 } 1138 } 1139 1140 if (engine->pub.enp_settings.es_send_prst 1141 && !(packet_in->pi_flags & PI_GQUIC) 1142 && HETY_NOT_SET == packet_in->pi_header_type) 1143 goto maybe_send_prst; 1144 1145 if (0 != maybe_grow_conn_heaps(engine)) 1146 return NULL; 1147 1148 const struct parse_funcs *pf; 1149 enum lsquic_version version; 1150 switch (version_matches(engine, packet_in, &version)) 1151 { 1152 case VER_UNSUPPORTED: 1153 if (engine->flags & ENG_SERVER) 1154 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1155 sa_local, sa_peer, peer_ctx); 1156 return NULL; 1157 case VER_NOT_SPECIFIED: 1158 maybe_send_prst: 1159 if ((engine->flags & ENG_SERVER) && 1160 engine->pub.enp_settings.es_send_prst) 1161 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1162 sa_local, sa_peer, peer_ctx); 1163 return NULL; 1164 case VER_SUPPORTED: 1165 pf = select_pf_by_ver(version); 1166 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1167 break; 1168 } 1169 1170 1171 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1172 { 1173 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1174 sa_peer->sa_family == AF_INET, NULL, packet_in_size); 1175 } 1176 else 1177 { 1178 conn = mini_conn_new(&engine->pub, packet_in, version); 1179 } 1180 if (!conn) 1181 return NULL; 1182 ++engine->mini_conns_count; 1183 ++engine->n_conns; 1184 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1185 { 1186 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1187 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1188 CID_BITS(cid)); 1189 destroy_conn(engine, conn, packet_in->pi_received); 1190 return NULL; 1191 } 1192 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1193 conn->cn_flags |= LSCONN_HASHED; 1194 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1195 conn->cn_last_sent = engine->last_sent; 1196 return conn; 1197} 1198 1199 1200lsquic_conn_t * 1201lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1202 const lsquic_cid_t *cid) 1203{ 1204 struct lsquic_hash_elem *el; 1205 lsquic_conn_t *conn = NULL; 1206 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1207 1208 if (el) 1209 conn = lsquic_hashelem_getdata(el); 1210 return conn; 1211} 1212 1213 1214#if !defined(NDEBUG) && __GNUC__ 1215__attribute__((weak)) 1216#endif 1217void 1218lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1219 lsquic_conn_t *conn) 1220{ 1221 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1222 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1223 { 1224 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1225 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1226 engine_incref_conn(conn, LSCONN_TICKABLE); 1227 } 1228} 1229 1230 1231void 1232lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1233 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1234{ 1235 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1236 if (conn->cn_flags & LSCONN_TICKABLE) 1237 { 1238 /* Optimization: no need to add the connection to the Advisory Tick 1239 * Time Queue: it is about to be ticked, after which it its next tick 1240 * time may be queried again. 1241 */; 1242 } 1243 else if (conn->cn_flags & LSCONN_ATTQ) 1244 { 1245 if (lsquic_conn_adv_time(conn) != tick_time) 1246 { 1247 attq_remove(engine->attq, conn); 1248 if (0 != attq_add(engine->attq, conn, tick_time, why)) 1249 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1250 } 1251 } 1252 else if (0 == attq_add(engine->attq, conn, tick_time, why)) 1253 engine_incref_conn(conn, LSCONN_ATTQ); 1254} 1255 1256 1257static struct lsquic_conn * 1258find_conn_by_srst (struct lsquic_engine *engine, 1259 const struct lsquic_packet_in *packet_in) 1260{ 1261 struct lsquic_hash_elem *el; 1262 struct lsquic_conn *conn; 1263 1264 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1265 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1266 return NULL; 1267 1268 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1269 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1270 IQUIC_SRESET_TOKEN_SZ); 1271 if (!el) 1272 return NULL; 1273 1274 conn = lsquic_hashelem_getdata(el); 1275 return conn; 1276} 1277 1278 1279/* Return 0 if packet is being processed by a real connection (mini or full), 1280 * otherwise return 1. 1281 */ 1282static int 1283process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1284 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1285 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1286{ 1287 lsquic_conn_t *conn; 1288 const unsigned char *packet_in_data; 1289 1290 if (lsquic_packet_in_is_gquic_prst(packet_in) 1291 && !engine->pub.enp_settings.es_honor_prst) 1292 { 1293 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1294 LSQ_DEBUG("public reset packet: discarding"); 1295 return 1; 1296 } 1297 1298 if (engine->flags & ENG_SERVER) 1299 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1300 sa_peer, peer_ctx, packet_in_size); 1301 else 1302 conn = find_conn(engine, packet_in, ppstate, sa_local); 1303 1304 if (!conn) 1305 { 1306 if (engine->pub.enp_settings.es_honor_prst 1307 && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */ 1308 && !(packet_in->pi_flags & PI_GQUIC) 1309 && engine->pub.enp_srst_hash 1310 && (conn = find_conn_by_srst(engine, packet_in))) 1311 { 1312 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1313 CID_BITS(lsquic_conn_log_cid(conn))); 1314 conn->cn_if->ci_stateless_reset(conn); 1315 if (!(conn->cn_flags & LSCONN_TICKABLE) 1316 && conn->cn_if->ci_is_tickable(conn)) 1317 { 1318 lsquic_mh_insert(&engine->conns_tickable, conn, 1319 conn->cn_last_ticked); 1320 engine_incref_conn(conn, LSCONN_TICKABLE); 1321 } 1322 /* Even though the connection processes this packet, we return 1323 * 1 so that the caller does not add reset packet's random 1324 * bytes to the list of valid CIDs. 1325 */ 1326 } 1327 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1328 return 1; 1329 } 1330 1331 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1332 { 1333 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1334 engine_incref_conn(conn, LSCONN_TICKABLE); 1335 } 1336 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1337 sa_local, sa_peer); 1338 lsquic_packet_in_upref(packet_in); 1339#if LOG_PACKET_CHECKSUM 1340 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1341 packet_in->pi_data_sz); 1342#endif 1343 /* Note on QLog: 1344 * For the PACKET_RX QLog event, we are interested in logging these things: 1345 * - raw packet (however it comes in, encrypted or not) 1346 * - frames (list of frame names) 1347 * - packet type and number 1348 * - packet rx timestamp 1349 * 1350 * Since only some of these items are available at this code 1351 * juncture, we will wait until after the packet has been 1352 * decrypted (if necessary) and parsed to call the log functions. 1353 * 1354 * Once the PACKET_RX event is finally logged, the timestamp 1355 * will come from packet_in->pi_received. For correct sequential 1356 * ordering of QLog events, be sure to process the QLogs downstream. 1357 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1358 */ 1359 packet_in_data = packet_in->pi_data; 1360 packet_in_size = packet_in->pi_data_sz; 1361 conn->cn_if->ci_packet_in(conn, packet_in); 1362 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1363 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1364 return 0; 1365} 1366 1367 1368void 1369lsquic_engine_destroy (lsquic_engine_t *engine) 1370{ 1371 struct lsquic_hash_elem *el; 1372 lsquic_conn_t *conn; 1373 1374 LSQ_DEBUG("destroying engine"); 1375#ifndef NDEBUG 1376 engine->flags |= ENG_DTOR; 1377#endif 1378 1379 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1380 { 1381 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1382 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1383 } 1384 1385 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1386 { 1387 assert(conn->cn_flags & LSCONN_TICKABLE); 1388 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1389 } 1390 1391 for (el = lsquic_hash_first(engine->conns_hash); el; 1392 el = lsquic_hash_next(engine->conns_hash)) 1393 { 1394 conn = lsquic_hashelem_getdata(el); 1395 force_close_conn(engine, conn); 1396 } 1397 lsquic_hash_destroy(engine->conns_hash); 1398 1399 assert(0 == engine->n_conns); 1400 assert(0 == engine->mini_conns_count); 1401 if (engine->pr_queue) 1402 prq_destroy(engine->pr_queue); 1403 if (engine->purga) 1404 lsquic_purga_destroy(engine->purga); 1405 attq_destroy(engine->attq); 1406 1407 assert(0 == lsquic_mh_count(&engine->conns_out)); 1408 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1409 if (engine->pub.enp_shi == &stock_shi) 1410 stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1411 lsquic_mm_cleanup(&engine->pub.enp_mm); 1412 free(engine->conns_tickable.mh_elems); 1413#ifndef NDEBUG 1414 if (engine->flags & ENG_LOSE_PACKETS) 1415 regfree(&engine->lose_packets_re); 1416#endif 1417 if (engine->pub.enp_tokgen) 1418 lsquic_tg_destroy(engine->pub.enp_tokgen); 1419#if LSQUIC_CONN_STATS 1420 if (engine->stats_fh) 1421 { 1422 const struct conn_stats *const stats = &engine->conn_stats_sum; 1423 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1424 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1425 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1426 fprintf(engine->stats_fh, "In:\n"); 1427 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1428 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1429 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1430 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1431 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1432 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1433 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1434 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1435 stats->in.headers_comp, stats->in.headers_uncomp, 1436 stats->in.headers_uncomp ? 1437 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1438 : 0); 1439 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1440 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1441 fprintf(engine->stats_fh, " ACK frames merged to new: %lu\n", stats->in.n_acks_merged[0]); 1442 fprintf(engine->stats_fh, " ACK frames merged to old: %lu\n", stats->in.n_acks_merged[1]); 1443 fprintf(engine->stats_fh, "Out:\n"); 1444 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1445 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1446 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1447 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1448 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1449 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1450 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1451 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1452 stats->out.headers_comp, stats->out.headers_uncomp, 1453 stats->out.headers_uncomp ? 1454 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1455 : 0); 1456 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1457 } 1458#endif 1459 if (engine->pub.enp_srst_hash) 1460 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1461#if LSQUIC_COUNT_ENGINE_CALLS 1462 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1463#endif 1464 if (engine->pub.enp_retry_aead_ctx) 1465 EVP_AEAD_CTX_cleanup(engine->pub.enp_retry_aead_ctx); 1466 free(engine); 1467} 1468 1469 1470static struct conn_cid_elem * 1471find_free_cce (struct lsquic_conn *conn) 1472{ 1473 struct conn_cid_elem *cce; 1474 1475 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1476 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1477 return cce; 1478 1479 return NULL; 1480} 1481 1482 1483static int 1484add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1485 const struct sockaddr *local_sa, void *peer_ctx) 1486{ 1487 struct conn_cid_elem *cce; 1488 1489 if (engine->flags & ENG_CONNS_BY_ADDR) 1490 { 1491 cce = find_free_cce(conn); 1492 if (!cce) 1493 { 1494 LSQ_ERROR("cannot find free CCE"); 1495 return -1; 1496 } 1497 cce->cce_port = sa2port(local_sa); 1498 cce->cce_flags = CCE_PORT; 1499 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1500 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1501 { 1502 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1503 return 0; 1504 } 1505 else 1506 return -1; 1507 1508 } 1509 else 1510 return insert_conn_into_hash(engine, conn, peer_ctx); 1511} 1512 1513 1514lsquic_conn_t * 1515lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1516 const struct sockaddr *local_sa, 1517 const struct sockaddr *peer_sa, 1518 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1519 const char *hostname, unsigned short max_packet_size, 1520 const unsigned char *zero_rtt, size_t zero_rtt_len, 1521 const unsigned char *token, size_t token_sz) 1522{ 1523 lsquic_conn_t *conn; 1524 unsigned flags, versions; 1525 int is_ipv4; 1526 1527 ENGINE_IN(engine); 1528 1529 if (engine->flags & ENG_SERVER) 1530 { 1531 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1532 goto err; 1533 } 1534 1535 if (engine->flags & ENG_CONNS_BY_ADDR 1536 && find_conn_by_addr(engine->conns_hash, local_sa)) 1537 { 1538 LSQ_ERROR("cannot have more than one connection on the same port"); 1539 goto err; 1540 } 1541 1542 if (0 != maybe_grow_conn_heaps(engine)) 1543 return NULL; 1544 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1545 is_ipv4 = peer_sa->sa_family == AF_INET; 1546 if (zero_rtt && zero_rtt_len) 1547 { 1548 version = lsquic_zero_rtt_version(zero_rtt, zero_rtt_len); 1549 if (version >= N_LSQVER) 1550 { 1551 LSQ_INFO("zero-rtt version is bad, won't use"); 1552 zero_rtt = NULL; 1553 zero_rtt_len = 0; 1554 } 1555 } 1556 if (version >= N_LSQVER) 1557 { 1558 if (version > N_LSQVER) 1559 LSQ_WARN("invalid version specified, engine will pick"); 1560 versions = engine->pub.enp_settings.es_versions; 1561 } 1562 else 1563 versions = 1u << version; 1564 if (versions & LSQUIC_IETF_VERSIONS) 1565 { 1566 if (version == LSQVER_ID24) 1567 conn = lsquic_id24_full_conn_client_new(&engine->pub, versions, 1568 flags, hostname, max_packet_size, 1569 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1570 else 1571 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1572 flags, hostname, max_packet_size, 1573 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1574 } 1575 else 1576 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1577 flags, hostname, max_packet_size, is_ipv4, 1578 zero_rtt, zero_rtt_len); 1579 if (!conn) 1580 goto err; 1581 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1582 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1583 lsquic_ver2str[conn->cn_version]); 1584 ++engine->n_conns; 1585 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1586 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1587 { 1588 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1589 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1590 CID_BITS(cid)); 1591 destroy_conn(engine, conn, lsquic_time_now()); 1592 goto err; 1593 } 1594 assert(!(conn->cn_flags & 1595 (CONN_REF_FLAGS 1596 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1597 callbacks */ 1598 ))); 1599 conn->cn_flags |= LSCONN_HASHED; 1600 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1601 engine_incref_conn(conn, LSCONN_TICKABLE); 1602 lsquic_conn_set_ctx(conn, conn_ctx); 1603 conn->cn_if->ci_client_call_on_new(conn); 1604 end: 1605 ENGINE_OUT(engine); 1606 return conn; 1607 err: 1608 conn = NULL; 1609 goto end; 1610} 1611 1612 1613static void 1614remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1615{ 1616 remove_all_cces_from_hash(engine->conns_hash, conn); 1617 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1618} 1619 1620 1621static void 1622refflags2str (enum lsquic_conn_flags flags, char s[6]) 1623{ 1624 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1625 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1626 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1627 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1628 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1629 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1630 *s = '\0'; 1631} 1632 1633 1634static void 1635engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1636{ 1637 char str[2][7]; 1638 assert(flag & CONN_REF_FLAGS); 1639 assert(!(conn->cn_flags & flag)); 1640 conn->cn_flags |= flag; 1641 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1642 CID_BITS(lsquic_conn_log_cid(conn)), 1643 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1644 (refflags2str(conn->cn_flags, str[1]), str[1])); 1645} 1646 1647 1648static lsquic_conn_t * 1649engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1650 enum lsquic_conn_flags flags) 1651{ 1652 char str[2][7]; 1653 lsquic_time_t now; 1654 assert(flags & CONN_REF_FLAGS); 1655 assert(conn->cn_flags & flags); 1656#ifndef NDEBUG 1657 if (flags & LSCONN_CLOSING) 1658 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1659#endif 1660 conn->cn_flags &= ~flags; 1661 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1662 CID_BITS(lsquic_conn_log_cid(conn)), 1663 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1664 (refflags2str(conn->cn_flags, str[1]), str[1])); 1665 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1666 { 1667 now = lsquic_time_now(); 1668 if (conn->cn_flags & LSCONN_MINI) 1669 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1670 else 1671 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1672 destroy_conn(engine, conn, now); 1673 return NULL; 1674 } 1675 else 1676 return conn; 1677} 1678 1679 1680/* This is not a general-purpose function. Only call from engine dtor. */ 1681static void 1682force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1683{ 1684 assert(engine->flags & ENG_DTOR); 1685 const enum lsquic_conn_flags flags = conn->cn_flags; 1686 assert(conn->cn_flags & CONN_REF_FLAGS); 1687 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1688 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1689 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1690 if (flags & LSCONN_ATTQ) 1691 { 1692 attq_remove(engine->attq, conn); 1693 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1694 } 1695 if (flags & LSCONN_HASHED) 1696 remove_conn_from_hash(engine, conn); 1697} 1698 1699 1700/* Iterator for tickable connections (those on the Tickable Queue). Before 1701 * a connection is returned, it is removed from the Advisory Tick Time queue 1702 * if necessary. 1703 */ 1704static lsquic_conn_t * 1705conn_iter_next_tickable (struct lsquic_engine *engine) 1706{ 1707 lsquic_conn_t *conn; 1708 1709 if (engine->flags & ENG_SERVER) 1710 while (1) 1711 { 1712 conn = lsquic_mh_pop(&engine->conns_tickable); 1713 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1714 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1715 else 1716 break; 1717 } 1718 else 1719 conn = lsquic_mh_pop(&engine->conns_tickable); 1720 1721 if (conn) 1722 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1723 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1724 { 1725 attq_remove(engine->attq, conn); 1726 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1727 } 1728 1729 return conn; 1730} 1731 1732 1733static void 1734cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1735 void *update_ctx) 1736{ 1737 cub->cub_update_cids = update; 1738 cub->cub_update_ctx = update_ctx; 1739 cub->cub_count = 0; 1740} 1741 1742 1743static void 1744cub_flush (struct cid_update_batch *cub) 1745{ 1746 if (cub->cub_count > 0 && cub->cub_update_cids) 1747 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1748 cub->cub_cids, cub->cub_count); 1749 cub->cub_count = 0; 1750} 1751 1752 1753static void 1754cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1755{ 1756 cub->cub_cids [ cub->cub_count ] = *cid; 1757 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1758 ++cub->cub_count; 1759 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1760 cub_flush(cub); 1761} 1762 1763 1764/* Process registered CIDs */ 1765static void 1766cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1767{ 1768 struct cce_cid_iter citer; 1769 struct conn_cid_elem *cce; 1770 void *peer_ctx; 1771 1772 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1773 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1774 if (cce->cce_flags & CCE_REG) 1775 cub_add(cub, &cce->cce_cid, peer_ctx); 1776} 1777 1778 1779static void 1780drop_all_mini_conns (lsquic_engine_t *engine) 1781{ 1782 struct lsquic_hash_elem *el; 1783 lsquic_conn_t *conn; 1784 struct cid_update_batch cub; 1785 1786 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1787 1788 for (el = lsquic_hash_first(engine->conns_hash); el; 1789 el = lsquic_hash_next(engine->conns_hash)) 1790 { 1791 conn = lsquic_hashelem_getdata(el); 1792 if (conn->cn_flags & LSCONN_MINI) 1793 { 1794 /* If promoted, why is it still in this hash? */ 1795 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1796 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1797 cub_add_cids_from_cces(&cub, conn); 1798 remove_conn_from_hash(engine, conn); 1799 } 1800 } 1801 1802 cub_flush(&cub); 1803} 1804 1805 1806void 1807lsquic_engine_process_conns (lsquic_engine_t *engine) 1808{ 1809 lsquic_conn_t *conn; 1810 lsquic_time_t now; 1811 1812 ENGINE_IN(engine); 1813 1814 now = lsquic_time_now(); 1815 while ((conn = attq_pop(engine->attq, now))) 1816 { 1817 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1818 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 1819 { 1820 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1821 engine_incref_conn(conn, LSCONN_TICKABLE); 1822 } 1823 } 1824 1825 process_connections(engine, conn_iter_next_tickable, now); 1826 ENGINE_OUT(engine); 1827} 1828 1829 1830static void 1831release_or_return_enc_data (struct lsquic_engine *engine, 1832 void (*pmi_rel_or_ret) (void *, void *, void *, char), 1833 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 1834{ 1835 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1836 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 1837 packet_out->po_flags &= ~PO_ENCRYPTED; 1838 packet_out->po_enc_data = NULL; 1839} 1840 1841 1842static void 1843release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1844 struct lsquic_packet_out *packet_out) 1845{ 1846 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 1847 conn, packet_out); 1848} 1849 1850 1851static void 1852return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1853 struct lsquic_packet_out *packet_out) 1854{ 1855 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 1856 conn, packet_out); 1857} 1858 1859 1860static int 1861copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 1862 struct lsquic_packet_out *packet_out) 1863{ 1864 int ipv6; 1865 1866 ipv6 = NP_IS_IPv6(packet_out->po_path); 1867 if (packet_out->po_flags & PO_ENCRYPTED) 1868 { 1869 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1870 && packet_out->po_data_sz == packet_out->po_enc_data_sz 1871 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 1872 packet_out->po_data_sz)) 1873 return 0; 1874 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1875 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 1876 goto copy; 1877 return_enc_data(engine, conn, packet_out); 1878 } 1879 1880 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 1881 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1882 packet_out->po_data_sz, ipv6); 1883 if (!packet_out->po_enc_data) 1884 { 1885 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 1886 "of size %hu", packet_out->po_data_sz); 1887 return -1; 1888 } 1889 1890 copy: 1891 memcpy(packet_out->po_enc_data, packet_out->po_data, 1892 packet_out->po_data_sz); 1893 packet_out->po_enc_data_sz = packet_out->po_data_sz; 1894 packet_out->po_sent_sz = packet_out->po_data_sz; 1895 packet_out->po_flags &= ~PO_IPv6; 1896 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 1897 1898 return 0; 1899} 1900 1901 1902STAILQ_HEAD(conns_stailq, lsquic_conn); 1903TAILQ_HEAD(conns_tailq, lsquic_conn); 1904 1905 1906struct conns_out_iter 1907{ 1908 struct min_heap *coi_heap; 1909 struct pr_queue *coi_prq; 1910 TAILQ_HEAD(, lsquic_conn) coi_active_list, 1911 coi_inactive_list; 1912 lsquic_conn_t *coi_next; 1913#ifndef NDEBUG 1914 lsquic_time_t coi_last_sent; 1915#endif 1916}; 1917 1918 1919static void 1920coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 1921{ 1922 iter->coi_heap = &engine->conns_out; 1923 iter->coi_prq = engine->pr_queue; 1924 iter->coi_next = NULL; 1925 TAILQ_INIT(&iter->coi_active_list); 1926 TAILQ_INIT(&iter->coi_inactive_list); 1927#ifndef NDEBUG 1928 iter->coi_last_sent = 0; 1929#endif 1930} 1931 1932 1933static lsquic_conn_t * 1934coi_next (struct conns_out_iter *iter) 1935{ 1936 lsquic_conn_t *conn; 1937 1938 if (lsquic_mh_count(iter->coi_heap) > 0) 1939 { 1940 conn = lsquic_mh_pop(iter->coi_heap); 1941 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1942 conn->cn_flags |= LSCONN_COI_ACTIVE; 1943#ifndef NDEBUG 1944 if (iter->coi_last_sent) 1945 assert(iter->coi_last_sent <= conn->cn_last_sent); 1946 iter->coi_last_sent = conn->cn_last_sent; 1947#endif 1948 return conn; 1949 } 1950 else if (iter->coi_prq && (conn = prq_next_conn(iter->coi_prq))) 1951 { 1952 return conn; 1953 } 1954 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 1955 { 1956 iter->coi_prq = NULL; /* Save function call in previous conditional */ 1957 conn = iter->coi_next; 1958 if (!conn) 1959 conn = TAILQ_FIRST(&iter->coi_active_list); 1960 if (conn) 1961 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 1962 return conn; 1963 } 1964 else 1965 return NULL; 1966} 1967 1968 1969static void 1970coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1971{ 1972 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 1973 { 1974 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 1975 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1976 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1977 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 1978 conn->cn_flags |= LSCONN_COI_INACTIVE; 1979 } 1980} 1981 1982 1983static void 1984coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1985{ 1986 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 1987 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 1988 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 1989 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1990 conn->cn_flags |= LSCONN_COI_ACTIVE; 1991} 1992 1993 1994static void 1995coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 1996{ 1997 lsquic_conn_t *conn; 1998 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 1999 { 2000 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2001 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2002 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 2003 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 2004 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 2005 else /* Closed connection gets one shot at sending packets */ 2006 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2007 } 2008 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 2009 { 2010 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2011 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2012 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2013 } 2014} 2015 2016 2017#ifndef NDEBUG 2018static void 2019lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 2020 unsigned n) 2021{ 2022 const lsquic_cid_t *cid; 2023 struct iovec *iov; 2024 unsigned i; 2025 char packno_str[22]; 2026 2027 for (i = 0; i < n; ++i) 2028 { 2029 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 2030 batch->packets[i]->po_packno); 2031 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 2032 { 2033 for (iov = batch->outs[i].iov; iov < 2034 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 2035 batch->outs[i].iov->iov_len -= 1; 2036 cid = lsquic_conn_log_cid(batch->conns[i]); 2037 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 2038 CID_BITS(cid)); 2039 } 2040 } 2041} 2042 2043 2044#endif 2045 2046 2047#ifdef NDEBUG 2048#define CONST_BATCH const 2049#else 2050#define CONST_BATCH 2051#endif 2052 2053 2054struct send_batch_ctx { 2055 struct conns_stailq *closed_conns; 2056 struct conns_tailq *ticked_conns; 2057 struct conns_out_iter *conns_iter; 2058 CONST_BATCH struct out_batch *batch; 2059}; 2060 2061 2062static void 2063close_conn_immediately (struct lsquic_engine *engine, 2064 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2065{ 2066 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2067 if (!(conn->cn_flags & LSCONN_CLOSING)) 2068 { 2069 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2070 engine_incref_conn(conn, LSCONN_CLOSING); 2071 if (conn->cn_flags & LSCONN_HASHED) 2072 remove_conn_from_hash(engine, conn); 2073 } 2074 if (conn->cn_flags & LSCONN_TICKED) 2075 { 2076 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2077 engine_decref_conn(engine, conn, LSCONN_TICKED); 2078 } 2079} 2080 2081 2082static void 2083close_conn_on_send_error (struct lsquic_engine *engine, 2084 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2085{ 2086 const struct out_batch *batch = sb_ctx->batch; 2087 struct lsquic_conn *const conn = batch->conns[n]; 2088 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2089 2090 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2091 "src: %s; dst: %s; errno: %d", 2092 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2093 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2094 CID_BITS(lsquic_conn_log_cid(conn)), 2095 SA2STR(batch->outs[n].local_sa, buf[0]), 2096 SA2STR(batch->outs[n].dest_sa, buf[1]), 2097 e_val); 2098 if (conn->cn_flags & LSCONN_EVANESCENT) 2099 lsquic_prq_drop(conn); 2100 else 2101 close_conn_immediately(engine, sb_ctx, conn); 2102} 2103 2104 2105static unsigned 2106send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2107 unsigned n_to_send) 2108{ 2109 int n_sent, i, e_val; 2110 lsquic_time_t now; 2111 unsigned off; 2112 size_t count; 2113 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2114 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2115 2116#ifndef NDEBUG 2117 if (engine->flags & ENG_LOSE_PACKETS) 2118 lose_matching_packets(engine, batch, n_to_send); 2119#endif 2120 /* Set sent time before the write to avoid underestimating RTT */ 2121 now = lsquic_time_now(); 2122 for (i = 0; i < (int) n_to_send; ++i) 2123 { 2124 off = batch->pack_off[i]; 2125 count = batch->outs[i].iovlen; 2126 assert(count > 0); 2127 packet_out = &batch->packets[off]; 2128 end = packet_out + count; 2129 do 2130 (*packet_out)->po_sent = now; 2131 while (++packet_out < end); 2132 } 2133 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs, 2134 n_to_send); 2135 e_val = errno; 2136 if (n_sent < (int) n_to_send) 2137 { 2138 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2139 engine->resume_sending_at = now + 1000000; 2140 LSQ_DEBUG("cannot send packets"); 2141 EV_LOG_GENERIC_EVENT("cannot send packets"); 2142 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2143 close_conn_on_send_error(engine, sb_ctx, 2144 n_sent < 0 ? 0 : n_sent, e_val); 2145 } 2146 if (n_sent >= 0) 2147 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, n_to_send); 2148 else 2149 { 2150 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2151 n_sent = 0; 2152 } 2153 if (n_sent > 0) 2154 engine->last_sent = now + n_sent; 2155 for (i = 0; i < n_sent; ++i) 2156 { 2157 eng_hist_inc(&engine->history, now, sl_packets_out); 2158 /* `i' is added to maintain relative order */ 2159 batch->conns[i]->cn_last_sent = now + i; 2160 2161 off = batch->pack_off[i]; 2162 count = batch->outs[i].iovlen; 2163 assert(count > 0); 2164 packet_out = &batch->packets[off]; 2165 end = packet_out + count; 2166 do 2167 { 2168#if LOG_PACKET_CHECKSUM 2169 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2170 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2171 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2172#endif 2173 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2174 *packet_out); 2175 /* Release packet out buffer as soon as the packet is sent 2176 * successfully. If not successfully sent, we hold on to 2177 * this buffer until the packet sending is attempted again 2178 * or until it times out and regenerated. 2179 */ 2180 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2181 release_enc_data(engine, batch->conns[i], *packet_out); 2182 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2183 *packet_out); 2184 } 2185 while (++packet_out < end); 2186 } 2187 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2188 for ( ; i < (int) n_to_send; ++i) 2189 { 2190 off = batch->pack_off[i]; 2191 count = batch->outs[i].iovlen; 2192 assert(count > 0); 2193 packet_out = &batch->packets[off]; 2194 end = packet_out + count; 2195 do 2196 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2197 *packet_out); 2198 while (++packet_out < end); 2199 } 2200 /* Return packets to the connection in reverse order so that the packet 2201 * ordering is maintained. 2202 */ 2203 for (i = (int) n_to_send - 1; i >= n_sent; --i) 2204 { 2205 off = batch->pack_off[i]; 2206 count = batch->outs[i].iovlen; 2207 assert(count > 0); 2208 packet_out = &batch->packets[off + count - 1]; 2209 end = &batch->packets[off - 1]; 2210 do 2211 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2212 *packet_out); 2213 while (--packet_out > end); 2214 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2215 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2216 } 2217 return n_sent; 2218} 2219 2220 2221/* Return 1 if went past deadline, 0 otherwise */ 2222static int 2223check_deadline (lsquic_engine_t *engine) 2224{ 2225 if (engine->pub.enp_settings.es_proc_time_thresh && 2226 lsquic_time_now() > engine->deadline) 2227 { 2228 LSQ_INFO("went past threshold of %u usec, stop sending", 2229 engine->pub.enp_settings.es_proc_time_thresh); 2230 engine->flags |= ENG_PAST_DEADLINE; 2231 return 1; 2232 } 2233 else 2234 return 0; 2235} 2236 2237 2238static size_t 2239iov_size (const struct iovec *iov, const struct iovec *const end) 2240{ 2241 size_t size; 2242 2243 assert(iov < end); 2244 2245 size = 0; 2246 do 2247 size += iov->iov_len; 2248 while (++iov < end); 2249 2250 return size; 2251} 2252 2253 2254static void 2255send_packets_out (struct lsquic_engine *engine, 2256 struct conns_tailq *ticked_conns, 2257 struct conns_stailq *closed_conns) 2258{ 2259 unsigned n, w, n_sent, n_batches_sent; 2260 lsquic_packet_out_t *packet_out; 2261 struct lsquic_packet_out **packet; 2262 lsquic_conn_t *conn; 2263 struct out_batch *const batch = &engine->out_batch; 2264 struct iovec *iov, *packet_iov; 2265 struct conns_out_iter conns_iter; 2266 int shrink, deadline_exceeded; 2267 const struct send_batch_ctx sb_ctx = { 2268 closed_conns, 2269 ticked_conns, 2270 &conns_iter, 2271 &engine->out_batch, 2272 }; 2273 2274 coi_init(&conns_iter, engine); 2275 n_batches_sent = 0; 2276 n_sent = 0, n = 0; 2277 shrink = 0; 2278 deadline_exceeded = 0; 2279 iov = batch->iov; 2280 packet = batch->packets; 2281 2282 while ((conn = coi_next(&conns_iter))) 2283 { 2284 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2285 if (!packet_out) { 2286 /* Evanescent connection always has a packet to send: */ 2287 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2288 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2289 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2290 CID_BITS(lsquic_conn_log_cid(conn))); 2291 coi_deactivate(&conns_iter, conn); 2292 continue; 2293 } 2294 batch->outs[n].iov = packet_iov = iov; 2295 next_coa: 2296 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2297 { 2298 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2299 &engine->pub, conn, packet_out)) 2300 { 2301 case ENCPA_NOMEM: 2302 /* Send what we have and wait for a more opportune moment */ 2303 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2304 goto end_for; 2305 case ENCPA_BADCRYPT: 2306 /* This is pretty bad: close connection immediately */ 2307 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2308 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2309 CID_BITS(lsquic_conn_log_cid(conn))); 2310 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2311 { 2312 close_conn_immediately(engine, &sb_ctx, conn); 2313 coi_deactivate(&conns_iter, conn); 2314 } 2315 continue; 2316 case ENCPA_OK: 2317 break; 2318 } 2319 } 2320 else if ((packet_out->po_flags & PO_NOENCRYPT) 2321 && engine->pub.enp_pmi != &stock_pmi) 2322 { 2323 if (0 != copy_packet(engine, conn, packet_out)) 2324 { 2325 /* Copy can only fail if packet could not be allocated */ 2326 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2327 goto end_for; 2328 } 2329 } 2330 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2331 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2332 if (packet_out->po_flags & PO_ENCRYPTED) 2333 { 2334 iov->iov_base = packet_out->po_enc_data; 2335 iov->iov_len = packet_out->po_enc_data_sz; 2336 } 2337 else 2338 { 2339 iov->iov_base = packet_out->po_data; 2340 iov->iov_len = packet_out->po_data_sz; 2341 } 2342 if (packet_iov == iov) 2343 { 2344 batch->pack_off[n] = packet - batch->packets; 2345 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2346 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2347 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2348 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2349 batch->conns [n] = conn; 2350 } 2351 *packet = packet_out; 2352 ++packet; 2353 ++iov; 2354 if ((conn->cn_flags & LSCONN_IETF) 2355 && ((1 << packet_out->po_header_type) 2356 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2357#ifndef NDEBUG 2358 && (engine->flags & ENG_COALESCE) 2359#endif 2360 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2361 { 2362 const size_t size = iov_size(packet_iov, iov); 2363 packet_out = conn->cn_if->ci_next_packet_to_send(conn, size); 2364 if (packet_out) 2365 goto next_coa; 2366 } 2367 batch->outs [n].iovlen = iov - packet_iov; 2368 ++n; 2369 if (n == engine->batch_size 2370 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2371 { 2372 w = send_batch(engine, &sb_ctx, n); 2373 n = 0; 2374 iov = batch->iov; 2375 packet = batch->packets; 2376 ++n_batches_sent; 2377 n_sent += w; 2378 if (w < engine->batch_size) 2379 { 2380 shrink = 1; 2381 break; 2382 } 2383 deadline_exceeded = check_deadline(engine); 2384 if (deadline_exceeded) 2385 break; 2386 grow_batch_size(engine); 2387 } 2388 } 2389 end_for: 2390 2391 if (n > 0) { 2392 w = send_batch(engine, &sb_ctx, n); 2393 n_sent += w; 2394 shrink = w < n; 2395 ++n_batches_sent; 2396 } 2397 2398 if (shrink) 2399 shrink_batch_size(engine); 2400 else if (n_batches_sent > 1) 2401 { 2402 deadline_exceeded = check_deadline(engine); 2403 if (!deadline_exceeded) 2404 grow_batch_size(engine); 2405 } 2406 2407 coi_reheap(&conns_iter, engine); 2408 2409 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2410} 2411 2412 2413int 2414lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2415{ 2416 return lsquic_mh_count(&engine->conns_out) > 0 2417 || (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2418 ; 2419} 2420 2421 2422static void 2423reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2424{ 2425 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2426 engine->flags &= ~ENG_PAST_DEADLINE; 2427} 2428 2429 2430void 2431lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2432{ 2433 lsquic_conn_t *conn; 2434 struct conns_stailq closed_conns; 2435 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2436 struct cid_update_batch cub; 2437 2438 ENGINE_IN(engine); 2439 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2440 STAILQ_INIT(&closed_conns); 2441 reset_deadline(engine, lsquic_time_now()); 2442 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2443 { 2444 LSQ_DEBUG("can send again"); 2445 EV_LOG_GENERIC_EVENT("can send again"); 2446 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2447 } 2448 2449 send_packets_out(engine, &ticked_conns, &closed_conns); 2450 2451 while ((conn = STAILQ_FIRST(&closed_conns))) { 2452 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2453 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2454 cub_add_cids_from_cces(&cub, conn); 2455 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2456 } 2457 2458 cub_flush(&cub); 2459 ENGINE_OUT(engine); 2460} 2461 2462 2463static lsquic_conn_t * 2464next_new_full_conn (struct conns_stailq *new_full_conns) 2465{ 2466 lsquic_conn_t *conn; 2467 2468 conn = STAILQ_FIRST(new_full_conns); 2469 if (conn) 2470 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2471 return conn; 2472} 2473 2474 2475static void 2476process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2477 lsquic_time_t now) 2478{ 2479 lsquic_conn_t *conn; 2480 enum tick_st tick_st; 2481 unsigned i, why; 2482 lsquic_time_t next_tick_time; 2483 struct conns_stailq closed_conns; 2484 struct conns_tailq ticked_conns; 2485 struct conns_stailq new_full_conns; 2486 struct cid_update_batch cub_old, cub_live; 2487 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2488 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2489 2490 eng_hist_tick(&engine->history, now); 2491 2492 STAILQ_INIT(&closed_conns); 2493 TAILQ_INIT(&ticked_conns); 2494 reset_deadline(engine, now); 2495 STAILQ_INIT(&new_full_conns); 2496 2497 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2498 && now > engine->resume_sending_at) 2499 { 2500 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2501 "timeout"); 2502 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2503 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2504 } 2505 2506 i = 0; 2507 while ((conn = next_conn(engine)) 2508 || (conn = next_new_full_conn(&new_full_conns))) 2509 { 2510 tick_st = conn->cn_if->ci_tick(conn, now); 2511 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2512 if (tick_st & TICK_PROMOTE) 2513 { 2514 lsquic_conn_t *new_conn; 2515 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2516 "scheduled for promotion"); 2517 assert(conn->cn_flags & LSCONN_MINI); 2518 new_conn = new_full_conn_server(engine, conn, now); 2519 if (new_conn) 2520 { 2521 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2522 new_conn->cn_last_sent = engine->last_sent; 2523 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2524 conn->cn_flags |= LSCONN_PROMOTED; 2525 } 2526 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2527 } 2528 if (tick_st & TICK_SEND) 2529 { 2530 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2531 { 2532 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2533 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2534 } 2535 } 2536 if (tick_st & TICK_CLOSE) 2537 { 2538 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2539 engine_incref_conn(conn, LSCONN_CLOSING); 2540 if (conn->cn_flags & LSCONN_HASHED) 2541 remove_conn_from_hash(engine, conn); 2542 } 2543 else 2544 { 2545 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2546 engine_incref_conn(conn, LSCONN_TICKED); 2547 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2548 && conn->cn_if->ci_report_live(conn, now)) 2549 cub_add_cids_from_cces(&cub_live, conn); 2550 } 2551 } 2552 2553 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2554 && lsquic_engine_has_unsent_packets(engine)) 2555 send_packets_out(engine, &ticked_conns, &closed_conns); 2556 2557 while ((conn = STAILQ_FIRST(&closed_conns))) { 2558 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2559 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2560 cub_add_cids_from_cces(&cub_old, conn); 2561 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2562 } 2563 2564 while ((conn = TAILQ_FIRST(&ticked_conns))) 2565 { 2566 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2567 engine_decref_conn(engine, conn, LSCONN_TICKED); 2568 if (!(conn->cn_flags & LSCONN_TICKABLE) 2569 && conn->cn_if->ci_is_tickable(conn)) 2570 { 2571 /* Floyd heapification is not faster, don't bother. */ 2572 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2573 engine_incref_conn(conn, LSCONN_TICKABLE); 2574 } 2575 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2576 { 2577 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2578 if (next_tick_time) 2579 { 2580 if (0 == attq_add(engine->attq, conn, next_tick_time, why)) 2581 engine_incref_conn(conn, LSCONN_ATTQ); 2582 } 2583 else 2584 /* In all other cases, the idle timeout would make the next 2585 * tick time non-zero: 2586 */ 2587 assert((conn->cn_flags & LSCONN_IETF) 2588 && engine->pub.enp_settings.es_idle_timeout == 0); 2589 } 2590 } 2591 2592 cub_flush(&engine->new_scids); 2593 cub_flush(&cub_live); 2594 cub_flush(&cub_old); 2595} 2596 2597 2598/* Return 0 if packet is being processed by a real connection, 1 if the 2599 * packet was processed, but not by a connection, and -1 on error. 2600 */ 2601int 2602lsquic_engine_packet_in (lsquic_engine_t *engine, 2603 const unsigned char *packet_in_data, size_t packet_in_size, 2604 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2605 void *peer_ctx, int ecn) 2606{ 2607 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2608 struct packin_parse_state ppstate; 2609 lsquic_packet_in_t *packet_in; 2610 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2611 int is_server, unsigned cid_len, struct packin_parse_state *); 2612 unsigned n_zeroes; 2613 int s; 2614 2615 ENGINE_CALLS_INCR(engine); 2616 2617 if (engine->flags & ENG_SERVER) 2618 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2619 else if (engine->flags & ENG_CONNS_BY_ADDR) 2620 { 2621 struct lsquic_hash_elem *el; 2622 const struct lsquic_conn *conn; 2623 el = find_conn_by_addr(engine->conns_hash, sa_local); 2624 if (!el) 2625 return -1; 2626 conn = lsquic_hashelem_getdata(el); 2627 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2628 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2629 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2630 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2631 else if (conn->cn_version == LSQVER_050) 2632 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 2633 else 2634 { 2635 assert(conn->cn_version == LSQVER_046 2636#if LSQUIC_USE_Q098 2637 || conn->cn_version == LSQVER_098 2638#endif 2639 2640 ); 2641 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2642 } 2643 } 2644 else 2645 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2646 2647 n_zeroes = 0; 2648 do 2649 { 2650 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 2651 if (!packet_in) 2652 return -1; 2653 /* Library does not modify packet_in_data, it is not referenced after 2654 * this function returns and subsequent release of pi_data is guarded 2655 * by PI_OWN_DATA flag. 2656 */ 2657 packet_in->pi_data = (unsigned char *) packet_in_data; 2658 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 2659 engine->flags & ENG_SERVER, 2660 engine->pub.enp_settings.es_scid_len, &ppstate)) 2661 { 2662 LSQ_DEBUG("Cannot parse incoming packet's header"); 2663 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 2664 s = 1; 2665 break; 2666 } 2667 2668 packet_in_data += packet_in->pi_data_sz; 2669 packet_in->pi_received = lsquic_time_now(); 2670 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 2671 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 2672 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 2673 peer_ctx, packet_in_size); 2674 n_zeroes += s == 0; 2675 } 2676 while (0 == s && packet_in_data < packet_end); 2677 2678 return n_zeroes > 0 ? 0 : s; 2679} 2680 2681 2682#if __GNUC__ && !defined(NDEBUG) 2683__attribute__((weak)) 2684#endif 2685unsigned 2686lsquic_engine_quic_versions (const lsquic_engine_t *engine) 2687{ 2688 return engine->pub.enp_settings.es_versions; 2689} 2690 2691 2692void 2693lsquic_engine_cooldown (lsquic_engine_t *engine) 2694{ 2695 struct lsquic_hash_elem *el; 2696 lsquic_conn_t *conn; 2697 2698 if (engine->flags & ENG_COOLDOWN) 2699 /* AFAICT, there is no harm in calling this function more than once, 2700 * but log it just in case, as it may indicate an error in the caller. 2701 */ 2702 LSQ_INFO("cooldown called again"); 2703 engine->flags |= ENG_COOLDOWN; 2704 LSQ_INFO("entering cooldown mode"); 2705 if (engine->flags & ENG_SERVER) 2706 drop_all_mini_conns(engine); 2707 for (el = lsquic_hash_first(engine->conns_hash); el; 2708 el = lsquic_hash_next(engine->conns_hash)) 2709 { 2710 conn = lsquic_hashelem_getdata(el); 2711 lsquic_conn_going_away(conn); 2712 } 2713} 2714 2715 2716int 2717lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 2718{ 2719 const struct attq_elem *next_attq; 2720 lsquic_time_t now, next_time; 2721#if LSQUIC_DEBUG_NEXT_ADV_TICK 2722 const struct lsquic_conn *conn; 2723 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 2724#endif 2725 2726 ENGINE_CALLS_INCR(engine); 2727 2728 if ((engine->flags & ENG_PAST_DEADLINE) 2729 && lsquic_mh_count(&engine->conns_out)) 2730 { 2731#if LSQUIC_DEBUG_NEXT_ADV_TICK 2732 conn = lsquic_mh_peek(&engine->conns_out); 2733 engine->last_logged_conn = 0; 2734 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 2735 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 2736 lsquic_mh_count(&engine->conns_out), 2737 lsquic_mh_count(&engine->conns_out) != 1, "s", 2738 CID_BITS(lsquic_conn_log_cid(conn))); 2739#endif 2740 *diff = 0; 2741 return 1; 2742 } 2743 2744 if (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2745 { 2746#if LSQUIC_DEBUG_NEXT_ADV_TICK 2747 engine->last_logged_conn = 0; 2748 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 2749#endif 2750 *diff = 0; 2751 return 1; 2752 } 2753 2754 if (lsquic_mh_count(&engine->conns_tickable)) 2755 { 2756#if LSQUIC_DEBUG_NEXT_ADV_TICK 2757 conn = lsquic_mh_peek(&engine->conns_tickable); 2758 engine->last_logged_conn = 0; 2759 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 2760 "connection%.*s (%"CID_FMT" first)", 2761 lsquic_mh_count(&engine->conns_tickable), 2762 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 2763 CID_BITS(lsquic_conn_log_cid(conn))); 2764#endif 2765 *diff = 0; 2766 return 1; 2767 } 2768 2769 next_attq = attq_next(engine->attq); 2770 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 2771 { 2772 if (next_attq) 2773 next_time = next_attq->ae_adv_time; 2774 else 2775 return 0; 2776 } 2777 else 2778 { 2779 if (next_attq) 2780 { 2781 next_time = next_attq->ae_adv_time; 2782 if (engine->resume_sending_at < next_time) 2783 { 2784 next_time = engine->resume_sending_at; 2785 next_attq = NULL; 2786 } 2787 } 2788 else 2789 next_time = engine->resume_sending_at; 2790 } 2791 2792 now = lsquic_time_now(); 2793 *diff = (int) ((int64_t) next_time - (int64_t) now); 2794#if LSQUIC_DEBUG_NEXT_ADV_TICK 2795 if (next_attq) 2796 { 2797 /* Deduplicate consecutive log messages about the same reason for the 2798 * same connection. 2799 * If diff is always zero or diff reset to a higher value, event is 2800 * still logged. 2801 */ 2802 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 2803 && (uintptr_t) next_attq->ae_conn 2804 == engine->last_logged_conn 2805 && *diff < engine->last_tick_diff)) 2806 { 2807 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 2808 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 2809 engine->last_tick_diff = *diff; 2810 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 2811 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 2812 lsquic_attq_why2str(next_attq->ae_why)); 2813 } 2814 } 2815 else 2816 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 2817#endif 2818 return 1; 2819} 2820 2821 2822unsigned 2823lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 2824{ 2825 lsquic_time_t now; 2826 ENGINE_CALLS_INCR(engine); 2827 now = lsquic_time_now(); 2828 if (from_now < 0) 2829 now -= from_now; 2830 else 2831 now += from_now; 2832 return attq_count_before(engine->attq, now); 2833} 2834 2835 2836int 2837lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 2838 struct lsquic_conn *conn, unsigned cce_idx) 2839{ 2840 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2841 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2842 void *peer_ctx; 2843 2844 assert(cce_idx < conn->cn_n_cces); 2845 assert(conn->cn_cces_mask & (1 << cce_idx)); 2846 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 2847 2848 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 2849 cce->cce_cid.len, conn, &cce->cce_hash_el)) 2850 { 2851 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 2852 CID_BITS(&cce->cce_cid)); 2853 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2854 cce->cce_flags |= CCE_REG; 2855 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 2856 return 0; 2857 } 2858 else 2859 { 2860 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 2861 CID_BITS(&cce->cce_cid)); 2862 return -1; 2863 } 2864} 2865 2866 2867void 2868lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 2869 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now) 2870{ 2871 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2872 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2873 void *peer_ctx; 2874 2875 assert(cce_idx < conn->cn_n_cces); 2876 2877 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 2878 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 2879 2880 if (engine->purga) 2881 { 2882 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2883 lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 2884 PUTY_CID_RETIRED, now); 2885 } 2886 conn->cn_cces_mask &= ~(1u << cce_idx); 2887 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 2888} 2889 2890 2891