lsquic_engine.c revision df25d34a
1/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#ifndef WIN32 17#include <sys/time.h> 18#include <netinet/in.h> 19#include <sys/types.h> 20#include <sys/stat.h> 21#include <fcntl.h> 22#include <unistd.h> 23#include <netdb.h> 24#endif 25 26#ifndef NDEBUG 27#include <sys/types.h> 28#include <regex.h> /* For code that loses packets */ 29#endif 30 31#if LOG_PACKET_CHECKSUM 32#include <zlib.h> 33#endif 34 35#include <openssl/aead.h> 36 37#include "lsquic.h" 38#include "lsquic_types.h" 39#include "lsquic_int_types.h" 40#include "lsquic_sizes.h" 41#include "lsquic_parse_common.h" 42#include "lsquic_parse.h" 43#include "lsquic_packet_in.h" 44#include "lsquic_packet_out.h" 45#include "lsquic_senhist.h" 46#include "lsquic_rtt.h" 47#include "lsquic_cubic.h" 48#include "lsquic_pacer.h" 49#include "lsquic_bw_sampler.h" 50#include "lsquic_minmax.h" 51#include "lsquic_bbr.h" 52#include "lsquic_send_ctl.h" 53#include "lsquic_set.h" 54#include "lsquic_conn_flow.h" 55#include "lsquic_sfcw.h" 56#include "lsquic_hash.h" 57#include "lsquic_conn.h" 58#include "lsquic_full_conn.h" 59#include "lsquic_util.h" 60#include "lsquic_qtags.h" 61#include "lsquic_enc_sess.h" 62#include "lsquic_mm.h" 63#include "lsquic_engine_public.h" 64#include "lsquic_eng_hist.h" 65#include "lsquic_ev_log.h" 66#include "lsquic_version.h" 67#include "lsquic_pr_queue.h" 68#include "lsquic_mini_conn.h" 69#include "lsquic_mini_conn_ietf.h" 70#include "lsquic_stock_shi.h" 71#include "lsquic_purga.h" 72#include "lsquic_tokgen.h" 73#include "lsquic_attq.h" 74#include "lsquic_min_heap.h" 75#include "lsquic_http1x_if.h" 76#include "lsquic_handshake.h" 77#include "lsquic_crand.h" 78#include "lsquic_ietf.h" 79 80#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 81#include "lsquic_logger.h" 82 83#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 84#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 85#endif 86 87#if LSQUIC_DEBUG_NEXT_ADV_TICK 88#include "lsquic_alarmset.h" 89#endif 90 91#define MIN(a, b) ((a) < (b) ? (a) : (b)) 92 93/* The batch of outgoing packets grows and shrinks dynamically */ 94#define MAX_OUT_BATCH_SIZE 1024 95#define MIN_OUT_BATCH_SIZE 4 96#define INITIAL_OUT_BATCH_SIZE 32 97 98struct out_batch 99{ 100 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 101 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 102 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 103 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 104 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 105}; 106 107typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 108 109static void 110process_connections (struct lsquic_engine *engine, conn_iter_f iter, 111 lsquic_time_t now); 112 113static void 114engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 115 116static lsquic_conn_t * 117engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 118 enum lsquic_conn_flags flag); 119 120static void 121force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 122 123#if LSQUIC_COUNT_ENGINE_CALLS 124#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 125#else 126#define ENGINE_CALLS_INCR(e) 127#endif 128 129/* Nested calls to LSQUIC are not supported */ 130#define ENGINE_IN(e) do { \ 131 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 132 (e)->pub.enp_flags |= ENPUB_PROC; \ 133 ENGINE_CALLS_INCR(e); \ 134} while (0) 135 136#define ENGINE_OUT(e) do { \ 137 assert((e)->pub.enp_flags & ENPUB_PROC); \ 138 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 139} while (0) 140 141/* A connection can be referenced from one of six places: 142 * 143 * 1. A hash is used to find connections in order to dispatch an incoming 144 * packet. Connections can be hashed by CIDs or by address. In the 145 * former case, each connection has one or more mappings in the hash 146 * table. IETF QUIC connections have up to eight (in our implementation) 147 * source CIDs and each of those would have a mapping. In client mode, 148 * depending on QUIC versions and options selected, it is may be 149 * necessary to hash connections by address, in which case incoming 150 * packets are delivered to connections based on the address. 151 * 152 * 2. Outgoing queue. 153 * 154 * 3. Tickable queue 155 * 156 * 4. Advisory Tick Time queue. 157 * 158 * 5. Closing connections queue. This is a transient queue -- it only 159 * exists for the duration of process_connections() function call. 160 * 161 * 6. Ticked connections queue. Another transient queue, similar to (5). 162 * 163 * The idea is to destroy the connection when it is no longer referenced. 164 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 165 * that case, the connection is referenced from two places: (2) and (5). 166 * After its packets are sent, it is only referenced in (5), and at the 167 * end of the function call, when it is removed from (5), reference count 168 * goes to zero and the connection is destroyed. If not all packets can 169 * be sent, at the end of the function call, the connection is referenced 170 * by (2) and will only be removed once all outgoing packets have been 171 * sent. 172 */ 173#define CONN_REF_FLAGS (LSCONN_HASHED \ 174 |LSCONN_HAS_OUTGOING \ 175 |LSCONN_TICKABLE \ 176 |LSCONN_TICKED \ 177 |LSCONN_CLOSING \ 178 |LSCONN_ATTQ) 179 180 181 182 183struct cid_update_batch 184{ 185 lsquic_cids_update_f cub_update_cids; 186 void *cub_update_ctx; 187 unsigned cub_count; 188 lsquic_cid_t cub_cids[20]; 189 void *cub_peer_ctxs[20]; 190}; 191 192static void 193cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 194 195 196struct lsquic_engine 197{ 198 struct lsquic_engine_public pub; 199 enum { 200 ENG_SERVER = LSENG_SERVER, 201 ENG_HTTP = LSENG_HTTP, 202 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 203 ENG_PAST_DEADLINE 204 = (1 << 8), /* Previous call to a processing 205 * function went past time threshold. 206 */ 207 ENG_CONNS_BY_ADDR 208 = (1 << 9), /* Connections are hashed by address */ 209#ifndef NDEBUG 210 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 211 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 212 ENG_DTOR = (1 << 26), /* Engine destructor */ 213#endif 214 } flags; 215 lsquic_packets_out_f packets_out; 216 void *packets_out_ctx; 217 lsquic_cids_update_f report_new_scids; 218 lsquic_cids_update_f report_live_scids; 219 lsquic_cids_update_f report_old_scids; 220 void *scids_ctx; 221 struct lsquic_hash *conns_hash; 222 struct min_heap conns_tickable; 223 struct min_heap conns_out; 224 struct eng_hist history; 225 unsigned batch_size; 226 struct pr_queue *pr_queue; 227 struct attq *attq; 228 /* Track time last time a packet was sent to give new connections 229 * priority lower than that of existing connections. 230 */ 231 lsquic_time_t last_sent; 232#ifndef NDEBUG 233 regex_t lose_packets_re; 234 const char *lose_packets_str; 235#endif 236 unsigned n_conns; 237 lsquic_time_t deadline; 238 lsquic_time_t resume_sending_at; 239 unsigned mini_conns_count; 240 struct lsquic_purga *purga; 241#if LSQUIC_CONN_STATS 242 struct { 243 unsigned conns; 244 } stats; 245 struct conn_stats conn_stats_sum; 246 FILE *stats_fh; 247#endif 248 struct cid_update_batch new_scids; 249 struct out_batch out_batch; 250#if LSQUIC_COUNT_ENGINE_CALLS 251 unsigned long n_engine_calls; 252#endif 253#if LSQUIC_DEBUG_NEXT_ADV_TICK 254 uintptr_t last_logged_conn; 255 unsigned last_logged_ae_why; 256 int last_tick_diff; 257#endif 258 struct crand crand; 259 EVP_AEAD_CTX retry_aead_ctx; 260}; 261 262 263void 264lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 265 unsigned flags) 266{ 267 memset(settings, 0, sizeof(*settings)); 268 settings->es_versions = LSQUIC_DF_VERSIONS; 269 if (flags & ENG_SERVER) 270 { 271 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 272 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 273 settings->es_init_max_data 274 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 275 settings->es_init_max_stream_data_bidi_remote 276 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 277 settings->es_init_max_stream_data_bidi_local 278 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 279 settings->es_init_max_stream_data_uni 280 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 281 settings->es_init_max_streams_uni 282 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 283 settings->es_ping_period = 0; 284 } 285 else 286 { 287 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 288 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 289 settings->es_init_max_data 290 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 291 settings->es_init_max_stream_data_bidi_remote 292 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 293 settings->es_init_max_stream_data_bidi_local 294 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 295 settings->es_init_max_stream_data_uni 296 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 297 settings->es_init_max_streams_uni 298 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 299 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 300 } 301 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 302 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 303 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 304 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 305 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 306 settings->es_max_header_list_size 307 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 308 settings->es_ua = LSQUIC_DF_UA; 309 settings->es_ecn = LSQUIC_DF_ECN; 310 311 settings->es_pdmd = QTAG_X509; 312 settings->es_aead = QTAG_AESG; 313 settings->es_kexs = QTAG_C255; 314 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 315 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 316 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 317 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 318 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 319 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 320 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 321 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 322 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 323 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 324 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 325 settings->es_sttl = LSQUIC_DF_STTL; 326 settings->es_init_max_streams_bidi 327 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 328 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 329 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 330 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 331 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 332 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 333 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 334 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 335 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 336 settings->es_spin = LSQUIC_DF_SPIN; 337} 338 339 340/* Note: if returning an error, err_buf must be valid if non-NULL */ 341int 342lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 343 unsigned flags, 344 char *err_buf, size_t err_buf_sz) 345{ 346 if (settings->es_cfcw < LSQUIC_MIN_FCW || 347 settings->es_sfcw < LSQUIC_MIN_FCW) 348 { 349 if (err_buf) 350 snprintf(err_buf, err_buf_sz, "%s", 351 "flow control window set too low"); 352 return -1; 353 } 354 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 355 { 356 if (err_buf) 357 snprintf(err_buf, err_buf_sz, "%s", 358 "No supported QUIC versions specified"); 359 return -1; 360 } 361 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 362 { 363 if (err_buf) 364 snprintf(err_buf, err_buf_sz, "%s", 365 "one or more unsupported QUIC version is specified"); 366 return -1; 367 } 368 if (flags & ENG_SERVER) 369 { 370 if (settings->es_handshake_to > 371 MAX_MINI_CONN_LIFESPAN_IN_USEC) 372 { 373 if (err_buf) 374 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 375 " usec is too large. The maximum for server is %u usec", 376 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 377 return -1; 378 } 379 } 380 if (settings->es_idle_timeout > 600) 381 { 382 if (err_buf) 383 snprintf(err_buf, err_buf_sz, "%s", 384 "The maximum value of idle timeout is 600 seconds"); 385 return -1; 386 } 387 if (settings->es_scid_len > MAX_CID_LEN) 388 { 389 if (err_buf) 390 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 391 "bytes long; it must be between 0 and %u.", 392 settings->es_scid_len, MAX_CID_LEN); 393 return -1; 394 } 395 396 if (settings->es_cc_algo > 2) 397 { 398 if (err_buf) 399 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 400 "algorithm value %u", settings->es_cc_algo); 401 return -1; 402 } 403 404 if (!(settings->es_ql_bits >= 0 && settings->es_ql_bits <= 2)) 405 { 406 if (err_buf) 407 snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ", 408 settings->es_ql_bits); 409 return -1; 410 } 411 412 if (!(settings->es_spin == 0 || settings->es_spin == 1)) 413 { 414 if (err_buf) 415 snprintf(err_buf, err_buf_sz, "Invalid spin value %d", 416 settings->es_spin); 417 return -1; 418 } 419 420 return 0; 421} 422 423 424static void 425free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 426{ 427 free(packet_data); 428} 429 430 431static void * 432malloc_buf (void *ctx, void *conn_ctx, unsigned short size, char is_ipv6) 433{ 434 return malloc(size); 435} 436 437 438static const struct lsquic_packout_mem_if stock_pmi = 439{ 440 malloc_buf, free_packet, free_packet, 441}; 442 443 444static int 445hash_conns_by_addr (const struct lsquic_engine *engine) 446{ 447 if (engine->flags & ENG_SERVER) 448 return 0; 449 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 450 return 1; 451 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 452 && engine->pub.enp_settings.es_support_tcid0) 453 return 1; 454 if (engine->pub.enp_settings.es_scid_len == 0) 455 return 1; 456 return 0; 457} 458 459 460lsquic_engine_t * 461lsquic_engine_new (unsigned flags, 462 const struct lsquic_engine_api *api) 463{ 464 lsquic_engine_t *engine; 465 char err_buf[100]; 466 467 if (!api->ea_packets_out) 468 { 469 LSQ_ERROR("packets_out callback is not specified"); 470 return NULL; 471 } 472 473 if (!api->ea_stream_if) 474 { 475 LSQ_ERROR("stream interface is not specified"); 476 return NULL; 477 } 478 479 if (api->ea_settings && 480 0 != lsquic_engine_check_settings(api->ea_settings, flags, 481 err_buf, sizeof(err_buf))) 482 { 483 LSQ_ERROR("cannot create engine: %s", err_buf); 484 return NULL; 485 } 486 487 engine = calloc(1, sizeof(*engine)); 488 if (!engine) 489 return NULL; 490 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 491 { 492 free(engine); 493 return NULL; 494 } 495 if (api->ea_settings) 496 engine->pub.enp_settings = *api->ea_settings; 497 else 498 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 499 int tag_buf_len; 500 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 501 sizeof(engine->pub.enp_ver_tags_buf), 502 engine->pub.enp_settings.es_versions); 503 if (tag_buf_len <= 0) 504 { 505 LSQ_ERROR("cannot generate version tags buffer"); 506 free(engine); 507 return NULL; 508 } 509 engine->pub.enp_ver_tags_len = tag_buf_len; 510 engine->pub.enp_flags = ENPUB_CAN_SEND; 511 engine->pub.enp_stream_if = api->ea_stream_if; 512 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 513 514 engine->flags = flags; 515#ifndef NDEBUG 516 engine->flags |= ENG_COALESCE; 517#endif 518 engine->packets_out = api->ea_packets_out; 519 engine->packets_out_ctx = api->ea_packets_out_ctx; 520 engine->report_new_scids = api->ea_new_scids; 521 engine->report_live_scids = api->ea_live_scids; 522 engine->report_old_scids = api->ea_old_scids; 523 engine->scids_ctx = api->ea_cids_update_ctx; 524 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 525 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 526 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 527 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 528 if (api->ea_shi) 529 { 530 engine->pub.enp_shi = api->ea_shi; 531 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 532 } 533 else 534 { 535 engine->pub.enp_shi = &stock_shi; 536 engine->pub.enp_shi_ctx = stock_shared_hash_new(); 537 if (!engine->pub.enp_shi_ctx) 538 { 539 free(engine); 540 return NULL; 541 } 542 } 543 if (api->ea_hsi_if) 544 { 545 engine->pub.enp_hsi_if = api->ea_hsi_if; 546 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 547 } 548 else 549 { 550 engine->pub.enp_hsi_if = lsquic_http1x_if; 551 engine->pub.enp_hsi_ctx = NULL; 552 } 553 if (api->ea_pmi) 554 { 555 engine->pub.enp_pmi = api->ea_pmi; 556 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 557 } 558 else 559 { 560 engine->pub.enp_pmi = &stock_pmi; 561 engine->pub.enp_pmi_ctx = NULL; 562 } 563 engine->pub.enp_verify_cert = api->ea_verify_cert; 564 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 565 engine->pub.enp_kli = api->ea_keylog_if; 566 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 567 engine->pub.enp_engine = engine; 568 if (hash_conns_by_addr(engine)) 569 engine->flags |= ENG_CONNS_BY_ADDR; 570 engine->conns_hash = lsquic_hash_create(); 571 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 572 if (!engine->pub.enp_tokgen) 573 return NULL; 574 engine->pub.enp_crand = &engine->crand; 575 if (flags & ENG_SERVER) 576 { 577 engine->pr_queue = prq_create( 578 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 579 &engine->pub); 580 if (!engine->pr_queue) 581 { 582 lsquic_tg_destroy(engine->pub.enp_tokgen); 583 return NULL; 584 } 585 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 586 engine->report_old_scids, engine->scids_ctx); 587 if (!engine->purga) 588 { 589 lsquic_tg_destroy(engine->pub.enp_tokgen); 590 prq_destroy(engine->pr_queue); 591 return NULL; 592 } 593 } 594 engine->attq = attq_create(); 595 eng_hist_init(&engine->history); 596 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 597 if (engine->pub.enp_settings.es_honor_prst) 598 { 599 engine->pub.enp_srst_hash = lsquic_hash_create(); 600 if (!engine->pub.enp_srst_hash) 601 { 602 lsquic_engine_destroy(engine); 603 return NULL; 604 } 605 } 606 607#ifndef NDEBUG 608 { 609 const char *env; 610 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 611 if (env) 612 { 613 if (0 != regcomp(&engine->lose_packets_re, env, 614 REG_EXTENDED|REG_NOSUB)) 615 { 616 LSQ_ERROR("could not compile lost packet regex `%s'", env); 617 return NULL; 618 } 619 engine->flags |= ENG_LOSE_PACKETS; 620 engine->lose_packets_str = env; 621 LSQ_WARN("will lose packets that match the following regex: %s", 622 env); 623 } 624 env = getenv("LSQUIC_COALESCE"); 625 if (env) 626 { 627 engine->flags &= ~ENG_COALESCE; 628 if (atoi(env)) 629 { 630 engine->flags |= ENG_COALESCE; 631 LSQ_NOTICE("will coalesce packets"); 632 } 633 else 634 LSQ_NOTICE("will not coalesce packets"); 635 } 636 } 637#endif 638#if LSQUIC_CONN_STATS 639 engine->stats_fh = api->ea_stats_fh; 640#endif 641 if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx, EVP_aead_aes_128_gcm(), 642 IETF_RETRY_KEY_BUF, IETF_RETRY_KEY_SZ, 16, NULL)) 643 { 644 LSQ_ERROR("could not initialize retry AEAD ctx"); 645 lsquic_engine_destroy(engine); 646 return NULL; 647 } 648 engine->pub.enp_retry_aead_ctx = &engine->retry_aead_ctx; 649 650 LSQ_INFO("instantiated engine"); 651 return engine; 652} 653 654 655#if LOG_PACKET_CHECKSUM 656static void 657log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 658 const unsigned char *buf, size_t bufsz) 659{ 660 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 661 (uint32_t) crc32(0, buf, bufsz)); 662} 663 664 665#endif 666 667 668static void 669grow_batch_size (struct lsquic_engine *engine) 670{ 671 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 672} 673 674 675static void 676shrink_batch_size (struct lsquic_engine *engine) 677{ 678 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 679} 680 681 682struct cce_cid_iter 683{ 684 const struct lsquic_conn *conn; 685 unsigned todo, n; 686}; 687 688 689static struct conn_cid_elem * 690cce_iter_next (struct cce_cid_iter *citer) 691{ 692 struct conn_cid_elem *cce; 693 694 while (citer->todo) 695 if (citer->todo & (1 << citer->n)) 696 { 697 citer->todo &= ~(1 << citer->n); 698 cce = &citer->conn->cn_cces[ citer->n++ ]; 699 if (!(cce->cce_flags & CCE_PORT)) 700 return cce; 701 } 702 else 703 ++citer->n; 704 705 return NULL; 706} 707 708 709static struct conn_cid_elem * 710cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 711{ 712 citer->conn = conn; 713 citer->todo = conn->cn_cces_mask; 714 citer->n = 0; 715 return cce_iter_next(citer); 716} 717 718 719#if LSQUIC_CONN_STATS 720void 721update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 722{ 723 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 724 const unsigned long *src; 725 const struct conn_stats *stats; 726 unsigned i; 727 728 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 729 { 730 ++engine->stats.conns; 731 src = (unsigned long *) stats; 732 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 733 dst[i] += src[i]; 734 } 735} 736 737 738#endif 739 740 741/* Wrapper to make sure important things occur before the connection is 742 * really destroyed. 743 */ 744static void 745destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 746 lsquic_time_t now) 747{ 748 struct cce_cid_iter citer; 749 const struct conn_cid_elem *cce; 750 lsquic_time_t drain_time; 751 struct purga_el *puel; 752 753 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 754 if (engine->purga 755 /* Blacklist all CIDs except for promoted mini connections */ 756 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 757 != (LSCONN_MINI|LSCONN_PROMOTED)) 758 { 759 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 760 && conn->cn_if->ci_drain_time && 761 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 762 { 763 for (cce = cce_iter_first(&citer, conn); cce; 764 cce = cce_iter_next(&citer)) 765 { 766 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 767 lsquic_conn_get_peer_ctx(conn, NULL), 768 PUTY_CONN_DRAIN, now); 769 if (puel) 770 puel->puel_time = now + drain_time; 771 } 772 } 773 else 774 { 775 for (cce = cce_iter_first(&citer, conn); cce; 776 cce = cce_iter_next(&citer)) 777 { 778 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 779 lsquic_conn_get_peer_ctx(conn, NULL), 780 PUTY_CONN_DELETED, now); 781 if (puel) 782 { 783 puel->puel_time = now; 784 puel->puel_count = 0; 785 } 786 } 787 } 788 } 789#if LSQUIC_CONN_STATS 790 update_stats_sum(engine, conn); 791#endif 792 --engine->n_conns; 793 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 794 conn->cn_if->ci_destroy(conn); 795} 796 797 798static int 799maybe_grow_conn_heaps (struct lsquic_engine *engine) 800{ 801 struct min_heap_elem *els; 802 unsigned count; 803 804 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 805 return 0; /* Nothing to do */ 806 807 if (lsquic_mh_nalloc(&engine->conns_tickable)) 808 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 809 else 810 count = 8; 811 812 els = malloc(sizeof(els[0]) * count); 813 if (!els) 814 { 815 LSQ_ERROR("%s: malloc failed", __func__); 816 return -1; 817 } 818 819 LSQ_DEBUG("grew heaps to %u elements", count / 2); 820 memcpy(&els[0], engine->conns_tickable.mh_elems, 821 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 822 memcpy(&els[count / 2], engine->conns_out.mh_elems, 823 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 824 free(engine->conns_tickable.mh_elems); 825 engine->conns_tickable.mh_elems = els; 826 engine->conns_out.mh_elems = &els[count / 2]; 827 engine->conns_tickable.mh_nalloc = count / 2; 828 engine->conns_out.mh_nalloc = count / 2; 829 return 0; 830} 831 832 833static void 834remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 835 unsigned todo) 836{ 837 unsigned n; 838 839 for (n = 0; todo; todo &= ~(1 << n++)) 840 if ((todo & (1 << n)) && 841 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 842 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 843} 844 845 846static void 847remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 848{ 849 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 850} 851 852 853static void 854cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 855 856 857static int 858insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 859 void *peer_ctx) 860{ 861 struct conn_cid_elem *cce; 862 unsigned todo, done, n; 863 864 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 865 if (todo & (1 << n)) 866 { 867 cce = &conn->cn_cces[n]; 868 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 869 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 870 cce->cce_cid.len, conn, &cce->cce_hash_el)) 871 done |= 1 << n; 872 else 873 goto err; 874 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 875 { 876 cce->cce_flags |= CCE_REG; 877 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 878 } 879 } 880 881 return 0; 882 883 err: 884 remove_cces_from_hash(engine->conns_hash, conn, done); 885 return -1; 886} 887 888 889static lsquic_conn_t * 890new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 891 lsquic_time_t now) 892{ 893 const lsquic_cid_t *cid; 894 server_conn_ctor_f ctor; 895 lsquic_conn_t *conn; 896 unsigned flags; 897 if (0 != maybe_grow_conn_heaps(engine)) 898 return NULL; 899 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 900 901 if (mini_conn->cn_flags & LSCONN_IETF) 902 { 903 if (mini_conn->cn_version == LSQVER_ID24) 904 ctor = lsquic_id24_full_conn_server_new; 905 else 906 ctor = lsquic_ietf_full_conn_server_new; 907 } 908 else 909 ctor = lsquic_gquic_full_conn_server_new; 910 911 conn = ctor(&engine->pub, flags, mini_conn); 912 if (!conn) 913 { 914 /* Otherwise, full_conn_server_new prints its own warnings */ 915 if (ENOMEM == errno) 916 { 917 cid = lsquic_conn_log_cid(mini_conn); 918 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 919 CID_BITS(cid), strerror(errno)); 920 } 921 return NULL; 922 } 923 ++engine->n_conns; 924 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 925 { 926 cid = lsquic_conn_log_cid(conn); 927 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 928 CID_BITS(cid)); 929 destroy_conn(engine, conn, now); 930 return NULL; 931 } 932 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 933 conn->cn_flags |= LSCONN_HASHED; 934 return conn; 935} 936 937 938static enum 939{ 940 VER_NOT_SPECIFIED, 941 VER_SUPPORTED, 942 VER_UNSUPPORTED, 943} 944 945 946version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 947 enum lsquic_version *pversion) 948{ 949 lsquic_ver_tag_t ver_tag; 950 enum lsquic_version version; 951 952 if (!packet_in->pi_quic_ver) 953 { 954 LSQ_DEBUG("packet does not specify version"); 955 return VER_NOT_SPECIFIED; 956 } 957 958 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 959 version = lsquic_tag2ver(ver_tag); 960 if (version < N_LSQVER) 961 { 962 if (engine->pub.enp_settings.es_versions & (1 << version)) 963 { 964 LSQ_DEBUG("client-supplied version %s is supported", 965 lsquic_ver2str[version]); 966 *pversion = version; 967 return VER_SUPPORTED; 968 } 969 else 970 LSQ_DEBUG("client-supplied version %s is not supported", 971 lsquic_ver2str[version]); 972 } 973 else 974 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 975 ver_tag); 976 977 return VER_UNSUPPORTED; 978} 979 980 981static void 982schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 983 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 984 const struct sockaddr *sa_peer, void *peer_ctx) 985{ 986 assert(engine->pr_queue); 987 if (0 == prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 988 sa_local, sa_peer)) 989 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 990 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 991 else 992 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 993} 994 995 996static unsigned short 997sa2port (const struct sockaddr *sa) 998{ 999 if (sa->sa_family == AF_INET) 1000 { 1001 struct sockaddr_in *const sa4 = (void *) sa; 1002 return sa4->sin_port; 1003 } 1004 else 1005 { 1006 struct sockaddr_in6 *const sa6 = (void *) sa; 1007 return sa6->sin6_port; 1008 } 1009} 1010 1011 1012static struct lsquic_hash_elem * 1013find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 1014{ 1015 unsigned short port; 1016 1017 port = sa2port(sa); 1018 return lsquic_hash_find(hash, &port, sizeof(port)); 1019} 1020 1021 1022static lsquic_conn_t * 1023find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1024 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 1025{ 1026 struct lsquic_hash_elem *el; 1027 lsquic_conn_t *conn; 1028 1029 if (engine->flags & ENG_CONNS_BY_ADDR) 1030 el = find_conn_by_addr(engine->conns_hash, sa_local); 1031 else if (packet_in->pi_flags & PI_CONN_ID) 1032 el = lsquic_hash_find(engine->conns_hash, 1033 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1034 else 1035 { 1036 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1037 return NULL; 1038 } 1039 1040 if (!el) 1041 return NULL; 1042 1043 conn = lsquic_hashelem_getdata(el); 1044 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1045 if ((engine->flags & ENG_CONNS_BY_ADDR) 1046 && !(conn->cn_flags & LSCONN_IETF) 1047 && (packet_in->pi_flags & PI_CONN_ID) 1048 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1049 { 1050 LSQ_DEBUG("connection IDs do not match"); 1051 return NULL; 1052 } 1053 1054 return conn; 1055} 1056 1057 1058static lsquic_conn_t * 1059find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1060 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1061 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1062{ 1063 struct lsquic_hash_elem *el; 1064 struct purga_el *puel; 1065 lsquic_conn_t *conn; 1066 1067 if (!(packet_in->pi_flags & PI_CONN_ID)) 1068 { 1069 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1070 return NULL; 1071 } 1072 el = lsquic_hash_find(engine->conns_hash, 1073 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1074 1075 if (el) 1076 { 1077 conn = lsquic_hashelem_getdata(el); 1078 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1079 return conn; 1080 } 1081 1082 if (engine->flags & ENG_COOLDOWN) 1083 { /* Do not create incoming connections during cooldown */ 1084 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1085 return NULL; 1086 } 1087 1088 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1089 { 1090 LSQ_DEBUG("reached limit of %u inchoate connections", 1091 engine->pub.enp_settings.es_max_inchoate); 1092 return NULL; 1093 } 1094 1095 1096 if (engine->purga 1097 && (puel = lsquic_purga_contains(engine->purga, 1098 &packet_in->pi_conn_id), puel)) 1099 { 1100 switch (puel->puel_type) 1101 { 1102 case PUTY_CID_RETIRED: 1103 LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet", 1104 CID_BITS(&packet_in->pi_conn_id)); 1105 return NULL; 1106 case PUTY_CONN_DRAIN: 1107 LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64, 1108 puel->puel_time, packet_in->pi_received); 1109 if (puel->puel_time > packet_in->pi_received) 1110 { 1111 LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet", 1112 CID_BITS(&packet_in->pi_conn_id)); 1113 return NULL; 1114 } 1115 LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted", 1116 CID_BITS(&packet_in->pi_conn_id)); 1117 puel->puel_type = PUTY_CONN_DELETED; 1118 puel->puel_count = 0; 1119 puel->puel_time = 0; 1120 /* fall-through */ 1121 case PUTY_CONN_DELETED: 1122 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1123 CID_BITS(&packet_in->pi_conn_id)); 1124 if (puel->puel_time < packet_in->pi_received) 1125 { 1126 puel->puel_time = packet_in->pi_received 1127 /* Exponential back-off */ 1128 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1129 ++puel->puel_count; 1130 goto maybe_send_prst; 1131 } 1132 return NULL; 1133 default: 1134 assert(0); 1135 return NULL; 1136 } 1137 } 1138 1139 if (engine->pub.enp_settings.es_send_prst 1140 && !(packet_in->pi_flags & PI_GQUIC) 1141 && HETY_NOT_SET == packet_in->pi_header_type) 1142 goto maybe_send_prst; 1143 1144 if (0 != maybe_grow_conn_heaps(engine)) 1145 return NULL; 1146 1147 const struct parse_funcs *pf; 1148 enum lsquic_version version; 1149 switch (version_matches(engine, packet_in, &version)) 1150 { 1151 case VER_UNSUPPORTED: 1152 if (engine->flags & ENG_SERVER) 1153 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1154 sa_local, sa_peer, peer_ctx); 1155 return NULL; 1156 case VER_NOT_SPECIFIED: 1157 maybe_send_prst: 1158 if ((engine->flags & ENG_SERVER) && 1159 engine->pub.enp_settings.es_send_prst) 1160 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1161 sa_local, sa_peer, peer_ctx); 1162 return NULL; 1163 case VER_SUPPORTED: 1164 pf = select_pf_by_ver(version); 1165 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1166 break; 1167 } 1168 1169 1170 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1171 { 1172 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1173 sa_peer->sa_family == AF_INET, NULL, packet_in_size); 1174 } 1175 else 1176 { 1177 conn = mini_conn_new(&engine->pub, packet_in, version); 1178 } 1179 if (!conn) 1180 return NULL; 1181 ++engine->mini_conns_count; 1182 ++engine->n_conns; 1183 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1184 { 1185 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1186 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1187 CID_BITS(cid)); 1188 destroy_conn(engine, conn, packet_in->pi_received); 1189 return NULL; 1190 } 1191 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1192 conn->cn_flags |= LSCONN_HASHED; 1193 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1194 conn->cn_last_sent = engine->last_sent; 1195 return conn; 1196} 1197 1198 1199lsquic_conn_t * 1200lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1201 const lsquic_cid_t *cid) 1202{ 1203 struct lsquic_hash_elem *el; 1204 lsquic_conn_t *conn = NULL; 1205 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1206 1207 if (el) 1208 conn = lsquic_hashelem_getdata(el); 1209 return conn; 1210} 1211 1212 1213#if !defined(NDEBUG) && __GNUC__ 1214__attribute__((weak)) 1215#endif 1216void 1217lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1218 lsquic_conn_t *conn) 1219{ 1220 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1221 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1222 { 1223 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1224 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1225 engine_incref_conn(conn, LSCONN_TICKABLE); 1226 } 1227} 1228 1229 1230void 1231lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1232 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1233{ 1234 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1235 if (conn->cn_flags & LSCONN_TICKABLE) 1236 { 1237 /* Optimization: no need to add the connection to the Advisory Tick 1238 * Time Queue: it is about to be ticked, after which it its next tick 1239 * time may be queried again. 1240 */; 1241 } 1242 else if (conn->cn_flags & LSCONN_ATTQ) 1243 { 1244 if (lsquic_conn_adv_time(conn) != tick_time) 1245 { 1246 attq_remove(engine->attq, conn); 1247 if (0 != attq_add(engine->attq, conn, tick_time, why)) 1248 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1249 } 1250 } 1251 else if (0 == attq_add(engine->attq, conn, tick_time, why)) 1252 engine_incref_conn(conn, LSCONN_ATTQ); 1253} 1254 1255 1256static struct lsquic_conn * 1257find_conn_by_srst (struct lsquic_engine *engine, 1258 const struct lsquic_packet_in *packet_in) 1259{ 1260 struct lsquic_hash_elem *el; 1261 struct lsquic_conn *conn; 1262 1263 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1264 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1265 return NULL; 1266 1267 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1268 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1269 IQUIC_SRESET_TOKEN_SZ); 1270 if (!el) 1271 return NULL; 1272 1273 conn = lsquic_hashelem_getdata(el); 1274 return conn; 1275} 1276 1277 1278/* Return 0 if packet is being processed by a real connection (mini or full), 1279 * otherwise return 1. 1280 */ 1281static int 1282process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1283 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1284 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1285{ 1286 lsquic_conn_t *conn; 1287 const unsigned char *packet_in_data; 1288 1289 if (lsquic_packet_in_is_gquic_prst(packet_in) 1290 && !engine->pub.enp_settings.es_honor_prst) 1291 { 1292 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1293 LSQ_DEBUG("public reset packet: discarding"); 1294 return 1; 1295 } 1296 1297 if (engine->flags & ENG_SERVER) 1298 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1299 sa_peer, peer_ctx, packet_in_size); 1300 else 1301 conn = find_conn(engine, packet_in, ppstate, sa_local); 1302 1303 if (!conn) 1304 { 1305 if (engine->pub.enp_settings.es_honor_prst 1306 && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */ 1307 && !(packet_in->pi_flags & PI_GQUIC) 1308 && engine->pub.enp_srst_hash 1309 && (conn = find_conn_by_srst(engine, packet_in))) 1310 { 1311 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1312 CID_BITS(lsquic_conn_log_cid(conn))); 1313 conn->cn_if->ci_stateless_reset(conn); 1314 if (!(conn->cn_flags & LSCONN_TICKABLE) 1315 && conn->cn_if->ci_is_tickable(conn)) 1316 { 1317 lsquic_mh_insert(&engine->conns_tickable, conn, 1318 conn->cn_last_ticked); 1319 engine_incref_conn(conn, LSCONN_TICKABLE); 1320 } 1321 /* Even though the connection processes this packet, we return 1322 * 1 so that the caller does not add reset packet's random 1323 * bytes to the list of valid CIDs. 1324 */ 1325 } 1326 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1327 return 1; 1328 } 1329 1330 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1331 { 1332 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1333 engine_incref_conn(conn, LSCONN_TICKABLE); 1334 } 1335 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1336 sa_local, sa_peer); 1337 lsquic_packet_in_upref(packet_in); 1338#if LOG_PACKET_CHECKSUM 1339 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1340 packet_in->pi_data_sz); 1341#endif 1342 /* Note on QLog: 1343 * For the PACKET_RX QLog event, we are interested in logging these things: 1344 * - raw packet (however it comes in, encrypted or not) 1345 * - frames (list of frame names) 1346 * - packet type and number 1347 * - packet rx timestamp 1348 * 1349 * Since only some of these items are available at this code 1350 * juncture, we will wait until after the packet has been 1351 * decrypted (if necessary) and parsed to call the log functions. 1352 * 1353 * Once the PACKET_RX event is finally logged, the timestamp 1354 * will come from packet_in->pi_received. For correct sequential 1355 * ordering of QLog events, be sure to process the QLogs downstream. 1356 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1357 */ 1358 packet_in_data = packet_in->pi_data; 1359 packet_in_size = packet_in->pi_data_sz; 1360 conn->cn_if->ci_packet_in(conn, packet_in); 1361 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1362 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1363 return 0; 1364} 1365 1366 1367void 1368lsquic_engine_destroy (lsquic_engine_t *engine) 1369{ 1370 struct lsquic_hash_elem *el; 1371 lsquic_conn_t *conn; 1372 1373 LSQ_DEBUG("destroying engine"); 1374#ifndef NDEBUG 1375 engine->flags |= ENG_DTOR; 1376#endif 1377 1378 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1379 { 1380 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1381 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1382 } 1383 1384 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1385 { 1386 assert(conn->cn_flags & LSCONN_TICKABLE); 1387 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1388 } 1389 1390 for (el = lsquic_hash_first(engine->conns_hash); el; 1391 el = lsquic_hash_next(engine->conns_hash)) 1392 { 1393 conn = lsquic_hashelem_getdata(el); 1394 force_close_conn(engine, conn); 1395 } 1396 lsquic_hash_destroy(engine->conns_hash); 1397 1398 assert(0 == engine->n_conns); 1399 assert(0 == engine->mini_conns_count); 1400 if (engine->pr_queue) 1401 prq_destroy(engine->pr_queue); 1402 if (engine->purga) 1403 lsquic_purga_destroy(engine->purga); 1404 attq_destroy(engine->attq); 1405 1406 assert(0 == lsquic_mh_count(&engine->conns_out)); 1407 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1408 if (engine->pub.enp_shi == &stock_shi) 1409 stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1410 lsquic_mm_cleanup(&engine->pub.enp_mm); 1411 free(engine->conns_tickable.mh_elems); 1412#ifndef NDEBUG 1413 if (engine->flags & ENG_LOSE_PACKETS) 1414 regfree(&engine->lose_packets_re); 1415#endif 1416 if (engine->pub.enp_tokgen) 1417 lsquic_tg_destroy(engine->pub.enp_tokgen); 1418#if LSQUIC_CONN_STATS 1419 if (engine->stats_fh) 1420 { 1421 const struct conn_stats *const stats = &engine->conn_stats_sum; 1422 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1423 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1424 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1425 fprintf(engine->stats_fh, "In:\n"); 1426 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1427 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1428 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1429 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1430 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1431 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1432 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1433 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1434 stats->in.headers_comp, stats->in.headers_uncomp, 1435 stats->in.headers_uncomp ? 1436 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1437 : 0); 1438 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1439 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1440 fprintf(engine->stats_fh, " ACK frames merged to new: %lu\n", stats->in.n_acks_merged[0]); 1441 fprintf(engine->stats_fh, " ACK frames merged to old: %lu\n", stats->in.n_acks_merged[1]); 1442 fprintf(engine->stats_fh, "Out:\n"); 1443 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1444 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1445 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1446 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1447 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1448 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1449 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1450 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1451 stats->out.headers_comp, stats->out.headers_uncomp, 1452 stats->out.headers_uncomp ? 1453 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1454 : 0); 1455 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1456 } 1457#endif 1458 if (engine->pub.enp_srst_hash) 1459 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1460#if LSQUIC_COUNT_ENGINE_CALLS 1461 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1462#endif 1463 if (engine->pub.enp_retry_aead_ctx) 1464 EVP_AEAD_CTX_cleanup(engine->pub.enp_retry_aead_ctx); 1465 free(engine); 1466} 1467 1468 1469static struct conn_cid_elem * 1470find_free_cce (struct lsquic_conn *conn) 1471{ 1472 struct conn_cid_elem *cce; 1473 1474 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1475 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1476 return cce; 1477 1478 return NULL; 1479} 1480 1481 1482static int 1483add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1484 const struct sockaddr *local_sa, void *peer_ctx) 1485{ 1486 struct conn_cid_elem *cce; 1487 1488 if (engine->flags & ENG_CONNS_BY_ADDR) 1489 { 1490 cce = find_free_cce(conn); 1491 if (!cce) 1492 { 1493 LSQ_ERROR("cannot find free CCE"); 1494 return -1; 1495 } 1496 cce->cce_port = sa2port(local_sa); 1497 cce->cce_flags = CCE_PORT; 1498 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1499 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1500 { 1501 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1502 return 0; 1503 } 1504 else 1505 return -1; 1506 1507 } 1508 else 1509 return insert_conn_into_hash(engine, conn, peer_ctx); 1510} 1511 1512 1513lsquic_conn_t * 1514lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1515 const struct sockaddr *local_sa, 1516 const struct sockaddr *peer_sa, 1517 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1518 const char *hostname, unsigned short max_packet_size, 1519 const unsigned char *zero_rtt, size_t zero_rtt_len, 1520 const unsigned char *token, size_t token_sz) 1521{ 1522 lsquic_conn_t *conn; 1523 unsigned flags, versions; 1524 int is_ipv4; 1525 1526 ENGINE_IN(engine); 1527 1528 if (engine->flags & ENG_SERVER) 1529 { 1530 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1531 goto err; 1532 } 1533 1534 if (engine->flags & ENG_CONNS_BY_ADDR 1535 && find_conn_by_addr(engine->conns_hash, local_sa)) 1536 { 1537 LSQ_ERROR("cannot have more than one connection on the same port"); 1538 goto err; 1539 } 1540 1541 if (0 != maybe_grow_conn_heaps(engine)) 1542 return NULL; 1543 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1544 is_ipv4 = peer_sa->sa_family == AF_INET; 1545 if (zero_rtt && zero_rtt_len) 1546 { 1547 version = lsquic_zero_rtt_version(zero_rtt, zero_rtt_len); 1548 if (version >= N_LSQVER) 1549 { 1550 LSQ_INFO("zero-rtt version is bad, won't use"); 1551 zero_rtt = NULL; 1552 zero_rtt_len = 0; 1553 } 1554 } 1555 if (version >= N_LSQVER) 1556 { 1557 if (version > N_LSQVER) 1558 LSQ_WARN("invalid version specified, engine will pick"); 1559 versions = engine->pub.enp_settings.es_versions; 1560 } 1561 else 1562 versions = 1u << version; 1563 if (versions & LSQUIC_IETF_VERSIONS) 1564 { 1565 if (version == LSQVER_ID24) 1566 conn = lsquic_id24_full_conn_client_new(&engine->pub, versions, 1567 flags, hostname, max_packet_size, 1568 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1569 else 1570 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1571 flags, hostname, max_packet_size, 1572 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1573 } 1574 else 1575 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1576 flags, hostname, max_packet_size, is_ipv4, 1577 zero_rtt, zero_rtt_len); 1578 if (!conn) 1579 goto err; 1580 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1581 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1582 lsquic_ver2str[conn->cn_version]); 1583 ++engine->n_conns; 1584 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1585 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1586 { 1587 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1588 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1589 CID_BITS(cid)); 1590 destroy_conn(engine, conn, lsquic_time_now()); 1591 goto err; 1592 } 1593 assert(!(conn->cn_flags & 1594 (CONN_REF_FLAGS 1595 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1596 callbacks */ 1597 ))); 1598 conn->cn_flags |= LSCONN_HASHED; 1599 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1600 engine_incref_conn(conn, LSCONN_TICKABLE); 1601 lsquic_conn_set_ctx(conn, conn_ctx); 1602 conn->cn_if->ci_client_call_on_new(conn); 1603 end: 1604 ENGINE_OUT(engine); 1605 return conn; 1606 err: 1607 conn = NULL; 1608 goto end; 1609} 1610 1611 1612static void 1613remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1614{ 1615 remove_all_cces_from_hash(engine->conns_hash, conn); 1616 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1617} 1618 1619 1620static void 1621refflags2str (enum lsquic_conn_flags flags, char s[6]) 1622{ 1623 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1624 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1625 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1626 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1627 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1628 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1629 *s = '\0'; 1630} 1631 1632 1633static void 1634engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1635{ 1636 char str[2][7]; 1637 assert(flag & CONN_REF_FLAGS); 1638 assert(!(conn->cn_flags & flag)); 1639 conn->cn_flags |= flag; 1640 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1641 CID_BITS(lsquic_conn_log_cid(conn)), 1642 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1643 (refflags2str(conn->cn_flags, str[1]), str[1])); 1644} 1645 1646 1647static lsquic_conn_t * 1648engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1649 enum lsquic_conn_flags flags) 1650{ 1651 char str[2][7]; 1652 lsquic_time_t now; 1653 assert(flags & CONN_REF_FLAGS); 1654 assert(conn->cn_flags & flags); 1655#ifndef NDEBUG 1656 if (flags & LSCONN_CLOSING) 1657 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1658#endif 1659 conn->cn_flags &= ~flags; 1660 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1661 CID_BITS(lsquic_conn_log_cid(conn)), 1662 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1663 (refflags2str(conn->cn_flags, str[1]), str[1])); 1664 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1665 { 1666 now = lsquic_time_now(); 1667 if (conn->cn_flags & LSCONN_MINI) 1668 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1669 else 1670 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1671 destroy_conn(engine, conn, now); 1672 return NULL; 1673 } 1674 else 1675 return conn; 1676} 1677 1678 1679/* This is not a general-purpose function. Only call from engine dtor. */ 1680static void 1681force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1682{ 1683 assert(engine->flags & ENG_DTOR); 1684 const enum lsquic_conn_flags flags = conn->cn_flags; 1685 assert(conn->cn_flags & CONN_REF_FLAGS); 1686 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1687 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1688 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1689 if (flags & LSCONN_ATTQ) 1690 { 1691 attq_remove(engine->attq, conn); 1692 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1693 } 1694 if (flags & LSCONN_HASHED) 1695 remove_conn_from_hash(engine, conn); 1696} 1697 1698 1699/* Iterator for tickable connections (those on the Tickable Queue). Before 1700 * a connection is returned, it is removed from the Advisory Tick Time queue 1701 * if necessary. 1702 */ 1703static lsquic_conn_t * 1704conn_iter_next_tickable (struct lsquic_engine *engine) 1705{ 1706 lsquic_conn_t *conn; 1707 1708 if (engine->flags & ENG_SERVER) 1709 while (1) 1710 { 1711 conn = lsquic_mh_pop(&engine->conns_tickable); 1712 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1713 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1714 else 1715 break; 1716 } 1717 else 1718 conn = lsquic_mh_pop(&engine->conns_tickable); 1719 1720 if (conn) 1721 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1722 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1723 { 1724 attq_remove(engine->attq, conn); 1725 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1726 } 1727 1728 return conn; 1729} 1730 1731 1732static void 1733cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1734 void *update_ctx) 1735{ 1736 cub->cub_update_cids = update; 1737 cub->cub_update_ctx = update_ctx; 1738 cub->cub_count = 0; 1739} 1740 1741 1742static void 1743cub_flush (struct cid_update_batch *cub) 1744{ 1745 if (cub->cub_count > 0 && cub->cub_update_cids) 1746 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1747 cub->cub_cids, cub->cub_count); 1748 cub->cub_count = 0; 1749} 1750 1751 1752static void 1753cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1754{ 1755 cub->cub_cids [ cub->cub_count ] = *cid; 1756 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1757 ++cub->cub_count; 1758 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1759 cub_flush(cub); 1760} 1761 1762 1763/* Process registered CIDs */ 1764static void 1765cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1766{ 1767 struct cce_cid_iter citer; 1768 struct conn_cid_elem *cce; 1769 void *peer_ctx; 1770 1771 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1772 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1773 if (cce->cce_flags & CCE_REG) 1774 cub_add(cub, &cce->cce_cid, peer_ctx); 1775} 1776 1777 1778static void 1779drop_all_mini_conns (lsquic_engine_t *engine) 1780{ 1781 struct lsquic_hash_elem *el; 1782 lsquic_conn_t *conn; 1783 struct cid_update_batch cub; 1784 1785 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1786 1787 for (el = lsquic_hash_first(engine->conns_hash); el; 1788 el = lsquic_hash_next(engine->conns_hash)) 1789 { 1790 conn = lsquic_hashelem_getdata(el); 1791 if (conn->cn_flags & LSCONN_MINI) 1792 { 1793 /* If promoted, why is it still in this hash? */ 1794 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1795 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1796 cub_add_cids_from_cces(&cub, conn); 1797 remove_conn_from_hash(engine, conn); 1798 } 1799 } 1800 1801 cub_flush(&cub); 1802} 1803 1804 1805void 1806lsquic_engine_process_conns (lsquic_engine_t *engine) 1807{ 1808 lsquic_conn_t *conn; 1809 lsquic_time_t now; 1810 1811 ENGINE_IN(engine); 1812 1813 now = lsquic_time_now(); 1814 while ((conn = attq_pop(engine->attq, now))) 1815 { 1816 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1817 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 1818 { 1819 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1820 engine_incref_conn(conn, LSCONN_TICKABLE); 1821 } 1822 } 1823 1824 process_connections(engine, conn_iter_next_tickable, now); 1825 ENGINE_OUT(engine); 1826} 1827 1828 1829static void 1830release_or_return_enc_data (struct lsquic_engine *engine, 1831 void (*pmi_rel_or_ret) (void *, void *, void *, char), 1832 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 1833{ 1834 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1835 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 1836 packet_out->po_flags &= ~PO_ENCRYPTED; 1837 packet_out->po_enc_data = NULL; 1838} 1839 1840 1841static void 1842release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1843 struct lsquic_packet_out *packet_out) 1844{ 1845 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 1846 conn, packet_out); 1847} 1848 1849 1850static void 1851return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1852 struct lsquic_packet_out *packet_out) 1853{ 1854 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 1855 conn, packet_out); 1856} 1857 1858 1859static int 1860copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 1861 struct lsquic_packet_out *packet_out) 1862{ 1863 int ipv6; 1864 1865 ipv6 = NP_IS_IPv6(packet_out->po_path); 1866 if (packet_out->po_flags & PO_ENCRYPTED) 1867 { 1868 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1869 && packet_out->po_data_sz == packet_out->po_enc_data_sz 1870 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 1871 packet_out->po_data_sz)) 1872 return 0; 1873 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1874 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 1875 goto copy; 1876 return_enc_data(engine, conn, packet_out); 1877 } 1878 1879 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 1880 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1881 packet_out->po_data_sz, ipv6); 1882 if (!packet_out->po_enc_data) 1883 { 1884 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 1885 "of size %hu", packet_out->po_data_sz); 1886 return -1; 1887 } 1888 1889 copy: 1890 memcpy(packet_out->po_enc_data, packet_out->po_data, 1891 packet_out->po_data_sz); 1892 packet_out->po_enc_data_sz = packet_out->po_data_sz; 1893 packet_out->po_sent_sz = packet_out->po_data_sz; 1894 packet_out->po_flags &= ~PO_IPv6; 1895 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 1896 1897 return 0; 1898} 1899 1900 1901STAILQ_HEAD(conns_stailq, lsquic_conn); 1902TAILQ_HEAD(conns_tailq, lsquic_conn); 1903 1904 1905struct conns_out_iter 1906{ 1907 struct min_heap *coi_heap; 1908 struct pr_queue *coi_prq; 1909 TAILQ_HEAD(, lsquic_conn) coi_active_list, 1910 coi_inactive_list; 1911 lsquic_conn_t *coi_next; 1912#ifndef NDEBUG 1913 lsquic_time_t coi_last_sent; 1914#endif 1915}; 1916 1917 1918static void 1919coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 1920{ 1921 iter->coi_heap = &engine->conns_out; 1922 iter->coi_prq = engine->pr_queue; 1923 iter->coi_next = NULL; 1924 TAILQ_INIT(&iter->coi_active_list); 1925 TAILQ_INIT(&iter->coi_inactive_list); 1926#ifndef NDEBUG 1927 iter->coi_last_sent = 0; 1928#endif 1929} 1930 1931 1932static lsquic_conn_t * 1933coi_next (struct conns_out_iter *iter) 1934{ 1935 lsquic_conn_t *conn; 1936 1937 if (lsquic_mh_count(iter->coi_heap) > 0) 1938 { 1939 conn = lsquic_mh_pop(iter->coi_heap); 1940 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1941 conn->cn_flags |= LSCONN_COI_ACTIVE; 1942#ifndef NDEBUG 1943 if (iter->coi_last_sent) 1944 assert(iter->coi_last_sent <= conn->cn_last_sent); 1945 iter->coi_last_sent = conn->cn_last_sent; 1946#endif 1947 return conn; 1948 } 1949 else if (iter->coi_prq && (conn = prq_next_conn(iter->coi_prq))) 1950 { 1951 return conn; 1952 } 1953 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 1954 { 1955 iter->coi_prq = NULL; /* Save function call in previous conditional */ 1956 conn = iter->coi_next; 1957 if (!conn) 1958 conn = TAILQ_FIRST(&iter->coi_active_list); 1959 if (conn) 1960 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 1961 return conn; 1962 } 1963 else 1964 return NULL; 1965} 1966 1967 1968static void 1969coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1970{ 1971 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 1972 { 1973 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 1974 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1975 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1976 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 1977 conn->cn_flags |= LSCONN_COI_INACTIVE; 1978 } 1979} 1980 1981 1982static void 1983coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1984{ 1985 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 1986 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 1987 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 1988 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1989 conn->cn_flags |= LSCONN_COI_ACTIVE; 1990} 1991 1992 1993static void 1994coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 1995{ 1996 lsquic_conn_t *conn; 1997 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 1998 { 1999 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2000 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2001 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 2002 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 2003 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 2004 else /* Closed connection gets one shot at sending packets */ 2005 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2006 } 2007 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 2008 { 2009 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2010 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2011 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2012 } 2013} 2014 2015 2016#ifndef NDEBUG 2017static void 2018lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 2019 unsigned n) 2020{ 2021 const lsquic_cid_t *cid; 2022 struct iovec *iov; 2023 unsigned i; 2024 char packno_str[22]; 2025 2026 for (i = 0; i < n; ++i) 2027 { 2028 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 2029 batch->packets[i]->po_packno); 2030 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 2031 { 2032 for (iov = batch->outs[i].iov; iov < 2033 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 2034 batch->outs[i].iov->iov_len -= 1; 2035 cid = lsquic_conn_log_cid(batch->conns[i]); 2036 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 2037 CID_BITS(cid)); 2038 } 2039 } 2040} 2041 2042 2043#endif 2044 2045 2046#ifdef NDEBUG 2047#define CONST_BATCH const 2048#else 2049#define CONST_BATCH 2050#endif 2051 2052 2053struct send_batch_ctx { 2054 struct conns_stailq *closed_conns; 2055 struct conns_tailq *ticked_conns; 2056 struct conns_out_iter *conns_iter; 2057 CONST_BATCH struct out_batch *batch; 2058}; 2059 2060 2061static void 2062close_conn_immediately (struct lsquic_engine *engine, 2063 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2064{ 2065 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2066 if (!(conn->cn_flags & LSCONN_CLOSING)) 2067 { 2068 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2069 engine_incref_conn(conn, LSCONN_CLOSING); 2070 if (conn->cn_flags & LSCONN_HASHED) 2071 remove_conn_from_hash(engine, conn); 2072 } 2073 if (conn->cn_flags & LSCONN_TICKED) 2074 { 2075 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2076 engine_decref_conn(engine, conn, LSCONN_TICKED); 2077 } 2078} 2079 2080 2081static void 2082close_conn_on_send_error (struct lsquic_engine *engine, 2083 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2084{ 2085 const struct out_batch *batch = sb_ctx->batch; 2086 struct lsquic_conn *const conn = batch->conns[n]; 2087 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2088 2089 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2090 "src: %s; dst: %s; errno: %d", 2091 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2092 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2093 CID_BITS(lsquic_conn_log_cid(conn)), 2094 SA2STR(batch->outs[n].local_sa, buf[0]), 2095 SA2STR(batch->outs[n].dest_sa, buf[1]), 2096 e_val); 2097 if (conn->cn_flags & LSCONN_EVANESCENT) 2098 lsquic_prq_drop(conn); 2099 else 2100 close_conn_immediately(engine, sb_ctx, conn); 2101} 2102 2103 2104static unsigned 2105send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2106 unsigned n_to_send) 2107{ 2108 int n_sent, i, e_val; 2109 lsquic_time_t now; 2110 unsigned off; 2111 size_t count; 2112 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2113 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2114 2115#ifndef NDEBUG 2116 if (engine->flags & ENG_LOSE_PACKETS) 2117 lose_matching_packets(engine, batch, n_to_send); 2118#endif 2119 /* Set sent time before the write to avoid underestimating RTT */ 2120 now = lsquic_time_now(); 2121 for (i = 0; i < (int) n_to_send; ++i) 2122 { 2123 off = batch->pack_off[i]; 2124 count = batch->outs[i].iovlen; 2125 assert(count > 0); 2126 packet_out = &batch->packets[off]; 2127 end = packet_out + count; 2128 do 2129 (*packet_out)->po_sent = now; 2130 while (++packet_out < end); 2131 } 2132 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs, 2133 n_to_send); 2134 e_val = errno; 2135 if (n_sent < (int) n_to_send) 2136 { 2137 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2138 engine->resume_sending_at = now + 1000000; 2139 LSQ_DEBUG("cannot send packets"); 2140 EV_LOG_GENERIC_EVENT("cannot send packets"); 2141 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2142 close_conn_on_send_error(engine, sb_ctx, 2143 n_sent < 0 ? 0 : n_sent, e_val); 2144 } 2145 if (n_sent >= 0) 2146 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, n_to_send); 2147 else 2148 { 2149 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2150 n_sent = 0; 2151 } 2152 if (n_sent > 0) 2153 engine->last_sent = now + n_sent; 2154 for (i = 0; i < n_sent; ++i) 2155 { 2156 eng_hist_inc(&engine->history, now, sl_packets_out); 2157 /* `i' is added to maintain relative order */ 2158 batch->conns[i]->cn_last_sent = now + i; 2159 2160 off = batch->pack_off[i]; 2161 count = batch->outs[i].iovlen; 2162 assert(count > 0); 2163 packet_out = &batch->packets[off]; 2164 end = packet_out + count; 2165 do 2166 { 2167#if LOG_PACKET_CHECKSUM 2168 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2169 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2170 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2171#endif 2172 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2173 *packet_out); 2174 /* Release packet out buffer as soon as the packet is sent 2175 * successfully. If not successfully sent, we hold on to 2176 * this buffer until the packet sending is attempted again 2177 * or until it times out and regenerated. 2178 */ 2179 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2180 release_enc_data(engine, batch->conns[i], *packet_out); 2181 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2182 *packet_out); 2183 } 2184 while (++packet_out < end); 2185 } 2186 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2187 for ( ; i < (int) n_to_send; ++i) 2188 { 2189 off = batch->pack_off[i]; 2190 count = batch->outs[i].iovlen; 2191 assert(count > 0); 2192 packet_out = &batch->packets[off]; 2193 end = packet_out + count; 2194 do 2195 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2196 *packet_out); 2197 while (++packet_out < end); 2198 } 2199 /* Return packets to the connection in reverse order so that the packet 2200 * ordering is maintained. 2201 */ 2202 for (i = (int) n_to_send - 1; i >= n_sent; --i) 2203 { 2204 off = batch->pack_off[i]; 2205 count = batch->outs[i].iovlen; 2206 assert(count > 0); 2207 packet_out = &batch->packets[off + count - 1]; 2208 end = &batch->packets[off - 1]; 2209 do 2210 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2211 *packet_out); 2212 while (--packet_out > end); 2213 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2214 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2215 } 2216 return n_sent; 2217} 2218 2219 2220/* Return 1 if went past deadline, 0 otherwise */ 2221static int 2222check_deadline (lsquic_engine_t *engine) 2223{ 2224 if (engine->pub.enp_settings.es_proc_time_thresh && 2225 lsquic_time_now() > engine->deadline) 2226 { 2227 LSQ_INFO("went past threshold of %u usec, stop sending", 2228 engine->pub.enp_settings.es_proc_time_thresh); 2229 engine->flags |= ENG_PAST_DEADLINE; 2230 return 1; 2231 } 2232 else 2233 return 0; 2234} 2235 2236 2237static size_t 2238iov_size (const struct iovec *iov, const struct iovec *const end) 2239{ 2240 size_t size; 2241 2242 assert(iov < end); 2243 2244 size = 0; 2245 do 2246 size += iov->iov_len; 2247 while (++iov < end); 2248 2249 return size; 2250} 2251 2252 2253static void 2254send_packets_out (struct lsquic_engine *engine, 2255 struct conns_tailq *ticked_conns, 2256 struct conns_stailq *closed_conns) 2257{ 2258 unsigned n, w, n_sent, n_batches_sent; 2259 lsquic_packet_out_t *packet_out; 2260 struct lsquic_packet_out **packet; 2261 lsquic_conn_t *conn; 2262 struct out_batch *const batch = &engine->out_batch; 2263 struct iovec *iov, *packet_iov; 2264 struct conns_out_iter conns_iter; 2265 int shrink, deadline_exceeded; 2266 const struct send_batch_ctx sb_ctx = { 2267 closed_conns, 2268 ticked_conns, 2269 &conns_iter, 2270 &engine->out_batch, 2271 }; 2272 2273 coi_init(&conns_iter, engine); 2274 n_batches_sent = 0; 2275 n_sent = 0, n = 0; 2276 shrink = 0; 2277 deadline_exceeded = 0; 2278 iov = batch->iov; 2279 packet = batch->packets; 2280 2281 while ((conn = coi_next(&conns_iter))) 2282 { 2283 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2284 if (!packet_out) { 2285 /* Evanescent connection always has a packet to send: */ 2286 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2287 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2288 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2289 CID_BITS(lsquic_conn_log_cid(conn))); 2290 coi_deactivate(&conns_iter, conn); 2291 continue; 2292 } 2293 batch->outs[n].iov = packet_iov = iov; 2294 next_coa: 2295 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2296 { 2297 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2298 &engine->pub, conn, packet_out)) 2299 { 2300 case ENCPA_NOMEM: 2301 /* Send what we have and wait for a more opportune moment */ 2302 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2303 goto end_for; 2304 case ENCPA_BADCRYPT: 2305 /* This is pretty bad: close connection immediately */ 2306 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2307 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2308 CID_BITS(lsquic_conn_log_cid(conn))); 2309 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2310 { 2311 close_conn_immediately(engine, &sb_ctx, conn); 2312 coi_deactivate(&conns_iter, conn); 2313 } 2314 continue; 2315 case ENCPA_OK: 2316 break; 2317 } 2318 } 2319 else if ((packet_out->po_flags & PO_NOENCRYPT) 2320 && engine->pub.enp_pmi != &stock_pmi) 2321 { 2322 if (0 != copy_packet(engine, conn, packet_out)) 2323 { 2324 /* Copy can only fail if packet could not be allocated */ 2325 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2326 goto end_for; 2327 } 2328 } 2329 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2330 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2331 if (packet_out->po_flags & PO_ENCRYPTED) 2332 { 2333 iov->iov_base = packet_out->po_enc_data; 2334 iov->iov_len = packet_out->po_enc_data_sz; 2335 } 2336 else 2337 { 2338 iov->iov_base = packet_out->po_data; 2339 iov->iov_len = packet_out->po_data_sz; 2340 } 2341 if (packet_iov == iov) 2342 { 2343 batch->pack_off[n] = packet - batch->packets; 2344 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2345 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2346 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2347 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2348 batch->conns [n] = conn; 2349 } 2350 *packet = packet_out; 2351 ++packet; 2352 ++iov; 2353 if ((conn->cn_flags & LSCONN_IETF) 2354 && ((1 << packet_out->po_header_type) 2355 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2356#ifndef NDEBUG 2357 && (engine->flags & ENG_COALESCE) 2358#endif 2359 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2360 { 2361 const size_t size = iov_size(packet_iov, iov); 2362 packet_out = conn->cn_if->ci_next_packet_to_send(conn, size); 2363 if (packet_out) 2364 goto next_coa; 2365 } 2366 batch->outs [n].iovlen = iov - packet_iov; 2367 ++n; 2368 if (n == engine->batch_size 2369 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2370 { 2371 w = send_batch(engine, &sb_ctx, n); 2372 n = 0; 2373 iov = batch->iov; 2374 packet = batch->packets; 2375 ++n_batches_sent; 2376 n_sent += w; 2377 if (w < engine->batch_size) 2378 { 2379 shrink = 1; 2380 break; 2381 } 2382 deadline_exceeded = check_deadline(engine); 2383 if (deadline_exceeded) 2384 break; 2385 grow_batch_size(engine); 2386 } 2387 } 2388 end_for: 2389 2390 if (n > 0) { 2391 w = send_batch(engine, &sb_ctx, n); 2392 n_sent += w; 2393 shrink = w < n; 2394 ++n_batches_sent; 2395 } 2396 2397 if (shrink) 2398 shrink_batch_size(engine); 2399 else if (n_batches_sent > 1) 2400 { 2401 deadline_exceeded = check_deadline(engine); 2402 if (!deadline_exceeded) 2403 grow_batch_size(engine); 2404 } 2405 2406 coi_reheap(&conns_iter, engine); 2407 2408 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2409} 2410 2411 2412int 2413lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2414{ 2415 return lsquic_mh_count(&engine->conns_out) > 0 2416 || (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2417 ; 2418} 2419 2420 2421static void 2422reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2423{ 2424 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2425 engine->flags &= ~ENG_PAST_DEADLINE; 2426} 2427 2428 2429void 2430lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2431{ 2432 lsquic_conn_t *conn; 2433 struct conns_stailq closed_conns; 2434 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2435 struct cid_update_batch cub; 2436 2437 ENGINE_IN(engine); 2438 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2439 STAILQ_INIT(&closed_conns); 2440 reset_deadline(engine, lsquic_time_now()); 2441 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2442 { 2443 LSQ_DEBUG("can send again"); 2444 EV_LOG_GENERIC_EVENT("can send again"); 2445 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2446 } 2447 2448 send_packets_out(engine, &ticked_conns, &closed_conns); 2449 2450 while ((conn = STAILQ_FIRST(&closed_conns))) { 2451 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2452 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2453 cub_add_cids_from_cces(&cub, conn); 2454 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2455 } 2456 2457 cub_flush(&cub); 2458 ENGINE_OUT(engine); 2459} 2460 2461 2462static lsquic_conn_t * 2463next_new_full_conn (struct conns_stailq *new_full_conns) 2464{ 2465 lsquic_conn_t *conn; 2466 2467 conn = STAILQ_FIRST(new_full_conns); 2468 if (conn) 2469 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2470 return conn; 2471} 2472 2473 2474static void 2475process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2476 lsquic_time_t now) 2477{ 2478 lsquic_conn_t *conn; 2479 enum tick_st tick_st; 2480 unsigned i, why; 2481 lsquic_time_t next_tick_time; 2482 struct conns_stailq closed_conns; 2483 struct conns_tailq ticked_conns; 2484 struct conns_stailq new_full_conns; 2485 struct cid_update_batch cub_old, cub_live; 2486 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2487 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2488 2489 eng_hist_tick(&engine->history, now); 2490 2491 STAILQ_INIT(&closed_conns); 2492 TAILQ_INIT(&ticked_conns); 2493 reset_deadline(engine, now); 2494 STAILQ_INIT(&new_full_conns); 2495 2496 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2497 && now > engine->resume_sending_at) 2498 { 2499 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2500 "timeout"); 2501 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2502 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2503 } 2504 2505 i = 0; 2506 while ((conn = next_conn(engine)) 2507 || (conn = next_new_full_conn(&new_full_conns))) 2508 { 2509 tick_st = conn->cn_if->ci_tick(conn, now); 2510 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2511 if (tick_st & TICK_PROMOTE) 2512 { 2513 lsquic_conn_t *new_conn; 2514 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2515 "scheduled for promotion"); 2516 assert(conn->cn_flags & LSCONN_MINI); 2517 new_conn = new_full_conn_server(engine, conn, now); 2518 if (new_conn) 2519 { 2520 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2521 new_conn->cn_last_sent = engine->last_sent; 2522 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2523 conn->cn_flags |= LSCONN_PROMOTED; 2524 } 2525 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2526 } 2527 if (tick_st & TICK_SEND) 2528 { 2529 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2530 { 2531 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2532 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2533 } 2534 } 2535 if (tick_st & TICK_CLOSE) 2536 { 2537 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2538 engine_incref_conn(conn, LSCONN_CLOSING); 2539 if (conn->cn_flags & LSCONN_HASHED) 2540 remove_conn_from_hash(engine, conn); 2541 } 2542 else 2543 { 2544 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2545 engine_incref_conn(conn, LSCONN_TICKED); 2546 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2547 && conn->cn_if->ci_report_live(conn, now)) 2548 cub_add_cids_from_cces(&cub_live, conn); 2549 } 2550 } 2551 2552 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2553 && lsquic_engine_has_unsent_packets(engine)) 2554 send_packets_out(engine, &ticked_conns, &closed_conns); 2555 2556 while ((conn = STAILQ_FIRST(&closed_conns))) { 2557 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2558 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2559 cub_add_cids_from_cces(&cub_old, conn); 2560 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2561 } 2562 2563 while ((conn = TAILQ_FIRST(&ticked_conns))) 2564 { 2565 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2566 engine_decref_conn(engine, conn, LSCONN_TICKED); 2567 if (!(conn->cn_flags & LSCONN_TICKABLE) 2568 && conn->cn_if->ci_is_tickable(conn)) 2569 { 2570 /* Floyd heapification is not faster, don't bother. */ 2571 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2572 engine_incref_conn(conn, LSCONN_TICKABLE); 2573 } 2574 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2575 { 2576 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2577 if (next_tick_time) 2578 { 2579 if (0 == attq_add(engine->attq, conn, next_tick_time, why)) 2580 engine_incref_conn(conn, LSCONN_ATTQ); 2581 } 2582 else 2583 /* In all other cases, the idle timeout would make the next 2584 * tick time non-zero: 2585 */ 2586 assert((conn->cn_flags & LSCONN_IETF) 2587 && engine->pub.enp_settings.es_idle_timeout == 0); 2588 } 2589 } 2590 2591 cub_flush(&engine->new_scids); 2592 cub_flush(&cub_live); 2593 cub_flush(&cub_old); 2594} 2595 2596 2597/* Return 0 if packet is being processed by a real connection, 1 if the 2598 * packet was processed, but not by a connection, and -1 on error. 2599 */ 2600int 2601lsquic_engine_packet_in (lsquic_engine_t *engine, 2602 const unsigned char *packet_in_data, size_t packet_in_size, 2603 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2604 void *peer_ctx, int ecn) 2605{ 2606 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2607 struct packin_parse_state ppstate; 2608 lsquic_packet_in_t *packet_in; 2609 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2610 int is_server, unsigned cid_len, struct packin_parse_state *); 2611 unsigned n_zeroes; 2612 int s; 2613 2614 ENGINE_CALLS_INCR(engine); 2615 2616 if (engine->flags & ENG_SERVER) 2617 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2618 else if (engine->flags & ENG_CONNS_BY_ADDR) 2619 { 2620 struct lsquic_hash_elem *el; 2621 const struct lsquic_conn *conn; 2622 el = find_conn_by_addr(engine->conns_hash, sa_local); 2623 if (!el) 2624 return -1; 2625 conn = lsquic_hashelem_getdata(el); 2626 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2627 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2628 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2629 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2630 else if (conn->cn_version == LSQVER_050) 2631 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 2632 else 2633 { 2634 assert(conn->cn_version == LSQVER_046 2635#if LSQUIC_USE_Q098 2636 || conn->cn_version == LSQVER_098 2637#endif 2638 2639 ); 2640 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2641 } 2642 } 2643 else 2644 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2645 2646 n_zeroes = 0; 2647 do 2648 { 2649 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 2650 if (!packet_in) 2651 return -1; 2652 /* Library does not modify packet_in_data, it is not referenced after 2653 * this function returns and subsequent release of pi_data is guarded 2654 * by PI_OWN_DATA flag. 2655 */ 2656 packet_in->pi_data = (unsigned char *) packet_in_data; 2657 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 2658 engine->flags & ENG_SERVER, 2659 engine->pub.enp_settings.es_scid_len, &ppstate)) 2660 { 2661 LSQ_DEBUG("Cannot parse incoming packet's header"); 2662 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 2663 s = 1; 2664 break; 2665 } 2666 2667 packet_in_data += packet_in->pi_data_sz; 2668 packet_in->pi_received = lsquic_time_now(); 2669 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 2670 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 2671 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 2672 peer_ctx, packet_in_size); 2673 n_zeroes += s == 0; 2674 } 2675 while (0 == s && packet_in_data < packet_end); 2676 2677 return n_zeroes > 0 ? 0 : s; 2678} 2679 2680 2681#if __GNUC__ && !defined(NDEBUG) 2682__attribute__((weak)) 2683#endif 2684unsigned 2685lsquic_engine_quic_versions (const lsquic_engine_t *engine) 2686{ 2687 return engine->pub.enp_settings.es_versions; 2688} 2689 2690 2691void 2692lsquic_engine_cooldown (lsquic_engine_t *engine) 2693{ 2694 struct lsquic_hash_elem *el; 2695 lsquic_conn_t *conn; 2696 2697 if (engine->flags & ENG_COOLDOWN) 2698 /* AFAICT, there is no harm in calling this function more than once, 2699 * but log it just in case, as it may indicate an error in the caller. 2700 */ 2701 LSQ_INFO("cooldown called again"); 2702 engine->flags |= ENG_COOLDOWN; 2703 LSQ_INFO("entering cooldown mode"); 2704 if (engine->flags & ENG_SERVER) 2705 drop_all_mini_conns(engine); 2706 for (el = lsquic_hash_first(engine->conns_hash); el; 2707 el = lsquic_hash_next(engine->conns_hash)) 2708 { 2709 conn = lsquic_hashelem_getdata(el); 2710 lsquic_conn_going_away(conn); 2711 } 2712} 2713 2714 2715int 2716lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 2717{ 2718 const struct attq_elem *next_attq; 2719 lsquic_time_t now, next_time; 2720#if LSQUIC_DEBUG_NEXT_ADV_TICK 2721 const struct lsquic_conn *conn; 2722 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 2723#endif 2724 2725 ENGINE_CALLS_INCR(engine); 2726 2727 if ((engine->flags & ENG_PAST_DEADLINE) 2728 && lsquic_mh_count(&engine->conns_out)) 2729 { 2730#if LSQUIC_DEBUG_NEXT_ADV_TICK 2731 conn = lsquic_mh_peek(&engine->conns_out); 2732 engine->last_logged_conn = 0; 2733 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 2734 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 2735 lsquic_mh_count(&engine->conns_out), 2736 lsquic_mh_count(&engine->conns_out) != 1, "s", 2737 CID_BITS(lsquic_conn_log_cid(conn))); 2738#endif 2739 *diff = 0; 2740 return 1; 2741 } 2742 2743 if (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2744 { 2745#if LSQUIC_DEBUG_NEXT_ADV_TICK 2746 engine->last_logged_conn = 0; 2747 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 2748#endif 2749 *diff = 0; 2750 return 1; 2751 } 2752 2753 if (lsquic_mh_count(&engine->conns_tickable)) 2754 { 2755#if LSQUIC_DEBUG_NEXT_ADV_TICK 2756 conn = lsquic_mh_peek(&engine->conns_tickable); 2757 engine->last_logged_conn = 0; 2758 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 2759 "connection%.*s (%"CID_FMT" first)", 2760 lsquic_mh_count(&engine->conns_tickable), 2761 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 2762 CID_BITS(lsquic_conn_log_cid(conn))); 2763#endif 2764 *diff = 0; 2765 return 1; 2766 } 2767 2768 next_attq = attq_next(engine->attq); 2769 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 2770 { 2771 if (next_attq) 2772 next_time = next_attq->ae_adv_time; 2773 else 2774 return 0; 2775 } 2776 else 2777 { 2778 if (next_attq) 2779 { 2780 next_time = next_attq->ae_adv_time; 2781 if (engine->resume_sending_at < next_time) 2782 { 2783 next_time = engine->resume_sending_at; 2784 next_attq = NULL; 2785 } 2786 } 2787 else 2788 next_time = engine->resume_sending_at; 2789 } 2790 2791 now = lsquic_time_now(); 2792 *diff = (int) ((int64_t) next_time - (int64_t) now); 2793#if LSQUIC_DEBUG_NEXT_ADV_TICK 2794 if (next_attq) 2795 { 2796 /* Deduplicate consecutive log messages about the same reason for the 2797 * same connection. 2798 * If diff is always zero or diff reset to a higher value, event is 2799 * still logged. 2800 */ 2801 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 2802 && (uintptr_t) next_attq->ae_conn 2803 == engine->last_logged_conn 2804 && *diff < engine->last_tick_diff)) 2805 { 2806 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 2807 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 2808 engine->last_tick_diff = *diff; 2809 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 2810 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 2811 lsquic_attq_why2str(next_attq->ae_why)); 2812 } 2813 } 2814 else 2815 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 2816#endif 2817 return 1; 2818} 2819 2820 2821unsigned 2822lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 2823{ 2824 lsquic_time_t now; 2825 ENGINE_CALLS_INCR(engine); 2826 now = lsquic_time_now(); 2827 if (from_now < 0) 2828 now -= from_now; 2829 else 2830 now += from_now; 2831 return attq_count_before(engine->attq, now); 2832} 2833 2834 2835int 2836lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 2837 struct lsquic_conn *conn, unsigned cce_idx) 2838{ 2839 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2840 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2841 void *peer_ctx; 2842 2843 assert(cce_idx < conn->cn_n_cces); 2844 assert(conn->cn_cces_mask & (1 << cce_idx)); 2845 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 2846 2847 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 2848 cce->cce_cid.len, conn, &cce->cce_hash_el)) 2849 { 2850 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 2851 CID_BITS(&cce->cce_cid)); 2852 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2853 cce->cce_flags |= CCE_REG; 2854 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 2855 return 0; 2856 } 2857 else 2858 { 2859 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 2860 CID_BITS(&cce->cce_cid)); 2861 return -1; 2862 } 2863} 2864 2865 2866void 2867lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 2868 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now) 2869{ 2870 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2871 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2872 void *peer_ctx; 2873 2874 assert(cce_idx < conn->cn_n_cces); 2875 2876 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 2877 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 2878 2879 if (engine->purga) 2880 { 2881 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2882 lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 2883 PUTY_CID_RETIRED, now); 2884 } 2885 conn->cn_cces_mask &= ~(1u << cce_idx); 2886 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 2887} 2888 2889 2890