lsquic_engine.c revision b86524a4
1/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#ifndef WIN32 17#include <sys/time.h> 18#include <netinet/in.h> 19#include <sys/types.h> 20#include <sys/stat.h> 21#include <fcntl.h> 22#include <unistd.h> 23#include <netdb.h> 24#endif 25 26#ifndef NDEBUG 27#include <sys/types.h> 28#include <regex.h> /* For code that loses packets */ 29#endif 30 31#if LOG_PACKET_CHECKSUM 32#include <zlib.h> 33#endif 34 35#include <openssl/aead.h> 36 37#include "lsquic.h" 38#include "lsquic_types.h" 39#include "lsquic_int_types.h" 40#include "lsquic_sizes.h" 41#include "lsquic_parse_common.h" 42#include "lsquic_parse.h" 43#include "lsquic_packet_in.h" 44#include "lsquic_packet_out.h" 45#include "lsquic_senhist.h" 46#include "lsquic_rtt.h" 47#include "lsquic_cubic.h" 48#include "lsquic_pacer.h" 49#include "lsquic_bw_sampler.h" 50#include "lsquic_minmax.h" 51#include "lsquic_bbr.h" 52#include "lsquic_send_ctl.h" 53#include "lsquic_set.h" 54#include "lsquic_conn_flow.h" 55#include "lsquic_sfcw.h" 56#include "lsquic_hash.h" 57#include "lsquic_conn.h" 58#include "lsquic_full_conn.h" 59#include "lsquic_util.h" 60#include "lsquic_qtags.h" 61#include "lsquic_enc_sess.h" 62#include "lsquic_mm.h" 63#include "lsquic_engine_public.h" 64#include "lsquic_eng_hist.h" 65#include "lsquic_ev_log.h" 66#include "lsquic_version.h" 67#include "lsquic_pr_queue.h" 68#include "lsquic_mini_conn.h" 69#include "lsquic_mini_conn_ietf.h" 70#include "lsquic_stock_shi.h" 71#include "lsquic_purga.h" 72#include "lsquic_tokgen.h" 73#include "lsquic_attq.h" 74#include "lsquic_min_heap.h" 75#include "lsquic_http1x_if.h" 76#include "lsquic_handshake.h" 77#include "lsquic_crand.h" 78#include "lsquic_ietf.h" 79 80#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 81#include "lsquic_logger.h" 82 83#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 84#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 85#endif 86 87#if LSQUIC_DEBUG_NEXT_ADV_TICK 88#include "lsquic_alarmset.h" 89#endif 90 91#define MIN(a, b) ((a) < (b) ? (a) : (b)) 92 93/* The batch of outgoing packets grows and shrinks dynamically */ 94#define MAX_OUT_BATCH_SIZE 1024 95#define MIN_OUT_BATCH_SIZE 4 96#define INITIAL_OUT_BATCH_SIZE 32 97 98struct out_batch 99{ 100 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 101 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 102 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 103 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 104 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 105}; 106 107typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 108 109static void 110process_connections (struct lsquic_engine *engine, conn_iter_f iter, 111 lsquic_time_t now); 112 113static void 114engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 115 116static lsquic_conn_t * 117engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 118 enum lsquic_conn_flags flag); 119 120static void 121force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 122 123#if LSQUIC_COUNT_ENGINE_CALLS 124#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 125#else 126#define ENGINE_CALLS_INCR(e) 127#endif 128 129/* Nested calls to LSQUIC are not supported */ 130#define ENGINE_IN(e) do { \ 131 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 132 (e)->pub.enp_flags |= ENPUB_PROC; \ 133 ENGINE_CALLS_INCR(e); \ 134} while (0) 135 136#define ENGINE_OUT(e) do { \ 137 assert((e)->pub.enp_flags & ENPUB_PROC); \ 138 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 139} while (0) 140 141/* A connection can be referenced from one of six places: 142 * 143 * 1. A hash is used to find connections in order to dispatch an incoming 144 * packet. Connections can be hashed by CIDs or by address. In the 145 * former case, each connection has one or more mappings in the hash 146 * table. IETF QUIC connections have up to eight (in our implementation) 147 * source CIDs and each of those would have a mapping. In client mode, 148 * depending on QUIC versions and options selected, it is may be 149 * necessary to hash connections by address, in which case incoming 150 * packets are delivered to connections based on the address. 151 * 152 * 2. Outgoing queue. 153 * 154 * 3. Tickable queue 155 * 156 * 4. Advisory Tick Time queue. 157 * 158 * 5. Closing connections queue. This is a transient queue -- it only 159 * exists for the duration of process_connections() function call. 160 * 161 * 6. Ticked connections queue. Another transient queue, similar to (5). 162 * 163 * The idea is to destroy the connection when it is no longer referenced. 164 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 165 * that case, the connection is referenced from two places: (2) and (5). 166 * After its packets are sent, it is only referenced in (5), and at the 167 * end of the function call, when it is removed from (5), reference count 168 * goes to zero and the connection is destroyed. If not all packets can 169 * be sent, at the end of the function call, the connection is referenced 170 * by (2) and will only be removed once all outgoing packets have been 171 * sent. 172 */ 173#define CONN_REF_FLAGS (LSCONN_HASHED \ 174 |LSCONN_HAS_OUTGOING \ 175 |LSCONN_TICKABLE \ 176 |LSCONN_TICKED \ 177 |LSCONN_CLOSING \ 178 |LSCONN_ATTQ) 179 180 181 182 183struct cid_update_batch 184{ 185 lsquic_cids_update_f cub_update_cids; 186 void *cub_update_ctx; 187 unsigned cub_count; 188 lsquic_cid_t cub_cids[20]; 189 void *cub_peer_ctxs[20]; 190}; 191 192static void 193cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 194 195 196struct lsquic_engine 197{ 198 struct lsquic_engine_public pub; 199 enum { 200 ENG_SERVER = LSENG_SERVER, 201 ENG_HTTP = LSENG_HTTP, 202 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 203 ENG_PAST_DEADLINE 204 = (1 << 8), /* Previous call to a processing 205 * function went past time threshold. 206 */ 207 ENG_CONNS_BY_ADDR 208 = (1 << 9), /* Connections are hashed by address */ 209#ifndef NDEBUG 210 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 211 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 212 ENG_DTOR = (1 << 26), /* Engine destructor */ 213#endif 214 } flags; 215 lsquic_packets_out_f packets_out; 216 void *packets_out_ctx; 217 lsquic_cids_update_f report_new_scids; 218 lsquic_cids_update_f report_live_scids; 219 lsquic_cids_update_f report_old_scids; 220 void *scids_ctx; 221 struct lsquic_hash *conns_hash; 222 struct min_heap conns_tickable; 223 struct min_heap conns_out; 224 struct eng_hist history; 225 unsigned batch_size; 226 struct pr_queue *pr_queue; 227 struct attq *attq; 228 /* Track time last time a packet was sent to give new connections 229 * priority lower than that of existing connections. 230 */ 231 lsquic_time_t last_sent; 232#ifndef NDEBUG 233 regex_t lose_packets_re; 234 const char *lose_packets_str; 235#endif 236 unsigned n_conns; 237 lsquic_time_t deadline; 238 lsquic_time_t resume_sending_at; 239 unsigned mini_conns_count; 240 struct lsquic_purga *purga; 241#if LSQUIC_CONN_STATS 242 struct { 243 unsigned conns; 244 } stats; 245 struct conn_stats conn_stats_sum; 246 FILE *stats_fh; 247#endif 248 struct cid_update_batch new_scids; 249 struct out_batch out_batch; 250#if LSQUIC_COUNT_ENGINE_CALLS 251 unsigned long n_engine_calls; 252#endif 253#if LSQUIC_DEBUG_NEXT_ADV_TICK 254 uintptr_t last_logged_conn; 255 unsigned last_logged_ae_why; 256 int last_tick_diff; 257#endif 258 struct crand crand; 259 EVP_AEAD_CTX retry_aead_ctx; 260}; 261 262 263void 264lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 265 unsigned flags) 266{ 267 memset(settings, 0, sizeof(*settings)); 268 settings->es_versions = LSQUIC_DF_VERSIONS; 269 if (flags & ENG_SERVER) 270 { 271 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 272 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 273 settings->es_init_max_data 274 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 275 settings->es_init_max_stream_data_bidi_remote 276 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 277 settings->es_init_max_stream_data_bidi_local 278 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 279 settings->es_init_max_stream_data_uni 280 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 281 settings->es_init_max_streams_uni 282 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 283 settings->es_ping_period = 0; 284 } 285 else 286 { 287 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 288 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 289 settings->es_init_max_data 290 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 291 settings->es_init_max_stream_data_bidi_remote 292 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 293 settings->es_init_max_stream_data_bidi_local 294 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 295 settings->es_init_max_stream_data_uni 296 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 297 settings->es_init_max_streams_uni 298 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 299 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 300 } 301 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 302 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 303 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 304 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 305 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 306 settings->es_max_header_list_size 307 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 308 settings->es_ua = LSQUIC_DF_UA; 309 settings->es_ecn = LSQUIC_DF_ECN; 310 311 settings->es_pdmd = QTAG_X509; 312 settings->es_aead = QTAG_AESG; 313 settings->es_kexs = QTAG_C255; 314 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 315 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 316 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 317 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 318 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 319 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 320 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 321 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 322 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 323 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 324 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 325 settings->es_sttl = LSQUIC_DF_STTL; 326 settings->es_init_max_streams_bidi 327 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 328 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 329 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 330 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 331 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 332 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 333 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 334 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 335 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 336 settings->es_spin = LSQUIC_DF_SPIN; 337} 338 339 340/* Note: if returning an error, err_buf must be valid if non-NULL */ 341int 342lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 343 unsigned flags, 344 char *err_buf, size_t err_buf_sz) 345{ 346 if (settings->es_cfcw < LSQUIC_MIN_FCW || 347 settings->es_sfcw < LSQUIC_MIN_FCW) 348 { 349 if (err_buf) 350 snprintf(err_buf, err_buf_sz, "%s", 351 "flow control window set too low"); 352 return -1; 353 } 354 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 355 { 356 if (err_buf) 357 snprintf(err_buf, err_buf_sz, "%s", 358 "No supported QUIC versions specified"); 359 return -1; 360 } 361 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 362 { 363 if (err_buf) 364 snprintf(err_buf, err_buf_sz, "%s", 365 "one or more unsupported QUIC version is specified"); 366 return -1; 367 } 368 if (flags & ENG_SERVER) 369 { 370 if (settings->es_handshake_to > 371 MAX_MINI_CONN_LIFESPAN_IN_USEC) 372 { 373 if (err_buf) 374 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 375 " usec is too large. The maximum for server is %u usec", 376 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 377 return -1; 378 } 379 } 380 if (settings->es_idle_timeout > 600) 381 { 382 if (err_buf) 383 snprintf(err_buf, err_buf_sz, "%s", 384 "The maximum value of idle timeout is 600 seconds"); 385 return -1; 386 } 387 if (settings->es_scid_len > MAX_CID_LEN) 388 { 389 if (err_buf) 390 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 391 "bytes long; it must be between 0 and %u.", 392 settings->es_scid_len, MAX_CID_LEN); 393 return -1; 394 } 395 396 if (settings->es_cc_algo > 2) 397 { 398 if (err_buf) 399 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 400 "algorithm value %u", settings->es_cc_algo); 401 return -1; 402 } 403 404 if (!(settings->es_ql_bits >= 0 && settings->es_ql_bits <= 2)) 405 { 406 if (err_buf) 407 snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ", 408 settings->es_ql_bits); 409 return -1; 410 } 411 412 if (!(settings->es_spin == 0 || settings->es_spin == 1)) 413 { 414 if (err_buf) 415 snprintf(err_buf, err_buf_sz, "Invalid spin value %d", 416 settings->es_spin); 417 return -1; 418 } 419 420 return 0; 421} 422 423 424static void 425free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 426{ 427 free(packet_data); 428} 429 430 431static void * 432malloc_buf (void *ctx, void *conn_ctx, unsigned short size, char is_ipv6) 433{ 434 return malloc(size); 435} 436 437 438static const struct lsquic_packout_mem_if stock_pmi = 439{ 440 malloc_buf, free_packet, free_packet, 441}; 442 443 444static int 445hash_conns_by_addr (const struct lsquic_engine *engine) 446{ 447 if (engine->flags & ENG_SERVER) 448 return 0; 449 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 450 return 1; 451 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 452 && engine->pub.enp_settings.es_support_tcid0) 453 return 1; 454 if (engine->pub.enp_settings.es_scid_len == 0) 455 return 1; 456 return 0; 457} 458 459 460lsquic_engine_t * 461lsquic_engine_new (unsigned flags, 462 const struct lsquic_engine_api *api) 463{ 464 lsquic_engine_t *engine; 465 char err_buf[100]; 466 467 if (!api->ea_packets_out) 468 { 469 LSQ_ERROR("packets_out callback is not specified"); 470 return NULL; 471 } 472 473 if (api->ea_settings && 474 0 != lsquic_engine_check_settings(api->ea_settings, flags, 475 err_buf, sizeof(err_buf))) 476 { 477 LSQ_ERROR("cannot create engine: %s", err_buf); 478 return NULL; 479 } 480 481 engine = calloc(1, sizeof(*engine)); 482 if (!engine) 483 return NULL; 484 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 485 { 486 free(engine); 487 return NULL; 488 } 489 if (api->ea_settings) 490 engine->pub.enp_settings = *api->ea_settings; 491 else 492 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 493 int tag_buf_len; 494 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 495 sizeof(engine->pub.enp_ver_tags_buf), 496 engine->pub.enp_settings.es_versions); 497 if (tag_buf_len <= 0) 498 { 499 LSQ_ERROR("cannot generate version tags buffer"); 500 free(engine); 501 return NULL; 502 } 503 engine->pub.enp_ver_tags_len = tag_buf_len; 504 engine->pub.enp_flags = ENPUB_CAN_SEND; 505 engine->pub.enp_stream_if = api->ea_stream_if; 506 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 507 508 engine->flags = flags; 509#ifndef NDEBUG 510 engine->flags |= ENG_COALESCE; 511#endif 512 engine->packets_out = api->ea_packets_out; 513 engine->packets_out_ctx = api->ea_packets_out_ctx; 514 engine->report_new_scids = api->ea_new_scids; 515 engine->report_live_scids = api->ea_live_scids; 516 engine->report_old_scids = api->ea_old_scids; 517 engine->scids_ctx = api->ea_cids_update_ctx; 518 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 519 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 520 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 521 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 522 if (api->ea_shi) 523 { 524 engine->pub.enp_shi = api->ea_shi; 525 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 526 } 527 else 528 { 529 engine->pub.enp_shi = &stock_shi; 530 engine->pub.enp_shi_ctx = stock_shared_hash_new(); 531 if (!engine->pub.enp_shi_ctx) 532 { 533 free(engine); 534 return NULL; 535 } 536 } 537 if (api->ea_hsi_if) 538 { 539 engine->pub.enp_hsi_if = api->ea_hsi_if; 540 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 541 } 542 else 543 { 544 engine->pub.enp_hsi_if = lsquic_http1x_if; 545 engine->pub.enp_hsi_ctx = NULL; 546 } 547 if (api->ea_pmi) 548 { 549 engine->pub.enp_pmi = api->ea_pmi; 550 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 551 } 552 else 553 { 554 engine->pub.enp_pmi = &stock_pmi; 555 engine->pub.enp_pmi_ctx = NULL; 556 } 557 engine->pub.enp_verify_cert = api->ea_verify_cert; 558 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 559 engine->pub.enp_kli = api->ea_keylog_if; 560 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 561 engine->pub.enp_engine = engine; 562 if (hash_conns_by_addr(engine)) 563 engine->flags |= ENG_CONNS_BY_ADDR; 564 engine->conns_hash = lsquic_hash_create(); 565 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 566 if (!engine->pub.enp_tokgen) 567 return NULL; 568 engine->pub.enp_crand = &engine->crand; 569 if (flags & ENG_SERVER) 570 { 571 engine->pr_queue = prq_create( 572 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 573 &engine->pub); 574 if (!engine->pr_queue) 575 { 576 lsquic_tg_destroy(engine->pub.enp_tokgen); 577 return NULL; 578 } 579 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 580 engine->report_old_scids, engine->scids_ctx); 581 if (!engine->purga) 582 { 583 lsquic_tg_destroy(engine->pub.enp_tokgen); 584 prq_destroy(engine->pr_queue); 585 return NULL; 586 } 587 } 588 engine->attq = attq_create(); 589 eng_hist_init(&engine->history); 590 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 591 if (engine->pub.enp_settings.es_honor_prst) 592 { 593 engine->pub.enp_srst_hash = lsquic_hash_create(); 594 if (!engine->pub.enp_srst_hash) 595 { 596 lsquic_engine_destroy(engine); 597 return NULL; 598 } 599 } 600 601#ifndef NDEBUG 602 { 603 const char *env; 604 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 605 if (env) 606 { 607 if (0 != regcomp(&engine->lose_packets_re, env, 608 REG_EXTENDED|REG_NOSUB)) 609 { 610 LSQ_ERROR("could not compile lost packet regex `%s'", env); 611 return NULL; 612 } 613 engine->flags |= ENG_LOSE_PACKETS; 614 engine->lose_packets_str = env; 615 LSQ_WARN("will lose packets that match the following regex: %s", 616 env); 617 } 618 env = getenv("LSQUIC_COALESCE"); 619 if (env) 620 { 621 engine->flags &= ~ENG_COALESCE; 622 if (atoi(env)) 623 { 624 engine->flags |= ENG_COALESCE; 625 LSQ_NOTICE("will coalesce packets"); 626 } 627 else 628 LSQ_NOTICE("will not coalesce packets"); 629 } 630 } 631#endif 632#if LSQUIC_CONN_STATS 633 engine->stats_fh = api->ea_stats_fh; 634#endif 635 if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx, EVP_aead_aes_128_gcm(), 636 IETF_RETRY_KEY_BUF, IETF_RETRY_KEY_SZ, 16, NULL)) 637 { 638 LSQ_ERROR("could not initialize retry AEAD ctx"); 639 lsquic_engine_destroy(engine); 640 return NULL; 641 } 642 engine->pub.enp_retry_aead_ctx = &engine->retry_aead_ctx; 643 644 LSQ_INFO("instantiated engine"); 645 return engine; 646} 647 648 649#if LOG_PACKET_CHECKSUM 650static void 651log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 652 const unsigned char *buf, size_t bufsz) 653{ 654 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 655 (uint32_t) crc32(0, buf, bufsz)); 656} 657 658 659#endif 660 661 662static void 663grow_batch_size (struct lsquic_engine *engine) 664{ 665 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 666} 667 668 669static void 670shrink_batch_size (struct lsquic_engine *engine) 671{ 672 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 673} 674 675 676struct cce_cid_iter 677{ 678 const struct lsquic_conn *conn; 679 unsigned todo, n; 680}; 681 682 683static struct conn_cid_elem * 684cce_iter_next (struct cce_cid_iter *citer) 685{ 686 struct conn_cid_elem *cce; 687 688 while (citer->todo) 689 if (citer->todo & (1 << citer->n)) 690 { 691 citer->todo &= ~(1 << citer->n); 692 cce = &citer->conn->cn_cces[ citer->n++ ]; 693 if (!(cce->cce_flags & CCE_PORT)) 694 return cce; 695 } 696 else 697 ++citer->n; 698 699 return NULL; 700} 701 702 703static struct conn_cid_elem * 704cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 705{ 706 citer->conn = conn; 707 citer->todo = conn->cn_cces_mask; 708 citer->n = 0; 709 return cce_iter_next(citer); 710} 711 712 713#if LSQUIC_CONN_STATS 714void 715update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 716{ 717 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 718 const unsigned long *src; 719 const struct conn_stats *stats; 720 unsigned i; 721 722 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 723 { 724 ++engine->stats.conns; 725 src = (unsigned long *) stats; 726 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 727 dst[i] += src[i]; 728 } 729} 730 731 732#endif 733 734 735/* Wrapper to make sure important things occur before the connection is 736 * really destroyed. 737 */ 738static void 739destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 740 lsquic_time_t now) 741{ 742 struct cce_cid_iter citer; 743 const struct conn_cid_elem *cce; 744 lsquic_time_t drain_time; 745 struct purga_el *puel; 746 747 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 748 if (engine->purga 749 /* Blacklist all CIDs except for promoted mini connections */ 750 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 751 != (LSCONN_MINI|LSCONN_PROMOTED)) 752 { 753 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 754 && conn->cn_if->ci_drain_time && 755 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 756 { 757 for (cce = cce_iter_first(&citer, conn); cce; 758 cce = cce_iter_next(&citer)) 759 { 760 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 761 lsquic_conn_get_peer_ctx(conn, NULL), 762 PUTY_CONN_DRAIN, now); 763 if (puel) 764 puel->puel_time = now + drain_time; 765 } 766 } 767 else 768 { 769 for (cce = cce_iter_first(&citer, conn); cce; 770 cce = cce_iter_next(&citer)) 771 { 772 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 773 lsquic_conn_get_peer_ctx(conn, NULL), 774 PUTY_CONN_DELETED, now); 775 if (puel) 776 { 777 puel->puel_time = now; 778 puel->puel_count = 0; 779 } 780 } 781 } 782 } 783#if LSQUIC_CONN_STATS 784 update_stats_sum(engine, conn); 785#endif 786 --engine->n_conns; 787 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 788 conn->cn_if->ci_destroy(conn); 789} 790 791 792static int 793maybe_grow_conn_heaps (struct lsquic_engine *engine) 794{ 795 struct min_heap_elem *els; 796 unsigned count; 797 798 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 799 return 0; /* Nothing to do */ 800 801 if (lsquic_mh_nalloc(&engine->conns_tickable)) 802 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 803 else 804 count = 8; 805 806 els = malloc(sizeof(els[0]) * count); 807 if (!els) 808 { 809 LSQ_ERROR("%s: malloc failed", __func__); 810 return -1; 811 } 812 813 LSQ_DEBUG("grew heaps to %u elements", count / 2); 814 memcpy(&els[0], engine->conns_tickable.mh_elems, 815 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 816 memcpy(&els[count / 2], engine->conns_out.mh_elems, 817 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 818 free(engine->conns_tickable.mh_elems); 819 engine->conns_tickable.mh_elems = els; 820 engine->conns_out.mh_elems = &els[count / 2]; 821 engine->conns_tickable.mh_nalloc = count / 2; 822 engine->conns_out.mh_nalloc = count / 2; 823 return 0; 824} 825 826 827static void 828remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 829 unsigned todo) 830{ 831 unsigned n; 832 833 for (n = 0; todo; todo &= ~(1 << n++)) 834 if ((todo & (1 << n)) && 835 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 836 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 837} 838 839 840static void 841remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 842{ 843 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 844} 845 846 847static void 848cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 849 850 851static int 852insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 853 void *peer_ctx) 854{ 855 struct conn_cid_elem *cce; 856 unsigned todo, done, n; 857 858 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 859 if (todo & (1 << n)) 860 { 861 cce = &conn->cn_cces[n]; 862 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 863 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 864 cce->cce_cid.len, conn, &cce->cce_hash_el)) 865 done |= 1 << n; 866 else 867 goto err; 868 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 869 { 870 cce->cce_flags |= CCE_REG; 871 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 872 } 873 } 874 875 return 0; 876 877 err: 878 remove_cces_from_hash(engine->conns_hash, conn, done); 879 return -1; 880} 881 882 883static lsquic_conn_t * 884new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 885 lsquic_time_t now) 886{ 887 const lsquic_cid_t *cid; 888 server_conn_ctor_f ctor; 889 lsquic_conn_t *conn; 890 unsigned flags; 891 if (0 != maybe_grow_conn_heaps(engine)) 892 return NULL; 893 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 894 895 if (mini_conn->cn_flags & LSCONN_IETF) 896 { 897 if (mini_conn->cn_version == LSQVER_ID24) 898 ctor = lsquic_id24_full_conn_server_new; 899 else 900 ctor = lsquic_ietf_full_conn_server_new; 901 } 902 else 903 ctor = lsquic_gquic_full_conn_server_new; 904 905 conn = ctor(&engine->pub, flags, mini_conn); 906 if (!conn) 907 { 908 /* Otherwise, full_conn_server_new prints its own warnings */ 909 if (ENOMEM == errno) 910 { 911 cid = lsquic_conn_log_cid(mini_conn); 912 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 913 CID_BITS(cid), strerror(errno)); 914 } 915 return NULL; 916 } 917 ++engine->n_conns; 918 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 919 { 920 cid = lsquic_conn_log_cid(conn); 921 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 922 CID_BITS(cid)); 923 destroy_conn(engine, conn, now); 924 return NULL; 925 } 926 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 927 conn->cn_flags |= LSCONN_HASHED; 928 return conn; 929} 930 931 932static enum 933{ 934 VER_NOT_SPECIFIED, 935 VER_SUPPORTED, 936 VER_UNSUPPORTED, 937} 938 939 940version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 941 enum lsquic_version *pversion) 942{ 943 lsquic_ver_tag_t ver_tag; 944 enum lsquic_version version; 945 946 if (!packet_in->pi_quic_ver) 947 { 948 LSQ_DEBUG("packet does not specify version"); 949 return VER_NOT_SPECIFIED; 950 } 951 952 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 953 version = lsquic_tag2ver(ver_tag); 954 if (version < N_LSQVER) 955 { 956 if (engine->pub.enp_settings.es_versions & (1 << version)) 957 { 958 LSQ_DEBUG("client-supplied version %s is supported", 959 lsquic_ver2str[version]); 960 *pversion = version; 961 return VER_SUPPORTED; 962 } 963 else 964 LSQ_DEBUG("client-supplied version %s is not supported", 965 lsquic_ver2str[version]); 966 } 967 else 968 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 969 ver_tag); 970 971 return VER_UNSUPPORTED; 972} 973 974 975static void 976schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 977 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 978 const struct sockaddr *sa_peer, void *peer_ctx) 979{ 980 assert(engine->pr_queue); 981 if (0 == prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 982 sa_local, sa_peer)) 983 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 984 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 985 else 986 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 987} 988 989 990static unsigned short 991sa2port (const struct sockaddr *sa) 992{ 993 if (sa->sa_family == AF_INET) 994 { 995 struct sockaddr_in *const sa4 = (void *) sa; 996 return sa4->sin_port; 997 } 998 else 999 { 1000 struct sockaddr_in6 *const sa6 = (void *) sa; 1001 return sa6->sin6_port; 1002 } 1003} 1004 1005 1006static struct lsquic_hash_elem * 1007find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 1008{ 1009 unsigned short port; 1010 1011 port = sa2port(sa); 1012 return lsquic_hash_find(hash, &port, sizeof(port)); 1013} 1014 1015 1016static lsquic_conn_t * 1017find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1018 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 1019{ 1020 struct lsquic_hash_elem *el; 1021 lsquic_conn_t *conn; 1022 1023 if (engine->flags & ENG_CONNS_BY_ADDR) 1024 el = find_conn_by_addr(engine->conns_hash, sa_local); 1025 else if (packet_in->pi_flags & PI_CONN_ID) 1026 el = lsquic_hash_find(engine->conns_hash, 1027 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1028 else 1029 { 1030 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1031 return NULL; 1032 } 1033 1034 if (!el) 1035 return NULL; 1036 1037 conn = lsquic_hashelem_getdata(el); 1038 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1039 if ((engine->flags & ENG_CONNS_BY_ADDR) 1040 && !(conn->cn_flags & LSCONN_IETF) 1041 && (packet_in->pi_flags & PI_CONN_ID) 1042 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1043 { 1044 LSQ_DEBUG("connection IDs do not match"); 1045 return NULL; 1046 } 1047 1048 return conn; 1049} 1050 1051 1052static lsquic_conn_t * 1053find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1054 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1055 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1056{ 1057 struct lsquic_hash_elem *el; 1058 struct purga_el *puel; 1059 lsquic_conn_t *conn; 1060 1061 if (!(packet_in->pi_flags & PI_CONN_ID)) 1062 { 1063 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1064 return NULL; 1065 } 1066 el = lsquic_hash_find(engine->conns_hash, 1067 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1068 1069 if (el) 1070 { 1071 conn = lsquic_hashelem_getdata(el); 1072 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1073 return conn; 1074 } 1075 1076 if (engine->flags & ENG_COOLDOWN) 1077 { /* Do not create incoming connections during cooldown */ 1078 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1079 return NULL; 1080 } 1081 1082 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1083 { 1084 LSQ_DEBUG("reached limit of %u inchoate connections", 1085 engine->pub.enp_settings.es_max_inchoate); 1086 return NULL; 1087 } 1088 1089 1090 if (engine->purga 1091 && (puel = lsquic_purga_contains(engine->purga, 1092 &packet_in->pi_conn_id), puel)) 1093 { 1094 switch (puel->puel_type) 1095 { 1096 case PUTY_CID_RETIRED: 1097 LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet", 1098 CID_BITS(&packet_in->pi_conn_id)); 1099 return NULL; 1100 case PUTY_CONN_DRAIN: 1101 LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64, 1102 puel->puel_time, packet_in->pi_received); 1103 if (puel->puel_time > packet_in->pi_received) 1104 { 1105 LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet", 1106 CID_BITS(&packet_in->pi_conn_id)); 1107 return NULL; 1108 } 1109 LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted", 1110 CID_BITS(&packet_in->pi_conn_id)); 1111 puel->puel_type = PUTY_CONN_DELETED; 1112 puel->puel_count = 0; 1113 puel->puel_time = 0; 1114 /* fall-through */ 1115 case PUTY_CONN_DELETED: 1116 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1117 CID_BITS(&packet_in->pi_conn_id)); 1118 if (puel->puel_time < packet_in->pi_received) 1119 { 1120 puel->puel_time = packet_in->pi_received 1121 /* Exponential back-off */ 1122 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1123 ++puel->puel_count; 1124 goto maybe_send_prst; 1125 } 1126 return NULL; 1127 default: 1128 assert(0); 1129 return NULL; 1130 } 1131 } 1132 1133 if (engine->pub.enp_settings.es_send_prst 1134 && !(packet_in->pi_flags & PI_GQUIC) 1135 && HETY_NOT_SET == packet_in->pi_header_type) 1136 goto maybe_send_prst; 1137 1138 if (0 != maybe_grow_conn_heaps(engine)) 1139 return NULL; 1140 1141 const struct parse_funcs *pf; 1142 enum lsquic_version version; 1143 switch (version_matches(engine, packet_in, &version)) 1144 { 1145 case VER_UNSUPPORTED: 1146 if (engine->flags & ENG_SERVER) 1147 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1148 sa_local, sa_peer, peer_ctx); 1149 return NULL; 1150 case VER_NOT_SPECIFIED: 1151 maybe_send_prst: 1152 if ((engine->flags & ENG_SERVER) && 1153 engine->pub.enp_settings.es_send_prst) 1154 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1155 sa_local, sa_peer, peer_ctx); 1156 return NULL; 1157 case VER_SUPPORTED: 1158 pf = select_pf_by_ver(version); 1159 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1160 break; 1161 } 1162 1163 1164 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1165 { 1166 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1167 sa_peer->sa_family == AF_INET, NULL, packet_in_size); 1168 } 1169 else 1170 { 1171 conn = mini_conn_new(&engine->pub, packet_in, version); 1172 } 1173 if (!conn) 1174 return NULL; 1175 ++engine->mini_conns_count; 1176 ++engine->n_conns; 1177 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1178 { 1179 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1180 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1181 CID_BITS(cid)); 1182 destroy_conn(engine, conn, packet_in->pi_received); 1183 return NULL; 1184 } 1185 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1186 conn->cn_flags |= LSCONN_HASHED; 1187 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1188 conn->cn_last_sent = engine->last_sent; 1189 return conn; 1190} 1191 1192 1193lsquic_conn_t * 1194lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1195 const lsquic_cid_t *cid) 1196{ 1197 struct lsquic_hash_elem *el; 1198 lsquic_conn_t *conn = NULL; 1199 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1200 1201 if (el) 1202 conn = lsquic_hashelem_getdata(el); 1203 return conn; 1204} 1205 1206 1207#if !defined(NDEBUG) && __GNUC__ 1208__attribute__((weak)) 1209#endif 1210void 1211lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1212 lsquic_conn_t *conn) 1213{ 1214 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1215 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1216 { 1217 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1218 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1219 engine_incref_conn(conn, LSCONN_TICKABLE); 1220 } 1221} 1222 1223 1224void 1225lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1226 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1227{ 1228 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1229 if (conn->cn_flags & LSCONN_TICKABLE) 1230 { 1231 /* Optimization: no need to add the connection to the Advisory Tick 1232 * Time Queue: it is about to be ticked, after which it its next tick 1233 * time may be queried again. 1234 */; 1235 } 1236 else if (conn->cn_flags & LSCONN_ATTQ) 1237 { 1238 if (lsquic_conn_adv_time(conn) != tick_time) 1239 { 1240 attq_remove(engine->attq, conn); 1241 if (0 != attq_add(engine->attq, conn, tick_time, why)) 1242 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1243 } 1244 } 1245 else if (0 == attq_add(engine->attq, conn, tick_time, why)) 1246 engine_incref_conn(conn, LSCONN_ATTQ); 1247} 1248 1249 1250static struct lsquic_conn * 1251find_conn_by_srst (struct lsquic_engine *engine, 1252 const struct lsquic_packet_in *packet_in) 1253{ 1254 struct lsquic_hash_elem *el; 1255 struct lsquic_conn *conn; 1256 1257 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1258 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1259 return NULL; 1260 1261 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1262 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1263 IQUIC_SRESET_TOKEN_SZ); 1264 if (!el) 1265 return NULL; 1266 1267 conn = lsquic_hashelem_getdata(el); 1268 return conn; 1269} 1270 1271 1272/* Return 0 if packet is being processed by a real connection (mini or full), 1273 * otherwise return 1. 1274 */ 1275static int 1276process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1277 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1278 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1279{ 1280 lsquic_conn_t *conn; 1281 const unsigned char *packet_in_data; 1282 1283 if (lsquic_packet_in_is_gquic_prst(packet_in) 1284 && !engine->pub.enp_settings.es_honor_prst) 1285 { 1286 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1287 LSQ_DEBUG("public reset packet: discarding"); 1288 return 1; 1289 } 1290 1291 if (engine->flags & ENG_SERVER) 1292 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1293 sa_peer, peer_ctx, packet_in_size); 1294 else 1295 conn = find_conn(engine, packet_in, ppstate, sa_local); 1296 1297 if (!conn) 1298 { 1299 if (engine->pub.enp_settings.es_honor_prst 1300 && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */ 1301 && !(packet_in->pi_flags & PI_GQUIC) 1302 && engine->pub.enp_srst_hash 1303 && (conn = find_conn_by_srst(engine, packet_in))) 1304 { 1305 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1306 CID_BITS(lsquic_conn_log_cid(conn))); 1307 conn->cn_if->ci_stateless_reset(conn); 1308 if (!(conn->cn_flags & LSCONN_TICKABLE) 1309 && conn->cn_if->ci_is_tickable(conn)) 1310 { 1311 lsquic_mh_insert(&engine->conns_tickable, conn, 1312 conn->cn_last_ticked); 1313 engine_incref_conn(conn, LSCONN_TICKABLE); 1314 } 1315 /* Even though the connection processes this packet, we return 1316 * 1 so that the caller does not add reset packet's random 1317 * bytes to the list of valid CIDs. 1318 */ 1319 } 1320 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1321 return 1; 1322 } 1323 1324 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1325 { 1326 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1327 engine_incref_conn(conn, LSCONN_TICKABLE); 1328 } 1329 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1330 sa_local, sa_peer); 1331 lsquic_packet_in_upref(packet_in); 1332#if LOG_PACKET_CHECKSUM 1333 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1334 packet_in->pi_data_sz); 1335#endif 1336 /* Note on QLog: 1337 * For the PACKET_RX QLog event, we are interested in logging these things: 1338 * - raw packet (however it comes in, encrypted or not) 1339 * - frames (list of frame names) 1340 * - packet type and number 1341 * - packet rx timestamp 1342 * 1343 * Since only some of these items are available at this code 1344 * juncture, we will wait until after the packet has been 1345 * decrypted (if necessary) and parsed to call the log functions. 1346 * 1347 * Once the PACKET_RX event is finally logged, the timestamp 1348 * will come from packet_in->pi_received. For correct sequential 1349 * ordering of QLog events, be sure to process the QLogs downstream. 1350 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1351 */ 1352 packet_in_data = packet_in->pi_data; 1353 packet_in_size = packet_in->pi_data_sz; 1354 conn->cn_if->ci_packet_in(conn, packet_in); 1355 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1356 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1357 return 0; 1358} 1359 1360 1361void 1362lsquic_engine_destroy (lsquic_engine_t *engine) 1363{ 1364 struct lsquic_hash_elem *el; 1365 lsquic_conn_t *conn; 1366 1367 LSQ_DEBUG("destroying engine"); 1368#ifndef NDEBUG 1369 engine->flags |= ENG_DTOR; 1370#endif 1371 1372 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1373 { 1374 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1375 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1376 } 1377 1378 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1379 { 1380 assert(conn->cn_flags & LSCONN_TICKABLE); 1381 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1382 } 1383 1384 for (el = lsquic_hash_first(engine->conns_hash); el; 1385 el = lsquic_hash_next(engine->conns_hash)) 1386 { 1387 conn = lsquic_hashelem_getdata(el); 1388 force_close_conn(engine, conn); 1389 } 1390 lsquic_hash_destroy(engine->conns_hash); 1391 1392 assert(0 == engine->n_conns); 1393 assert(0 == engine->mini_conns_count); 1394 if (engine->pr_queue) 1395 prq_destroy(engine->pr_queue); 1396 if (engine->purga) 1397 lsquic_purga_destroy(engine->purga); 1398 attq_destroy(engine->attq); 1399 1400 assert(0 == lsquic_mh_count(&engine->conns_out)); 1401 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1402 if (engine->pub.enp_shi == &stock_shi) 1403 stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1404 lsquic_mm_cleanup(&engine->pub.enp_mm); 1405 free(engine->conns_tickable.mh_elems); 1406#ifndef NDEBUG 1407 if (engine->flags & ENG_LOSE_PACKETS) 1408 regfree(&engine->lose_packets_re); 1409#endif 1410 if (engine->pub.enp_tokgen) 1411 lsquic_tg_destroy(engine->pub.enp_tokgen); 1412#if LSQUIC_CONN_STATS 1413 if (engine->stats_fh) 1414 { 1415 const struct conn_stats *const stats = &engine->conn_stats_sum; 1416 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1417 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1418 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1419 fprintf(engine->stats_fh, "In:\n"); 1420 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1421 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1422 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1423 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1424 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1425 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1426 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1427 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1428 stats->in.headers_comp, stats->in.headers_uncomp, 1429 stats->in.headers_uncomp ? 1430 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1431 : 0); 1432 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1433 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1434 fprintf(engine->stats_fh, " ACK frames merged to new: %lu\n", stats->in.n_acks_merged[0]); 1435 fprintf(engine->stats_fh, " ACK frames merged to old: %lu\n", stats->in.n_acks_merged[1]); 1436 fprintf(engine->stats_fh, "Out:\n"); 1437 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1438 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1439 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1440 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1441 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1442 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1443 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1444 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1445 stats->out.headers_comp, stats->out.headers_uncomp, 1446 stats->out.headers_uncomp ? 1447 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1448 : 0); 1449 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1450 } 1451#endif 1452 if (engine->pub.enp_srst_hash) 1453 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1454#if LSQUIC_COUNT_ENGINE_CALLS 1455 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1456#endif 1457 if (engine->pub.enp_retry_aead_ctx) 1458 EVP_AEAD_CTX_cleanup(engine->pub.enp_retry_aead_ctx); 1459 free(engine); 1460} 1461 1462 1463static struct conn_cid_elem * 1464find_free_cce (struct lsquic_conn *conn) 1465{ 1466 struct conn_cid_elem *cce; 1467 1468 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1469 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1470 return cce; 1471 1472 return NULL; 1473} 1474 1475 1476static int 1477add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1478 const struct sockaddr *local_sa, void *peer_ctx) 1479{ 1480 struct conn_cid_elem *cce; 1481 1482 if (engine->flags & ENG_CONNS_BY_ADDR) 1483 { 1484 cce = find_free_cce(conn); 1485 if (!cce) 1486 { 1487 LSQ_ERROR("cannot find free CCE"); 1488 return -1; 1489 } 1490 cce->cce_port = sa2port(local_sa); 1491 cce->cce_flags = CCE_PORT; 1492 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1493 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1494 { 1495 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1496 return 0; 1497 } 1498 else 1499 return -1; 1500 1501 } 1502 else 1503 return insert_conn_into_hash(engine, conn, peer_ctx); 1504} 1505 1506 1507lsquic_conn_t * 1508lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1509 const struct sockaddr *local_sa, 1510 const struct sockaddr *peer_sa, 1511 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1512 const char *hostname, unsigned short max_packet_size, 1513 const unsigned char *zero_rtt, size_t zero_rtt_len, 1514 const unsigned char *token, size_t token_sz) 1515{ 1516 lsquic_conn_t *conn; 1517 unsigned flags, versions; 1518 int is_ipv4; 1519 1520 ENGINE_IN(engine); 1521 1522 if (engine->flags & ENG_SERVER) 1523 { 1524 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1525 goto err; 1526 } 1527 1528 if (engine->flags & ENG_CONNS_BY_ADDR 1529 && find_conn_by_addr(engine->conns_hash, local_sa)) 1530 { 1531 LSQ_ERROR("cannot have more than one connection on the same port"); 1532 goto err; 1533 } 1534 1535 if (0 != maybe_grow_conn_heaps(engine)) 1536 return NULL; 1537 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1538 is_ipv4 = peer_sa->sa_family == AF_INET; 1539 if (zero_rtt && zero_rtt_len) 1540 { 1541 version = lsquic_zero_rtt_version(zero_rtt, zero_rtt_len); 1542 if (version >= N_LSQVER) 1543 { 1544 LSQ_INFO("zero-rtt version is bad, won't use"); 1545 zero_rtt = NULL; 1546 zero_rtt_len = 0; 1547 } 1548 } 1549 if (version >= N_LSQVER) 1550 { 1551 if (version > N_LSQVER) 1552 LSQ_WARN("invalid version specified, engine will pick"); 1553 versions = engine->pub.enp_settings.es_versions; 1554 } 1555 else 1556 versions = 1u << version; 1557 if (versions & LSQUIC_IETF_VERSIONS) 1558 { 1559 if (version == LSQVER_ID24) 1560 conn = lsquic_id24_full_conn_client_new(&engine->pub, versions, 1561 flags, hostname, max_packet_size, 1562 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1563 else 1564 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1565 flags, hostname, max_packet_size, 1566 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1567 } 1568 else 1569 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1570 flags, hostname, max_packet_size, is_ipv4, 1571 zero_rtt, zero_rtt_len); 1572 if (!conn) 1573 goto err; 1574 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1575 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1576 lsquic_ver2str[conn->cn_version]); 1577 ++engine->n_conns; 1578 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1579 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1580 { 1581 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1582 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1583 CID_BITS(cid)); 1584 destroy_conn(engine, conn, lsquic_time_now()); 1585 goto err; 1586 } 1587 assert(!(conn->cn_flags & 1588 (CONN_REF_FLAGS 1589 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1590 callbacks */ 1591 ))); 1592 conn->cn_flags |= LSCONN_HASHED; 1593 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1594 engine_incref_conn(conn, LSCONN_TICKABLE); 1595 lsquic_conn_set_ctx(conn, conn_ctx); 1596 conn->cn_if->ci_client_call_on_new(conn); 1597 end: 1598 ENGINE_OUT(engine); 1599 return conn; 1600 err: 1601 conn = NULL; 1602 goto end; 1603} 1604 1605 1606static void 1607remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1608{ 1609 remove_all_cces_from_hash(engine->conns_hash, conn); 1610 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1611} 1612 1613 1614static void 1615refflags2str (enum lsquic_conn_flags flags, char s[6]) 1616{ 1617 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1618 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1619 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1620 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1621 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1622 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1623 *s = '\0'; 1624} 1625 1626 1627static void 1628engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1629{ 1630 char str[2][7]; 1631 assert(flag & CONN_REF_FLAGS); 1632 assert(!(conn->cn_flags & flag)); 1633 conn->cn_flags |= flag; 1634 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1635 CID_BITS(lsquic_conn_log_cid(conn)), 1636 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1637 (refflags2str(conn->cn_flags, str[1]), str[1])); 1638} 1639 1640 1641static lsquic_conn_t * 1642engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1643 enum lsquic_conn_flags flags) 1644{ 1645 char str[2][7]; 1646 lsquic_time_t now; 1647 assert(flags & CONN_REF_FLAGS); 1648 assert(conn->cn_flags & flags); 1649#ifndef NDEBUG 1650 if (flags & LSCONN_CLOSING) 1651 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1652#endif 1653 conn->cn_flags &= ~flags; 1654 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1655 CID_BITS(lsquic_conn_log_cid(conn)), 1656 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1657 (refflags2str(conn->cn_flags, str[1]), str[1])); 1658 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1659 { 1660 now = lsquic_time_now(); 1661 if (conn->cn_flags & LSCONN_MINI) 1662 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1663 else 1664 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1665 destroy_conn(engine, conn, now); 1666 return NULL; 1667 } 1668 else 1669 return conn; 1670} 1671 1672 1673/* This is not a general-purpose function. Only call from engine dtor. */ 1674static void 1675force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1676{ 1677 assert(engine->flags & ENG_DTOR); 1678 const enum lsquic_conn_flags flags = conn->cn_flags; 1679 assert(conn->cn_flags & CONN_REF_FLAGS); 1680 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1681 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1682 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1683 if (flags & LSCONN_ATTQ) 1684 { 1685 attq_remove(engine->attq, conn); 1686 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1687 } 1688 if (flags & LSCONN_HASHED) 1689 remove_conn_from_hash(engine, conn); 1690} 1691 1692 1693/* Iterator for tickable connections (those on the Tickable Queue). Before 1694 * a connection is returned, it is removed from the Advisory Tick Time queue 1695 * if necessary. 1696 */ 1697static lsquic_conn_t * 1698conn_iter_next_tickable (struct lsquic_engine *engine) 1699{ 1700 lsquic_conn_t *conn; 1701 1702 if (engine->flags & ENG_SERVER) 1703 while (1) 1704 { 1705 conn = lsquic_mh_pop(&engine->conns_tickable); 1706 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1707 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1708 else 1709 break; 1710 } 1711 else 1712 conn = lsquic_mh_pop(&engine->conns_tickable); 1713 1714 if (conn) 1715 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1716 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1717 { 1718 attq_remove(engine->attq, conn); 1719 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1720 } 1721 1722 return conn; 1723} 1724 1725 1726static void 1727cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1728 void *update_ctx) 1729{ 1730 cub->cub_update_cids = update; 1731 cub->cub_update_ctx = update_ctx; 1732 cub->cub_count = 0; 1733} 1734 1735 1736static void 1737cub_flush (struct cid_update_batch *cub) 1738{ 1739 if (cub->cub_count > 0 && cub->cub_update_cids) 1740 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1741 cub->cub_cids, cub->cub_count); 1742 cub->cub_count = 0; 1743} 1744 1745 1746static void 1747cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1748{ 1749 cub->cub_cids [ cub->cub_count ] = *cid; 1750 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1751 ++cub->cub_count; 1752 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1753 cub_flush(cub); 1754} 1755 1756 1757/* Process registered CIDs */ 1758static void 1759cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1760{ 1761 struct cce_cid_iter citer; 1762 struct conn_cid_elem *cce; 1763 void *peer_ctx; 1764 1765 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1766 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1767 if (cce->cce_flags & CCE_REG) 1768 cub_add(cub, &cce->cce_cid, peer_ctx); 1769} 1770 1771 1772static void 1773drop_all_mini_conns (lsquic_engine_t *engine) 1774{ 1775 struct lsquic_hash_elem *el; 1776 lsquic_conn_t *conn; 1777 struct cid_update_batch cub; 1778 1779 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1780 1781 for (el = lsquic_hash_first(engine->conns_hash); el; 1782 el = lsquic_hash_next(engine->conns_hash)) 1783 { 1784 conn = lsquic_hashelem_getdata(el); 1785 if (conn->cn_flags & LSCONN_MINI) 1786 { 1787 /* If promoted, why is it still in this hash? */ 1788 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1789 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1790 cub_add_cids_from_cces(&cub, conn); 1791 remove_conn_from_hash(engine, conn); 1792 } 1793 } 1794 1795 cub_flush(&cub); 1796} 1797 1798 1799void 1800lsquic_engine_process_conns (lsquic_engine_t *engine) 1801{ 1802 lsquic_conn_t *conn; 1803 lsquic_time_t now; 1804 1805 ENGINE_IN(engine); 1806 1807 now = lsquic_time_now(); 1808 while ((conn = attq_pop(engine->attq, now))) 1809 { 1810 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1811 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 1812 { 1813 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1814 engine_incref_conn(conn, LSCONN_TICKABLE); 1815 } 1816 } 1817 1818 process_connections(engine, conn_iter_next_tickable, now); 1819 ENGINE_OUT(engine); 1820} 1821 1822 1823static void 1824release_or_return_enc_data (struct lsquic_engine *engine, 1825 void (*pmi_rel_or_ret) (void *, void *, void *, char), 1826 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 1827{ 1828 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1829 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 1830 packet_out->po_flags &= ~PO_ENCRYPTED; 1831 packet_out->po_enc_data = NULL; 1832} 1833 1834 1835static void 1836release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1837 struct lsquic_packet_out *packet_out) 1838{ 1839 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 1840 conn, packet_out); 1841} 1842 1843 1844static void 1845return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1846 struct lsquic_packet_out *packet_out) 1847{ 1848 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 1849 conn, packet_out); 1850} 1851 1852 1853static int 1854copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 1855 struct lsquic_packet_out *packet_out) 1856{ 1857 int ipv6; 1858 1859 ipv6 = NP_IS_IPv6(packet_out->po_path); 1860 if (packet_out->po_flags & PO_ENCRYPTED) 1861 { 1862 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1863 && packet_out->po_data_sz == packet_out->po_enc_data_sz 1864 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 1865 packet_out->po_data_sz)) 1866 return 0; 1867 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1868 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 1869 goto copy; 1870 return_enc_data(engine, conn, packet_out); 1871 } 1872 1873 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 1874 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1875 packet_out->po_data_sz, ipv6); 1876 if (!packet_out->po_enc_data) 1877 { 1878 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 1879 "of size %hu", packet_out->po_data_sz); 1880 return -1; 1881 } 1882 1883 copy: 1884 memcpy(packet_out->po_enc_data, packet_out->po_data, 1885 packet_out->po_data_sz); 1886 packet_out->po_enc_data_sz = packet_out->po_data_sz; 1887 packet_out->po_sent_sz = packet_out->po_data_sz; 1888 packet_out->po_flags &= ~PO_IPv6; 1889 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 1890 1891 return 0; 1892} 1893 1894 1895STAILQ_HEAD(conns_stailq, lsquic_conn); 1896TAILQ_HEAD(conns_tailq, lsquic_conn); 1897 1898 1899struct conns_out_iter 1900{ 1901 struct min_heap *coi_heap; 1902 struct pr_queue *coi_prq; 1903 TAILQ_HEAD(, lsquic_conn) coi_active_list, 1904 coi_inactive_list; 1905 lsquic_conn_t *coi_next; 1906#ifndef NDEBUG 1907 lsquic_time_t coi_last_sent; 1908#endif 1909}; 1910 1911 1912static void 1913coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 1914{ 1915 iter->coi_heap = &engine->conns_out; 1916 iter->coi_prq = engine->pr_queue; 1917 iter->coi_next = NULL; 1918 TAILQ_INIT(&iter->coi_active_list); 1919 TAILQ_INIT(&iter->coi_inactive_list); 1920#ifndef NDEBUG 1921 iter->coi_last_sent = 0; 1922#endif 1923} 1924 1925 1926static lsquic_conn_t * 1927coi_next (struct conns_out_iter *iter) 1928{ 1929 lsquic_conn_t *conn; 1930 1931 if (lsquic_mh_count(iter->coi_heap) > 0) 1932 { 1933 conn = lsquic_mh_pop(iter->coi_heap); 1934 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1935 conn->cn_flags |= LSCONN_COI_ACTIVE; 1936#ifndef NDEBUG 1937 if (iter->coi_last_sent) 1938 assert(iter->coi_last_sent <= conn->cn_last_sent); 1939 iter->coi_last_sent = conn->cn_last_sent; 1940#endif 1941 return conn; 1942 } 1943 else if (iter->coi_prq && (conn = prq_next_conn(iter->coi_prq))) 1944 { 1945 return conn; 1946 } 1947 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 1948 { 1949 iter->coi_prq = NULL; /* Save function call in previous conditional */ 1950 conn = iter->coi_next; 1951 if (!conn) 1952 conn = TAILQ_FIRST(&iter->coi_active_list); 1953 if (conn) 1954 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 1955 return conn; 1956 } 1957 else 1958 return NULL; 1959} 1960 1961 1962static void 1963coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1964{ 1965 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 1966 { 1967 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 1968 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1969 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1970 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 1971 conn->cn_flags |= LSCONN_COI_INACTIVE; 1972 } 1973} 1974 1975 1976static void 1977coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1978{ 1979 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 1980 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 1981 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 1982 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1983 conn->cn_flags |= LSCONN_COI_ACTIVE; 1984} 1985 1986 1987static void 1988coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 1989{ 1990 lsquic_conn_t *conn; 1991 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 1992 { 1993 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1994 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1995 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 1996 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 1997 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 1998 else /* Closed connection gets one shot at sending packets */ 1999 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2000 } 2001 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 2002 { 2003 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2004 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2005 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2006 } 2007} 2008 2009 2010#ifndef NDEBUG 2011static void 2012lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 2013 unsigned n) 2014{ 2015 const lsquic_cid_t *cid; 2016 struct iovec *iov; 2017 unsigned i; 2018 char packno_str[22]; 2019 2020 for (i = 0; i < n; ++i) 2021 { 2022 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 2023 batch->packets[i]->po_packno); 2024 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 2025 { 2026 for (iov = batch->outs[i].iov; iov < 2027 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 2028 batch->outs[i].iov->iov_len -= 1; 2029 cid = lsquic_conn_log_cid(batch->conns[i]); 2030 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 2031 CID_BITS(cid)); 2032 } 2033 } 2034} 2035 2036 2037#endif 2038 2039 2040#ifdef NDEBUG 2041#define CONST_BATCH const 2042#else 2043#define CONST_BATCH 2044#endif 2045 2046 2047struct send_batch_ctx { 2048 struct conns_stailq *closed_conns; 2049 struct conns_tailq *ticked_conns; 2050 struct conns_out_iter *conns_iter; 2051 CONST_BATCH struct out_batch *batch; 2052}; 2053 2054 2055static void 2056close_conn_immediately (struct lsquic_engine *engine, 2057 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2058{ 2059 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2060 if (!(conn->cn_flags & LSCONN_CLOSING)) 2061 { 2062 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2063 engine_incref_conn(conn, LSCONN_CLOSING); 2064 if (conn->cn_flags & LSCONN_HASHED) 2065 remove_conn_from_hash(engine, conn); 2066 } 2067 if (conn->cn_flags & LSCONN_TICKED) 2068 { 2069 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2070 engine_decref_conn(engine, conn, LSCONN_TICKED); 2071 } 2072} 2073 2074 2075static void 2076close_conn_on_send_error (struct lsquic_engine *engine, 2077 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2078{ 2079 const struct out_batch *batch = sb_ctx->batch; 2080 struct lsquic_conn *const conn = batch->conns[n]; 2081 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2082 2083 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2084 "src: %s; dst: %s; errno: %d", 2085 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2086 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2087 CID_BITS(lsquic_conn_log_cid(conn)), 2088 SA2STR(batch->outs[n].local_sa, buf[0]), 2089 SA2STR(batch->outs[n].dest_sa, buf[1]), 2090 e_val); 2091 if (conn->cn_flags & LSCONN_EVANESCENT) 2092 lsquic_prq_drop(conn); 2093 else 2094 close_conn_immediately(engine, sb_ctx, conn); 2095} 2096 2097 2098static unsigned 2099send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2100 unsigned n_to_send) 2101{ 2102 int n_sent, i, e_val; 2103 lsquic_time_t now; 2104 unsigned off; 2105 size_t count; 2106 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2107 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2108 2109#ifndef NDEBUG 2110 if (engine->flags & ENG_LOSE_PACKETS) 2111 lose_matching_packets(engine, batch, n_to_send); 2112#endif 2113 /* Set sent time before the write to avoid underestimating RTT */ 2114 now = lsquic_time_now(); 2115 for (i = 0; i < (int) n_to_send; ++i) 2116 { 2117 off = batch->pack_off[i]; 2118 count = batch->outs[i].iovlen; 2119 assert(count > 0); 2120 packet_out = &batch->packets[off]; 2121 end = packet_out + count; 2122 do 2123 (*packet_out)->po_sent = now; 2124 while (++packet_out < end); 2125 } 2126 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs, 2127 n_to_send); 2128 e_val = errno; 2129 if (n_sent < (int) n_to_send) 2130 { 2131 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2132 engine->resume_sending_at = now + 1000000; 2133 LSQ_DEBUG("cannot send packets"); 2134 EV_LOG_GENERIC_EVENT("cannot send packets"); 2135 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2136 close_conn_on_send_error(engine, sb_ctx, 2137 n_sent < 0 ? 0 : n_sent, e_val); 2138 } 2139 if (n_sent >= 0) 2140 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, n_to_send); 2141 else 2142 { 2143 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2144 n_sent = 0; 2145 } 2146 if (n_sent > 0) 2147 engine->last_sent = now + n_sent; 2148 for (i = 0; i < n_sent; ++i) 2149 { 2150 eng_hist_inc(&engine->history, now, sl_packets_out); 2151 /* `i' is added to maintain relative order */ 2152 batch->conns[i]->cn_last_sent = now + i; 2153 2154 off = batch->pack_off[i]; 2155 count = batch->outs[i].iovlen; 2156 assert(count > 0); 2157 packet_out = &batch->packets[off]; 2158 end = packet_out + count; 2159 do 2160 { 2161#if LOG_PACKET_CHECKSUM 2162 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2163 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2164 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2165#endif 2166 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2167 *packet_out); 2168 /* Release packet out buffer as soon as the packet is sent 2169 * successfully. If not successfully sent, we hold on to 2170 * this buffer until the packet sending is attempted again 2171 * or until it times out and regenerated. 2172 */ 2173 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2174 release_enc_data(engine, batch->conns[i], *packet_out); 2175 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2176 *packet_out); 2177 } 2178 while (++packet_out < end); 2179 } 2180 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2181 for ( ; i < (int) n_to_send; ++i) 2182 { 2183 off = batch->pack_off[i]; 2184 count = batch->outs[i].iovlen; 2185 assert(count > 0); 2186 packet_out = &batch->packets[off]; 2187 end = packet_out + count; 2188 do 2189 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2190 *packet_out); 2191 while (++packet_out < end); 2192 } 2193 /* Return packets to the connection in reverse order so that the packet 2194 * ordering is maintained. 2195 */ 2196 for (i = (int) n_to_send - 1; i >= n_sent; --i) 2197 { 2198 off = batch->pack_off[i]; 2199 count = batch->outs[i].iovlen; 2200 assert(count > 0); 2201 packet_out = &batch->packets[off + count - 1]; 2202 end = &batch->packets[off - 1]; 2203 do 2204 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2205 *packet_out); 2206 while (--packet_out > end); 2207 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2208 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2209 } 2210 return n_sent; 2211} 2212 2213 2214/* Return 1 if went past deadline, 0 otherwise */ 2215static int 2216check_deadline (lsquic_engine_t *engine) 2217{ 2218 if (engine->pub.enp_settings.es_proc_time_thresh && 2219 lsquic_time_now() > engine->deadline) 2220 { 2221 LSQ_INFO("went past threshold of %u usec, stop sending", 2222 engine->pub.enp_settings.es_proc_time_thresh); 2223 engine->flags |= ENG_PAST_DEADLINE; 2224 return 1; 2225 } 2226 else 2227 return 0; 2228} 2229 2230 2231static size_t 2232iov_size (const struct iovec *iov, const struct iovec *const end) 2233{ 2234 size_t size; 2235 2236 assert(iov < end); 2237 2238 size = 0; 2239 do 2240 size += iov->iov_len; 2241 while (++iov < end); 2242 2243 return size; 2244} 2245 2246 2247static void 2248send_packets_out (struct lsquic_engine *engine, 2249 struct conns_tailq *ticked_conns, 2250 struct conns_stailq *closed_conns) 2251{ 2252 unsigned n, w, n_sent, n_batches_sent; 2253 lsquic_packet_out_t *packet_out; 2254 struct lsquic_packet_out **packet; 2255 lsquic_conn_t *conn; 2256 struct out_batch *const batch = &engine->out_batch; 2257 struct iovec *iov, *packet_iov; 2258 struct conns_out_iter conns_iter; 2259 int shrink, deadline_exceeded; 2260 const struct send_batch_ctx sb_ctx = { 2261 closed_conns, 2262 ticked_conns, 2263 &conns_iter, 2264 &engine->out_batch, 2265 }; 2266 2267 coi_init(&conns_iter, engine); 2268 n_batches_sent = 0; 2269 n_sent = 0, n = 0; 2270 shrink = 0; 2271 deadline_exceeded = 0; 2272 iov = batch->iov; 2273 packet = batch->packets; 2274 2275 while ((conn = coi_next(&conns_iter))) 2276 { 2277 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2278 if (!packet_out) { 2279 /* Evanescent connection always has a packet to send: */ 2280 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2281 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2282 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2283 CID_BITS(lsquic_conn_log_cid(conn))); 2284 coi_deactivate(&conns_iter, conn); 2285 continue; 2286 } 2287 batch->outs[n].iov = packet_iov = iov; 2288 next_coa: 2289 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2290 { 2291 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2292 &engine->pub, conn, packet_out)) 2293 { 2294 case ENCPA_NOMEM: 2295 /* Send what we have and wait for a more opportune moment */ 2296 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2297 goto end_for; 2298 case ENCPA_BADCRYPT: 2299 /* This is pretty bad: close connection immediately */ 2300 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2301 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2302 CID_BITS(lsquic_conn_log_cid(conn))); 2303 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2304 { 2305 close_conn_immediately(engine, &sb_ctx, conn); 2306 coi_deactivate(&conns_iter, conn); 2307 } 2308 continue; 2309 case ENCPA_OK: 2310 break; 2311 } 2312 } 2313 else if ((packet_out->po_flags & PO_NOENCRYPT) 2314 && engine->pub.enp_pmi != &stock_pmi) 2315 { 2316 if (0 != copy_packet(engine, conn, packet_out)) 2317 { 2318 /* Copy can only fail if packet could not be allocated */ 2319 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2320 goto end_for; 2321 } 2322 } 2323 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2324 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2325 if (packet_out->po_flags & PO_ENCRYPTED) 2326 { 2327 iov->iov_base = packet_out->po_enc_data; 2328 iov->iov_len = packet_out->po_enc_data_sz; 2329 } 2330 else 2331 { 2332 iov->iov_base = packet_out->po_data; 2333 iov->iov_len = packet_out->po_data_sz; 2334 } 2335 if (packet_iov == iov) 2336 { 2337 batch->pack_off[n] = packet - batch->packets; 2338 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2339 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2340 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2341 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2342 batch->conns [n] = conn; 2343 } 2344 *packet = packet_out; 2345 ++packet; 2346 ++iov; 2347 if ((conn->cn_flags & LSCONN_IETF) 2348 && ((1 << packet_out->po_header_type) 2349 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2350#ifndef NDEBUG 2351 && (engine->flags & ENG_COALESCE) 2352#endif 2353 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2354 { 2355 const size_t size = iov_size(packet_iov, iov); 2356 packet_out = conn->cn_if->ci_next_packet_to_send(conn, size); 2357 if (packet_out) 2358 goto next_coa; 2359 } 2360 batch->outs [n].iovlen = iov - packet_iov; 2361 ++n; 2362 if (n == engine->batch_size 2363 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2364 { 2365 w = send_batch(engine, &sb_ctx, n); 2366 n = 0; 2367 iov = batch->iov; 2368 packet = batch->packets; 2369 ++n_batches_sent; 2370 n_sent += w; 2371 if (w < engine->batch_size) 2372 { 2373 shrink = 1; 2374 break; 2375 } 2376 deadline_exceeded = check_deadline(engine); 2377 if (deadline_exceeded) 2378 break; 2379 grow_batch_size(engine); 2380 } 2381 } 2382 end_for: 2383 2384 if (n > 0) { 2385 w = send_batch(engine, &sb_ctx, n); 2386 n_sent += w; 2387 shrink = w < n; 2388 ++n_batches_sent; 2389 } 2390 2391 if (shrink) 2392 shrink_batch_size(engine); 2393 else if (n_batches_sent > 1) 2394 { 2395 deadline_exceeded = check_deadline(engine); 2396 if (!deadline_exceeded) 2397 grow_batch_size(engine); 2398 } 2399 2400 coi_reheap(&conns_iter, engine); 2401 2402 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2403} 2404 2405 2406int 2407lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2408{ 2409 return lsquic_mh_count(&engine->conns_out) > 0 2410 || (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2411 ; 2412} 2413 2414 2415static void 2416reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2417{ 2418 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2419 engine->flags &= ~ENG_PAST_DEADLINE; 2420} 2421 2422 2423void 2424lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2425{ 2426 lsquic_conn_t *conn; 2427 struct conns_stailq closed_conns; 2428 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2429 struct cid_update_batch cub; 2430 2431 ENGINE_IN(engine); 2432 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2433 STAILQ_INIT(&closed_conns); 2434 reset_deadline(engine, lsquic_time_now()); 2435 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2436 { 2437 LSQ_DEBUG("can send again"); 2438 EV_LOG_GENERIC_EVENT("can send again"); 2439 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2440 } 2441 2442 send_packets_out(engine, &ticked_conns, &closed_conns); 2443 2444 while ((conn = STAILQ_FIRST(&closed_conns))) { 2445 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2446 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2447 cub_add_cids_from_cces(&cub, conn); 2448 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2449 } 2450 2451 cub_flush(&cub); 2452 ENGINE_OUT(engine); 2453} 2454 2455 2456static lsquic_conn_t * 2457next_new_full_conn (struct conns_stailq *new_full_conns) 2458{ 2459 lsquic_conn_t *conn; 2460 2461 conn = STAILQ_FIRST(new_full_conns); 2462 if (conn) 2463 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2464 return conn; 2465} 2466 2467 2468static void 2469process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2470 lsquic_time_t now) 2471{ 2472 lsquic_conn_t *conn; 2473 enum tick_st tick_st; 2474 unsigned i, why; 2475 lsquic_time_t next_tick_time; 2476 struct conns_stailq closed_conns; 2477 struct conns_tailq ticked_conns; 2478 struct conns_stailq new_full_conns; 2479 struct cid_update_batch cub_old, cub_live; 2480 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2481 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2482 2483 eng_hist_tick(&engine->history, now); 2484 2485 STAILQ_INIT(&closed_conns); 2486 TAILQ_INIT(&ticked_conns); 2487 reset_deadline(engine, now); 2488 STAILQ_INIT(&new_full_conns); 2489 2490 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2491 && now > engine->resume_sending_at) 2492 { 2493 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2494 "timeout"); 2495 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2496 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2497 } 2498 2499 i = 0; 2500 while ((conn = next_conn(engine)) 2501 || (conn = next_new_full_conn(&new_full_conns))) 2502 { 2503 tick_st = conn->cn_if->ci_tick(conn, now); 2504 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2505 if (tick_st & TICK_PROMOTE) 2506 { 2507 lsquic_conn_t *new_conn; 2508 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2509 "scheduled for promotion"); 2510 assert(conn->cn_flags & LSCONN_MINI); 2511 new_conn = new_full_conn_server(engine, conn, now); 2512 if (new_conn) 2513 { 2514 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2515 new_conn->cn_last_sent = engine->last_sent; 2516 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2517 conn->cn_flags |= LSCONN_PROMOTED; 2518 } 2519 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2520 } 2521 if (tick_st & TICK_SEND) 2522 { 2523 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2524 { 2525 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2526 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2527 } 2528 } 2529 if (tick_st & TICK_CLOSE) 2530 { 2531 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2532 engine_incref_conn(conn, LSCONN_CLOSING); 2533 if (conn->cn_flags & LSCONN_HASHED) 2534 remove_conn_from_hash(engine, conn); 2535 } 2536 else 2537 { 2538 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2539 engine_incref_conn(conn, LSCONN_TICKED); 2540 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2541 && conn->cn_if->ci_report_live(conn, now)) 2542 cub_add_cids_from_cces(&cub_live, conn); 2543 } 2544 } 2545 2546 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2547 && lsquic_engine_has_unsent_packets(engine)) 2548 send_packets_out(engine, &ticked_conns, &closed_conns); 2549 2550 while ((conn = STAILQ_FIRST(&closed_conns))) { 2551 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2552 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2553 cub_add_cids_from_cces(&cub_old, conn); 2554 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2555 } 2556 2557 while ((conn = TAILQ_FIRST(&ticked_conns))) 2558 { 2559 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2560 engine_decref_conn(engine, conn, LSCONN_TICKED); 2561 if (!(conn->cn_flags & LSCONN_TICKABLE) 2562 && conn->cn_if->ci_is_tickable(conn)) 2563 { 2564 /* Floyd heapification is not faster, don't bother. */ 2565 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2566 engine_incref_conn(conn, LSCONN_TICKABLE); 2567 } 2568 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2569 { 2570 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2571 if (next_tick_time) 2572 { 2573 if (0 == attq_add(engine->attq, conn, next_tick_time, why)) 2574 engine_incref_conn(conn, LSCONN_ATTQ); 2575 } 2576 else 2577 /* In all other cases, the idle timeout would make the next 2578 * tick time non-zero: 2579 */ 2580 assert((conn->cn_flags & LSCONN_IETF) 2581 && engine->pub.enp_settings.es_idle_timeout == 0); 2582 } 2583 } 2584 2585 cub_flush(&engine->new_scids); 2586 cub_flush(&cub_live); 2587 cub_flush(&cub_old); 2588} 2589 2590 2591/* Return 0 if packet is being processed by a real connection, 1 if the 2592 * packet was processed, but not by a connection, and -1 on error. 2593 */ 2594int 2595lsquic_engine_packet_in (lsquic_engine_t *engine, 2596 const unsigned char *packet_in_data, size_t packet_in_size, 2597 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2598 void *peer_ctx, int ecn) 2599{ 2600 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2601 struct packin_parse_state ppstate; 2602 lsquic_packet_in_t *packet_in; 2603 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2604 int is_server, unsigned cid_len, struct packin_parse_state *); 2605 unsigned n_zeroes; 2606 int s; 2607 2608 ENGINE_CALLS_INCR(engine); 2609 2610 if (engine->flags & ENG_SERVER) 2611 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2612 else if (engine->flags & ENG_CONNS_BY_ADDR) 2613 { 2614 struct lsquic_hash_elem *el; 2615 const struct lsquic_conn *conn; 2616 el = find_conn_by_addr(engine->conns_hash, sa_local); 2617 if (!el) 2618 return -1; 2619 conn = lsquic_hashelem_getdata(el); 2620 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2621 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2622 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2623 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2624 else if (conn->cn_version == LSQVER_050) 2625 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 2626 else 2627 { 2628 assert(conn->cn_version == LSQVER_046 2629#if LSQUIC_USE_Q098 2630 || conn->cn_version == LSQVER_098 2631#endif 2632 2633 ); 2634 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2635 } 2636 } 2637 else 2638 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2639 2640 n_zeroes = 0; 2641 do 2642 { 2643 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 2644 if (!packet_in) 2645 return -1; 2646 /* Library does not modify packet_in_data, it is not referenced after 2647 * this function returns and subsequent release of pi_data is guarded 2648 * by PI_OWN_DATA flag. 2649 */ 2650 packet_in->pi_data = (unsigned char *) packet_in_data; 2651 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 2652 engine->flags & ENG_SERVER, 2653 engine->pub.enp_settings.es_scid_len, &ppstate)) 2654 { 2655 LSQ_DEBUG("Cannot parse incoming packet's header"); 2656 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 2657 s = 1; 2658 break; 2659 } 2660 2661 packet_in_data += packet_in->pi_data_sz; 2662 packet_in->pi_received = lsquic_time_now(); 2663 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 2664 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 2665 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 2666 peer_ctx, packet_in_size); 2667 n_zeroes += s == 0; 2668 } 2669 while (0 == s && packet_in_data < packet_end); 2670 2671 return n_zeroes > 0 ? 0 : s; 2672} 2673 2674 2675#if __GNUC__ && !defined(NDEBUG) 2676__attribute__((weak)) 2677#endif 2678unsigned 2679lsquic_engine_quic_versions (const lsquic_engine_t *engine) 2680{ 2681 return engine->pub.enp_settings.es_versions; 2682} 2683 2684 2685void 2686lsquic_engine_cooldown (lsquic_engine_t *engine) 2687{ 2688 struct lsquic_hash_elem *el; 2689 lsquic_conn_t *conn; 2690 2691 if (engine->flags & ENG_COOLDOWN) 2692 /* AFAICT, there is no harm in calling this function more than once, 2693 * but log it just in case, as it may indicate an error in the caller. 2694 */ 2695 LSQ_INFO("cooldown called again"); 2696 engine->flags |= ENG_COOLDOWN; 2697 LSQ_INFO("entering cooldown mode"); 2698 if (engine->flags & ENG_SERVER) 2699 drop_all_mini_conns(engine); 2700 for (el = lsquic_hash_first(engine->conns_hash); el; 2701 el = lsquic_hash_next(engine->conns_hash)) 2702 { 2703 conn = lsquic_hashelem_getdata(el); 2704 lsquic_conn_going_away(conn); 2705 } 2706} 2707 2708 2709int 2710lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 2711{ 2712 const struct attq_elem *next_attq; 2713 lsquic_time_t now, next_time; 2714#if LSQUIC_DEBUG_NEXT_ADV_TICK 2715 const struct lsquic_conn *conn; 2716 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 2717#endif 2718 2719 ENGINE_CALLS_INCR(engine); 2720 2721 if ((engine->flags & ENG_PAST_DEADLINE) 2722 && lsquic_mh_count(&engine->conns_out)) 2723 { 2724#if LSQUIC_DEBUG_NEXT_ADV_TICK 2725 conn = lsquic_mh_peek(&engine->conns_out); 2726 engine->last_logged_conn = 0; 2727 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 2728 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 2729 lsquic_mh_count(&engine->conns_out), 2730 lsquic_mh_count(&engine->conns_out) != 1, "s", 2731 CID_BITS(lsquic_conn_log_cid(conn))); 2732#endif 2733 *diff = 0; 2734 return 1; 2735 } 2736 2737 if (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2738 { 2739#if LSQUIC_DEBUG_NEXT_ADV_TICK 2740 engine->last_logged_conn = 0; 2741 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 2742#endif 2743 *diff = 0; 2744 return 1; 2745 } 2746 2747 if (lsquic_mh_count(&engine->conns_tickable)) 2748 { 2749#if LSQUIC_DEBUG_NEXT_ADV_TICK 2750 conn = lsquic_mh_peek(&engine->conns_tickable); 2751 engine->last_logged_conn = 0; 2752 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 2753 "connection%.*s (%"CID_FMT" first)", 2754 lsquic_mh_count(&engine->conns_tickable), 2755 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 2756 CID_BITS(lsquic_conn_log_cid(conn))); 2757#endif 2758 *diff = 0; 2759 return 1; 2760 } 2761 2762 next_attq = attq_next(engine->attq); 2763 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 2764 { 2765 if (next_attq) 2766 next_time = next_attq->ae_adv_time; 2767 else 2768 return 0; 2769 } 2770 else 2771 { 2772 if (next_attq) 2773 { 2774 next_time = next_attq->ae_adv_time; 2775 if (engine->resume_sending_at < next_time) 2776 { 2777 next_time = engine->resume_sending_at; 2778 next_attq = NULL; 2779 } 2780 } 2781 else 2782 next_time = engine->resume_sending_at; 2783 } 2784 2785 now = lsquic_time_now(); 2786 *diff = (int) ((int64_t) next_time - (int64_t) now); 2787#if LSQUIC_DEBUG_NEXT_ADV_TICK 2788 if (next_attq) 2789 { 2790 /* Deduplicate consecutive log messages about the same reason for the 2791 * same connection. 2792 * If diff is always zero or diff reset to a higher value, event is 2793 * still logged. 2794 */ 2795 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 2796 && (uintptr_t) next_attq->ae_conn 2797 == engine->last_logged_conn 2798 && *diff < engine->last_tick_diff)) 2799 { 2800 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 2801 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 2802 engine->last_tick_diff = *diff; 2803 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 2804 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 2805 lsquic_attq_why2str(next_attq->ae_why)); 2806 } 2807 } 2808 else 2809 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 2810#endif 2811 return 1; 2812} 2813 2814 2815unsigned 2816lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 2817{ 2818 lsquic_time_t now; 2819 ENGINE_CALLS_INCR(engine); 2820 now = lsquic_time_now(); 2821 if (from_now < 0) 2822 now -= from_now; 2823 else 2824 now += from_now; 2825 return attq_count_before(engine->attq, now); 2826} 2827 2828 2829int 2830lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 2831 struct lsquic_conn *conn, unsigned cce_idx) 2832{ 2833 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2834 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2835 void *peer_ctx; 2836 2837 assert(cce_idx < conn->cn_n_cces); 2838 assert(conn->cn_cces_mask & (1 << cce_idx)); 2839 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 2840 2841 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 2842 cce->cce_cid.len, conn, &cce->cce_hash_el)) 2843 { 2844 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 2845 CID_BITS(&cce->cce_cid)); 2846 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2847 cce->cce_flags |= CCE_REG; 2848 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 2849 return 0; 2850 } 2851 else 2852 { 2853 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 2854 CID_BITS(&cce->cce_cid)); 2855 return -1; 2856 } 2857} 2858 2859 2860void 2861lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 2862 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now) 2863{ 2864 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2865 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2866 void *peer_ctx; 2867 2868 assert(cce_idx < conn->cn_n_cces); 2869 2870 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 2871 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 2872 2873 if (engine->purga) 2874 { 2875 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2876 lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 2877 PUTY_CID_RETIRED, now); 2878 } 2879 conn->cn_cces_mask &= ~(1u << cce_idx); 2880 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 2881} 2882 2883 2884