lsquic_engine.c revision de46bf2f
1/* Copyright (c) 2017 - 2019 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#include <arpa/inet.h> 17#ifndef WIN32 18#include <sys/time.h> 19#include <netinet/in.h> 20#include <sys/types.h> 21#include <sys/stat.h> 22#include <fcntl.h> 23#include <unistd.h> 24#include <netdb.h> 25#endif 26 27#ifndef NDEBUG 28#include <sys/types.h> 29#include <regex.h> /* For code that loses packets */ 30#endif 31 32#if LOG_PACKET_CHECKSUM 33#include <zlib.h> 34#endif 35 36#include <openssl/aead.h> 37 38#include "lsquic.h" 39#include "lsquic_types.h" 40#include "lsquic_int_types.h" 41#include "lsquic_sizes.h" 42#include "lsquic_parse_common.h" 43#include "lsquic_parse.h" 44#include "lsquic_packet_in.h" 45#include "lsquic_packet_out.h" 46#include "lsquic_senhist.h" 47#include "lsquic_rtt.h" 48#include "lsquic_cubic.h" 49#include "lsquic_pacer.h" 50#include "lsquic_bw_sampler.h" 51#include "lsquic_minmax.h" 52#include "lsquic_bbr.h" 53#include "lsquic_send_ctl.h" 54#include "lsquic_set.h" 55#include "lsquic_conn_flow.h" 56#include "lsquic_sfcw.h" 57#include "lsquic_hash.h" 58#include "lsquic_conn.h" 59#include "lsquic_full_conn.h" 60#include "lsquic_util.h" 61#include "lsquic_qtags.h" 62#include "lsquic_enc_sess.h" 63#include "lsquic_mm.h" 64#include "lsquic_engine_public.h" 65#include "lsquic_eng_hist.h" 66#include "lsquic_ev_log.h" 67#include "lsquic_version.h" 68#include "lsquic_pr_queue.h" 69#include "lsquic_mini_conn.h" 70#include "lsquic_mini_conn_ietf.h" 71#include "lsquic_stock_shi.h" 72#include "lsquic_purga.h" 73#include "lsquic_tokgen.h" 74#include "lsquic_attq.h" 75#include "lsquic_min_heap.h" 76#include "lsquic_http1x_if.h" 77#include "lsquic_parse_common.h" 78#include "lsquic_handshake.h" 79 80#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 81#include "lsquic_logger.h" 82 83#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 84#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 85#endif 86 87#if LSQUIC_DEBUG_NEXT_ADV_TICK 88#include "lsquic_alarmset.h" 89#endif 90 91#define MIN(a, b) ((a) < (b) ? (a) : (b)) 92 93/* The batch of outgoing packets grows and shrinks dynamically */ 94#define MAX_OUT_BATCH_SIZE 1024 95#define MIN_OUT_BATCH_SIZE 4 96#define INITIAL_OUT_BATCH_SIZE 32 97 98struct out_batch 99{ 100 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 101 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 102 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 103 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 104 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 105}; 106 107typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 108 109static void 110process_connections (struct lsquic_engine *engine, conn_iter_f iter, 111 lsquic_time_t now); 112 113static void 114engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 115 116static lsquic_conn_t * 117engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 118 enum lsquic_conn_flags flag); 119 120static void 121force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 122 123#if LSQUIC_COUNT_ENGINE_CALLS 124#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 125#else 126#define ENGINE_CALLS_INCR(e) 127#endif 128 129/* Nested calls to LSQUIC are not supported */ 130#define ENGINE_IN(e) do { \ 131 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 132 (e)->pub.enp_flags |= ENPUB_PROC; \ 133 ENGINE_CALLS_INCR(e); \ 134} while (0) 135 136#define ENGINE_OUT(e) do { \ 137 assert((e)->pub.enp_flags & ENPUB_PROC); \ 138 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 139} while (0) 140 141/* A connection can be referenced from one of six places: 142 * 143 * 1. A hash. The engine maintains two hash tables -- one for full, and 144 * one for mini connections. A connection starts its life in one of 145 * those. 146 * 147 * 2. Outgoing queue. 148 * 149 * 3. Tickable queue 150 * 151 * 4. Advisory Tick Time queue. 152 * 153 * 5. Closing connections queue. This is a transient queue -- it only 154 * exists for the duration of process_connections() function call. 155 * 156 * 6. Ticked connections queue. Another transient queue, similar to (5). 157 * 158 * The idea is to destroy the connection when it is no longer referenced. 159 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 160 * that case, the connection is referenced from two places: (2) and (5). 161 * After its packets are sent, it is only referenced in (5), and at the 162 * end of the function call, when it is removed from (5), reference count 163 * goes to zero and the connection is destroyed. If not all packets can 164 * be sent, at the end of the function call, the connection is referenced 165 * by (2) and will only be removed once all outgoing packets have been 166 * sent. 167 */ 168#define CONN_REF_FLAGS (LSCONN_HASHED \ 169 |LSCONN_HAS_OUTGOING \ 170 |LSCONN_TICKABLE \ 171 |LSCONN_TICKED \ 172 |LSCONN_CLOSING \ 173 |LSCONN_ATTQ) 174 175 176 177 178struct cid_update_batch 179{ 180 lsquic_cids_update_f cub_update_cids; 181 void *cub_update_ctx; 182 unsigned cub_count; 183 lsquic_cid_t cub_cids[20]; 184 void *cub_peer_ctxs[20]; 185}; 186 187static void 188cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 189 190 191struct lsquic_engine 192{ 193 struct lsquic_engine_public pub; 194 enum { 195 ENG_SERVER = LSENG_SERVER, 196 ENG_HTTP = LSENG_HTTP, 197 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 198 ENG_PAST_DEADLINE 199 = (1 << 8), /* Previous call to a processing 200 * function went past time threshold. 201 */ 202 ENG_CONNS_BY_ADDR 203 = (1 << 9), /* Connections are hashed by address */ 204#ifndef NDEBUG 205 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 206 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 207 ENG_DTOR = (1 << 26), /* Engine destructor */ 208#endif 209 } flags; 210 lsquic_packets_out_f packets_out; 211 void *packets_out_ctx; 212 lsquic_cids_update_f report_new_scids; 213 lsquic_cids_update_f report_live_scids; 214 lsquic_cids_update_f report_old_scids; 215 void *scids_ctx; 216 struct lsquic_hash *conns_hash; 217 struct min_heap conns_tickable; 218 struct min_heap conns_out; 219 /* Use a union because only one iterator is being used at any one time */ 220 union { 221 struct { 222 struct cert_susp_head *head; 223 } resumed; 224 struct lsquic_conn *one_conn; 225 } iter_state; 226 struct eng_hist history; 227 unsigned batch_size; 228 struct pr_queue *pr_queue; 229 struct attq *attq; 230 /* Track time last time a packet was sent to give new connections 231 * priority lower than that of existing connections. 232 */ 233 lsquic_time_t last_sent; 234#ifndef NDEBUG 235 regex_t lose_packets_re; 236 const char *lose_packets_str; 237#endif 238 unsigned n_conns; 239 lsquic_time_t deadline; 240 lsquic_time_t resume_sending_at; 241 unsigned mini_conns_count; 242 struct lsquic_purga *purga; 243#if LSQUIC_CONN_STATS 244 struct { 245 unsigned conns; 246 } stats; 247 struct conn_stats conn_stats_sum; 248 FILE *stats_fh; 249#endif 250 struct cid_update_batch new_scids; 251 struct out_batch out_batch; 252#if LSQUIC_COUNT_ENGINE_CALLS 253 unsigned long n_engine_calls; 254#endif 255#if LSQUIC_DEBUG_NEXT_ADV_TICK 256 uintptr_t last_logged_conn; 257 unsigned last_logged_ae_why; 258 int last_tick_diff; 259#endif 260}; 261 262 263void 264lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 265 unsigned flags) 266{ 267 memset(settings, 0, sizeof(*settings)); 268 settings->es_versions = LSQUIC_DF_VERSIONS; 269 if (flags & ENG_SERVER) 270 { 271 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 272 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 273 settings->es_init_max_data 274 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 275 settings->es_init_max_stream_data_bidi_remote 276 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 277 settings->es_init_max_stream_data_bidi_local 278 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 279 settings->es_init_max_stream_data_uni 280 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 281 settings->es_init_max_streams_uni 282 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 283 settings->es_ping_period = 0; 284 } 285 else 286 { 287 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 288 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 289 settings->es_init_max_data 290 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 291 settings->es_init_max_stream_data_bidi_remote 292 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 293 settings->es_init_max_stream_data_bidi_local 294 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 295 settings->es_init_max_stream_data_uni 296 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 297 settings->es_init_max_streams_uni 298 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 299 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 300 } 301 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 302 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 303 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 304 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 305 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 306 settings->es_max_header_list_size 307 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 308 settings->es_ua = LSQUIC_DF_UA; 309 settings->es_ecn = LSQUIC_DF_ECN; 310 311 settings->es_pdmd = QTAG_X509; 312 settings->es_aead = QTAG_AESG; 313 settings->es_kexs = QTAG_C255; 314 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 315 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 316 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 317 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 318 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 319 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 320 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 321 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 322 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 323 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 324 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 325 settings->es_sttl = LSQUIC_DF_STTL; 326 settings->es_init_max_streams_bidi 327 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 328 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 329 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 330 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 331 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 332 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 333 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 334 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 335 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 336} 337 338 339/* Note: if returning an error, err_buf must be valid if non-NULL */ 340int 341lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 342 unsigned flags, 343 char *err_buf, size_t err_buf_sz) 344{ 345 if (settings->es_cfcw < LSQUIC_MIN_FCW || 346 settings->es_sfcw < LSQUIC_MIN_FCW) 347 { 348 if (err_buf) 349 snprintf(err_buf, err_buf_sz, "%s", 350 "flow control window set too low"); 351 return -1; 352 } 353 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 354 { 355 if (err_buf) 356 snprintf(err_buf, err_buf_sz, "%s", 357 "No supported QUIC versions specified"); 358 return -1; 359 } 360 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 361 { 362 if (err_buf) 363 snprintf(err_buf, err_buf_sz, "%s", 364 "one or more unsupported QUIC version is specified"); 365 return -1; 366 } 367 if (flags & ENG_SERVER) 368 { 369 if (settings->es_handshake_to > 370 MAX_MINI_CONN_LIFESPAN_IN_USEC) 371 { 372 if (err_buf) 373 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 374 " usec is too large. The maximum for server is %u usec", 375 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 376 return -1; 377 } 378 } 379 if (settings->es_idle_timeout > 600) 380 { 381 if (err_buf) 382 snprintf(err_buf, err_buf_sz, "%s", 383 "The maximum value of idle timeout is 600 seconds"); 384 return -1; 385 } 386 if (!(!(flags & ENG_SERVER) && settings->es_scid_len == 0) 387 && (settings->es_scid_len < 4 || settings->es_scid_len > 18)) 388 { 389 if (err_buf) 390 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 391 "bytes long; it must be between 4 and 18.", 392 settings->es_scid_len); 393 return -1; 394 } 395 396 if (settings->es_cc_algo > 2) 397 { 398 if (err_buf) 399 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 400 "algorithm value %u", settings->es_cc_algo); 401 return -1; 402 } 403 return 0; 404} 405 406 407static void 408free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 409{ 410 free(packet_data); 411} 412 413 414static void * 415malloc_buf (void *ctx, void *conn_ctx, unsigned short size, char is_ipv6) 416{ 417 return malloc(size); 418} 419 420 421static const struct lsquic_packout_mem_if stock_pmi = 422{ 423 malloc_buf, free_packet, free_packet, 424}; 425 426 427static int 428hash_conns_by_addr (const struct lsquic_engine *engine) 429{ 430 if (engine->flags & ENG_SERVER) 431 return 0; 432 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 433 return 1; 434 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 435 && engine->pub.enp_settings.es_support_tcid0) 436 return 1; 437 if (engine->pub.enp_settings.es_scid_len == 0) 438 return 1; 439 return 0; 440} 441 442 443lsquic_engine_t * 444lsquic_engine_new (unsigned flags, 445 const struct lsquic_engine_api *api) 446{ 447 lsquic_engine_t *engine; 448 char err_buf[100]; 449 450 if (!api->ea_packets_out) 451 { 452 LSQ_ERROR("packets_out callback is not specified"); 453 return NULL; 454 } 455 456 if (api->ea_settings && 457 0 != lsquic_engine_check_settings(api->ea_settings, flags, 458 err_buf, sizeof(err_buf))) 459 { 460 LSQ_ERROR("cannot create engine: %s", err_buf); 461 return NULL; 462 } 463 464 engine = calloc(1, sizeof(*engine)); 465 if (!engine) 466 return NULL; 467 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 468 { 469 free(engine); 470 return NULL; 471 } 472 if (api->ea_settings) 473 engine->pub.enp_settings = *api->ea_settings; 474 else 475 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 476 int tag_buf_len; 477 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 478 sizeof(engine->pub.enp_ver_tags_buf), 479 engine->pub.enp_settings.es_versions); 480 if (tag_buf_len <= 0) 481 { 482 LSQ_ERROR("cannot generate version tags buffer"); 483 free(engine); 484 return NULL; 485 } 486 engine->pub.enp_ver_tags_len = tag_buf_len; 487 engine->pub.enp_flags = ENPUB_CAN_SEND; 488 engine->pub.enp_stream_if = api->ea_stream_if; 489 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 490 491 engine->flags = flags; 492#ifndef NDEBUG 493 engine->flags |= ENG_COALESCE; 494#endif 495 engine->packets_out = api->ea_packets_out; 496 engine->packets_out_ctx = api->ea_packets_out_ctx; 497 engine->report_new_scids = api->ea_new_scids; 498 engine->report_live_scids = api->ea_live_scids; 499 engine->report_old_scids = api->ea_old_scids; 500 engine->scids_ctx = api->ea_cids_update_ctx; 501 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 502 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 503 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 504 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 505 if (api->ea_shi) 506 { 507 engine->pub.enp_shi = api->ea_shi; 508 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 509 } 510 else 511 { 512 engine->pub.enp_shi = &stock_shi; 513 engine->pub.enp_shi_ctx = stock_shared_hash_new(); 514 if (!engine->pub.enp_shi_ctx) 515 { 516 free(engine); 517 return NULL; 518 } 519 } 520 if (api->ea_hsi_if) 521 { 522 engine->pub.enp_hsi_if = api->ea_hsi_if; 523 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 524 } 525 else 526 { 527 engine->pub.enp_hsi_if = lsquic_http1x_if; 528 engine->pub.enp_hsi_ctx = NULL; 529 } 530 if (api->ea_pmi) 531 { 532 engine->pub.enp_pmi = api->ea_pmi; 533 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 534 } 535 else 536 { 537 engine->pub.enp_pmi = &stock_pmi; 538 engine->pub.enp_pmi_ctx = NULL; 539 } 540 engine->pub.enp_verify_cert = api->ea_verify_cert; 541 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 542 engine->pub.enp_kli = api->ea_keylog_if; 543 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 544 engine->pub.enp_engine = engine; 545 if (hash_conns_by_addr(engine)) 546 engine->flags |= ENG_CONNS_BY_ADDR; 547 engine->conns_hash = lsquic_hash_create(); 548 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 549 if (!engine->pub.enp_tokgen) 550 return NULL; 551 if (flags & ENG_SERVER) 552 { 553 engine->pr_queue = prq_create( 554 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 555 &engine->pub); 556 if (!engine->pr_queue) 557 { 558 lsquic_tg_destroy(engine->pub.enp_tokgen); 559 return NULL; 560 } 561 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 562 engine->report_old_scids, engine->scids_ctx); 563 if (!engine->purga) 564 { 565 lsquic_tg_destroy(engine->pub.enp_tokgen); 566 prq_destroy(engine->pr_queue); 567 return NULL; 568 } 569 } 570 engine->attq = attq_create(); 571 eng_hist_init(&engine->history); 572 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 573 if (engine->pub.enp_settings.es_honor_prst) 574 { 575 engine->pub.enp_srst_hash = lsquic_hash_create(); 576 if (!engine->pub.enp_srst_hash) 577 { 578 lsquic_engine_destroy(engine); 579 return NULL; 580 } 581 } 582 583#ifndef NDEBUG 584 { 585 const char *env; 586 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 587 if (env) 588 { 589 if (0 != regcomp(&engine->lose_packets_re, env, 590 REG_EXTENDED|REG_NOSUB)) 591 { 592 LSQ_ERROR("could not compile lost packet regex `%s'", env); 593 return NULL; 594 } 595 engine->flags |= ENG_LOSE_PACKETS; 596 engine->lose_packets_str = env; 597 LSQ_WARN("will lose packets that match the following regex: %s", 598 env); 599 } 600 env = getenv("LSQUIC_COALESCE"); 601 if (env) 602 { 603 engine->flags &= ~ENG_COALESCE; 604 if (atoi(env)) 605 { 606 engine->flags |= ENG_COALESCE; 607 LSQ_NOTICE("will coalesce packets"); 608 } 609 else 610 LSQ_NOTICE("will not coalesce packets"); 611 } 612 } 613#endif 614#if LSQUIC_CONN_STATS 615 engine->stats_fh = api->ea_stats_fh; 616#endif 617 618 LSQ_INFO("instantiated engine"); 619 return engine; 620} 621 622 623#if LOG_PACKET_CHECKSUM 624static void 625log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 626 const unsigned char *buf, size_t bufsz) 627{ 628 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 629 (uint32_t) crc32(0, buf, bufsz)); 630} 631 632 633#endif 634 635 636static void 637grow_batch_size (struct lsquic_engine *engine) 638{ 639 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 640} 641 642 643static void 644shrink_batch_size (struct lsquic_engine *engine) 645{ 646 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 647} 648 649 650struct cce_cid_iter 651{ 652 const struct lsquic_conn *conn; 653 unsigned todo, n; 654}; 655 656 657static struct conn_cid_elem * 658cce_iter_next (struct cce_cid_iter *citer) 659{ 660 struct conn_cid_elem *cce; 661 662 while (citer->todo) 663 if (citer->todo & (1 << citer->n)) 664 { 665 citer->todo &= ~(1 << citer->n); 666 cce = &citer->conn->cn_cces[ citer->n++ ]; 667 if (!(cce->cce_flags & CCE_PORT)) 668 return cce; 669 } 670 else 671 ++citer->n; 672 673 return NULL; 674} 675 676 677static struct conn_cid_elem * 678cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 679{ 680 citer->conn = conn; 681 citer->todo = conn->cn_cces_mask; 682 citer->n = 0; 683 return cce_iter_next(citer); 684} 685 686 687#if LSQUIC_CONN_STATS 688void 689update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 690{ 691 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 692 const unsigned long *src; 693 const struct conn_stats *stats; 694 unsigned i; 695 696 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 697 { 698 ++engine->stats.conns; 699 src = (unsigned long *) stats; 700 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 701 dst[i] += src[i]; 702 } 703} 704 705 706#endif 707 708 709/* Wrapper to make sure important things occur before the connection is 710 * really destroyed. 711 */ 712static void 713destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 714 lsquic_time_t now) 715{ 716 struct cce_cid_iter citer; 717 const struct conn_cid_elem *cce; 718 lsquic_time_t drain_time; 719 struct purga_el *puel; 720 721 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 722 if (engine->purga 723 /* Blacklist all CIDs except for promoted mini connections */ 724 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 725 != (LSCONN_MINI|LSCONN_PROMOTED)) 726 { 727 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 728 && conn->cn_if->ci_drain_time && 729 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 730 { 731 for (cce = cce_iter_first(&citer, conn); cce; 732 cce = cce_iter_next(&citer)) 733 { 734 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 735 lsquic_conn_get_peer_ctx(conn, NULL), 736 PUTY_CONN_DRAIN, now); 737 if (puel) 738 puel->puel_time = now + drain_time; 739 } 740 } 741 else 742 { 743 for (cce = cce_iter_first(&citer, conn); cce; 744 cce = cce_iter_next(&citer)) 745 { 746 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 747 lsquic_conn_get_peer_ctx(conn, NULL), 748 PUTY_CONN_DELETED, now); 749 if (puel) 750 { 751 puel->puel_time = now; 752 puel->puel_count = 0; 753 } 754 } 755 } 756 } 757#if LSQUIC_CONN_STATS 758 update_stats_sum(engine, conn); 759#endif 760 --engine->n_conns; 761 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 762 conn->cn_if->ci_destroy(conn); 763} 764 765 766static int 767maybe_grow_conn_heaps (struct lsquic_engine *engine) 768{ 769 struct min_heap_elem *els; 770 unsigned count; 771 772 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 773 return 0; /* Nothing to do */ 774 775 if (lsquic_mh_nalloc(&engine->conns_tickable)) 776 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 777 else 778 count = 8; 779 780 els = malloc(sizeof(els[0]) * count); 781 if (!els) 782 { 783 LSQ_ERROR("%s: malloc failed", __func__); 784 return -1; 785 } 786 787 LSQ_DEBUG("grew heaps to %u elements", count / 2); 788 memcpy(&els[0], engine->conns_tickable.mh_elems, 789 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 790 memcpy(&els[count / 2], engine->conns_out.mh_elems, 791 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 792 free(engine->conns_tickable.mh_elems); 793 engine->conns_tickable.mh_elems = els; 794 engine->conns_out.mh_elems = &els[count / 2]; 795 engine->conns_tickable.mh_nalloc = count / 2; 796 engine->conns_out.mh_nalloc = count / 2; 797 return 0; 798} 799 800 801static void 802remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 803 unsigned todo) 804{ 805 unsigned n; 806 807 for (n = 0; todo; todo &= ~(1 << n++)) 808 if ((todo & (1 << n)) && 809 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 810 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 811} 812 813 814static void 815remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 816{ 817 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 818} 819 820 821static void 822cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 823 824 825static int 826insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 827 void *peer_ctx) 828{ 829 struct conn_cid_elem *cce; 830 unsigned todo, done, n; 831 832 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 833 if (todo & (1 << n)) 834 { 835 cce = &conn->cn_cces[n]; 836 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 837 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 838 cce->cce_cid.len, conn, &cce->cce_hash_el)) 839 done |= 1 << n; 840 else 841 goto err; 842 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 843 { 844 cce->cce_flags |= CCE_REG; 845 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 846 } 847 } 848 849 return 0; 850 851 err: 852 remove_cces_from_hash(engine->conns_hash, conn, done); 853 return -1; 854} 855 856 857static lsquic_conn_t * 858new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 859 lsquic_time_t now) 860{ 861 const lsquic_cid_t *cid; 862 server_conn_ctor_f ctor; 863 lsquic_conn_t *conn; 864 unsigned flags; 865 if (0 != maybe_grow_conn_heaps(engine)) 866 return NULL; 867 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 868 869 if (mini_conn->cn_flags & LSCONN_IETF) 870 ctor = lsquic_ietf_full_conn_server_new; 871 else 872 ctor = lsquic_gquic_full_conn_server_new; 873 874 conn = ctor(&engine->pub, flags, mini_conn); 875 if (!conn) 876 { 877 /* Otherwise, full_conn_server_new prints its own warnings */ 878 if (ENOMEM == errno) 879 { 880 cid = lsquic_conn_log_cid(mini_conn); 881 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 882 CID_BITS(cid), strerror(errno)); 883 } 884 return NULL; 885 } 886 ++engine->n_conns; 887 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 888 { 889 cid = lsquic_conn_log_cid(conn); 890 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 891 CID_BITS(cid)); 892 destroy_conn(engine, conn, now); 893 return NULL; 894 } 895 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 896 conn->cn_flags |= LSCONN_HASHED; 897 return conn; 898} 899 900 901static enum 902{ 903 VER_NOT_SPECIFIED, 904 VER_SUPPORTED, 905 VER_UNSUPPORTED, 906} 907 908 909version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 910 enum lsquic_version *pversion) 911{ 912 lsquic_ver_tag_t ver_tag; 913 enum lsquic_version version; 914 915 if (!packet_in->pi_quic_ver) 916 { 917 LSQ_DEBUG("packet does not specify version"); 918 return VER_NOT_SPECIFIED; 919 } 920 921 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 922 version = lsquic_tag2ver(ver_tag); 923 if (version < N_LSQVER) 924 { 925 if (engine->pub.enp_settings.es_versions & (1 << version)) 926 { 927 LSQ_DEBUG("client-supplied version %s is supported", 928 lsquic_ver2str[version]); 929 *pversion = version; 930 return VER_SUPPORTED; 931 } 932 else 933 LSQ_DEBUG("client-supplied version %s is not supported", 934 lsquic_ver2str[version]); 935 } 936 else 937 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 938 ver_tag); 939 940 return VER_UNSUPPORTED; 941} 942 943 944static void 945schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 946 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 947 const struct sockaddr *sa_peer, void *peer_ctx) 948{ 949 assert(engine->pr_queue); 950 if (0 == prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 951 sa_local, sa_peer)) 952 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 953 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 954 else 955 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 956} 957 958 959static unsigned short 960sa2port (const struct sockaddr *sa) 961{ 962 if (sa->sa_family == AF_INET) 963 { 964 struct sockaddr_in *const sa4 = (void *) sa; 965 return sa4->sin_port; 966 } 967 else 968 { 969 struct sockaddr_in6 *const sa6 = (void *) sa; 970 return sa6->sin6_port; 971 } 972} 973 974 975static struct lsquic_hash_elem * 976find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 977{ 978 unsigned short port; 979 980 port = sa2port(sa); 981 return lsquic_hash_find(hash, &port, sizeof(port)); 982} 983 984 985static lsquic_conn_t * 986find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 987 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 988{ 989 struct lsquic_hash_elem *el; 990 lsquic_conn_t *conn; 991 992 if (engine->flags & ENG_CONNS_BY_ADDR) 993 el = find_conn_by_addr(engine->conns_hash, sa_local); 994 else if (packet_in->pi_flags & PI_CONN_ID) 995 el = lsquic_hash_find(engine->conns_hash, 996 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 997 else 998 { 999 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1000 return NULL; 1001 } 1002 1003 if (!el) 1004 return NULL; 1005 1006 conn = lsquic_hashelem_getdata(el); 1007 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1008 if ((engine->flags & ENG_CONNS_BY_ADDR) 1009 && !(conn->cn_flags & LSCONN_IETF) 1010 && (packet_in->pi_flags & PI_CONN_ID) 1011 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1012 { 1013 LSQ_DEBUG("connection IDs do not match"); 1014 return NULL; 1015 } 1016 1017 return conn; 1018} 1019 1020 1021static lsquic_conn_t * 1022find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1023 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1024 const struct sockaddr *sa_peer, void *peer_ctx) 1025{ 1026 struct lsquic_hash_elem *el; 1027 struct purga_el *puel; 1028 lsquic_conn_t *conn; 1029 1030 if (!(packet_in->pi_flags & PI_CONN_ID)) 1031 { 1032 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1033 return NULL; 1034 } 1035 el = lsquic_hash_find(engine->conns_hash, 1036 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1037 1038 if (el) 1039 { 1040 conn = lsquic_hashelem_getdata(el); 1041 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1042 return conn; 1043 } 1044 1045 if (engine->flags & ENG_COOLDOWN) 1046 { /* Do not create incoming connections during cooldown */ 1047 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1048 return NULL; 1049 } 1050 1051 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1052 { 1053 LSQ_DEBUG("reached limit of %u inchoate connections", 1054 engine->pub.enp_settings.es_max_inchoate); 1055 return NULL; 1056 } 1057 1058 1059 if (engine->purga 1060 && (puel = lsquic_purga_contains(engine->purga, 1061 &packet_in->pi_conn_id), puel)) 1062 { 1063 switch (puel->puel_type) 1064 { 1065 case PUTY_CID_RETIRED: 1066 LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet", 1067 CID_BITS(&packet_in->pi_conn_id)); 1068 return NULL; 1069 case PUTY_CONN_DRAIN: 1070 LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64, 1071 puel->puel_time, packet_in->pi_received); 1072 if (puel->puel_time > packet_in->pi_received) 1073 { 1074 LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet", 1075 CID_BITS(&packet_in->pi_conn_id)); 1076 return NULL; 1077 } 1078 LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted", 1079 CID_BITS(&packet_in->pi_conn_id)); 1080 puel->puel_type = PUTY_CONN_DELETED; 1081 puel->puel_count = 0; 1082 puel->puel_time = 0; 1083 /* fall-through */ 1084 case PUTY_CONN_DELETED: 1085 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1086 CID_BITS(&packet_in->pi_conn_id)); 1087 if (puel->puel_time < packet_in->pi_received) 1088 { 1089 puel->puel_time = packet_in->pi_received 1090 /* Exponential back-off */ 1091 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1092 ++puel->puel_count; 1093 goto maybe_send_prst; 1094 } 1095 return NULL; 1096 default: 1097 assert(0); 1098 return NULL; 1099 } 1100 } 1101 1102 if (engine->pub.enp_settings.es_send_prst 1103 && !(packet_in->pi_flags & PI_GQUIC) 1104 && HETY_NOT_SET == packet_in->pi_header_type) 1105 goto maybe_send_prst; 1106 1107 if (0 != maybe_grow_conn_heaps(engine)) 1108 return NULL; 1109 1110 const struct parse_funcs *pf; 1111 enum lsquic_version version; 1112 switch (version_matches(engine, packet_in, &version)) 1113 { 1114 case VER_UNSUPPORTED: 1115 if (engine->flags & ENG_SERVER) 1116 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1117 sa_local, sa_peer, peer_ctx); 1118 return NULL; 1119 case VER_NOT_SPECIFIED: 1120 maybe_send_prst: 1121 if ((engine->flags & ENG_SERVER) && 1122 engine->pub.enp_settings.es_send_prst) 1123 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1124 sa_local, sa_peer, peer_ctx); 1125 return NULL; 1126 case VER_SUPPORTED: 1127 pf = select_pf_by_ver(version); 1128 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1129 break; 1130 } 1131 1132 1133 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1134 { 1135 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1136 sa_peer->sa_family == AF_INET, NULL); 1137 } 1138 else 1139 { 1140 conn = mini_conn_new(&engine->pub, packet_in, version); 1141 } 1142 if (!conn) 1143 return NULL; 1144 ++engine->mini_conns_count; 1145 ++engine->n_conns; 1146 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1147 { 1148 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1149 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1150 CID_BITS(cid)); 1151 destroy_conn(engine, conn, packet_in->pi_received); 1152 return NULL; 1153 } 1154 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1155 conn->cn_flags |= LSCONN_HASHED; 1156 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1157 conn->cn_last_sent = engine->last_sent; 1158 return conn; 1159} 1160 1161 1162lsquic_conn_t * 1163lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1164 const lsquic_cid_t *cid) 1165{ 1166 struct lsquic_hash_elem *el; 1167 lsquic_conn_t *conn = NULL; 1168 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1169 1170 if (el) 1171 conn = lsquic_hashelem_getdata(el); 1172 return conn; 1173} 1174 1175 1176#if !defined(NDEBUG) && __GNUC__ 1177__attribute__((weak)) 1178#endif 1179void 1180lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1181 lsquic_conn_t *conn) 1182{ 1183 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1184 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1185 { 1186 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1187 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1188 engine_incref_conn(conn, LSCONN_TICKABLE); 1189 } 1190} 1191 1192 1193void 1194lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1195 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1196{ 1197 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1198 if (conn->cn_flags & LSCONN_TICKABLE) 1199 { 1200 /* Optimization: no need to add the connection to the Advisory Tick 1201 * Time Queue: it is about to be ticked, after which it its next tick 1202 * time may be queried again. 1203 */; 1204 } 1205 else if (conn->cn_flags & LSCONN_ATTQ) 1206 { 1207 if (lsquic_conn_adv_time(conn) != tick_time) 1208 { 1209 attq_remove(engine->attq, conn); 1210 if (0 != attq_add(engine->attq, conn, tick_time, why)) 1211 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1212 } 1213 } 1214 else if (0 == attq_add(engine->attq, conn, tick_time, why)) 1215 engine_incref_conn(conn, LSCONN_ATTQ); 1216} 1217 1218 1219static struct lsquic_conn * 1220find_conn_by_srst (struct lsquic_engine *engine, 1221 const struct lsquic_packet_in *packet_in) 1222{ 1223 struct lsquic_hash_elem *el; 1224 struct lsquic_conn *conn; 1225 1226 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1227 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1228 return NULL; 1229 1230 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1231 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1232 IQUIC_SRESET_TOKEN_SZ); 1233 if (!el) 1234 return NULL; 1235 1236 conn = lsquic_hashelem_getdata(el); 1237 return conn; 1238} 1239 1240 1241/* Return 0 if packet is being processed by a real connection (mini or full), 1242 * otherwise return 1. 1243 */ 1244static int 1245process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1246 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1247 const struct sockaddr *sa_peer, void *peer_ctx, int full_udp_packet) 1248{ 1249 lsquic_conn_t *conn; 1250 const unsigned char *packet_in_data; 1251 size_t packet_in_size; 1252 1253 if (lsquic_packet_in_is_gquic_prst(packet_in) 1254 && !engine->pub.enp_settings.es_honor_prst) 1255 { 1256 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1257 LSQ_DEBUG("public reset packet: discarding"); 1258 return 1; 1259 } 1260 1261 if (engine->flags & ENG_SERVER) 1262 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1263 sa_peer, peer_ctx); 1264 else 1265 conn = find_conn(engine, packet_in, ppstate, sa_local); 1266 1267 if (!conn) 1268 { 1269 if (engine->pub.enp_settings.es_honor_prst 1270 && full_udp_packet 1271 && !(packet_in->pi_flags & PI_GQUIC) 1272 && engine->pub.enp_srst_hash 1273 && (conn = find_conn_by_srst(engine, packet_in))) 1274 { 1275 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1276 CID_BITS(lsquic_conn_log_cid(conn))); 1277 conn->cn_if->ci_stateless_reset(conn); 1278 if (!(conn->cn_flags & LSCONN_TICKABLE) 1279 && conn->cn_if->ci_is_tickable(conn)) 1280 { 1281 lsquic_mh_insert(&engine->conns_tickable, conn, 1282 conn->cn_last_ticked); 1283 engine_incref_conn(conn, LSCONN_TICKABLE); 1284 } 1285 /* Even though the connection processes this packet, we return 1286 * 1 so that the caller does not add reset packet's random 1287 * bytes to the list of valid CIDs. 1288 */ 1289 } 1290 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1291 return 1; 1292 } 1293 1294 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1295 { 1296 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1297 engine_incref_conn(conn, LSCONN_TICKABLE); 1298 } 1299 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1300 sa_local, sa_peer); 1301 lsquic_packet_in_upref(packet_in); 1302#if LOG_PACKET_CHECKSUM 1303 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1304 packet_in->pi_data_sz); 1305#endif 1306 /* Note on QLog: 1307 * For the PACKET_RX QLog event, we are interested in logging these things: 1308 * - raw packet (however it comes in, encrypted or not) 1309 * - frames (list of frame names) 1310 * - packet type and number 1311 * - packet rx timestamp 1312 * 1313 * Since only some of these items are available at this code 1314 * juncture, we will wait until after the packet has been 1315 * decrypted (if necessary) and parsed to call the log functions. 1316 * 1317 * Once the PACKET_RX event is finally logged, the timestamp 1318 * will come from packet_in->pi_received. For correct sequential 1319 * ordering of QLog events, be sure to process the QLogs downstream. 1320 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1321 */ 1322 packet_in_data = packet_in->pi_data; 1323 packet_in_size = packet_in->pi_data_sz; 1324 conn->cn_if->ci_packet_in(conn, packet_in); 1325 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1326 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1327 return 0; 1328} 1329 1330 1331void 1332lsquic_engine_destroy (lsquic_engine_t *engine) 1333{ 1334 struct lsquic_hash_elem *el; 1335 lsquic_conn_t *conn; 1336 1337 LSQ_DEBUG("destroying engine"); 1338#ifndef NDEBUG 1339 engine->flags |= ENG_DTOR; 1340#endif 1341 1342 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1343 { 1344 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1345 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1346 } 1347 1348 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1349 { 1350 assert(conn->cn_flags & LSCONN_TICKABLE); 1351 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1352 } 1353 1354 for (el = lsquic_hash_first(engine->conns_hash); el; 1355 el = lsquic_hash_next(engine->conns_hash)) 1356 { 1357 conn = lsquic_hashelem_getdata(el); 1358 force_close_conn(engine, conn); 1359 } 1360 lsquic_hash_destroy(engine->conns_hash); 1361 1362 assert(0 == engine->n_conns); 1363 assert(0 == engine->mini_conns_count); 1364 if (engine->pr_queue) 1365 prq_destroy(engine->pr_queue); 1366 if (engine->purga) 1367 lsquic_purga_destroy(engine->purga); 1368 attq_destroy(engine->attq); 1369 1370 assert(0 == lsquic_mh_count(&engine->conns_out)); 1371 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1372 if (engine->pub.enp_shi == &stock_shi) 1373 stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1374 lsquic_mm_cleanup(&engine->pub.enp_mm); 1375 free(engine->conns_tickable.mh_elems); 1376#ifndef NDEBUG 1377 if (engine->flags & ENG_LOSE_PACKETS) 1378 regfree(&engine->lose_packets_re); 1379#endif 1380 if (engine->pub.enp_tokgen) 1381 lsquic_tg_destroy(engine->pub.enp_tokgen); 1382#if LSQUIC_CONN_STATS 1383 if (engine->stats_fh) 1384 { 1385 const struct conn_stats *const stats = &engine->conn_stats_sum; 1386 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1387 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1388 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1389 fprintf(engine->stats_fh, "In:\n"); 1390 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1391 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1392 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1393 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1394 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1395 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1396 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1397 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1398 stats->in.headers_comp, stats->in.headers_uncomp, 1399 stats->in.headers_uncomp ? 1400 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1401 : 0); 1402 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1403 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1404 fprintf(engine->stats_fh, " ACK frames merged to new: %lu\n", stats->in.n_acks_merged[0]); 1405 fprintf(engine->stats_fh, " ACK frames merged to old: %lu\n", stats->in.n_acks_merged[1]); 1406 fprintf(engine->stats_fh, "Out:\n"); 1407 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1408 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1409 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1410 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1411 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1412 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1413 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1414 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1415 stats->out.headers_comp, stats->out.headers_uncomp, 1416 stats->out.headers_uncomp ? 1417 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1418 : 0); 1419 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1420 } 1421#endif 1422 if (engine->pub.enp_srst_hash) 1423 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1424#if LSQUIC_COUNT_ENGINE_CALLS 1425 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1426#endif 1427 free(engine); 1428} 1429 1430 1431static struct conn_cid_elem * 1432find_free_cce (struct lsquic_conn *conn) 1433{ 1434 struct conn_cid_elem *cce; 1435 1436 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1437 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1438 return cce; 1439 1440 return NULL; 1441} 1442 1443 1444static int 1445add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1446 const struct sockaddr *local_sa, void *peer_ctx) 1447{ 1448 struct conn_cid_elem *cce; 1449 1450 if (engine->flags & ENG_CONNS_BY_ADDR) 1451 { 1452 cce = find_free_cce(conn); 1453 if (!cce) 1454 { 1455 LSQ_ERROR("cannot find free CCE"); 1456 return -1; 1457 } 1458 cce->cce_port = sa2port(local_sa); 1459 cce->cce_flags = CCE_PORT; 1460 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1461 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1462 { 1463 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1464 return 0; 1465 } 1466 else 1467 return -1; 1468 1469 } 1470 else 1471 return insert_conn_into_hash(engine, conn, peer_ctx); 1472} 1473 1474 1475lsquic_conn_t * 1476lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1477 const struct sockaddr *local_sa, 1478 const struct sockaddr *peer_sa, 1479 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1480 const char *hostname, unsigned short max_packet_size, 1481 const unsigned char *zero_rtt, size_t zero_rtt_len, 1482 const unsigned char *token, size_t token_sz) 1483{ 1484 lsquic_conn_t *conn; 1485 unsigned flags, versions; 1486 int is_ipv4; 1487 1488 ENGINE_IN(engine); 1489 1490 if (engine->flags & ENG_SERVER) 1491 { 1492 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1493 goto err; 1494 } 1495 1496 if (engine->flags & ENG_CONNS_BY_ADDR 1497 && find_conn_by_addr(engine->conns_hash, local_sa)) 1498 { 1499 LSQ_ERROR("cannot have more than one connection on the same port"); 1500 goto err; 1501 } 1502 1503 if (0 != maybe_grow_conn_heaps(engine)) 1504 return NULL; 1505 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1506 is_ipv4 = peer_sa->sa_family == AF_INET; 1507 if (zero_rtt && zero_rtt_len) 1508 { 1509 version = lsquic_zero_rtt_version(zero_rtt, zero_rtt_len); 1510 if (version >= N_LSQVER) 1511 { 1512 LSQ_INFO("zero-rtt version is bad, won't use"); 1513 zero_rtt = NULL; 1514 zero_rtt_len = 0; 1515 } 1516 } 1517 if (version >= N_LSQVER) 1518 { 1519 if (version > N_LSQVER) 1520 LSQ_WARN("invalid version specified, engine will pick"); 1521 versions = engine->pub.enp_settings.es_versions; 1522 } 1523 else 1524 versions = 1u << version; 1525 if (versions & LSQUIC_IETF_VERSIONS) 1526 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1527 flags, hostname, max_packet_size, 1528 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1529 else 1530 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1531 flags, hostname, max_packet_size, is_ipv4, 1532 zero_rtt, zero_rtt_len); 1533 if (!conn) 1534 goto err; 1535 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1536 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1537 lsquic_ver2str[conn->cn_version]); 1538 ++engine->n_conns; 1539 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1540 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1541 { 1542 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1543 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1544 CID_BITS(cid)); 1545 destroy_conn(engine, conn, lsquic_time_now()); 1546 goto err; 1547 } 1548 assert(!(conn->cn_flags & 1549 (CONN_REF_FLAGS 1550 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1551 callbacks */ 1552 ))); 1553 conn->cn_flags |= LSCONN_HASHED; 1554 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1555 engine_incref_conn(conn, LSCONN_TICKABLE); 1556 lsquic_conn_set_ctx(conn, conn_ctx); 1557 conn->cn_if->ci_client_call_on_new(conn); 1558 end: 1559 ENGINE_OUT(engine); 1560 return conn; 1561 err: 1562 conn = NULL; 1563 goto end; 1564} 1565 1566 1567static void 1568remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1569{ 1570 remove_all_cces_from_hash(engine->conns_hash, conn); 1571 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1572} 1573 1574 1575static void 1576refflags2str (enum lsquic_conn_flags flags, char s[6]) 1577{ 1578 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1579 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1580 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1581 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1582 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1583 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1584 *s = '\0'; 1585} 1586 1587 1588static void 1589engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1590{ 1591 char str[2][7]; 1592 assert(flag & CONN_REF_FLAGS); 1593 assert(!(conn->cn_flags & flag)); 1594 conn->cn_flags |= flag; 1595 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1596 CID_BITS(lsquic_conn_log_cid(conn)), 1597 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1598 (refflags2str(conn->cn_flags, str[1]), str[1])); 1599} 1600 1601 1602static lsquic_conn_t * 1603engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1604 enum lsquic_conn_flags flags) 1605{ 1606 char str[2][7]; 1607 lsquic_time_t now; 1608 assert(flags & CONN_REF_FLAGS); 1609 assert(conn->cn_flags & flags); 1610#ifndef NDEBUG 1611 if (flags & LSCONN_CLOSING) 1612 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1613#endif 1614 conn->cn_flags &= ~flags; 1615 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1616 CID_BITS(lsquic_conn_log_cid(conn)), 1617 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1618 (refflags2str(conn->cn_flags, str[1]), str[1])); 1619 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1620 { 1621 now = lsquic_time_now(); 1622 if (conn->cn_flags & LSCONN_MINI) 1623 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1624 else 1625 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1626 destroy_conn(engine, conn, now); 1627 return NULL; 1628 } 1629 else 1630 return conn; 1631} 1632 1633 1634/* This is not a general-purpose function. Only call from engine dtor. */ 1635static void 1636force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1637{ 1638 assert(engine->flags & ENG_DTOR); 1639 const enum lsquic_conn_flags flags = conn->cn_flags; 1640 assert(conn->cn_flags & CONN_REF_FLAGS); 1641 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1642 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1643 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1644 if (flags & LSCONN_ATTQ) 1645 { 1646 attq_remove(engine->attq, conn); 1647 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1648 } 1649 if (flags & LSCONN_HASHED) 1650 remove_conn_from_hash(engine, conn); 1651} 1652 1653 1654/* Iterator for tickable connections (those on the Tickable Queue). Before 1655 * a connection is returned, it is removed from the Advisory Tick Time queue 1656 * if necessary. 1657 */ 1658static lsquic_conn_t * 1659conn_iter_next_tickable (struct lsquic_engine *engine) 1660{ 1661 lsquic_conn_t *conn; 1662 1663 if (engine->flags & ENG_SERVER) 1664 while (1) 1665 { 1666 conn = lsquic_mh_pop(&engine->conns_tickable); 1667 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1668 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1669 else 1670 break; 1671 } 1672 else 1673 conn = lsquic_mh_pop(&engine->conns_tickable); 1674 1675 if (conn) 1676 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1677 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1678 { 1679 attq_remove(engine->attq, conn); 1680 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1681 } 1682 1683 return conn; 1684} 1685 1686 1687static void 1688cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1689 void *update_ctx) 1690{ 1691 cub->cub_update_cids = update; 1692 cub->cub_update_ctx = update_ctx; 1693 cub->cub_count = 0; 1694} 1695 1696 1697static void 1698cub_flush (struct cid_update_batch *cub) 1699{ 1700 if (cub->cub_count > 0 && cub->cub_update_cids) 1701 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1702 cub->cub_cids, cub->cub_count); 1703 cub->cub_count = 0; 1704} 1705 1706 1707static void 1708cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1709{ 1710 cub->cub_cids [ cub->cub_count ] = *cid; 1711 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1712 ++cub->cub_count; 1713 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1714 cub_flush(cub); 1715} 1716 1717 1718/* Process registered CIDs */ 1719static void 1720cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1721{ 1722 struct cce_cid_iter citer; 1723 struct conn_cid_elem *cce; 1724 void *peer_ctx; 1725 1726 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1727 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1728 if (cce->cce_flags & CCE_REG) 1729 cub_add(cub, &cce->cce_cid, peer_ctx); 1730} 1731 1732 1733static void 1734drop_all_mini_conns (lsquic_engine_t *engine) 1735{ 1736 struct lsquic_hash_elem *el; 1737 lsquic_conn_t *conn; 1738 struct cid_update_batch cub; 1739 1740 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1741 1742 for (el = lsquic_hash_first(engine->conns_hash); el; 1743 el = lsquic_hash_next(engine->conns_hash)) 1744 { 1745 conn = lsquic_hashelem_getdata(el); 1746 if (conn->cn_flags & LSCONN_MINI) 1747 { 1748 /* If promoted, why is it still in this hash? */ 1749 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1750 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1751 cub_add_cids_from_cces(&cub, conn); 1752 remove_conn_from_hash(engine, conn); 1753 } 1754 } 1755 1756 cub_flush(&cub); 1757} 1758 1759 1760void 1761lsquic_engine_process_conns (lsquic_engine_t *engine) 1762{ 1763 lsquic_conn_t *conn; 1764 lsquic_time_t now; 1765 1766 ENGINE_IN(engine); 1767 1768 now = lsquic_time_now(); 1769 while ((conn = attq_pop(engine->attq, now))) 1770 { 1771 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1772 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 1773 { 1774 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1775 engine_incref_conn(conn, LSCONN_TICKABLE); 1776 } 1777 } 1778 1779 process_connections(engine, conn_iter_next_tickable, now); 1780 ENGINE_OUT(engine); 1781} 1782 1783 1784static void 1785release_or_return_enc_data (struct lsquic_engine *engine, 1786 void (*pmi_rel_or_ret) (void *, void *, void *, char), 1787 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 1788{ 1789 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1790 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 1791 packet_out->po_flags &= ~PO_ENCRYPTED; 1792 packet_out->po_enc_data = NULL; 1793} 1794 1795 1796static void 1797release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1798 struct lsquic_packet_out *packet_out) 1799{ 1800 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 1801 conn, packet_out); 1802} 1803 1804 1805static void 1806return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1807 struct lsquic_packet_out *packet_out) 1808{ 1809 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 1810 conn, packet_out); 1811} 1812 1813 1814static int 1815copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 1816 struct lsquic_packet_out *packet_out) 1817{ 1818 int ipv6; 1819 1820 ipv6 = NP_IS_IPv6(packet_out->po_path); 1821 if (packet_out->po_flags & PO_ENCRYPTED) 1822 { 1823 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1824 && packet_out->po_data_sz == packet_out->po_enc_data_sz 1825 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 1826 packet_out->po_data_sz)) 1827 return 0; 1828 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1829 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 1830 goto copy; 1831 return_enc_data(engine, conn, packet_out); 1832 } 1833 1834 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 1835 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1836 packet_out->po_data_sz, ipv6); 1837 if (!packet_out->po_enc_data) 1838 { 1839 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 1840 "of size %hu", packet_out->po_data_sz); 1841 return -1; 1842 } 1843 1844 copy: 1845 memcpy(packet_out->po_enc_data, packet_out->po_data, 1846 packet_out->po_data_sz); 1847 packet_out->po_enc_data_sz = packet_out->po_data_sz; 1848 packet_out->po_sent_sz = packet_out->po_data_sz; 1849 packet_out->po_flags &= ~PO_IPv6; 1850 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 1851 1852 return 0; 1853} 1854 1855 1856STAILQ_HEAD(conns_stailq, lsquic_conn); 1857TAILQ_HEAD(conns_tailq, lsquic_conn); 1858 1859 1860struct conns_out_iter 1861{ 1862 struct min_heap *coi_heap; 1863 struct pr_queue *coi_prq; 1864 TAILQ_HEAD(, lsquic_conn) coi_active_list, 1865 coi_inactive_list; 1866 lsquic_conn_t *coi_next; 1867#ifndef NDEBUG 1868 lsquic_time_t coi_last_sent; 1869#endif 1870}; 1871 1872 1873static void 1874coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 1875{ 1876 iter->coi_heap = &engine->conns_out; 1877 iter->coi_prq = engine->pr_queue; 1878 iter->coi_next = NULL; 1879 TAILQ_INIT(&iter->coi_active_list); 1880 TAILQ_INIT(&iter->coi_inactive_list); 1881#ifndef NDEBUG 1882 iter->coi_last_sent = 0; 1883#endif 1884} 1885 1886 1887static lsquic_conn_t * 1888coi_next (struct conns_out_iter *iter) 1889{ 1890 lsquic_conn_t *conn; 1891 1892 if (lsquic_mh_count(iter->coi_heap) > 0) 1893 { 1894 conn = lsquic_mh_pop(iter->coi_heap); 1895 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1896 conn->cn_flags |= LSCONN_COI_ACTIVE; 1897#ifndef NDEBUG 1898 if (iter->coi_last_sent) 1899 assert(iter->coi_last_sent <= conn->cn_last_sent); 1900 iter->coi_last_sent = conn->cn_last_sent; 1901#endif 1902 return conn; 1903 } 1904 else if (iter->coi_prq && (conn = prq_next_conn(iter->coi_prq))) 1905 { 1906 return conn; 1907 } 1908 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 1909 { 1910 iter->coi_prq = NULL; /* Save function call in previous conditional */ 1911 conn = iter->coi_next; 1912 if (!conn) 1913 conn = TAILQ_FIRST(&iter->coi_active_list); 1914 if (conn) 1915 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 1916 return conn; 1917 } 1918 else 1919 return NULL; 1920} 1921 1922 1923static void 1924coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1925{ 1926 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 1927 { 1928 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 1929 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1930 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1931 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 1932 conn->cn_flags |= LSCONN_COI_INACTIVE; 1933 } 1934} 1935 1936 1937static void 1938coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1939{ 1940 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 1941 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 1942 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 1943 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1944 conn->cn_flags |= LSCONN_COI_ACTIVE; 1945} 1946 1947 1948static void 1949coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 1950{ 1951 lsquic_conn_t *conn; 1952 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 1953 { 1954 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1955 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1956 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 1957 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 1958 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 1959 else /* Closed connection gets one shot at sending packets */ 1960 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1961 } 1962 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 1963 { 1964 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 1965 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 1966 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1967 } 1968} 1969 1970 1971#ifndef NDEBUG 1972static void 1973lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 1974 unsigned n) 1975{ 1976 const lsquic_cid_t *cid; 1977 struct iovec *iov; 1978 unsigned i; 1979 char packno_str[22]; 1980 1981 for (i = 0; i < n; ++i) 1982 { 1983 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 1984 batch->packets[i]->po_packno); 1985 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 1986 { 1987 for (iov = batch->outs[i].iov; iov < 1988 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 1989 batch->outs[i].iov->iov_len -= 1; 1990 cid = lsquic_conn_log_cid(batch->conns[i]); 1991 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 1992 CID_BITS(cid)); 1993 } 1994 } 1995} 1996 1997 1998#endif 1999 2000 2001#ifdef NDEBUG 2002#define CONST_BATCH const 2003#else 2004#define CONST_BATCH 2005#endif 2006 2007 2008static void 2009sockaddr2str (const struct sockaddr *addr, char *buf, size_t sz) 2010{ 2011 unsigned short port; 2012 int len; 2013 2014 switch (addr->sa_family) 2015 { 2016 case AF_INET: 2017 port = ((struct sockaddr_in *) addr)->sin_port; 2018 if (!inet_ntop(AF_INET, &((struct sockaddr_in *) addr)->sin_addr, 2019 buf, sz)) 2020 buf[0] = '\0'; 2021 break; 2022 case AF_INET6: 2023 port = ((struct sockaddr_in6 *) addr)->sin6_port; 2024 if (!inet_ntop(AF_INET6, &((struct sockaddr_in6 *) addr)->sin6_addr, 2025 buf, sz)) 2026 buf[0] = '\0'; 2027 break; 2028 default: 2029 port = 0; 2030 (void) snprintf(buf, sz, "<invalid family %d>", addr->sa_family); 2031 break; 2032 } 2033 2034 len = strlen(buf); 2035 if (len < (int) sz) 2036 snprintf(buf + len, sz - (size_t) len, ":%hu", port); 2037} 2038 2039 2040struct send_batch_ctx { 2041 struct conns_stailq *closed_conns; 2042 struct conns_tailq *ticked_conns; 2043 struct conns_out_iter *conns_iter; 2044 CONST_BATCH struct out_batch *batch; 2045}; 2046 2047 2048static void 2049close_conn_immediately (struct lsquic_engine *engine, 2050 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2051{ 2052 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2053 if (!(conn->cn_flags & LSCONN_CLOSING)) 2054 { 2055 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2056 engine_incref_conn(conn, LSCONN_CLOSING); 2057 if (conn->cn_flags & LSCONN_HASHED) 2058 remove_conn_from_hash(engine, conn); 2059 } 2060 if (conn->cn_flags & LSCONN_TICKED) 2061 { 2062 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2063 engine_decref_conn(engine, conn, LSCONN_TICKED); 2064 } 2065} 2066 2067 2068static void 2069close_conn_on_send_error (struct lsquic_engine *engine, 2070 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2071{ 2072 const struct out_batch *batch = sb_ctx->batch; 2073 struct lsquic_conn *const conn = batch->conns[n]; 2074 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2075 2076 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2077 "src: %s; dst: %s; errno: %d", 2078 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2079 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2080 CID_BITS(lsquic_conn_log_cid(conn)), 2081 (sockaddr2str(batch->outs[n].local_sa, buf[0], sizeof(buf[0])), buf[0]), 2082 (sockaddr2str(batch->outs[n].dest_sa, buf[1], sizeof(buf[1])), buf[1]), 2083 e_val); 2084 if (conn->cn_flags & LSCONN_EVANESCENT) 2085 lsquic_prq_drop(conn); 2086 else 2087 close_conn_immediately(engine, sb_ctx, conn); 2088} 2089 2090 2091static unsigned 2092send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2093 unsigned n_to_send) 2094{ 2095 int n_sent, i, e_val; 2096 lsquic_time_t now; 2097 unsigned off; 2098 size_t count; 2099 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2100 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2101 2102#ifndef NDEBUG 2103 if (engine->flags & ENG_LOSE_PACKETS) 2104 lose_matching_packets(engine, batch, n_to_send); 2105#endif 2106 /* Set sent time before the write to avoid underestimating RTT */ 2107 now = lsquic_time_now(); 2108 for (i = 0; i < (int) n_to_send; ++i) 2109 { 2110 off = batch->pack_off[i]; 2111 count = batch->outs[i].iovlen; 2112 assert(count > 0); 2113 packet_out = &batch->packets[off]; 2114 end = packet_out + count; 2115 do 2116 (*packet_out)->po_sent = now; 2117 while (++packet_out < end); 2118 } 2119 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs, 2120 n_to_send); 2121 e_val = errno; 2122 if (n_sent < (int) n_to_send) 2123 { 2124 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2125 engine->resume_sending_at = now + 1000000; 2126 LSQ_DEBUG("cannot send packets"); 2127 EV_LOG_GENERIC_EVENT("cannot send packets"); 2128 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2129 close_conn_on_send_error(engine, sb_ctx, 2130 n_sent < 0 ? 0 : n_sent, e_val); 2131 } 2132 if (n_sent >= 0) 2133 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, n_to_send); 2134 else 2135 { 2136 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2137 n_sent = 0; 2138 } 2139 if (n_sent > 0) 2140 engine->last_sent = now + n_sent; 2141 for (i = 0; i < n_sent; ++i) 2142 { 2143 eng_hist_inc(&engine->history, now, sl_packets_out); 2144 /* `i' is added to maintain relative order */ 2145 batch->conns[i]->cn_last_sent = now + i; 2146 2147 off = batch->pack_off[i]; 2148 count = batch->outs[i].iovlen; 2149 assert(count > 0); 2150 packet_out = &batch->packets[off]; 2151 end = packet_out + count; 2152 do 2153 { 2154#if LOG_PACKET_CHECKSUM 2155 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2156 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2157 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2158#endif 2159 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2160 *packet_out); 2161 /* Release packet out buffer as soon as the packet is sent 2162 * successfully. If not successfully sent, we hold on to 2163 * this buffer until the packet sending is attempted again 2164 * or until it times out and regenerated. 2165 */ 2166 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2167 release_enc_data(engine, batch->conns[i], *packet_out); 2168 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2169 *packet_out); 2170 } 2171 while (++packet_out < end); 2172 } 2173 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2174 for ( ; i < (int) n_to_send; ++i) 2175 { 2176 off = batch->pack_off[i]; 2177 count = batch->outs[i].iovlen; 2178 assert(count > 0); 2179 packet_out = &batch->packets[off]; 2180 end = packet_out + count; 2181 do 2182 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2183 *packet_out); 2184 while (++packet_out < end); 2185 } 2186 /* Return packets to the connection in reverse order so that the packet 2187 * ordering is maintained. 2188 */ 2189 for (i = (int) n_to_send - 1; i >= n_sent; --i) 2190 { 2191 off = batch->pack_off[i]; 2192 count = batch->outs[i].iovlen; 2193 assert(count > 0); 2194 packet_out = &batch->packets[off + count - 1]; 2195 end = &batch->packets[off - 1]; 2196 do 2197 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2198 *packet_out); 2199 while (--packet_out > end); 2200 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2201 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2202 } 2203 return n_sent; 2204} 2205 2206 2207/* Return 1 if went past deadline, 0 otherwise */ 2208static int 2209check_deadline (lsquic_engine_t *engine) 2210{ 2211 if (engine->pub.enp_settings.es_proc_time_thresh && 2212 lsquic_time_now() > engine->deadline) 2213 { 2214 LSQ_INFO("went past threshold of %u usec, stop sending", 2215 engine->pub.enp_settings.es_proc_time_thresh); 2216 engine->flags |= ENG_PAST_DEADLINE; 2217 return 1; 2218 } 2219 else 2220 return 0; 2221} 2222 2223 2224static size_t 2225iov_size (const struct iovec *iov, const struct iovec *const end) 2226{ 2227 size_t size; 2228 2229 assert(iov < end); 2230 2231 size = 0; 2232 do 2233 size += iov->iov_len; 2234 while (++iov < end); 2235 2236 return size; 2237} 2238 2239 2240static void 2241send_packets_out (struct lsquic_engine *engine, 2242 struct conns_tailq *ticked_conns, 2243 struct conns_stailq *closed_conns) 2244{ 2245 unsigned n, w, n_sent, n_batches_sent; 2246 lsquic_packet_out_t *packet_out; 2247 struct lsquic_packet_out **packet; 2248 lsquic_conn_t *conn; 2249 struct out_batch *const batch = &engine->out_batch; 2250 struct iovec *iov, *packet_iov; 2251 struct conns_out_iter conns_iter; 2252 int shrink, deadline_exceeded; 2253 const struct send_batch_ctx sb_ctx = { 2254 closed_conns, 2255 ticked_conns, 2256 &conns_iter, 2257 &engine->out_batch, 2258 }; 2259 2260 coi_init(&conns_iter, engine); 2261 n_batches_sent = 0; 2262 n_sent = 0, n = 0; 2263 shrink = 0; 2264 deadline_exceeded = 0; 2265 iov = batch->iov; 2266 packet = batch->packets; 2267 2268 while ((conn = coi_next(&conns_iter))) 2269 { 2270 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2271 if (!packet_out) { 2272 /* Evanescent connection always has a packet to send: */ 2273 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2274 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2275 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2276 CID_BITS(lsquic_conn_log_cid(conn))); 2277 coi_deactivate(&conns_iter, conn); 2278 continue; 2279 } 2280 batch->outs[n].iov = packet_iov = iov; 2281 next_coa: 2282 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2283 { 2284 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2285 &engine->pub, conn, packet_out)) 2286 { 2287 case ENCPA_NOMEM: 2288 /* Send what we have and wait for a more opportune moment */ 2289 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2290 goto end_for; 2291 case ENCPA_BADCRYPT: 2292 /* This is pretty bad: close connection immediately */ 2293 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2294 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2295 CID_BITS(lsquic_conn_log_cid(conn))); 2296 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2297 { 2298 close_conn_immediately(engine, &sb_ctx, conn); 2299 coi_deactivate(&conns_iter, conn); 2300 } 2301 continue; 2302 case ENCPA_OK: 2303 break; 2304 } 2305 } 2306 else if ((packet_out->po_flags & PO_NOENCRYPT) 2307 && engine->pub.enp_pmi != &stock_pmi) 2308 { 2309 if (0 != copy_packet(engine, conn, packet_out)) 2310 { 2311 /* Copy can only fail if packet could not be allocated */ 2312 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2313 goto end_for; 2314 } 2315 } 2316 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2317 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2318 if (packet_out->po_flags & PO_ENCRYPTED) 2319 { 2320 iov->iov_base = packet_out->po_enc_data; 2321 iov->iov_len = packet_out->po_enc_data_sz; 2322 } 2323 else 2324 { 2325 iov->iov_base = packet_out->po_data; 2326 iov->iov_len = packet_out->po_data_sz; 2327 } 2328 if (packet_iov == iov) 2329 { 2330 batch->pack_off[n] = packet - batch->packets; 2331 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2332 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2333 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2334 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2335 batch->conns [n] = conn; 2336 } 2337 *packet = packet_out; 2338 ++packet; 2339 ++iov; 2340 if ((conn->cn_flags & LSCONN_IETF) 2341 && ((1 << packet_out->po_header_type) 2342 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2343#ifndef NDEBUG 2344 && (engine->flags & ENG_COALESCE) 2345#endif 2346 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2347 { 2348 const size_t size = iov_size(packet_iov, iov); 2349 packet_out = conn->cn_if->ci_next_packet_to_send(conn, size); 2350 if (packet_out) 2351 goto next_coa; 2352 } 2353 batch->outs [n].iovlen = iov - packet_iov; 2354 ++n; 2355 if (n == engine->batch_size 2356 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2357 { 2358 w = send_batch(engine, &sb_ctx, n); 2359 n = 0; 2360 iov = batch->iov; 2361 packet = batch->packets; 2362 ++n_batches_sent; 2363 n_sent += w; 2364 if (w < engine->batch_size) 2365 { 2366 shrink = 1; 2367 break; 2368 } 2369 deadline_exceeded = check_deadline(engine); 2370 if (deadline_exceeded) 2371 break; 2372 grow_batch_size(engine); 2373 } 2374 } 2375 end_for: 2376 2377 if (n > 0) { 2378 w = send_batch(engine, &sb_ctx, n); 2379 n_sent += w; 2380 shrink = w < n; 2381 ++n_batches_sent; 2382 } 2383 2384 if (shrink) 2385 shrink_batch_size(engine); 2386 else if (n_batches_sent > 1) 2387 { 2388 deadline_exceeded = check_deadline(engine); 2389 if (!deadline_exceeded) 2390 grow_batch_size(engine); 2391 } 2392 2393 coi_reheap(&conns_iter, engine); 2394 2395 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2396} 2397 2398 2399int 2400lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2401{ 2402 return lsquic_mh_count(&engine->conns_out) > 0 2403 || (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2404 ; 2405} 2406 2407 2408static void 2409reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2410{ 2411 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2412 engine->flags &= ~ENG_PAST_DEADLINE; 2413} 2414 2415 2416void 2417lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2418{ 2419 lsquic_conn_t *conn; 2420 struct conns_stailq closed_conns; 2421 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2422 struct cid_update_batch cub; 2423 2424 ENGINE_IN(engine); 2425 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2426 STAILQ_INIT(&closed_conns); 2427 reset_deadline(engine, lsquic_time_now()); 2428 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2429 { 2430 LSQ_DEBUG("can send again"); 2431 EV_LOG_GENERIC_EVENT("can send again"); 2432 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2433 } 2434 2435 send_packets_out(engine, &ticked_conns, &closed_conns); 2436 2437 while ((conn = STAILQ_FIRST(&closed_conns))) { 2438 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2439 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2440 cub_add_cids_from_cces(&cub, conn); 2441 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2442 } 2443 2444 cub_flush(&cub); 2445 ENGINE_OUT(engine); 2446} 2447 2448 2449static lsquic_conn_t * 2450next_new_full_conn (struct conns_stailq *new_full_conns) 2451{ 2452 lsquic_conn_t *conn; 2453 2454 conn = STAILQ_FIRST(new_full_conns); 2455 if (conn) 2456 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2457 return conn; 2458} 2459 2460 2461static void 2462process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2463 lsquic_time_t now) 2464{ 2465 lsquic_conn_t *conn; 2466 enum tick_st tick_st; 2467 unsigned i, why; 2468 lsquic_time_t next_tick_time; 2469 struct conns_stailq closed_conns; 2470 struct conns_tailq ticked_conns; 2471 struct conns_stailq new_full_conns; 2472 struct cid_update_batch cub_old, cub_live; 2473 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2474 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2475 2476 eng_hist_tick(&engine->history, now); 2477 2478 STAILQ_INIT(&closed_conns); 2479 TAILQ_INIT(&ticked_conns); 2480 reset_deadline(engine, now); 2481 STAILQ_INIT(&new_full_conns); 2482 2483 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2484 && now > engine->resume_sending_at) 2485 { 2486 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2487 "timeout"); 2488 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2489 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2490 } 2491 2492 i = 0; 2493 while ((conn = next_conn(engine)) 2494 || (conn = next_new_full_conn(&new_full_conns))) 2495 { 2496 tick_st = conn->cn_if->ci_tick(conn, now); 2497 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2498 if (tick_st & TICK_PROMOTE) 2499 { 2500 lsquic_conn_t *new_conn; 2501 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2502 "scheduled for promotion"); 2503 assert(conn->cn_flags & LSCONN_MINI); 2504 new_conn = new_full_conn_server(engine, conn, now); 2505 if (new_conn) 2506 { 2507 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2508 new_conn->cn_last_sent = engine->last_sent; 2509 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2510 } 2511 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2512 conn->cn_flags |= LSCONN_PROMOTED; 2513 } 2514 if (tick_st & TICK_SEND) 2515 { 2516 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2517 { 2518 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2519 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2520 } 2521 } 2522 if (tick_st & TICK_CLOSE) 2523 { 2524 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2525 engine_incref_conn(conn, LSCONN_CLOSING); 2526 if (conn->cn_flags & LSCONN_HASHED) 2527 remove_conn_from_hash(engine, conn); 2528 } 2529 else 2530 { 2531 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2532 engine_incref_conn(conn, LSCONN_TICKED); 2533 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2534 && conn->cn_if->ci_report_live(conn, now)) 2535 cub_add_cids_from_cces(&cub_live, conn); 2536 } 2537 } 2538 2539 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2540 && lsquic_engine_has_unsent_packets(engine)) 2541 send_packets_out(engine, &ticked_conns, &closed_conns); 2542 2543 while ((conn = STAILQ_FIRST(&closed_conns))) { 2544 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2545 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2546 cub_add_cids_from_cces(&cub_old, conn); 2547 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2548 } 2549 2550 while ((conn = TAILQ_FIRST(&ticked_conns))) 2551 { 2552 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2553 engine_decref_conn(engine, conn, LSCONN_TICKED); 2554 if (!(conn->cn_flags & LSCONN_TICKABLE) 2555 && conn->cn_if->ci_is_tickable(conn)) 2556 { 2557 /* Floyd heapification is not faster, don't bother. */ 2558 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2559 engine_incref_conn(conn, LSCONN_TICKABLE); 2560 } 2561 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2562 { 2563 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2564 if (next_tick_time) 2565 { 2566 if (0 == attq_add(engine->attq, conn, next_tick_time, why)) 2567 engine_incref_conn(conn, LSCONN_ATTQ); 2568 } 2569 else 2570 assert(0); 2571 } 2572 } 2573 2574 cub_flush(&engine->new_scids); 2575 cub_flush(&cub_live); 2576 cub_flush(&cub_old); 2577} 2578 2579 2580/* Return 0 if packet is being processed by a real connection, 1 if the 2581 * packet was processed, but not by a connection, and -1 on error. 2582 */ 2583int 2584lsquic_engine_packet_in (lsquic_engine_t *engine, 2585 const unsigned char *packet_in_data, size_t packet_in_size, 2586 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2587 void *peer_ctx, int ecn) 2588{ 2589 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2590 struct packin_parse_state ppstate; 2591 lsquic_packet_in_t *packet_in; 2592 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2593 int is_server, unsigned cid_len, struct packin_parse_state *); 2594 unsigned n_zeroes; 2595 int s; 2596 2597 ENGINE_CALLS_INCR(engine); 2598 2599 if (engine->flags & ENG_SERVER) 2600 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2601 else 2602 if (engine->flags & ENG_CONNS_BY_ADDR) 2603 { 2604 struct lsquic_hash_elem *el; 2605 const struct lsquic_conn *conn; 2606 el = find_conn_by_addr(engine->conns_hash, sa_local); 2607 if (!el) 2608 return -1; 2609 conn = lsquic_hashelem_getdata(el); 2610 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2611 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2612 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2613 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2614 else if (conn->cn_version == LSQVER_050) 2615 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 2616 else 2617 { 2618 assert(conn->cn_version == LSQVER_046 2619#if LSQUIC_USE_Q098 2620 || conn->cn_version == LSQVER_098 2621#endif 2622 2623 ); 2624 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2625 } 2626 } 2627 else 2628 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2629 2630 n_zeroes = 0; 2631 do 2632 { 2633 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 2634 if (!packet_in) 2635 return -1; 2636 /* Library does not modify packet_in_data, it is not referenced after 2637 * this function returns and subsequent release of pi_data is guarded 2638 * by PI_OWN_DATA flag. 2639 */ 2640 packet_in->pi_data = (unsigned char *) packet_in_data; 2641 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 2642 engine->flags & ENG_SERVER, 2643 engine->pub.enp_settings.es_scid_len, &ppstate)) 2644 { 2645 LSQ_DEBUG("Cannot parse incoming packet's header"); 2646 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 2647 errno = EINVAL; 2648 return -1; 2649 } 2650 2651 packet_in_data += packet_in->pi_data_sz; 2652 packet_in->pi_received = lsquic_time_now(); 2653 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 2654 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 2655 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 2656 peer_ctx, packet_in->pi_data_sz == packet_in_size); 2657 n_zeroes += s == 0; 2658 } 2659 while (0 == s && packet_in_data < packet_end); 2660 2661 return n_zeroes > 0 ? 0 : s; 2662} 2663 2664 2665#if __GNUC__ && !defined(NDEBUG) 2666__attribute__((weak)) 2667#endif 2668unsigned 2669lsquic_engine_quic_versions (const lsquic_engine_t *engine) 2670{ 2671 return engine->pub.enp_settings.es_versions; 2672} 2673 2674 2675void 2676lsquic_engine_cooldown (lsquic_engine_t *engine) 2677{ 2678 struct lsquic_hash_elem *el; 2679 lsquic_conn_t *conn; 2680 2681 if (engine->flags & ENG_COOLDOWN) 2682 /* AFAICT, there is no harm in calling this function more than once, 2683 * but log it just in case, as it may indicate an error in the caller. 2684 */ 2685 LSQ_INFO("cooldown called again"); 2686 engine->flags |= ENG_COOLDOWN; 2687 LSQ_INFO("entering cooldown mode"); 2688 if (engine->flags & ENG_SERVER) 2689 drop_all_mini_conns(engine); 2690 for (el = lsquic_hash_first(engine->conns_hash); el; 2691 el = lsquic_hash_next(engine->conns_hash)) 2692 { 2693 conn = lsquic_hashelem_getdata(el); 2694 lsquic_conn_going_away(conn); 2695 } 2696} 2697 2698 2699int 2700lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 2701{ 2702 const struct attq_elem *next_attq; 2703 lsquic_time_t now, next_time; 2704#if LSQUIC_DEBUG_NEXT_ADV_TICK 2705 const struct lsquic_conn *conn; 2706 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 2707#endif 2708 2709 ENGINE_CALLS_INCR(engine); 2710 2711 if ((engine->flags & ENG_PAST_DEADLINE) 2712 && lsquic_mh_count(&engine->conns_out)) 2713 { 2714#if LSQUIC_DEBUG_NEXT_ADV_TICK 2715 conn = lsquic_mh_peek(&engine->conns_out); 2716 engine->last_logged_conn = 0; 2717 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 2718 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 2719 lsquic_mh_count(&engine->conns_out), 2720 lsquic_mh_count(&engine->conns_out) != 1, "s", 2721 CID_BITS(lsquic_conn_log_cid(conn))); 2722#endif 2723 *diff = 0; 2724 return 1; 2725 } 2726 2727 if (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2728 { 2729#if LSQUIC_DEBUG_NEXT_ADV_TICK 2730 engine->last_logged_conn = 0; 2731 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 2732#endif 2733 *diff = 0; 2734 return 1; 2735 } 2736 2737 if (lsquic_mh_count(&engine->conns_tickable)) 2738 { 2739#if LSQUIC_DEBUG_NEXT_ADV_TICK 2740 conn = lsquic_mh_peek(&engine->conns_tickable); 2741 engine->last_logged_conn = 0; 2742 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 2743 "connection%.*s (%"CID_FMT" first)", 2744 lsquic_mh_count(&engine->conns_tickable), 2745 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 2746 CID_BITS(lsquic_conn_log_cid(conn))); 2747#endif 2748 *diff = 0; 2749 return 1; 2750 } 2751 2752 next_attq = attq_next(engine->attq); 2753 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 2754 { 2755 if (next_attq) 2756 next_time = next_attq->ae_adv_time; 2757 else 2758 return 0; 2759 } 2760 else 2761 { 2762 if (next_attq) 2763 { 2764 next_time = next_attq->ae_adv_time; 2765 if (engine->resume_sending_at < next_time) 2766 { 2767 next_time = engine->resume_sending_at; 2768 next_attq = NULL; 2769 } 2770 } 2771 else 2772 next_time = engine->resume_sending_at; 2773 } 2774 2775 now = lsquic_time_now(); 2776 *diff = (int) ((int64_t) next_time - (int64_t) now); 2777#if LSQUIC_DEBUG_NEXT_ADV_TICK 2778 if (next_attq) 2779 { 2780 /* Deduplicate consecutive log messages about the same reason for the 2781 * same connection. 2782 * If diff is always zero or diff reset to a higher value, event is 2783 * still logged. 2784 */ 2785 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 2786 && (uintptr_t) next_attq->ae_conn 2787 == engine->last_logged_conn 2788 && *diff < engine->last_tick_diff)) 2789 { 2790 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 2791 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 2792 engine->last_tick_diff = *diff; 2793 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 2794 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 2795 lsquic_attq_why2str(next_attq->ae_why)); 2796 } 2797 } 2798 else 2799 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 2800#endif 2801 return 1; 2802} 2803 2804 2805unsigned 2806lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 2807{ 2808 lsquic_time_t now; 2809 ENGINE_CALLS_INCR(engine); 2810 now = lsquic_time_now(); 2811 if (from_now < 0) 2812 now -= from_now; 2813 else 2814 now += from_now; 2815 return attq_count_before(engine->attq, now); 2816} 2817 2818 2819int 2820lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 2821 struct lsquic_conn *conn, unsigned cce_idx) 2822{ 2823 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2824 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2825 void *peer_ctx; 2826 2827 assert(cce_idx < conn->cn_n_cces); 2828 assert(conn->cn_cces_mask & (1 << cce_idx)); 2829 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 2830 2831 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 2832 cce->cce_cid.len, conn, &cce->cce_hash_el)) 2833 { 2834 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 2835 CID_BITS(&cce->cce_cid)); 2836 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2837 cce->cce_flags |= CCE_REG; 2838 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 2839 return 0; 2840 } 2841 else 2842 { 2843 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 2844 CID_BITS(&cce->cce_cid)); 2845 return -1; 2846 } 2847} 2848 2849 2850void 2851lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 2852 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now) 2853{ 2854 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2855 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2856 void *peer_ctx; 2857 2858 assert(cce_idx < conn->cn_n_cces); 2859 2860 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 2861 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 2862 2863 if (engine->purga) 2864 { 2865 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2866 lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 2867 PUTY_CID_RETIRED, now); 2868 } 2869 conn->cn_cces_mask &= ~(1u << cce_idx); 2870 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 2871} 2872 2873 2874