lsquic_engine.c revision 4ab453a1
1/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#include <arpa/inet.h> 17#ifndef WIN32 18#include <sys/time.h> 19#include <netinet/in.h> 20#include <sys/types.h> 21#include <sys/stat.h> 22#include <fcntl.h> 23#include <unistd.h> 24#include <netdb.h> 25#endif 26 27#ifndef NDEBUG 28#include <sys/types.h> 29#include <regex.h> /* For code that loses packets */ 30#endif 31 32#if LOG_PACKET_CHECKSUM 33#include <zlib.h> 34#endif 35 36#include <openssl/aead.h> 37 38#include "lsquic.h" 39#include "lsquic_types.h" 40#include "lsquic_int_types.h" 41#include "lsquic_sizes.h" 42#include "lsquic_parse_common.h" 43#include "lsquic_parse.h" 44#include "lsquic_packet_in.h" 45#include "lsquic_packet_out.h" 46#include "lsquic_senhist.h" 47#include "lsquic_rtt.h" 48#include "lsquic_cubic.h" 49#include "lsquic_pacer.h" 50#include "lsquic_bw_sampler.h" 51#include "lsquic_minmax.h" 52#include "lsquic_bbr.h" 53#include "lsquic_send_ctl.h" 54#include "lsquic_set.h" 55#include "lsquic_conn_flow.h" 56#include "lsquic_sfcw.h" 57#include "lsquic_hash.h" 58#include "lsquic_conn.h" 59#include "lsquic_full_conn.h" 60#include "lsquic_util.h" 61#include "lsquic_qtags.h" 62#include "lsquic_enc_sess.h" 63#include "lsquic_mm.h" 64#include "lsquic_engine_public.h" 65#include "lsquic_eng_hist.h" 66#include "lsquic_ev_log.h" 67#include "lsquic_version.h" 68#include "lsquic_pr_queue.h" 69#include "lsquic_mini_conn.h" 70#include "lsquic_mini_conn_ietf.h" 71#include "lsquic_stock_shi.h" 72#include "lsquic_purga.h" 73#include "lsquic_tokgen.h" 74#include "lsquic_attq.h" 75#include "lsquic_min_heap.h" 76#include "lsquic_http1x_if.h" 77#include "lsquic_parse_common.h" 78#include "lsquic_handshake.h" 79#include "lsquic_crand.h" 80#include "lsquic_ietf.h" 81 82#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 83#include "lsquic_logger.h" 84 85#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 86#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 87#endif 88 89#if LSQUIC_DEBUG_NEXT_ADV_TICK 90#include "lsquic_alarmset.h" 91#endif 92 93#define MIN(a, b) ((a) < (b) ? (a) : (b)) 94 95/* The batch of outgoing packets grows and shrinks dynamically */ 96#define MAX_OUT_BATCH_SIZE 1024 97#define MIN_OUT_BATCH_SIZE 4 98#define INITIAL_OUT_BATCH_SIZE 32 99 100struct out_batch 101{ 102 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 103 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 104 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 105 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 106 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 107}; 108 109typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 110 111static void 112process_connections (struct lsquic_engine *engine, conn_iter_f iter, 113 lsquic_time_t now); 114 115static void 116engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 117 118static lsquic_conn_t * 119engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 120 enum lsquic_conn_flags flag); 121 122static void 123force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 124 125#if LSQUIC_COUNT_ENGINE_CALLS 126#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 127#else 128#define ENGINE_CALLS_INCR(e) 129#endif 130 131/* Nested calls to LSQUIC are not supported */ 132#define ENGINE_IN(e) do { \ 133 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 134 (e)->pub.enp_flags |= ENPUB_PROC; \ 135 ENGINE_CALLS_INCR(e); \ 136} while (0) 137 138#define ENGINE_OUT(e) do { \ 139 assert((e)->pub.enp_flags & ENPUB_PROC); \ 140 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 141} while (0) 142 143/* A connection can be referenced from one of six places: 144 * 145 * 1. A hash. The engine maintains two hash tables -- one for full, and 146 * one for mini connections. A connection starts its life in one of 147 * those. 148 * 149 * 2. Outgoing queue. 150 * 151 * 3. Tickable queue 152 * 153 * 4. Advisory Tick Time queue. 154 * 155 * 5. Closing connections queue. This is a transient queue -- it only 156 * exists for the duration of process_connections() function call. 157 * 158 * 6. Ticked connections queue. Another transient queue, similar to (5). 159 * 160 * The idea is to destroy the connection when it is no longer referenced. 161 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 162 * that case, the connection is referenced from two places: (2) and (5). 163 * After its packets are sent, it is only referenced in (5), and at the 164 * end of the function call, when it is removed from (5), reference count 165 * goes to zero and the connection is destroyed. If not all packets can 166 * be sent, at the end of the function call, the connection is referenced 167 * by (2) and will only be removed once all outgoing packets have been 168 * sent. 169 */ 170#define CONN_REF_FLAGS (LSCONN_HASHED \ 171 |LSCONN_HAS_OUTGOING \ 172 |LSCONN_TICKABLE \ 173 |LSCONN_TICKED \ 174 |LSCONN_CLOSING \ 175 |LSCONN_ATTQ) 176 177 178 179 180struct cid_update_batch 181{ 182 lsquic_cids_update_f cub_update_cids; 183 void *cub_update_ctx; 184 unsigned cub_count; 185 lsquic_cid_t cub_cids[20]; 186 void *cub_peer_ctxs[20]; 187}; 188 189static void 190cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 191 192 193struct lsquic_engine 194{ 195 struct lsquic_engine_public pub; 196 enum { 197 ENG_SERVER = LSENG_SERVER, 198 ENG_HTTP = LSENG_HTTP, 199 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 200 ENG_PAST_DEADLINE 201 = (1 << 8), /* Previous call to a processing 202 * function went past time threshold. 203 */ 204 ENG_CONNS_BY_ADDR 205 = (1 << 9), /* Connections are hashed by address */ 206#ifndef NDEBUG 207 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 208 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 209 ENG_DTOR = (1 << 26), /* Engine destructor */ 210#endif 211 } flags; 212 lsquic_packets_out_f packets_out; 213 void *packets_out_ctx; 214 lsquic_cids_update_f report_new_scids; 215 lsquic_cids_update_f report_live_scids; 216 lsquic_cids_update_f report_old_scids; 217 void *scids_ctx; 218 struct lsquic_hash *conns_hash; 219 struct min_heap conns_tickable; 220 struct min_heap conns_out; 221 /* Use a union because only one iterator is being used at any one time */ 222 union { 223 struct { 224 struct cert_susp_head *head; 225 } resumed; 226 struct lsquic_conn *one_conn; 227 } iter_state; 228 struct eng_hist history; 229 unsigned batch_size; 230 struct pr_queue *pr_queue; 231 struct attq *attq; 232 /* Track time last time a packet was sent to give new connections 233 * priority lower than that of existing connections. 234 */ 235 lsquic_time_t last_sent; 236#ifndef NDEBUG 237 regex_t lose_packets_re; 238 const char *lose_packets_str; 239#endif 240 unsigned n_conns; 241 lsquic_time_t deadline; 242 lsquic_time_t resume_sending_at; 243 unsigned mini_conns_count; 244 struct lsquic_purga *purga; 245#if LSQUIC_CONN_STATS 246 struct { 247 unsigned conns; 248 } stats; 249 struct conn_stats conn_stats_sum; 250 FILE *stats_fh; 251#endif 252 struct cid_update_batch new_scids; 253 struct out_batch out_batch; 254#if LSQUIC_COUNT_ENGINE_CALLS 255 unsigned long n_engine_calls; 256#endif 257#if LSQUIC_DEBUG_NEXT_ADV_TICK 258 uintptr_t last_logged_conn; 259 unsigned last_logged_ae_why; 260 int last_tick_diff; 261#endif 262 struct crand crand; 263 EVP_AEAD_CTX retry_aead_ctx; 264}; 265 266 267void 268lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 269 unsigned flags) 270{ 271 memset(settings, 0, sizeof(*settings)); 272 settings->es_versions = LSQUIC_DF_VERSIONS; 273 if (flags & ENG_SERVER) 274 { 275 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 276 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 277 settings->es_init_max_data 278 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 279 settings->es_init_max_stream_data_bidi_remote 280 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 281 settings->es_init_max_stream_data_bidi_local 282 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 283 settings->es_init_max_stream_data_uni 284 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 285 settings->es_init_max_streams_uni 286 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 287 settings->es_ping_period = 0; 288 } 289 else 290 { 291 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 292 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 293 settings->es_init_max_data 294 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 295 settings->es_init_max_stream_data_bidi_remote 296 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 297 settings->es_init_max_stream_data_bidi_local 298 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 299 settings->es_init_max_stream_data_uni 300 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 301 settings->es_init_max_streams_uni 302 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 303 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 304 } 305 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 306 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 307 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 308 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 309 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 310 settings->es_max_header_list_size 311 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 312 settings->es_ua = LSQUIC_DF_UA; 313 settings->es_ecn = LSQUIC_DF_ECN; 314 315 settings->es_pdmd = QTAG_X509; 316 settings->es_aead = QTAG_AESG; 317 settings->es_kexs = QTAG_C255; 318 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 319 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 320 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 321 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 322 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 323 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 324 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 325 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 326 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 327 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 328 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 329 settings->es_sttl = LSQUIC_DF_STTL; 330 settings->es_init_max_streams_bidi 331 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 332 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 333 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 334 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 335 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 336 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 337 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 338 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 339 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 340 settings->es_spin = LSQUIC_DF_SPIN; 341} 342 343 344/* Note: if returning an error, err_buf must be valid if non-NULL */ 345int 346lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 347 unsigned flags, 348 char *err_buf, size_t err_buf_sz) 349{ 350 if (settings->es_cfcw < LSQUIC_MIN_FCW || 351 settings->es_sfcw < LSQUIC_MIN_FCW) 352 { 353 if (err_buf) 354 snprintf(err_buf, err_buf_sz, "%s", 355 "flow control window set too low"); 356 return -1; 357 } 358 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 359 { 360 if (err_buf) 361 snprintf(err_buf, err_buf_sz, "%s", 362 "No supported QUIC versions specified"); 363 return -1; 364 } 365 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 366 { 367 if (err_buf) 368 snprintf(err_buf, err_buf_sz, "%s", 369 "one or more unsupported QUIC version is specified"); 370 return -1; 371 } 372 if (flags & ENG_SERVER) 373 { 374 if (settings->es_handshake_to > 375 MAX_MINI_CONN_LIFESPAN_IN_USEC) 376 { 377 if (err_buf) 378 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 379 " usec is too large. The maximum for server is %u usec", 380 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 381 return -1; 382 } 383 } 384 if (settings->es_idle_timeout > 600) 385 { 386 if (err_buf) 387 snprintf(err_buf, err_buf_sz, "%s", 388 "The maximum value of idle timeout is 600 seconds"); 389 return -1; 390 } 391 if (settings->es_scid_len > MAX_CID_LEN) 392 { 393 if (err_buf) 394 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 395 "bytes long; it must be between 0 and %u.", 396 settings->es_scid_len, MAX_CID_LEN); 397 return -1; 398 } 399 400 if (settings->es_cc_algo > 2) 401 { 402 if (err_buf) 403 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 404 "algorithm value %u", settings->es_cc_algo); 405 return -1; 406 } 407 408 if (!(settings->es_ql_bits >= -1 && settings->es_ql_bits <= 2)) 409 { 410 if (err_buf) 411 snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ", 412 settings->es_ql_bits); 413 return -1; 414 } 415 416 if (!(settings->es_spin == 0 || settings->es_spin == 1)) 417 { 418 if (err_buf) 419 snprintf(err_buf, err_buf_sz, "Invalid spin value %d", 420 settings->es_spin); 421 return -1; 422 } 423 424 return 0; 425} 426 427 428static void 429free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 430{ 431 free(packet_data); 432} 433 434 435static void * 436malloc_buf (void *ctx, void *conn_ctx, unsigned short size, char is_ipv6) 437{ 438 return malloc(size); 439} 440 441 442static const struct lsquic_packout_mem_if stock_pmi = 443{ 444 malloc_buf, free_packet, free_packet, 445}; 446 447 448static int 449hash_conns_by_addr (const struct lsquic_engine *engine) 450{ 451 if (engine->flags & ENG_SERVER) 452 return 0; 453 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 454 return 1; 455 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 456 && engine->pub.enp_settings.es_support_tcid0) 457 return 1; 458 if (engine->pub.enp_settings.es_scid_len == 0) 459 return 1; 460 return 0; 461} 462 463 464lsquic_engine_t * 465lsquic_engine_new (unsigned flags, 466 const struct lsquic_engine_api *api) 467{ 468 lsquic_engine_t *engine; 469 char err_buf[100]; 470 471 if (!api->ea_packets_out) 472 { 473 LSQ_ERROR("packets_out callback is not specified"); 474 return NULL; 475 } 476 477 if (api->ea_settings && 478 0 != lsquic_engine_check_settings(api->ea_settings, flags, 479 err_buf, sizeof(err_buf))) 480 { 481 LSQ_ERROR("cannot create engine: %s", err_buf); 482 return NULL; 483 } 484 485 engine = calloc(1, sizeof(*engine)); 486 if (!engine) 487 return NULL; 488 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 489 { 490 free(engine); 491 return NULL; 492 } 493 if (api->ea_settings) 494 engine->pub.enp_settings = *api->ea_settings; 495 else 496 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 497 int tag_buf_len; 498 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 499 sizeof(engine->pub.enp_ver_tags_buf), 500 engine->pub.enp_settings.es_versions); 501 if (tag_buf_len <= 0) 502 { 503 LSQ_ERROR("cannot generate version tags buffer"); 504 free(engine); 505 return NULL; 506 } 507 engine->pub.enp_ver_tags_len = tag_buf_len; 508 engine->pub.enp_flags = ENPUB_CAN_SEND; 509 engine->pub.enp_stream_if = api->ea_stream_if; 510 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 511 512 engine->flags = flags; 513#ifndef NDEBUG 514 engine->flags |= ENG_COALESCE; 515#endif 516 engine->packets_out = api->ea_packets_out; 517 engine->packets_out_ctx = api->ea_packets_out_ctx; 518 engine->report_new_scids = api->ea_new_scids; 519 engine->report_live_scids = api->ea_live_scids; 520 engine->report_old_scids = api->ea_old_scids; 521 engine->scids_ctx = api->ea_cids_update_ctx; 522 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 523 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 524 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 525 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 526 if (api->ea_shi) 527 { 528 engine->pub.enp_shi = api->ea_shi; 529 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 530 } 531 else 532 { 533 engine->pub.enp_shi = &stock_shi; 534 engine->pub.enp_shi_ctx = stock_shared_hash_new(); 535 if (!engine->pub.enp_shi_ctx) 536 { 537 free(engine); 538 return NULL; 539 } 540 } 541 if (api->ea_hsi_if) 542 { 543 engine->pub.enp_hsi_if = api->ea_hsi_if; 544 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 545 } 546 else 547 { 548 engine->pub.enp_hsi_if = lsquic_http1x_if; 549 engine->pub.enp_hsi_ctx = NULL; 550 } 551 if (api->ea_pmi) 552 { 553 engine->pub.enp_pmi = api->ea_pmi; 554 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 555 } 556 else 557 { 558 engine->pub.enp_pmi = &stock_pmi; 559 engine->pub.enp_pmi_ctx = NULL; 560 } 561 engine->pub.enp_verify_cert = api->ea_verify_cert; 562 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 563 engine->pub.enp_kli = api->ea_keylog_if; 564 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 565 engine->pub.enp_engine = engine; 566 if (hash_conns_by_addr(engine)) 567 engine->flags |= ENG_CONNS_BY_ADDR; 568 engine->conns_hash = lsquic_hash_create(); 569 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 570 if (!engine->pub.enp_tokgen) 571 return NULL; 572 engine->pub.enp_crand = &engine->crand; 573 if (flags & ENG_SERVER) 574 { 575 engine->pr_queue = prq_create( 576 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 577 &engine->pub); 578 if (!engine->pr_queue) 579 { 580 lsquic_tg_destroy(engine->pub.enp_tokgen); 581 return NULL; 582 } 583 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 584 engine->report_old_scids, engine->scids_ctx); 585 if (!engine->purga) 586 { 587 lsquic_tg_destroy(engine->pub.enp_tokgen); 588 prq_destroy(engine->pr_queue); 589 return NULL; 590 } 591 } 592 engine->attq = attq_create(); 593 eng_hist_init(&engine->history); 594 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 595 if (engine->pub.enp_settings.es_honor_prst) 596 { 597 engine->pub.enp_srst_hash = lsquic_hash_create(); 598 if (!engine->pub.enp_srst_hash) 599 { 600 lsquic_engine_destroy(engine); 601 return NULL; 602 } 603 } 604 605#ifndef NDEBUG 606 { 607 const char *env; 608 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 609 if (env) 610 { 611 if (0 != regcomp(&engine->lose_packets_re, env, 612 REG_EXTENDED|REG_NOSUB)) 613 { 614 LSQ_ERROR("could not compile lost packet regex `%s'", env); 615 return NULL; 616 } 617 engine->flags |= ENG_LOSE_PACKETS; 618 engine->lose_packets_str = env; 619 LSQ_WARN("will lose packets that match the following regex: %s", 620 env); 621 } 622 env = getenv("LSQUIC_COALESCE"); 623 if (env) 624 { 625 engine->flags &= ~ENG_COALESCE; 626 if (atoi(env)) 627 { 628 engine->flags |= ENG_COALESCE; 629 LSQ_NOTICE("will coalesce packets"); 630 } 631 else 632 LSQ_NOTICE("will not coalesce packets"); 633 } 634 } 635#endif 636#if LSQUIC_CONN_STATS 637 engine->stats_fh = api->ea_stats_fh; 638#endif 639 if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx, EVP_aead_aes_128_gcm(), 640 IETF_RETRY_KEY_BUF, IETF_RETRY_KEY_SZ, 16, NULL)) 641 { 642 LSQ_ERROR("could not initialize retry AEAD ctx"); 643 lsquic_engine_destroy(engine); 644 return NULL; 645 } 646 engine->pub.enp_retry_aead_ctx = &engine->retry_aead_ctx; 647 648 LSQ_INFO("instantiated engine"); 649 return engine; 650} 651 652 653#if LOG_PACKET_CHECKSUM 654static void 655log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 656 const unsigned char *buf, size_t bufsz) 657{ 658 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 659 (uint32_t) crc32(0, buf, bufsz)); 660} 661 662 663#endif 664 665 666static void 667grow_batch_size (struct lsquic_engine *engine) 668{ 669 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 670} 671 672 673static void 674shrink_batch_size (struct lsquic_engine *engine) 675{ 676 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 677} 678 679 680struct cce_cid_iter 681{ 682 const struct lsquic_conn *conn; 683 unsigned todo, n; 684}; 685 686 687static struct conn_cid_elem * 688cce_iter_next (struct cce_cid_iter *citer) 689{ 690 struct conn_cid_elem *cce; 691 692 while (citer->todo) 693 if (citer->todo & (1 << citer->n)) 694 { 695 citer->todo &= ~(1 << citer->n); 696 cce = &citer->conn->cn_cces[ citer->n++ ]; 697 if (!(cce->cce_flags & CCE_PORT)) 698 return cce; 699 } 700 else 701 ++citer->n; 702 703 return NULL; 704} 705 706 707static struct conn_cid_elem * 708cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 709{ 710 citer->conn = conn; 711 citer->todo = conn->cn_cces_mask; 712 citer->n = 0; 713 return cce_iter_next(citer); 714} 715 716 717#if LSQUIC_CONN_STATS 718void 719update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 720{ 721 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 722 const unsigned long *src; 723 const struct conn_stats *stats; 724 unsigned i; 725 726 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 727 { 728 ++engine->stats.conns; 729 src = (unsigned long *) stats; 730 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 731 dst[i] += src[i]; 732 } 733} 734 735 736#endif 737 738 739/* Wrapper to make sure important things occur before the connection is 740 * really destroyed. 741 */ 742static void 743destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 744 lsquic_time_t now) 745{ 746 struct cce_cid_iter citer; 747 const struct conn_cid_elem *cce; 748 lsquic_time_t drain_time; 749 struct purga_el *puel; 750 751 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 752 if (engine->purga 753 /* Blacklist all CIDs except for promoted mini connections */ 754 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 755 != (LSCONN_MINI|LSCONN_PROMOTED)) 756 { 757 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 758 && conn->cn_if->ci_drain_time && 759 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 760 { 761 for (cce = cce_iter_first(&citer, conn); cce; 762 cce = cce_iter_next(&citer)) 763 { 764 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 765 lsquic_conn_get_peer_ctx(conn, NULL), 766 PUTY_CONN_DRAIN, now); 767 if (puel) 768 puel->puel_time = now + drain_time; 769 } 770 } 771 else 772 { 773 for (cce = cce_iter_first(&citer, conn); cce; 774 cce = cce_iter_next(&citer)) 775 { 776 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 777 lsquic_conn_get_peer_ctx(conn, NULL), 778 PUTY_CONN_DELETED, now); 779 if (puel) 780 { 781 puel->puel_time = now; 782 puel->puel_count = 0; 783 } 784 } 785 } 786 } 787#if LSQUIC_CONN_STATS 788 update_stats_sum(engine, conn); 789#endif 790 --engine->n_conns; 791 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 792 conn->cn_if->ci_destroy(conn); 793} 794 795 796static int 797maybe_grow_conn_heaps (struct lsquic_engine *engine) 798{ 799 struct min_heap_elem *els; 800 unsigned count; 801 802 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 803 return 0; /* Nothing to do */ 804 805 if (lsquic_mh_nalloc(&engine->conns_tickable)) 806 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 807 else 808 count = 8; 809 810 els = malloc(sizeof(els[0]) * count); 811 if (!els) 812 { 813 LSQ_ERROR("%s: malloc failed", __func__); 814 return -1; 815 } 816 817 LSQ_DEBUG("grew heaps to %u elements", count / 2); 818 memcpy(&els[0], engine->conns_tickable.mh_elems, 819 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 820 memcpy(&els[count / 2], engine->conns_out.mh_elems, 821 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 822 free(engine->conns_tickable.mh_elems); 823 engine->conns_tickable.mh_elems = els; 824 engine->conns_out.mh_elems = &els[count / 2]; 825 engine->conns_tickable.mh_nalloc = count / 2; 826 engine->conns_out.mh_nalloc = count / 2; 827 return 0; 828} 829 830 831static void 832remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 833 unsigned todo) 834{ 835 unsigned n; 836 837 for (n = 0; todo; todo &= ~(1 << n++)) 838 if ((todo & (1 << n)) && 839 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 840 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 841} 842 843 844static void 845remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 846{ 847 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 848} 849 850 851static void 852cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 853 854 855static int 856insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 857 void *peer_ctx) 858{ 859 struct conn_cid_elem *cce; 860 unsigned todo, done, n; 861 862 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 863 if (todo & (1 << n)) 864 { 865 cce = &conn->cn_cces[n]; 866 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 867 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 868 cce->cce_cid.len, conn, &cce->cce_hash_el)) 869 done |= 1 << n; 870 else 871 goto err; 872 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 873 { 874 cce->cce_flags |= CCE_REG; 875 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 876 } 877 } 878 879 return 0; 880 881 err: 882 remove_cces_from_hash(engine->conns_hash, conn, done); 883 return -1; 884} 885 886 887static lsquic_conn_t * 888new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 889 lsquic_time_t now) 890{ 891 const lsquic_cid_t *cid; 892 server_conn_ctor_f ctor; 893 lsquic_conn_t *conn; 894 unsigned flags; 895 if (0 != maybe_grow_conn_heaps(engine)) 896 return NULL; 897 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 898 899 if (mini_conn->cn_flags & LSCONN_IETF) 900 { 901 if (mini_conn->cn_version == LSQVER_ID24) 902 ctor = lsquic_id24_full_conn_server_new; 903 else 904 ctor = lsquic_ietf_full_conn_server_new; 905 } 906 else 907 ctor = lsquic_gquic_full_conn_server_new; 908 909 conn = ctor(&engine->pub, flags, mini_conn); 910 if (!conn) 911 { 912 /* Otherwise, full_conn_server_new prints its own warnings */ 913 if (ENOMEM == errno) 914 { 915 cid = lsquic_conn_log_cid(mini_conn); 916 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 917 CID_BITS(cid), strerror(errno)); 918 } 919 return NULL; 920 } 921 ++engine->n_conns; 922 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 923 { 924 cid = lsquic_conn_log_cid(conn); 925 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 926 CID_BITS(cid)); 927 destroy_conn(engine, conn, now); 928 return NULL; 929 } 930 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 931 conn->cn_flags |= LSCONN_HASHED; 932 return conn; 933} 934 935 936static enum 937{ 938 VER_NOT_SPECIFIED, 939 VER_SUPPORTED, 940 VER_UNSUPPORTED, 941} 942 943 944version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 945 enum lsquic_version *pversion) 946{ 947 lsquic_ver_tag_t ver_tag; 948 enum lsquic_version version; 949 950 if (!packet_in->pi_quic_ver) 951 { 952 LSQ_DEBUG("packet does not specify version"); 953 return VER_NOT_SPECIFIED; 954 } 955 956 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 957 version = lsquic_tag2ver(ver_tag); 958 if (version < N_LSQVER) 959 { 960 if (engine->pub.enp_settings.es_versions & (1 << version)) 961 { 962 LSQ_DEBUG("client-supplied version %s is supported", 963 lsquic_ver2str[version]); 964 *pversion = version; 965 return VER_SUPPORTED; 966 } 967 else 968 LSQ_DEBUG("client-supplied version %s is not supported", 969 lsquic_ver2str[version]); 970 } 971 else 972 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 973 ver_tag); 974 975 return VER_UNSUPPORTED; 976} 977 978 979static void 980schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 981 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 982 const struct sockaddr *sa_peer, void *peer_ctx) 983{ 984 assert(engine->pr_queue); 985 if (0 == prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 986 sa_local, sa_peer)) 987 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 988 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 989 else 990 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 991} 992 993 994static unsigned short 995sa2port (const struct sockaddr *sa) 996{ 997 if (sa->sa_family == AF_INET) 998 { 999 struct sockaddr_in *const sa4 = (void *) sa; 1000 return sa4->sin_port; 1001 } 1002 else 1003 { 1004 struct sockaddr_in6 *const sa6 = (void *) sa; 1005 return sa6->sin6_port; 1006 } 1007} 1008 1009 1010static struct lsquic_hash_elem * 1011find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 1012{ 1013 unsigned short port; 1014 1015 port = sa2port(sa); 1016 return lsquic_hash_find(hash, &port, sizeof(port)); 1017} 1018 1019 1020static lsquic_conn_t * 1021find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1022 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 1023{ 1024 struct lsquic_hash_elem *el; 1025 lsquic_conn_t *conn; 1026 1027 if (engine->flags & ENG_CONNS_BY_ADDR) 1028 el = find_conn_by_addr(engine->conns_hash, sa_local); 1029 else if (packet_in->pi_flags & PI_CONN_ID) 1030 el = lsquic_hash_find(engine->conns_hash, 1031 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1032 else 1033 { 1034 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1035 return NULL; 1036 } 1037 1038 if (!el) 1039 return NULL; 1040 1041 conn = lsquic_hashelem_getdata(el); 1042 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1043 if ((engine->flags & ENG_CONNS_BY_ADDR) 1044 && !(conn->cn_flags & LSCONN_IETF) 1045 && (packet_in->pi_flags & PI_CONN_ID) 1046 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1047 { 1048 LSQ_DEBUG("connection IDs do not match"); 1049 return NULL; 1050 } 1051 1052 return conn; 1053} 1054 1055 1056static lsquic_conn_t * 1057find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1058 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1059 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1060{ 1061 struct lsquic_hash_elem *el; 1062 struct purga_el *puel; 1063 lsquic_conn_t *conn; 1064 1065 if (!(packet_in->pi_flags & PI_CONN_ID)) 1066 { 1067 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1068 return NULL; 1069 } 1070 el = lsquic_hash_find(engine->conns_hash, 1071 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1072 1073 if (el) 1074 { 1075 conn = lsquic_hashelem_getdata(el); 1076 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1077 return conn; 1078 } 1079 1080 if (engine->flags & ENG_COOLDOWN) 1081 { /* Do not create incoming connections during cooldown */ 1082 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1083 return NULL; 1084 } 1085 1086 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1087 { 1088 LSQ_DEBUG("reached limit of %u inchoate connections", 1089 engine->pub.enp_settings.es_max_inchoate); 1090 return NULL; 1091 } 1092 1093 1094 if (engine->purga 1095 && (puel = lsquic_purga_contains(engine->purga, 1096 &packet_in->pi_conn_id), puel)) 1097 { 1098 switch (puel->puel_type) 1099 { 1100 case PUTY_CID_RETIRED: 1101 LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet", 1102 CID_BITS(&packet_in->pi_conn_id)); 1103 return NULL; 1104 case PUTY_CONN_DRAIN: 1105 LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64, 1106 puel->puel_time, packet_in->pi_received); 1107 if (puel->puel_time > packet_in->pi_received) 1108 { 1109 LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet", 1110 CID_BITS(&packet_in->pi_conn_id)); 1111 return NULL; 1112 } 1113 LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted", 1114 CID_BITS(&packet_in->pi_conn_id)); 1115 puel->puel_type = PUTY_CONN_DELETED; 1116 puel->puel_count = 0; 1117 puel->puel_time = 0; 1118 /* fall-through */ 1119 case PUTY_CONN_DELETED: 1120 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1121 CID_BITS(&packet_in->pi_conn_id)); 1122 if (puel->puel_time < packet_in->pi_received) 1123 { 1124 puel->puel_time = packet_in->pi_received 1125 /* Exponential back-off */ 1126 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1127 ++puel->puel_count; 1128 goto maybe_send_prst; 1129 } 1130 return NULL; 1131 default: 1132 assert(0); 1133 return NULL; 1134 } 1135 } 1136 1137 if (engine->pub.enp_settings.es_send_prst 1138 && !(packet_in->pi_flags & PI_GQUIC) 1139 && HETY_NOT_SET == packet_in->pi_header_type) 1140 goto maybe_send_prst; 1141 1142 if (0 != maybe_grow_conn_heaps(engine)) 1143 return NULL; 1144 1145 const struct parse_funcs *pf; 1146 enum lsquic_version version; 1147 switch (version_matches(engine, packet_in, &version)) 1148 { 1149 case VER_UNSUPPORTED: 1150 if (engine->flags & ENG_SERVER) 1151 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1152 sa_local, sa_peer, peer_ctx); 1153 return NULL; 1154 case VER_NOT_SPECIFIED: 1155 maybe_send_prst: 1156 if ((engine->flags & ENG_SERVER) && 1157 engine->pub.enp_settings.es_send_prst) 1158 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1159 sa_local, sa_peer, peer_ctx); 1160 return NULL; 1161 case VER_SUPPORTED: 1162 pf = select_pf_by_ver(version); 1163 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1164 break; 1165 } 1166 1167 1168 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1169 { 1170 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1171 sa_peer->sa_family == AF_INET, NULL, packet_in_size); 1172 } 1173 else 1174 { 1175 conn = mini_conn_new(&engine->pub, packet_in, version); 1176 } 1177 if (!conn) 1178 return NULL; 1179 ++engine->mini_conns_count; 1180 ++engine->n_conns; 1181 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1182 { 1183 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1184 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1185 CID_BITS(cid)); 1186 destroy_conn(engine, conn, packet_in->pi_received); 1187 return NULL; 1188 } 1189 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1190 conn->cn_flags |= LSCONN_HASHED; 1191 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1192 conn->cn_last_sent = engine->last_sent; 1193 return conn; 1194} 1195 1196 1197lsquic_conn_t * 1198lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1199 const lsquic_cid_t *cid) 1200{ 1201 struct lsquic_hash_elem *el; 1202 lsquic_conn_t *conn = NULL; 1203 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1204 1205 if (el) 1206 conn = lsquic_hashelem_getdata(el); 1207 return conn; 1208} 1209 1210 1211#if !defined(NDEBUG) && __GNUC__ 1212__attribute__((weak)) 1213#endif 1214void 1215lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1216 lsquic_conn_t *conn) 1217{ 1218 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1219 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1220 { 1221 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1222 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1223 engine_incref_conn(conn, LSCONN_TICKABLE); 1224 } 1225} 1226 1227 1228void 1229lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1230 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1231{ 1232 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1233 if (conn->cn_flags & LSCONN_TICKABLE) 1234 { 1235 /* Optimization: no need to add the connection to the Advisory Tick 1236 * Time Queue: it is about to be ticked, after which it its next tick 1237 * time may be queried again. 1238 */; 1239 } 1240 else if (conn->cn_flags & LSCONN_ATTQ) 1241 { 1242 if (lsquic_conn_adv_time(conn) != tick_time) 1243 { 1244 attq_remove(engine->attq, conn); 1245 if (0 != attq_add(engine->attq, conn, tick_time, why)) 1246 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1247 } 1248 } 1249 else if (0 == attq_add(engine->attq, conn, tick_time, why)) 1250 engine_incref_conn(conn, LSCONN_ATTQ); 1251} 1252 1253 1254static struct lsquic_conn * 1255find_conn_by_srst (struct lsquic_engine *engine, 1256 const struct lsquic_packet_in *packet_in) 1257{ 1258 struct lsquic_hash_elem *el; 1259 struct lsquic_conn *conn; 1260 1261 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1262 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1263 return NULL; 1264 1265 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1266 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1267 IQUIC_SRESET_TOKEN_SZ); 1268 if (!el) 1269 return NULL; 1270 1271 conn = lsquic_hashelem_getdata(el); 1272 return conn; 1273} 1274 1275 1276/* Return 0 if packet is being processed by a real connection (mini or full), 1277 * otherwise return 1. 1278 */ 1279static int 1280process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1281 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1282 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1283{ 1284 lsquic_conn_t *conn; 1285 const unsigned char *packet_in_data; 1286 1287 if (lsquic_packet_in_is_gquic_prst(packet_in) 1288 && !engine->pub.enp_settings.es_honor_prst) 1289 { 1290 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1291 LSQ_DEBUG("public reset packet: discarding"); 1292 return 1; 1293 } 1294 1295 if (engine->flags & ENG_SERVER) 1296 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1297 sa_peer, peer_ctx, packet_in_size); 1298 else 1299 conn = find_conn(engine, packet_in, ppstate, sa_local); 1300 1301 if (!conn) 1302 { 1303 if (engine->pub.enp_settings.es_honor_prst 1304 && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */ 1305 && !(packet_in->pi_flags & PI_GQUIC) 1306 && engine->pub.enp_srst_hash 1307 && (conn = find_conn_by_srst(engine, packet_in))) 1308 { 1309 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1310 CID_BITS(lsquic_conn_log_cid(conn))); 1311 conn->cn_if->ci_stateless_reset(conn); 1312 if (!(conn->cn_flags & LSCONN_TICKABLE) 1313 && conn->cn_if->ci_is_tickable(conn)) 1314 { 1315 lsquic_mh_insert(&engine->conns_tickable, conn, 1316 conn->cn_last_ticked); 1317 engine_incref_conn(conn, LSCONN_TICKABLE); 1318 } 1319 /* Even though the connection processes this packet, we return 1320 * 1 so that the caller does not add reset packet's random 1321 * bytes to the list of valid CIDs. 1322 */ 1323 } 1324 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1325 return 1; 1326 } 1327 1328 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1329 { 1330 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1331 engine_incref_conn(conn, LSCONN_TICKABLE); 1332 } 1333 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1334 sa_local, sa_peer); 1335 lsquic_packet_in_upref(packet_in); 1336#if LOG_PACKET_CHECKSUM 1337 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1338 packet_in->pi_data_sz); 1339#endif 1340 /* Note on QLog: 1341 * For the PACKET_RX QLog event, we are interested in logging these things: 1342 * - raw packet (however it comes in, encrypted or not) 1343 * - frames (list of frame names) 1344 * - packet type and number 1345 * - packet rx timestamp 1346 * 1347 * Since only some of these items are available at this code 1348 * juncture, we will wait until after the packet has been 1349 * decrypted (if necessary) and parsed to call the log functions. 1350 * 1351 * Once the PACKET_RX event is finally logged, the timestamp 1352 * will come from packet_in->pi_received. For correct sequential 1353 * ordering of QLog events, be sure to process the QLogs downstream. 1354 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1355 */ 1356 packet_in_data = packet_in->pi_data; 1357 packet_in_size = packet_in->pi_data_sz; 1358 conn->cn_if->ci_packet_in(conn, packet_in); 1359 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1360 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1361 return 0; 1362} 1363 1364 1365void 1366lsquic_engine_destroy (lsquic_engine_t *engine) 1367{ 1368 struct lsquic_hash_elem *el; 1369 lsquic_conn_t *conn; 1370 1371 LSQ_DEBUG("destroying engine"); 1372#ifndef NDEBUG 1373 engine->flags |= ENG_DTOR; 1374#endif 1375 1376 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1377 { 1378 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1379 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1380 } 1381 1382 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1383 { 1384 assert(conn->cn_flags & LSCONN_TICKABLE); 1385 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1386 } 1387 1388 for (el = lsquic_hash_first(engine->conns_hash); el; 1389 el = lsquic_hash_next(engine->conns_hash)) 1390 { 1391 conn = lsquic_hashelem_getdata(el); 1392 force_close_conn(engine, conn); 1393 } 1394 lsquic_hash_destroy(engine->conns_hash); 1395 1396 assert(0 == engine->n_conns); 1397 assert(0 == engine->mini_conns_count); 1398 if (engine->pr_queue) 1399 prq_destroy(engine->pr_queue); 1400 if (engine->purga) 1401 lsquic_purga_destroy(engine->purga); 1402 attq_destroy(engine->attq); 1403 1404 assert(0 == lsquic_mh_count(&engine->conns_out)); 1405 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1406 if (engine->pub.enp_shi == &stock_shi) 1407 stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1408 lsquic_mm_cleanup(&engine->pub.enp_mm); 1409 free(engine->conns_tickable.mh_elems); 1410#ifndef NDEBUG 1411 if (engine->flags & ENG_LOSE_PACKETS) 1412 regfree(&engine->lose_packets_re); 1413#endif 1414 if (engine->pub.enp_tokgen) 1415 lsquic_tg_destroy(engine->pub.enp_tokgen); 1416#if LSQUIC_CONN_STATS 1417 if (engine->stats_fh) 1418 { 1419 const struct conn_stats *const stats = &engine->conn_stats_sum; 1420 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1421 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1422 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1423 fprintf(engine->stats_fh, "In:\n"); 1424 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1425 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1426 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1427 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1428 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1429 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1430 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1431 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1432 stats->in.headers_comp, stats->in.headers_uncomp, 1433 stats->in.headers_uncomp ? 1434 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1435 : 0); 1436 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1437 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1438 fprintf(engine->stats_fh, " ACK frames merged to new: %lu\n", stats->in.n_acks_merged[0]); 1439 fprintf(engine->stats_fh, " ACK frames merged to old: %lu\n", stats->in.n_acks_merged[1]); 1440 fprintf(engine->stats_fh, "Out:\n"); 1441 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1442 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1443 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1444 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1445 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1446 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1447 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1448 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1449 stats->out.headers_comp, stats->out.headers_uncomp, 1450 stats->out.headers_uncomp ? 1451 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1452 : 0); 1453 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1454 } 1455#endif 1456 if (engine->pub.enp_srst_hash) 1457 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1458#if LSQUIC_COUNT_ENGINE_CALLS 1459 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1460#endif 1461 if (engine->pub.enp_retry_aead_ctx) 1462 EVP_AEAD_CTX_cleanup(engine->pub.enp_retry_aead_ctx); 1463 free(engine); 1464} 1465 1466 1467static struct conn_cid_elem * 1468find_free_cce (struct lsquic_conn *conn) 1469{ 1470 struct conn_cid_elem *cce; 1471 1472 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1473 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1474 return cce; 1475 1476 return NULL; 1477} 1478 1479 1480static int 1481add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1482 const struct sockaddr *local_sa, void *peer_ctx) 1483{ 1484 struct conn_cid_elem *cce; 1485 1486 if (engine->flags & ENG_CONNS_BY_ADDR) 1487 { 1488 cce = find_free_cce(conn); 1489 if (!cce) 1490 { 1491 LSQ_ERROR("cannot find free CCE"); 1492 return -1; 1493 } 1494 cce->cce_port = sa2port(local_sa); 1495 cce->cce_flags = CCE_PORT; 1496 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1497 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1498 { 1499 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1500 return 0; 1501 } 1502 else 1503 return -1; 1504 1505 } 1506 else 1507 return insert_conn_into_hash(engine, conn, peer_ctx); 1508} 1509 1510 1511lsquic_conn_t * 1512lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1513 const struct sockaddr *local_sa, 1514 const struct sockaddr *peer_sa, 1515 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1516 const char *hostname, unsigned short max_packet_size, 1517 const unsigned char *zero_rtt, size_t zero_rtt_len, 1518 const unsigned char *token, size_t token_sz) 1519{ 1520 lsquic_conn_t *conn; 1521 unsigned flags, versions; 1522 int is_ipv4; 1523 1524 ENGINE_IN(engine); 1525 1526 if (engine->flags & ENG_SERVER) 1527 { 1528 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1529 goto err; 1530 } 1531 1532 if (engine->flags & ENG_CONNS_BY_ADDR 1533 && find_conn_by_addr(engine->conns_hash, local_sa)) 1534 { 1535 LSQ_ERROR("cannot have more than one connection on the same port"); 1536 goto err; 1537 } 1538 1539 if (0 != maybe_grow_conn_heaps(engine)) 1540 return NULL; 1541 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1542 is_ipv4 = peer_sa->sa_family == AF_INET; 1543 if (zero_rtt && zero_rtt_len) 1544 { 1545 version = lsquic_zero_rtt_version(zero_rtt, zero_rtt_len); 1546 if (version >= N_LSQVER) 1547 { 1548 LSQ_INFO("zero-rtt version is bad, won't use"); 1549 zero_rtt = NULL; 1550 zero_rtt_len = 0; 1551 } 1552 } 1553 if (version >= N_LSQVER) 1554 { 1555 if (version > N_LSQVER) 1556 LSQ_WARN("invalid version specified, engine will pick"); 1557 versions = engine->pub.enp_settings.es_versions; 1558 } 1559 else 1560 versions = 1u << version; 1561 if (versions & LSQUIC_IETF_VERSIONS) 1562 { 1563 if (version == LSQVER_ID24) 1564 conn = lsquic_id24_full_conn_client_new(&engine->pub, versions, 1565 flags, hostname, max_packet_size, 1566 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1567 else 1568 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1569 flags, hostname, max_packet_size, 1570 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1571 } 1572 else 1573 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1574 flags, hostname, max_packet_size, is_ipv4, 1575 zero_rtt, zero_rtt_len); 1576 if (!conn) 1577 goto err; 1578 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1579 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1580 lsquic_ver2str[conn->cn_version]); 1581 ++engine->n_conns; 1582 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1583 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1584 { 1585 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1586 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1587 CID_BITS(cid)); 1588 destroy_conn(engine, conn, lsquic_time_now()); 1589 goto err; 1590 } 1591 assert(!(conn->cn_flags & 1592 (CONN_REF_FLAGS 1593 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1594 callbacks */ 1595 ))); 1596 conn->cn_flags |= LSCONN_HASHED; 1597 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1598 engine_incref_conn(conn, LSCONN_TICKABLE); 1599 lsquic_conn_set_ctx(conn, conn_ctx); 1600 conn->cn_if->ci_client_call_on_new(conn); 1601 end: 1602 ENGINE_OUT(engine); 1603 return conn; 1604 err: 1605 conn = NULL; 1606 goto end; 1607} 1608 1609 1610static void 1611remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1612{ 1613 remove_all_cces_from_hash(engine->conns_hash, conn); 1614 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1615} 1616 1617 1618static void 1619refflags2str (enum lsquic_conn_flags flags, char s[6]) 1620{ 1621 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1622 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1623 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1624 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1625 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1626 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1627 *s = '\0'; 1628} 1629 1630 1631static void 1632engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1633{ 1634 char str[2][7]; 1635 assert(flag & CONN_REF_FLAGS); 1636 assert(!(conn->cn_flags & flag)); 1637 conn->cn_flags |= flag; 1638 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1639 CID_BITS(lsquic_conn_log_cid(conn)), 1640 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1641 (refflags2str(conn->cn_flags, str[1]), str[1])); 1642} 1643 1644 1645static lsquic_conn_t * 1646engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1647 enum lsquic_conn_flags flags) 1648{ 1649 char str[2][7]; 1650 lsquic_time_t now; 1651 assert(flags & CONN_REF_FLAGS); 1652 assert(conn->cn_flags & flags); 1653#ifndef NDEBUG 1654 if (flags & LSCONN_CLOSING) 1655 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1656#endif 1657 conn->cn_flags &= ~flags; 1658 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1659 CID_BITS(lsquic_conn_log_cid(conn)), 1660 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1661 (refflags2str(conn->cn_flags, str[1]), str[1])); 1662 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1663 { 1664 now = lsquic_time_now(); 1665 if (conn->cn_flags & LSCONN_MINI) 1666 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1667 else 1668 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1669 destroy_conn(engine, conn, now); 1670 return NULL; 1671 } 1672 else 1673 return conn; 1674} 1675 1676 1677/* This is not a general-purpose function. Only call from engine dtor. */ 1678static void 1679force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1680{ 1681 assert(engine->flags & ENG_DTOR); 1682 const enum lsquic_conn_flags flags = conn->cn_flags; 1683 assert(conn->cn_flags & CONN_REF_FLAGS); 1684 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1685 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1686 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1687 if (flags & LSCONN_ATTQ) 1688 { 1689 attq_remove(engine->attq, conn); 1690 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1691 } 1692 if (flags & LSCONN_HASHED) 1693 remove_conn_from_hash(engine, conn); 1694} 1695 1696 1697/* Iterator for tickable connections (those on the Tickable Queue). Before 1698 * a connection is returned, it is removed from the Advisory Tick Time queue 1699 * if necessary. 1700 */ 1701static lsquic_conn_t * 1702conn_iter_next_tickable (struct lsquic_engine *engine) 1703{ 1704 lsquic_conn_t *conn; 1705 1706 if (engine->flags & ENG_SERVER) 1707 while (1) 1708 { 1709 conn = lsquic_mh_pop(&engine->conns_tickable); 1710 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1711 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1712 else 1713 break; 1714 } 1715 else 1716 conn = lsquic_mh_pop(&engine->conns_tickable); 1717 1718 if (conn) 1719 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1720 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1721 { 1722 attq_remove(engine->attq, conn); 1723 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1724 } 1725 1726 return conn; 1727} 1728 1729 1730static void 1731cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1732 void *update_ctx) 1733{ 1734 cub->cub_update_cids = update; 1735 cub->cub_update_ctx = update_ctx; 1736 cub->cub_count = 0; 1737} 1738 1739 1740static void 1741cub_flush (struct cid_update_batch *cub) 1742{ 1743 if (cub->cub_count > 0 && cub->cub_update_cids) 1744 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1745 cub->cub_cids, cub->cub_count); 1746 cub->cub_count = 0; 1747} 1748 1749 1750static void 1751cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1752{ 1753 cub->cub_cids [ cub->cub_count ] = *cid; 1754 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1755 ++cub->cub_count; 1756 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1757 cub_flush(cub); 1758} 1759 1760 1761/* Process registered CIDs */ 1762static void 1763cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1764{ 1765 struct cce_cid_iter citer; 1766 struct conn_cid_elem *cce; 1767 void *peer_ctx; 1768 1769 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1770 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1771 if (cce->cce_flags & CCE_REG) 1772 cub_add(cub, &cce->cce_cid, peer_ctx); 1773} 1774 1775 1776static void 1777drop_all_mini_conns (lsquic_engine_t *engine) 1778{ 1779 struct lsquic_hash_elem *el; 1780 lsquic_conn_t *conn; 1781 struct cid_update_batch cub; 1782 1783 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1784 1785 for (el = lsquic_hash_first(engine->conns_hash); el; 1786 el = lsquic_hash_next(engine->conns_hash)) 1787 { 1788 conn = lsquic_hashelem_getdata(el); 1789 if (conn->cn_flags & LSCONN_MINI) 1790 { 1791 /* If promoted, why is it still in this hash? */ 1792 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1793 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1794 cub_add_cids_from_cces(&cub, conn); 1795 remove_conn_from_hash(engine, conn); 1796 } 1797 } 1798 1799 cub_flush(&cub); 1800} 1801 1802 1803void 1804lsquic_engine_process_conns (lsquic_engine_t *engine) 1805{ 1806 lsquic_conn_t *conn; 1807 lsquic_time_t now; 1808 1809 ENGINE_IN(engine); 1810 1811 now = lsquic_time_now(); 1812 while ((conn = attq_pop(engine->attq, now))) 1813 { 1814 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1815 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 1816 { 1817 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1818 engine_incref_conn(conn, LSCONN_TICKABLE); 1819 } 1820 } 1821 1822 process_connections(engine, conn_iter_next_tickable, now); 1823 ENGINE_OUT(engine); 1824} 1825 1826 1827static void 1828release_or_return_enc_data (struct lsquic_engine *engine, 1829 void (*pmi_rel_or_ret) (void *, void *, void *, char), 1830 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 1831{ 1832 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1833 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 1834 packet_out->po_flags &= ~PO_ENCRYPTED; 1835 packet_out->po_enc_data = NULL; 1836} 1837 1838 1839static void 1840release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1841 struct lsquic_packet_out *packet_out) 1842{ 1843 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 1844 conn, packet_out); 1845} 1846 1847 1848static void 1849return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1850 struct lsquic_packet_out *packet_out) 1851{ 1852 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 1853 conn, packet_out); 1854} 1855 1856 1857static int 1858copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 1859 struct lsquic_packet_out *packet_out) 1860{ 1861 int ipv6; 1862 1863 ipv6 = NP_IS_IPv6(packet_out->po_path); 1864 if (packet_out->po_flags & PO_ENCRYPTED) 1865 { 1866 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1867 && packet_out->po_data_sz == packet_out->po_enc_data_sz 1868 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 1869 packet_out->po_data_sz)) 1870 return 0; 1871 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1872 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 1873 goto copy; 1874 return_enc_data(engine, conn, packet_out); 1875 } 1876 1877 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 1878 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1879 packet_out->po_data_sz, ipv6); 1880 if (!packet_out->po_enc_data) 1881 { 1882 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 1883 "of size %hu", packet_out->po_data_sz); 1884 return -1; 1885 } 1886 1887 copy: 1888 memcpy(packet_out->po_enc_data, packet_out->po_data, 1889 packet_out->po_data_sz); 1890 packet_out->po_enc_data_sz = packet_out->po_data_sz; 1891 packet_out->po_sent_sz = packet_out->po_data_sz; 1892 packet_out->po_flags &= ~PO_IPv6; 1893 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 1894 1895 return 0; 1896} 1897 1898 1899STAILQ_HEAD(conns_stailq, lsquic_conn); 1900TAILQ_HEAD(conns_tailq, lsquic_conn); 1901 1902 1903struct conns_out_iter 1904{ 1905 struct min_heap *coi_heap; 1906 struct pr_queue *coi_prq; 1907 TAILQ_HEAD(, lsquic_conn) coi_active_list, 1908 coi_inactive_list; 1909 lsquic_conn_t *coi_next; 1910#ifndef NDEBUG 1911 lsquic_time_t coi_last_sent; 1912#endif 1913}; 1914 1915 1916static void 1917coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 1918{ 1919 iter->coi_heap = &engine->conns_out; 1920 iter->coi_prq = engine->pr_queue; 1921 iter->coi_next = NULL; 1922 TAILQ_INIT(&iter->coi_active_list); 1923 TAILQ_INIT(&iter->coi_inactive_list); 1924#ifndef NDEBUG 1925 iter->coi_last_sent = 0; 1926#endif 1927} 1928 1929 1930static lsquic_conn_t * 1931coi_next (struct conns_out_iter *iter) 1932{ 1933 lsquic_conn_t *conn; 1934 1935 if (lsquic_mh_count(iter->coi_heap) > 0) 1936 { 1937 conn = lsquic_mh_pop(iter->coi_heap); 1938 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1939 conn->cn_flags |= LSCONN_COI_ACTIVE; 1940#ifndef NDEBUG 1941 if (iter->coi_last_sent) 1942 assert(iter->coi_last_sent <= conn->cn_last_sent); 1943 iter->coi_last_sent = conn->cn_last_sent; 1944#endif 1945 return conn; 1946 } 1947 else if (iter->coi_prq && (conn = prq_next_conn(iter->coi_prq))) 1948 { 1949 return conn; 1950 } 1951 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 1952 { 1953 iter->coi_prq = NULL; /* Save function call in previous conditional */ 1954 conn = iter->coi_next; 1955 if (!conn) 1956 conn = TAILQ_FIRST(&iter->coi_active_list); 1957 if (conn) 1958 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 1959 return conn; 1960 } 1961 else 1962 return NULL; 1963} 1964 1965 1966static void 1967coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1968{ 1969 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 1970 { 1971 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 1972 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1973 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1974 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 1975 conn->cn_flags |= LSCONN_COI_INACTIVE; 1976 } 1977} 1978 1979 1980static void 1981coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1982{ 1983 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 1984 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 1985 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 1986 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1987 conn->cn_flags |= LSCONN_COI_ACTIVE; 1988} 1989 1990 1991static void 1992coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 1993{ 1994 lsquic_conn_t *conn; 1995 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 1996 { 1997 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1998 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1999 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 2000 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 2001 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 2002 else /* Closed connection gets one shot at sending packets */ 2003 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2004 } 2005 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 2006 { 2007 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2008 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2009 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2010 } 2011} 2012 2013 2014#ifndef NDEBUG 2015static void 2016lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 2017 unsigned n) 2018{ 2019 const lsquic_cid_t *cid; 2020 struct iovec *iov; 2021 unsigned i; 2022 char packno_str[22]; 2023 2024 for (i = 0; i < n; ++i) 2025 { 2026 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 2027 batch->packets[i]->po_packno); 2028 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 2029 { 2030 for (iov = batch->outs[i].iov; iov < 2031 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 2032 batch->outs[i].iov->iov_len -= 1; 2033 cid = lsquic_conn_log_cid(batch->conns[i]); 2034 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 2035 CID_BITS(cid)); 2036 } 2037 } 2038} 2039 2040 2041#endif 2042 2043 2044#ifdef NDEBUG 2045#define CONST_BATCH const 2046#else 2047#define CONST_BATCH 2048#endif 2049 2050 2051static void 2052sockaddr2str (const struct sockaddr *addr, char *buf, size_t sz) 2053{ 2054 unsigned short port; 2055 int len; 2056 2057 switch (addr->sa_family) 2058 { 2059 case AF_INET: 2060 port = ntohs(((struct sockaddr_in *) addr)->sin_port); 2061 if (!inet_ntop(AF_INET, &((struct sockaddr_in *) addr)->sin_addr, 2062 buf, sz)) 2063 buf[0] = '\0'; 2064 break; 2065 case AF_INET6: 2066 port = ntohs(((struct sockaddr_in6 *) addr)->sin6_port); 2067 if (!inet_ntop(AF_INET6, &((struct sockaddr_in6 *) addr)->sin6_addr, 2068 buf, sz)) 2069 buf[0] = '\0'; 2070 break; 2071 default: 2072 port = 0; 2073 (void) snprintf(buf, sz, "<invalid family %d>", addr->sa_family); 2074 break; 2075 } 2076 2077 len = strlen(buf); 2078 if (len < (int) sz) 2079 snprintf(buf + len, sz - (size_t) len, ":%hu", port); 2080} 2081 2082 2083struct send_batch_ctx { 2084 struct conns_stailq *closed_conns; 2085 struct conns_tailq *ticked_conns; 2086 struct conns_out_iter *conns_iter; 2087 CONST_BATCH struct out_batch *batch; 2088}; 2089 2090 2091static void 2092close_conn_immediately (struct lsquic_engine *engine, 2093 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2094{ 2095 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2096 if (!(conn->cn_flags & LSCONN_CLOSING)) 2097 { 2098 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2099 engine_incref_conn(conn, LSCONN_CLOSING); 2100 if (conn->cn_flags & LSCONN_HASHED) 2101 remove_conn_from_hash(engine, conn); 2102 } 2103 if (conn->cn_flags & LSCONN_TICKED) 2104 { 2105 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2106 engine_decref_conn(engine, conn, LSCONN_TICKED); 2107 } 2108} 2109 2110 2111static void 2112close_conn_on_send_error (struct lsquic_engine *engine, 2113 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2114{ 2115 const struct out_batch *batch = sb_ctx->batch; 2116 struct lsquic_conn *const conn = batch->conns[n]; 2117 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2118 2119 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2120 "src: %s; dst: %s; errno: %d", 2121 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2122 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2123 CID_BITS(lsquic_conn_log_cid(conn)), 2124 (sockaddr2str(batch->outs[n].local_sa, buf[0], sizeof(buf[0])), buf[0]), 2125 (sockaddr2str(batch->outs[n].dest_sa, buf[1], sizeof(buf[1])), buf[1]), 2126 e_val); 2127 if (conn->cn_flags & LSCONN_EVANESCENT) 2128 lsquic_prq_drop(conn); 2129 else 2130 close_conn_immediately(engine, sb_ctx, conn); 2131} 2132 2133 2134static unsigned 2135send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2136 unsigned n_to_send) 2137{ 2138 int n_sent, i, e_val; 2139 lsquic_time_t now; 2140 unsigned off; 2141 size_t count; 2142 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2143 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2144 2145#ifndef NDEBUG 2146 if (engine->flags & ENG_LOSE_PACKETS) 2147 lose_matching_packets(engine, batch, n_to_send); 2148#endif 2149 /* Set sent time before the write to avoid underestimating RTT */ 2150 now = lsquic_time_now(); 2151 for (i = 0; i < (int) n_to_send; ++i) 2152 { 2153 off = batch->pack_off[i]; 2154 count = batch->outs[i].iovlen; 2155 assert(count > 0); 2156 packet_out = &batch->packets[off]; 2157 end = packet_out + count; 2158 do 2159 (*packet_out)->po_sent = now; 2160 while (++packet_out < end); 2161 } 2162 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs, 2163 n_to_send); 2164 e_val = errno; 2165 if (n_sent < (int) n_to_send) 2166 { 2167 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2168 engine->resume_sending_at = now + 1000000; 2169 LSQ_DEBUG("cannot send packets"); 2170 EV_LOG_GENERIC_EVENT("cannot send packets"); 2171 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2172 close_conn_on_send_error(engine, sb_ctx, 2173 n_sent < 0 ? 0 : n_sent, e_val); 2174 } 2175 if (n_sent >= 0) 2176 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, n_to_send); 2177 else 2178 { 2179 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2180 n_sent = 0; 2181 } 2182 if (n_sent > 0) 2183 engine->last_sent = now + n_sent; 2184 for (i = 0; i < n_sent; ++i) 2185 { 2186 eng_hist_inc(&engine->history, now, sl_packets_out); 2187 /* `i' is added to maintain relative order */ 2188 batch->conns[i]->cn_last_sent = now + i; 2189 2190 off = batch->pack_off[i]; 2191 count = batch->outs[i].iovlen; 2192 assert(count > 0); 2193 packet_out = &batch->packets[off]; 2194 end = packet_out + count; 2195 do 2196 { 2197#if LOG_PACKET_CHECKSUM 2198 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2199 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2200 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2201#endif 2202 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2203 *packet_out); 2204 /* Release packet out buffer as soon as the packet is sent 2205 * successfully. If not successfully sent, we hold on to 2206 * this buffer until the packet sending is attempted again 2207 * or until it times out and regenerated. 2208 */ 2209 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2210 release_enc_data(engine, batch->conns[i], *packet_out); 2211 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2212 *packet_out); 2213 } 2214 while (++packet_out < end); 2215 } 2216 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2217 for ( ; i < (int) n_to_send; ++i) 2218 { 2219 off = batch->pack_off[i]; 2220 count = batch->outs[i].iovlen; 2221 assert(count > 0); 2222 packet_out = &batch->packets[off]; 2223 end = packet_out + count; 2224 do 2225 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2226 *packet_out); 2227 while (++packet_out < end); 2228 } 2229 /* Return packets to the connection in reverse order so that the packet 2230 * ordering is maintained. 2231 */ 2232 for (i = (int) n_to_send - 1; i >= n_sent; --i) 2233 { 2234 off = batch->pack_off[i]; 2235 count = batch->outs[i].iovlen; 2236 assert(count > 0); 2237 packet_out = &batch->packets[off + count - 1]; 2238 end = &batch->packets[off - 1]; 2239 do 2240 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2241 *packet_out); 2242 while (--packet_out > end); 2243 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2244 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2245 } 2246 return n_sent; 2247} 2248 2249 2250/* Return 1 if went past deadline, 0 otherwise */ 2251static int 2252check_deadline (lsquic_engine_t *engine) 2253{ 2254 if (engine->pub.enp_settings.es_proc_time_thresh && 2255 lsquic_time_now() > engine->deadline) 2256 { 2257 LSQ_INFO("went past threshold of %u usec, stop sending", 2258 engine->pub.enp_settings.es_proc_time_thresh); 2259 engine->flags |= ENG_PAST_DEADLINE; 2260 return 1; 2261 } 2262 else 2263 return 0; 2264} 2265 2266 2267static size_t 2268iov_size (const struct iovec *iov, const struct iovec *const end) 2269{ 2270 size_t size; 2271 2272 assert(iov < end); 2273 2274 size = 0; 2275 do 2276 size += iov->iov_len; 2277 while (++iov < end); 2278 2279 return size; 2280} 2281 2282 2283static void 2284send_packets_out (struct lsquic_engine *engine, 2285 struct conns_tailq *ticked_conns, 2286 struct conns_stailq *closed_conns) 2287{ 2288 unsigned n, w, n_sent, n_batches_sent; 2289 lsquic_packet_out_t *packet_out; 2290 struct lsquic_packet_out **packet; 2291 lsquic_conn_t *conn; 2292 struct out_batch *const batch = &engine->out_batch; 2293 struct iovec *iov, *packet_iov; 2294 struct conns_out_iter conns_iter; 2295 int shrink, deadline_exceeded; 2296 const struct send_batch_ctx sb_ctx = { 2297 closed_conns, 2298 ticked_conns, 2299 &conns_iter, 2300 &engine->out_batch, 2301 }; 2302 2303 coi_init(&conns_iter, engine); 2304 n_batches_sent = 0; 2305 n_sent = 0, n = 0; 2306 shrink = 0; 2307 deadline_exceeded = 0; 2308 iov = batch->iov; 2309 packet = batch->packets; 2310 2311 while ((conn = coi_next(&conns_iter))) 2312 { 2313 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2314 if (!packet_out) { 2315 /* Evanescent connection always has a packet to send: */ 2316 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2317 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2318 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2319 CID_BITS(lsquic_conn_log_cid(conn))); 2320 coi_deactivate(&conns_iter, conn); 2321 continue; 2322 } 2323 batch->outs[n].iov = packet_iov = iov; 2324 next_coa: 2325 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2326 { 2327 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2328 &engine->pub, conn, packet_out)) 2329 { 2330 case ENCPA_NOMEM: 2331 /* Send what we have and wait for a more opportune moment */ 2332 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2333 goto end_for; 2334 case ENCPA_BADCRYPT: 2335 /* This is pretty bad: close connection immediately */ 2336 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2337 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2338 CID_BITS(lsquic_conn_log_cid(conn))); 2339 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2340 { 2341 close_conn_immediately(engine, &sb_ctx, conn); 2342 coi_deactivate(&conns_iter, conn); 2343 } 2344 continue; 2345 case ENCPA_OK: 2346 break; 2347 } 2348 } 2349 else if ((packet_out->po_flags & PO_NOENCRYPT) 2350 && engine->pub.enp_pmi != &stock_pmi) 2351 { 2352 if (0 != copy_packet(engine, conn, packet_out)) 2353 { 2354 /* Copy can only fail if packet could not be allocated */ 2355 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2356 goto end_for; 2357 } 2358 } 2359 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2360 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2361 if (packet_out->po_flags & PO_ENCRYPTED) 2362 { 2363 iov->iov_base = packet_out->po_enc_data; 2364 iov->iov_len = packet_out->po_enc_data_sz; 2365 } 2366 else 2367 { 2368 iov->iov_base = packet_out->po_data; 2369 iov->iov_len = packet_out->po_data_sz; 2370 } 2371 if (packet_iov == iov) 2372 { 2373 batch->pack_off[n] = packet - batch->packets; 2374 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2375 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2376 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2377 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2378 batch->conns [n] = conn; 2379 } 2380 *packet = packet_out; 2381 ++packet; 2382 ++iov; 2383 if ((conn->cn_flags & LSCONN_IETF) 2384 && ((1 << packet_out->po_header_type) 2385 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2386#ifndef NDEBUG 2387 && (engine->flags & ENG_COALESCE) 2388#endif 2389 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2390 { 2391 const size_t size = iov_size(packet_iov, iov); 2392 packet_out = conn->cn_if->ci_next_packet_to_send(conn, size); 2393 if (packet_out) 2394 goto next_coa; 2395 } 2396 batch->outs [n].iovlen = iov - packet_iov; 2397 ++n; 2398 if (n == engine->batch_size 2399 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2400 { 2401 w = send_batch(engine, &sb_ctx, n); 2402 n = 0; 2403 iov = batch->iov; 2404 packet = batch->packets; 2405 ++n_batches_sent; 2406 n_sent += w; 2407 if (w < engine->batch_size) 2408 { 2409 shrink = 1; 2410 break; 2411 } 2412 deadline_exceeded = check_deadline(engine); 2413 if (deadline_exceeded) 2414 break; 2415 grow_batch_size(engine); 2416 } 2417 } 2418 end_for: 2419 2420 if (n > 0) { 2421 w = send_batch(engine, &sb_ctx, n); 2422 n_sent += w; 2423 shrink = w < n; 2424 ++n_batches_sent; 2425 } 2426 2427 if (shrink) 2428 shrink_batch_size(engine); 2429 else if (n_batches_sent > 1) 2430 { 2431 deadline_exceeded = check_deadline(engine); 2432 if (!deadline_exceeded) 2433 grow_batch_size(engine); 2434 } 2435 2436 coi_reheap(&conns_iter, engine); 2437 2438 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2439} 2440 2441 2442int 2443lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2444{ 2445 return lsquic_mh_count(&engine->conns_out) > 0 2446 || (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2447 ; 2448} 2449 2450 2451static void 2452reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2453{ 2454 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2455 engine->flags &= ~ENG_PAST_DEADLINE; 2456} 2457 2458 2459void 2460lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2461{ 2462 lsquic_conn_t *conn; 2463 struct conns_stailq closed_conns; 2464 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2465 struct cid_update_batch cub; 2466 2467 ENGINE_IN(engine); 2468 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2469 STAILQ_INIT(&closed_conns); 2470 reset_deadline(engine, lsquic_time_now()); 2471 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2472 { 2473 LSQ_DEBUG("can send again"); 2474 EV_LOG_GENERIC_EVENT("can send again"); 2475 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2476 } 2477 2478 send_packets_out(engine, &ticked_conns, &closed_conns); 2479 2480 while ((conn = STAILQ_FIRST(&closed_conns))) { 2481 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2482 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2483 cub_add_cids_from_cces(&cub, conn); 2484 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2485 } 2486 2487 cub_flush(&cub); 2488 ENGINE_OUT(engine); 2489} 2490 2491 2492static lsquic_conn_t * 2493next_new_full_conn (struct conns_stailq *new_full_conns) 2494{ 2495 lsquic_conn_t *conn; 2496 2497 conn = STAILQ_FIRST(new_full_conns); 2498 if (conn) 2499 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2500 return conn; 2501} 2502 2503 2504static void 2505process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2506 lsquic_time_t now) 2507{ 2508 lsquic_conn_t *conn; 2509 enum tick_st tick_st; 2510 unsigned i, why; 2511 lsquic_time_t next_tick_time; 2512 struct conns_stailq closed_conns; 2513 struct conns_tailq ticked_conns; 2514 struct conns_stailq new_full_conns; 2515 struct cid_update_batch cub_old, cub_live; 2516 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2517 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2518 2519 eng_hist_tick(&engine->history, now); 2520 2521 STAILQ_INIT(&closed_conns); 2522 TAILQ_INIT(&ticked_conns); 2523 reset_deadline(engine, now); 2524 STAILQ_INIT(&new_full_conns); 2525 2526 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2527 && now > engine->resume_sending_at) 2528 { 2529 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2530 "timeout"); 2531 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2532 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2533 } 2534 2535 i = 0; 2536 while ((conn = next_conn(engine)) 2537 || (conn = next_new_full_conn(&new_full_conns))) 2538 { 2539 tick_st = conn->cn_if->ci_tick(conn, now); 2540 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2541 if (tick_st & TICK_PROMOTE) 2542 { 2543 lsquic_conn_t *new_conn; 2544 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2545 "scheduled for promotion"); 2546 assert(conn->cn_flags & LSCONN_MINI); 2547 new_conn = new_full_conn_server(engine, conn, now); 2548 if (new_conn) 2549 { 2550 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2551 new_conn->cn_last_sent = engine->last_sent; 2552 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2553 conn->cn_flags |= LSCONN_PROMOTED; 2554 } 2555 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2556 } 2557 if (tick_st & TICK_SEND) 2558 { 2559 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2560 { 2561 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2562 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2563 } 2564 } 2565 if (tick_st & TICK_CLOSE) 2566 { 2567 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2568 engine_incref_conn(conn, LSCONN_CLOSING); 2569 if (conn->cn_flags & LSCONN_HASHED) 2570 remove_conn_from_hash(engine, conn); 2571 } 2572 else 2573 { 2574 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2575 engine_incref_conn(conn, LSCONN_TICKED); 2576 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2577 && conn->cn_if->ci_report_live(conn, now)) 2578 cub_add_cids_from_cces(&cub_live, conn); 2579 } 2580 } 2581 2582 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2583 && lsquic_engine_has_unsent_packets(engine)) 2584 send_packets_out(engine, &ticked_conns, &closed_conns); 2585 2586 while ((conn = STAILQ_FIRST(&closed_conns))) { 2587 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2588 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2589 cub_add_cids_from_cces(&cub_old, conn); 2590 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2591 } 2592 2593 while ((conn = TAILQ_FIRST(&ticked_conns))) 2594 { 2595 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2596 engine_decref_conn(engine, conn, LSCONN_TICKED); 2597 if (!(conn->cn_flags & LSCONN_TICKABLE) 2598 && conn->cn_if->ci_is_tickable(conn)) 2599 { 2600 /* Floyd heapification is not faster, don't bother. */ 2601 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2602 engine_incref_conn(conn, LSCONN_TICKABLE); 2603 } 2604 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2605 { 2606 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2607 if (next_tick_time) 2608 { 2609 if (0 == attq_add(engine->attq, conn, next_tick_time, why)) 2610 engine_incref_conn(conn, LSCONN_ATTQ); 2611 } 2612 else 2613 /* In all other cases, the idle timeout would make the next 2614 * tick time non-zero: 2615 */ 2616 assert((conn->cn_flags & LSCONN_IETF) 2617 && engine->pub.enp_settings.es_idle_timeout == 0); 2618 } 2619 } 2620 2621 cub_flush(&engine->new_scids); 2622 cub_flush(&cub_live); 2623 cub_flush(&cub_old); 2624} 2625 2626 2627/* Return 0 if packet is being processed by a real connection, 1 if the 2628 * packet was processed, but not by a connection, and -1 on error. 2629 */ 2630int 2631lsquic_engine_packet_in (lsquic_engine_t *engine, 2632 const unsigned char *packet_in_data, size_t packet_in_size, 2633 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2634 void *peer_ctx, int ecn) 2635{ 2636 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2637 struct packin_parse_state ppstate; 2638 lsquic_packet_in_t *packet_in; 2639 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2640 int is_server, unsigned cid_len, struct packin_parse_state *); 2641 unsigned n_zeroes; 2642 int s; 2643 2644 ENGINE_CALLS_INCR(engine); 2645 2646 if (engine->flags & ENG_SERVER) 2647 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2648 else 2649 if (engine->flags & ENG_CONNS_BY_ADDR) 2650 { 2651 struct lsquic_hash_elem *el; 2652 const struct lsquic_conn *conn; 2653 el = find_conn_by_addr(engine->conns_hash, sa_local); 2654 if (!el) 2655 return -1; 2656 conn = lsquic_hashelem_getdata(el); 2657 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2658 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2659 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2660 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2661 else if (conn->cn_version == LSQVER_050) 2662 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 2663 else 2664 { 2665 assert(conn->cn_version == LSQVER_046 2666#if LSQUIC_USE_Q098 2667 || conn->cn_version == LSQVER_098 2668#endif 2669 2670 ); 2671 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2672 } 2673 } 2674 else 2675 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2676 2677 n_zeroes = 0; 2678 do 2679 { 2680 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 2681 if (!packet_in) 2682 return -1; 2683 /* Library does not modify packet_in_data, it is not referenced after 2684 * this function returns and subsequent release of pi_data is guarded 2685 * by PI_OWN_DATA flag. 2686 */ 2687 packet_in->pi_data = (unsigned char *) packet_in_data; 2688 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 2689 engine->flags & ENG_SERVER, 2690 engine->pub.enp_settings.es_scid_len, &ppstate)) 2691 { 2692 LSQ_DEBUG("Cannot parse incoming packet's header"); 2693 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 2694 errno = EINVAL; 2695 return -1; 2696 } 2697 2698 packet_in_data += packet_in->pi_data_sz; 2699 packet_in->pi_received = lsquic_time_now(); 2700 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 2701 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 2702 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 2703 peer_ctx, packet_in_size); 2704 n_zeroes += s == 0; 2705 } 2706 while (0 == s && packet_in_data < packet_end); 2707 2708 return n_zeroes > 0 ? 0 : s; 2709} 2710 2711 2712#if __GNUC__ && !defined(NDEBUG) 2713__attribute__((weak)) 2714#endif 2715unsigned 2716lsquic_engine_quic_versions (const lsquic_engine_t *engine) 2717{ 2718 return engine->pub.enp_settings.es_versions; 2719} 2720 2721 2722void 2723lsquic_engine_cooldown (lsquic_engine_t *engine) 2724{ 2725 struct lsquic_hash_elem *el; 2726 lsquic_conn_t *conn; 2727 2728 if (engine->flags & ENG_COOLDOWN) 2729 /* AFAICT, there is no harm in calling this function more than once, 2730 * but log it just in case, as it may indicate an error in the caller. 2731 */ 2732 LSQ_INFO("cooldown called again"); 2733 engine->flags |= ENG_COOLDOWN; 2734 LSQ_INFO("entering cooldown mode"); 2735 if (engine->flags & ENG_SERVER) 2736 drop_all_mini_conns(engine); 2737 for (el = lsquic_hash_first(engine->conns_hash); el; 2738 el = lsquic_hash_next(engine->conns_hash)) 2739 { 2740 conn = lsquic_hashelem_getdata(el); 2741 lsquic_conn_going_away(conn); 2742 } 2743} 2744 2745 2746int 2747lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 2748{ 2749 const struct attq_elem *next_attq; 2750 lsquic_time_t now, next_time; 2751#if LSQUIC_DEBUG_NEXT_ADV_TICK 2752 const struct lsquic_conn *conn; 2753 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 2754#endif 2755 2756 ENGINE_CALLS_INCR(engine); 2757 2758 if ((engine->flags & ENG_PAST_DEADLINE) 2759 && lsquic_mh_count(&engine->conns_out)) 2760 { 2761#if LSQUIC_DEBUG_NEXT_ADV_TICK 2762 conn = lsquic_mh_peek(&engine->conns_out); 2763 engine->last_logged_conn = 0; 2764 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 2765 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 2766 lsquic_mh_count(&engine->conns_out), 2767 lsquic_mh_count(&engine->conns_out) != 1, "s", 2768 CID_BITS(lsquic_conn_log_cid(conn))); 2769#endif 2770 *diff = 0; 2771 return 1; 2772 } 2773 2774 if (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2775 { 2776#if LSQUIC_DEBUG_NEXT_ADV_TICK 2777 engine->last_logged_conn = 0; 2778 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 2779#endif 2780 *diff = 0; 2781 return 1; 2782 } 2783 2784 if (lsquic_mh_count(&engine->conns_tickable)) 2785 { 2786#if LSQUIC_DEBUG_NEXT_ADV_TICK 2787 conn = lsquic_mh_peek(&engine->conns_tickable); 2788 engine->last_logged_conn = 0; 2789 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 2790 "connection%.*s (%"CID_FMT" first)", 2791 lsquic_mh_count(&engine->conns_tickable), 2792 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 2793 CID_BITS(lsquic_conn_log_cid(conn))); 2794#endif 2795 *diff = 0; 2796 return 1; 2797 } 2798 2799 next_attq = attq_next(engine->attq); 2800 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 2801 { 2802 if (next_attq) 2803 next_time = next_attq->ae_adv_time; 2804 else 2805 return 0; 2806 } 2807 else 2808 { 2809 if (next_attq) 2810 { 2811 next_time = next_attq->ae_adv_time; 2812 if (engine->resume_sending_at < next_time) 2813 { 2814 next_time = engine->resume_sending_at; 2815 next_attq = NULL; 2816 } 2817 } 2818 else 2819 next_time = engine->resume_sending_at; 2820 } 2821 2822 now = lsquic_time_now(); 2823 *diff = (int) ((int64_t) next_time - (int64_t) now); 2824#if LSQUIC_DEBUG_NEXT_ADV_TICK 2825 if (next_attq) 2826 { 2827 /* Deduplicate consecutive log messages about the same reason for the 2828 * same connection. 2829 * If diff is always zero or diff reset to a higher value, event is 2830 * still logged. 2831 */ 2832 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 2833 && (uintptr_t) next_attq->ae_conn 2834 == engine->last_logged_conn 2835 && *diff < engine->last_tick_diff)) 2836 { 2837 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 2838 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 2839 engine->last_tick_diff = *diff; 2840 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 2841 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 2842 lsquic_attq_why2str(next_attq->ae_why)); 2843 } 2844 } 2845 else 2846 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 2847#endif 2848 return 1; 2849} 2850 2851 2852unsigned 2853lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 2854{ 2855 lsquic_time_t now; 2856 ENGINE_CALLS_INCR(engine); 2857 now = lsquic_time_now(); 2858 if (from_now < 0) 2859 now -= from_now; 2860 else 2861 now += from_now; 2862 return attq_count_before(engine->attq, now); 2863} 2864 2865 2866int 2867lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 2868 struct lsquic_conn *conn, unsigned cce_idx) 2869{ 2870 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2871 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2872 void *peer_ctx; 2873 2874 assert(cce_idx < conn->cn_n_cces); 2875 assert(conn->cn_cces_mask & (1 << cce_idx)); 2876 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 2877 2878 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 2879 cce->cce_cid.len, conn, &cce->cce_hash_el)) 2880 { 2881 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 2882 CID_BITS(&cce->cce_cid)); 2883 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2884 cce->cce_flags |= CCE_REG; 2885 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 2886 return 0; 2887 } 2888 else 2889 { 2890 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 2891 CID_BITS(&cce->cce_cid)); 2892 return -1; 2893 } 2894} 2895 2896 2897void 2898lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 2899 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now) 2900{ 2901 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2902 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2903 void *peer_ctx; 2904 2905 assert(cce_idx < conn->cn_n_cces); 2906 2907 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 2908 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 2909 2910 if (engine->purga) 2911 { 2912 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2913 lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 2914 PUTY_CID_RETIRED, now); 2915 } 2916 conn->cn_cces_mask &= ~(1u << cce_idx); 2917 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 2918} 2919 2920 2921