lsquic_engine.c revision 7ee41525
1/* Copyright (c) 2017 - 2019 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#include <arpa/inet.h> 17#ifndef WIN32 18#include <sys/time.h> 19#include <netinet/in.h> 20#include <sys/types.h> 21#include <sys/stat.h> 22#include <fcntl.h> 23#include <unistd.h> 24#include <netdb.h> 25#endif 26 27#ifndef NDEBUG 28#include <sys/types.h> 29#include <regex.h> /* For code that loses packets */ 30#endif 31 32#if LOG_PACKET_CHECKSUM 33#include <zlib.h> 34#endif 35 36#include <openssl/aead.h> 37 38#include "lsquic.h" 39#include "lsquic_types.h" 40#include "lsquic_int_types.h" 41#include "lsquic_sizes.h" 42#include "lsquic_parse_common.h" 43#include "lsquic_parse.h" 44#include "lsquic_packet_in.h" 45#include "lsquic_packet_out.h" 46#include "lsquic_senhist.h" 47#include "lsquic_rtt.h" 48#include "lsquic_cubic.h" 49#include "lsquic_pacer.h" 50#include "lsquic_bw_sampler.h" 51#include "lsquic_minmax.h" 52#include "lsquic_bbr.h" 53#include "lsquic_send_ctl.h" 54#include "lsquic_set.h" 55#include "lsquic_conn_flow.h" 56#include "lsquic_sfcw.h" 57#include "lsquic_hash.h" 58#include "lsquic_conn.h" 59#include "lsquic_full_conn.h" 60#include "lsquic_util.h" 61#include "lsquic_qtags.h" 62#include "lsquic_enc_sess.h" 63#include "lsquic_mm.h" 64#include "lsquic_engine_public.h" 65#include "lsquic_eng_hist.h" 66#include "lsquic_ev_log.h" 67#include "lsquic_version.h" 68#include "lsquic_pr_queue.h" 69#include "lsquic_mini_conn.h" 70#include "lsquic_mini_conn_ietf.h" 71#include "lsquic_stock_shi.h" 72#include "lsquic_purga.h" 73#include "lsquic_tokgen.h" 74#include "lsquic_attq.h" 75#include "lsquic_min_heap.h" 76#include "lsquic_http1x_if.h" 77#include "lsquic_parse_common.h" 78#include "lsquic_handshake.h" 79 80#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 81#include "lsquic_logger.h" 82 83#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 84#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 85#endif 86 87#define MIN(a, b) ((a) < (b) ? (a) : (b)) 88 89/* The batch of outgoing packets grows and shrinks dynamically */ 90#define MAX_OUT_BATCH_SIZE 1024 91#define MIN_OUT_BATCH_SIZE 4 92#define INITIAL_OUT_BATCH_SIZE 32 93 94struct out_batch 95{ 96 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 97 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 98 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 99 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 100 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 101}; 102 103typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 104 105static void 106process_connections (struct lsquic_engine *engine, conn_iter_f iter, 107 lsquic_time_t now); 108 109static void 110engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 111 112static lsquic_conn_t * 113engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 114 enum lsquic_conn_flags flag); 115 116static void 117force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 118 119#if LSQUIC_COUNT_ENGINE_CALLS 120#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 121#else 122#define ENGINE_CALLS_INCR(e) 123#endif 124 125/* Nested calls to LSQUIC are not supported */ 126#define ENGINE_IN(e) do { \ 127 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 128 (e)->pub.enp_flags |= ENPUB_PROC; \ 129 ENGINE_CALLS_INCR(e); \ 130} while (0) 131 132#define ENGINE_OUT(e) do { \ 133 assert((e)->pub.enp_flags & ENPUB_PROC); \ 134 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 135} while (0) 136 137/* A connection can be referenced from one of six places: 138 * 139 * 1. A hash. The engine maintains two hash tables -- one for full, and 140 * one for mini connections. A connection starts its life in one of 141 * those. 142 * 143 * 2. Outgoing queue. 144 * 145 * 3. Tickable queue 146 * 147 * 4. Advisory Tick Time queue. 148 * 149 * 5. Closing connections queue. This is a transient queue -- it only 150 * exists for the duration of process_connections() function call. 151 * 152 * 6. Ticked connections queue. Another transient queue, similar to (5). 153 * 154 * The idea is to destroy the connection when it is no longer referenced. 155 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 156 * that case, the connection is referenced from two places: (2) and (5). 157 * After its packets are sent, it is only referenced in (5), and at the 158 * end of the function call, when it is removed from (5), reference count 159 * goes to zero and the connection is destroyed. If not all packets can 160 * be sent, at the end of the function call, the connection is referenced 161 * by (2) and will only be removed once all outgoing packets have been 162 * sent. 163 */ 164#define CONN_REF_FLAGS (LSCONN_HASHED \ 165 |LSCONN_HAS_OUTGOING \ 166 |LSCONN_TICKABLE \ 167 |LSCONN_TICKED \ 168 |LSCONN_CLOSING \ 169 |LSCONN_ATTQ) 170 171 172 173 174struct cid_update_batch 175{ 176 lsquic_cids_update_f cub_update_cids; 177 void *cub_update_ctx; 178 unsigned cub_count; 179 lsquic_cid_t cub_cids[20]; 180 void *cub_peer_ctxs[20]; 181}; 182 183static void 184cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 185 186 187struct lsquic_engine 188{ 189 struct lsquic_engine_public pub; 190 enum { 191 ENG_SERVER = LSENG_SERVER, 192 ENG_HTTP = LSENG_HTTP, 193 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 194 ENG_PAST_DEADLINE 195 = (1 << 8), /* Previous call to a processing 196 * function went past time threshold. 197 */ 198 ENG_CONNS_BY_ADDR 199 = (1 << 9), /* Connections are hashed by address */ 200#ifndef NDEBUG 201 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 202 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 203 ENG_DTOR = (1 << 26), /* Engine destructor */ 204#endif 205 } flags; 206 lsquic_packets_out_f packets_out; 207 void *packets_out_ctx; 208 lsquic_cids_update_f report_new_scids; 209 lsquic_cids_update_f report_live_scids; 210 lsquic_cids_update_f report_old_scids; 211 void *scids_ctx; 212 struct lsquic_hash *conns_hash; 213 struct min_heap conns_tickable; 214 struct min_heap conns_out; 215 /* Use a union because only one iterator is being used at any one time */ 216 union { 217 struct { 218 struct cert_susp_head *head; 219 } resumed; 220 struct lsquic_conn *one_conn; 221 } iter_state; 222 struct eng_hist history; 223 unsigned batch_size; 224 struct pr_queue *pr_queue; 225 struct attq *attq; 226 /* Track time last time a packet was sent to give new connections 227 * priority lower than that of existing connections. 228 */ 229 lsquic_time_t last_sent; 230#ifndef NDEBUG 231 regex_t lose_packets_re; 232 const char *lose_packets_str; 233#endif 234 unsigned n_conns; 235 lsquic_time_t deadline; 236 lsquic_time_t resume_sending_at; 237 unsigned mini_conns_count; 238 struct lsquic_purga *purga; 239#if LSQUIC_CONN_STATS 240 struct { 241 unsigned conns; 242 } stats; 243 struct conn_stats conn_stats_sum; 244 FILE *stats_fh; 245#endif 246 struct cid_update_batch new_scids; 247 struct out_batch out_batch; 248#if LSQUIC_COUNT_ENGINE_CALLS 249 unsigned long n_engine_calls; 250#endif 251}; 252 253 254void 255lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 256 unsigned flags) 257{ 258 memset(settings, 0, sizeof(*settings)); 259 settings->es_versions = LSQUIC_DF_VERSIONS; 260 if (flags & ENG_SERVER) 261 { 262 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 263 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 264 settings->es_init_max_data 265 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 266 settings->es_init_max_stream_data_bidi_remote 267 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 268 settings->es_init_max_stream_data_bidi_local 269 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 270 settings->es_init_max_stream_data_uni 271 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 272 settings->es_init_max_streams_uni 273 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 274 settings->es_ping_period = 0; 275 } 276 else 277 { 278 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 279 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 280 settings->es_init_max_data 281 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 282 settings->es_init_max_stream_data_bidi_remote 283 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 284 settings->es_init_max_stream_data_bidi_local 285 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 286 settings->es_init_max_stream_data_uni 287 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 288 settings->es_init_max_streams_uni 289 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 290 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 291 } 292 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 293 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 294 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 295 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 296 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 297 settings->es_max_header_list_size 298 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 299 settings->es_ua = LSQUIC_DF_UA; 300 settings->es_ecn = LSQUIC_DF_ECN; 301 302 settings->es_pdmd = QTAG_X509; 303 settings->es_aead = QTAG_AESG; 304 settings->es_kexs = QTAG_C255; 305 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 306 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 307 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 308 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 309 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 310 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 311 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 312 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 313 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 314 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 315 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 316 settings->es_sttl = LSQUIC_DF_STTL; 317 settings->es_init_max_streams_bidi 318 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 319 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 320 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 321 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 322 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 323 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 324 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 325 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 326 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 327} 328 329 330/* Note: if returning an error, err_buf must be valid if non-NULL */ 331int 332lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 333 unsigned flags, 334 char *err_buf, size_t err_buf_sz) 335{ 336 if (settings->es_cfcw < LSQUIC_MIN_FCW || 337 settings->es_sfcw < LSQUIC_MIN_FCW) 338 { 339 if (err_buf) 340 snprintf(err_buf, err_buf_sz, "%s", 341 "flow control window set too low"); 342 return -1; 343 } 344 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 345 { 346 if (err_buf) 347 snprintf(err_buf, err_buf_sz, "%s", 348 "No supported QUIC versions specified"); 349 return -1; 350 } 351 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 352 { 353 if (err_buf) 354 snprintf(err_buf, err_buf_sz, "%s", 355 "one or more unsupported QUIC version is specified"); 356 return -1; 357 } 358 if (flags & ENG_SERVER) 359 { 360 if (settings->es_handshake_to > 361 MAX_MINI_CONN_LIFESPAN_IN_USEC) 362 { 363 if (err_buf) 364 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 365 " usec is too large. The maximum for server is %u usec", 366 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 367 return -1; 368 } 369 } 370 if (settings->es_idle_timeout > 600) 371 { 372 if (err_buf) 373 snprintf(err_buf, err_buf_sz, "%s", 374 "The maximum value of idle timeout is 600 seconds"); 375 return -1; 376 } 377 if (!(!(flags & ENG_SERVER) && settings->es_scid_len == 0) 378 && (settings->es_scid_len < 4 || settings->es_scid_len > 18)) 379 { 380 if (err_buf) 381 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 382 "bytes long; it must be between 4 and 18.", 383 settings->es_scid_len); 384 return -1; 385 } 386 387 if (settings->es_cc_algo > 2) 388 { 389 if (err_buf) 390 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 391 "algorithm value %u", settings->es_cc_algo); 392 return -1; 393 } 394 return 0; 395} 396 397 398static void 399free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 400{ 401 free(packet_data); 402} 403 404 405static void * 406malloc_buf (void *ctx, void *conn_ctx, unsigned short size, char is_ipv6) 407{ 408 return malloc(size); 409} 410 411 412static const struct lsquic_packout_mem_if stock_pmi = 413{ 414 malloc_buf, free_packet, free_packet, 415}; 416 417 418static int 419hash_conns_by_addr (const struct lsquic_engine *engine) 420{ 421 if (engine->flags & ENG_SERVER) 422 return 0; 423 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 424 return 1; 425 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 426 && engine->pub.enp_settings.es_support_tcid0) 427 return 1; 428 if (engine->pub.enp_settings.es_scid_len == 0) 429 return 1; 430 return 0; 431} 432 433 434lsquic_engine_t * 435lsquic_engine_new (unsigned flags, 436 const struct lsquic_engine_api *api) 437{ 438 lsquic_engine_t *engine; 439 char err_buf[100]; 440 441 if (!api->ea_packets_out) 442 { 443 LSQ_ERROR("packets_out callback is not specified"); 444 return NULL; 445 } 446 447 if (api->ea_settings && 448 0 != lsquic_engine_check_settings(api->ea_settings, flags, 449 err_buf, sizeof(err_buf))) 450 { 451 LSQ_ERROR("cannot create engine: %s", err_buf); 452 return NULL; 453 } 454 455 engine = calloc(1, sizeof(*engine)); 456 if (!engine) 457 return NULL; 458 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 459 { 460 free(engine); 461 return NULL; 462 } 463 if (api->ea_settings) 464 engine->pub.enp_settings = *api->ea_settings; 465 else 466 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 467 int tag_buf_len; 468 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 469 sizeof(engine->pub.enp_ver_tags_buf), 470 engine->pub.enp_settings.es_versions); 471 if (tag_buf_len <= 0) 472 { 473 LSQ_ERROR("cannot generate version tags buffer"); 474 free(engine); 475 return NULL; 476 } 477 engine->pub.enp_ver_tags_len = tag_buf_len; 478 engine->pub.enp_flags = ENPUB_CAN_SEND; 479 engine->pub.enp_stream_if = api->ea_stream_if; 480 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 481 482 engine->flags = flags; 483#ifndef NDEBUG 484 engine->flags |= ENG_COALESCE; 485#endif 486 engine->packets_out = api->ea_packets_out; 487 engine->packets_out_ctx = api->ea_packets_out_ctx; 488 engine->report_new_scids = api->ea_new_scids; 489 engine->report_live_scids = api->ea_live_scids; 490 engine->report_old_scids = api->ea_old_scids; 491 engine->scids_ctx = api->ea_cids_update_ctx; 492 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 493 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 494 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 495 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 496 if (api->ea_shi) 497 { 498 engine->pub.enp_shi = api->ea_shi; 499 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 500 } 501 else 502 { 503 engine->pub.enp_shi = &stock_shi; 504 engine->pub.enp_shi_ctx = stock_shared_hash_new(); 505 if (!engine->pub.enp_shi_ctx) 506 { 507 free(engine); 508 return NULL; 509 } 510 } 511 if (api->ea_hsi_if) 512 { 513 engine->pub.enp_hsi_if = api->ea_hsi_if; 514 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 515 } 516 else 517 { 518 engine->pub.enp_hsi_if = lsquic_http1x_if; 519 engine->pub.enp_hsi_ctx = NULL; 520 } 521 if (api->ea_pmi) 522 { 523 engine->pub.enp_pmi = api->ea_pmi; 524 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 525 } 526 else 527 { 528 engine->pub.enp_pmi = &stock_pmi; 529 engine->pub.enp_pmi_ctx = NULL; 530 } 531 engine->pub.enp_verify_cert = api->ea_verify_cert; 532 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 533 engine->pub.enp_kli = api->ea_keylog_if; 534 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 535 engine->pub.enp_engine = engine; 536 if (hash_conns_by_addr(engine)) 537 engine->flags |= ENG_CONNS_BY_ADDR; 538 engine->conns_hash = lsquic_hash_create(); 539 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 540 if (!engine->pub.enp_tokgen) 541 return NULL; 542 if (flags & ENG_SERVER) 543 { 544 engine->pr_queue = prq_create( 545 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 546 &engine->pub); 547 if (!engine->pr_queue) 548 { 549 lsquic_tg_destroy(engine->pub.enp_tokgen); 550 return NULL; 551 } 552 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 553 engine->report_old_scids, engine->scids_ctx); 554 if (!engine->purga) 555 { 556 lsquic_tg_destroy(engine->pub.enp_tokgen); 557 prq_destroy(engine->pr_queue); 558 return NULL; 559 } 560 } 561 engine->attq = attq_create(); 562 eng_hist_init(&engine->history); 563 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 564 if (engine->pub.enp_settings.es_honor_prst) 565 { 566 engine->pub.enp_srst_hash = lsquic_hash_create(); 567 if (!engine->pub.enp_srst_hash) 568 { 569 lsquic_engine_destroy(engine); 570 return NULL; 571 } 572 } 573 574#ifndef NDEBUG 575 { 576 const char *env; 577 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 578 if (env) 579 { 580 if (0 != regcomp(&engine->lose_packets_re, env, 581 REG_EXTENDED|REG_NOSUB)) 582 { 583 LSQ_ERROR("could not compile lost packet regex `%s'", env); 584 return NULL; 585 } 586 engine->flags |= ENG_LOSE_PACKETS; 587 engine->lose_packets_str = env; 588 LSQ_WARN("will lose packets that match the following regex: %s", 589 env); 590 } 591 env = getenv("LSQUIC_COALESCE"); 592 if (env) 593 { 594 engine->flags &= ~ENG_COALESCE; 595 if (atoi(env)) 596 { 597 engine->flags |= ENG_COALESCE; 598 LSQ_NOTICE("will coalesce packets"); 599 } 600 else 601 LSQ_NOTICE("will not coalesce packets"); 602 } 603 } 604#endif 605#if LSQUIC_CONN_STATS 606 engine->stats_fh = api->ea_stats_fh; 607#endif 608 609 LSQ_INFO("instantiated engine"); 610 return engine; 611} 612 613 614#if LOG_PACKET_CHECKSUM 615static void 616log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 617 const unsigned char *buf, size_t bufsz) 618{ 619 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 620 (uint32_t) crc32(0, buf, bufsz)); 621} 622 623 624#endif 625 626 627static void 628grow_batch_size (struct lsquic_engine *engine) 629{ 630 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 631} 632 633 634static void 635shrink_batch_size (struct lsquic_engine *engine) 636{ 637 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 638} 639 640 641struct cce_cid_iter 642{ 643 const struct lsquic_conn *conn; 644 unsigned todo, n; 645}; 646 647 648static struct conn_cid_elem * 649cce_iter_next (struct cce_cid_iter *citer) 650{ 651 struct conn_cid_elem *cce; 652 653 while (citer->todo) 654 if (citer->todo & (1 << citer->n)) 655 { 656 citer->todo &= ~(1 << citer->n); 657 cce = &citer->conn->cn_cces[ citer->n++ ]; 658 if (!(cce->cce_flags & CCE_PORT)) 659 return cce; 660 } 661 else 662 ++citer->n; 663 664 return NULL; 665} 666 667 668static struct conn_cid_elem * 669cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 670{ 671 citer->conn = conn; 672 citer->todo = conn->cn_cces_mask; 673 citer->n = 0; 674 return cce_iter_next(citer); 675} 676 677 678#if LSQUIC_CONN_STATS 679void 680update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 681{ 682 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 683 const unsigned long *src; 684 const struct conn_stats *stats; 685 unsigned i; 686 687 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 688 { 689 ++engine->stats.conns; 690 src = (unsigned long *) stats; 691 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 692 dst[i] += src[i]; 693 } 694} 695 696 697#endif 698 699 700/* Wrapper to make sure important things occur before the connection is 701 * really destroyed. 702 */ 703static void 704destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 705 lsquic_time_t now) 706{ 707 struct cce_cid_iter citer; 708 const struct conn_cid_elem *cce; 709 lsquic_time_t drain_time; 710 struct purga_el *puel; 711 712 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 713 if (engine->purga 714 /* Blacklist all CIDs except for promoted mini connections */ 715 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 716 != (LSCONN_MINI|LSCONN_PROMOTED)) 717 { 718 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 719 && conn->cn_if->ci_drain_time && 720 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 721 { 722 for (cce = cce_iter_first(&citer, conn); cce; 723 cce = cce_iter_next(&citer)) 724 { 725 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 726 lsquic_conn_get_peer_ctx(conn, NULL), 727 PUTY_CONN_DRAIN, now); 728 if (puel) 729 puel->puel_time = now + drain_time; 730 } 731 } 732 else 733 { 734 for (cce = cce_iter_first(&citer, conn); cce; 735 cce = cce_iter_next(&citer)) 736 { 737 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 738 lsquic_conn_get_peer_ctx(conn, NULL), 739 PUTY_CONN_DELETED, now); 740 if (puel) 741 { 742 puel->puel_time = now; 743 puel->puel_count = 0; 744 } 745 } 746 } 747 } 748#if LSQUIC_CONN_STATS 749 update_stats_sum(engine, conn); 750#endif 751 --engine->n_conns; 752 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 753 conn->cn_if->ci_destroy(conn); 754} 755 756 757static int 758maybe_grow_conn_heaps (struct lsquic_engine *engine) 759{ 760 struct min_heap_elem *els; 761 unsigned count; 762 763 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 764 return 0; /* Nothing to do */ 765 766 if (lsquic_mh_nalloc(&engine->conns_tickable)) 767 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 768 else 769 count = 8; 770 771 els = malloc(sizeof(els[0]) * count); 772 if (!els) 773 { 774 LSQ_ERROR("%s: malloc failed", __func__); 775 return -1; 776 } 777 778 LSQ_DEBUG("grew heaps to %u elements", count / 2); 779 memcpy(&els[0], engine->conns_tickable.mh_elems, 780 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 781 memcpy(&els[count / 2], engine->conns_out.mh_elems, 782 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 783 free(engine->conns_tickable.mh_elems); 784 engine->conns_tickable.mh_elems = els; 785 engine->conns_out.mh_elems = &els[count / 2]; 786 engine->conns_tickable.mh_nalloc = count / 2; 787 engine->conns_out.mh_nalloc = count / 2; 788 return 0; 789} 790 791 792static void 793remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 794 unsigned todo) 795{ 796 unsigned n; 797 798 for (n = 0; todo; todo &= ~(1 << n++)) 799 if ((todo & (1 << n)) && 800 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 801 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 802} 803 804 805static void 806remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 807{ 808 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 809} 810 811 812static void 813cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 814 815 816static int 817insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 818 void *peer_ctx) 819{ 820 struct conn_cid_elem *cce; 821 unsigned todo, done, n; 822 823 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 824 if (todo & (1 << n)) 825 { 826 cce = &conn->cn_cces[n]; 827 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 828 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 829 cce->cce_cid.len, conn, &cce->cce_hash_el)) 830 done |= 1 << n; 831 else 832 goto err; 833 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 834 { 835 cce->cce_flags |= CCE_REG; 836 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 837 } 838 } 839 840 return 0; 841 842 err: 843 remove_cces_from_hash(engine->conns_hash, conn, done); 844 return -1; 845} 846 847 848static lsquic_conn_t * 849new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 850 lsquic_time_t now) 851{ 852 const lsquic_cid_t *cid; 853 server_conn_ctor_f ctor; 854 lsquic_conn_t *conn; 855 unsigned flags; 856 if (0 != maybe_grow_conn_heaps(engine)) 857 return NULL; 858 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 859 860 if (mini_conn->cn_flags & LSCONN_IETF) 861 ctor = lsquic_ietf_full_conn_server_new; 862 else 863 ctor = lsquic_gquic_full_conn_server_new; 864 865 conn = ctor(&engine->pub, flags, mini_conn); 866 if (!conn) 867 { 868 /* Otherwise, full_conn_server_new prints its own warnings */ 869 if (ENOMEM == errno) 870 { 871 cid = lsquic_conn_log_cid(mini_conn); 872 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 873 CID_BITS(cid), strerror(errno)); 874 } 875 return NULL; 876 } 877 ++engine->n_conns; 878 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 879 { 880 cid = lsquic_conn_log_cid(conn); 881 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 882 CID_BITS(cid)); 883 destroy_conn(engine, conn, now); 884 return NULL; 885 } 886 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 887 conn->cn_flags |= LSCONN_HASHED; 888 return conn; 889} 890 891 892static enum 893{ 894 VER_NOT_SPECIFIED, 895 VER_SUPPORTED, 896 VER_UNSUPPORTED, 897} 898 899 900version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 901 enum lsquic_version *pversion) 902{ 903 lsquic_ver_tag_t ver_tag; 904 enum lsquic_version version; 905 906 if (!packet_in->pi_quic_ver) 907 { 908 LSQ_DEBUG("packet does not specify version"); 909 return VER_NOT_SPECIFIED; 910 } 911 912 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 913 version = lsquic_tag2ver(ver_tag); 914 if (version < N_LSQVER) 915 { 916 if (engine->pub.enp_settings.es_versions & (1 << version)) 917 { 918 LSQ_DEBUG("client-supplied version %s is supported", 919 lsquic_ver2str[version]); 920 *pversion = version; 921 return VER_SUPPORTED; 922 } 923 else 924 LSQ_DEBUG("client-supplied version %s is not supported", 925 lsquic_ver2str[version]); 926 } 927 else 928 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 929 ver_tag); 930 931 return VER_UNSUPPORTED; 932} 933 934 935static void 936schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 937 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 938 const struct sockaddr *sa_peer, void *peer_ctx) 939{ 940 assert(engine->pr_queue); 941 if (0 == prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 942 sa_local, sa_peer)) 943 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 944 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 945 else 946 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 947} 948 949 950static unsigned short 951sa2port (const struct sockaddr *sa) 952{ 953 if (sa->sa_family == AF_INET) 954 { 955 struct sockaddr_in *const sa4 = (void *) sa; 956 return sa4->sin_port; 957 } 958 else 959 { 960 struct sockaddr_in6 *const sa6 = (void *) sa; 961 return sa6->sin6_port; 962 } 963} 964 965 966static struct lsquic_hash_elem * 967find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 968{ 969 unsigned short port; 970 971 port = sa2port(sa); 972 return lsquic_hash_find(hash, &port, sizeof(port)); 973} 974 975 976static lsquic_conn_t * 977find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 978 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 979{ 980 struct lsquic_hash_elem *el; 981 lsquic_conn_t *conn; 982 983 if (engine->flags & ENG_CONNS_BY_ADDR) 984 el = find_conn_by_addr(engine->conns_hash, sa_local); 985 else if (packet_in->pi_flags & PI_CONN_ID) 986 el = lsquic_hash_find(engine->conns_hash, 987 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 988 else 989 { 990 LSQ_DEBUG("packet header does not have connection ID: discarding"); 991 return NULL; 992 } 993 994 if (!el) 995 return NULL; 996 997 conn = lsquic_hashelem_getdata(el); 998 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 999 if ((engine->flags & ENG_CONNS_BY_ADDR) 1000 && !(conn->cn_flags & LSCONN_IETF) 1001 && (packet_in->pi_flags & PI_CONN_ID) 1002 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1003 { 1004 LSQ_DEBUG("connection IDs do not match"); 1005 return NULL; 1006 } 1007 1008 return conn; 1009} 1010 1011 1012static lsquic_conn_t * 1013find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1014 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1015 const struct sockaddr *sa_peer, void *peer_ctx) 1016{ 1017 struct lsquic_hash_elem *el; 1018 struct purga_el *puel; 1019 lsquic_conn_t *conn; 1020 1021 if (!(packet_in->pi_flags & PI_CONN_ID)) 1022 { 1023 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1024 return NULL; 1025 } 1026 el = lsquic_hash_find(engine->conns_hash, 1027 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1028 1029 if (el) 1030 { 1031 conn = lsquic_hashelem_getdata(el); 1032 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1033 return conn; 1034 } 1035 1036 if (engine->flags & ENG_COOLDOWN) 1037 { /* Do not create incoming connections during cooldown */ 1038 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1039 return NULL; 1040 } 1041 1042 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1043 { 1044 LSQ_DEBUG("reached limit of %u inchoate connections", 1045 engine->pub.enp_settings.es_max_inchoate); 1046 return NULL; 1047 } 1048 1049 1050 if (engine->purga 1051 && (puel = lsquic_purga_contains(engine->purga, 1052 &packet_in->pi_conn_id), puel)) 1053 { 1054 switch (puel->puel_type) 1055 { 1056 case PUTY_CID_RETIRED: 1057 LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet", 1058 CID_BITS(&packet_in->pi_conn_id)); 1059 return NULL; 1060 case PUTY_CONN_DRAIN: 1061 LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64, 1062 puel->puel_time, packet_in->pi_received); 1063 if (puel->puel_time > packet_in->pi_received) 1064 { 1065 LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet", 1066 CID_BITS(&packet_in->pi_conn_id)); 1067 return NULL; 1068 } 1069 LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted", 1070 CID_BITS(&packet_in->pi_conn_id)); 1071 puel->puel_type = PUTY_CONN_DELETED; 1072 puel->puel_count = 0; 1073 puel->puel_time = 0; 1074 /* fall-through */ 1075 case PUTY_CONN_DELETED: 1076 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1077 CID_BITS(&packet_in->pi_conn_id)); 1078 if (puel->puel_time < packet_in->pi_received) 1079 { 1080 puel->puel_time = packet_in->pi_received 1081 /* Exponential back-off */ 1082 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1083 ++puel->puel_count; 1084 goto maybe_send_prst; 1085 } 1086 return NULL; 1087 default: 1088 assert(0); 1089 return NULL; 1090 } 1091 } 1092 1093 if (engine->pub.enp_settings.es_send_prst 1094 && !(packet_in->pi_flags & PI_GQUIC) 1095 && HETY_NOT_SET == packet_in->pi_header_type) 1096 goto maybe_send_prst; 1097 1098 if (0 != maybe_grow_conn_heaps(engine)) 1099 return NULL; 1100 1101 const struct parse_funcs *pf; 1102 enum lsquic_version version; 1103 switch (version_matches(engine, packet_in, &version)) 1104 { 1105 case VER_UNSUPPORTED: 1106 if (engine->flags & ENG_SERVER) 1107 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1108 sa_local, sa_peer, peer_ctx); 1109 return NULL; 1110 case VER_NOT_SPECIFIED: 1111 maybe_send_prst: 1112 if ((engine->flags & ENG_SERVER) && 1113 engine->pub.enp_settings.es_send_prst) 1114 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1115 sa_local, sa_peer, peer_ctx); 1116 return NULL; 1117 case VER_SUPPORTED: 1118 pf = select_pf_by_ver(version); 1119 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1120 break; 1121 } 1122 1123 1124 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1125 { 1126 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1127 sa_peer->sa_family == AF_INET, NULL); 1128 } 1129 else 1130 { 1131 conn = mini_conn_new(&engine->pub, packet_in, version); 1132 } 1133 if (!conn) 1134 return NULL; 1135 ++engine->mini_conns_count; 1136 ++engine->n_conns; 1137 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1138 { 1139 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1140 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1141 CID_BITS(cid)); 1142 destroy_conn(engine, conn, packet_in->pi_received); 1143 return NULL; 1144 } 1145 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1146 conn->cn_flags |= LSCONN_HASHED; 1147 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1148 conn->cn_last_sent = engine->last_sent; 1149 return conn; 1150} 1151 1152 1153lsquic_conn_t * 1154lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1155 const lsquic_cid_t *cid) 1156{ 1157 struct lsquic_hash_elem *el; 1158 lsquic_conn_t *conn = NULL; 1159 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1160 1161 if (el) 1162 conn = lsquic_hashelem_getdata(el); 1163 return conn; 1164} 1165 1166 1167#if !defined(NDEBUG) && __GNUC__ 1168__attribute__((weak)) 1169#endif 1170void 1171lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1172 lsquic_conn_t *conn) 1173{ 1174 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1175 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1176 { 1177 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1178 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1179 engine_incref_conn(conn, LSCONN_TICKABLE); 1180 } 1181} 1182 1183 1184void 1185lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1186 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1187{ 1188 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1189 if (conn->cn_flags & LSCONN_TICKABLE) 1190 { 1191 /* Optimization: no need to add the connection to the Advisory Tick 1192 * Time Queue: it is about to be ticked, after which it its next tick 1193 * time may be queried again. 1194 */; 1195 } 1196 else if (conn->cn_flags & LSCONN_ATTQ) 1197 { 1198 if (lsquic_conn_adv_time(conn) != tick_time) 1199 { 1200 attq_remove(engine->attq, conn); 1201 if (0 != attq_add(engine->attq, conn, tick_time, why)) 1202 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1203 } 1204 } 1205 else if (0 == attq_add(engine->attq, conn, tick_time, why)) 1206 engine_incref_conn(conn, LSCONN_ATTQ); 1207} 1208 1209 1210static struct lsquic_conn * 1211find_conn_by_srst (struct lsquic_engine *engine, 1212 const struct lsquic_packet_in *packet_in) 1213{ 1214 struct lsquic_hash_elem *el; 1215 struct lsquic_conn *conn; 1216 1217 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1218 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1219 return NULL; 1220 1221 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1222 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1223 IQUIC_SRESET_TOKEN_SZ); 1224 if (!el) 1225 return NULL; 1226 1227 conn = lsquic_hashelem_getdata(el); 1228 return conn; 1229} 1230 1231 1232/* Return 0 if packet is being processed by a real connection (mini or full), 1233 * otherwise return 1. 1234 */ 1235static int 1236process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1237 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1238 const struct sockaddr *sa_peer, void *peer_ctx, int full_udp_packet) 1239{ 1240 lsquic_conn_t *conn; 1241 const unsigned char *packet_in_data; 1242 size_t packet_in_size; 1243 1244 if (lsquic_packet_in_is_gquic_prst(packet_in) 1245 && !engine->pub.enp_settings.es_honor_prst) 1246 { 1247 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1248 LSQ_DEBUG("public reset packet: discarding"); 1249 return 1; 1250 } 1251 1252 if (engine->flags & ENG_SERVER) 1253 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1254 sa_peer, peer_ctx); 1255 else 1256 conn = find_conn(engine, packet_in, ppstate, sa_local); 1257 1258 if (!conn) 1259 { 1260 if (engine->pub.enp_settings.es_honor_prst 1261 && full_udp_packet 1262 && !(packet_in->pi_flags & PI_GQUIC) 1263 && engine->pub.enp_srst_hash 1264 && (conn = find_conn_by_srst(engine, packet_in))) 1265 { 1266 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1267 CID_BITS(lsquic_conn_log_cid(conn))); 1268 conn->cn_if->ci_stateless_reset(conn); 1269 if (!(conn->cn_flags & LSCONN_TICKABLE) 1270 && conn->cn_if->ci_is_tickable(conn)) 1271 { 1272 lsquic_mh_insert(&engine->conns_tickable, conn, 1273 conn->cn_last_ticked); 1274 engine_incref_conn(conn, LSCONN_TICKABLE); 1275 } 1276 /* Even though the connection processes this packet, we return 1277 * 1 so that the caller does not add reset packet's random 1278 * bytes to the list of valid CIDs. 1279 */ 1280 } 1281 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1282 return 1; 1283 } 1284 1285 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1286 { 1287 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1288 engine_incref_conn(conn, LSCONN_TICKABLE); 1289 } 1290 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1291 sa_local, sa_peer); 1292 lsquic_packet_in_upref(packet_in); 1293#if LOG_PACKET_CHECKSUM 1294 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1295 packet_in->pi_data_sz); 1296#endif 1297 /* Note on QLog: 1298 * For the PACKET_RX QLog event, we are interested in logging these things: 1299 * - raw packet (however it comes in, encrypted or not) 1300 * - frames (list of frame names) 1301 * - packet type and number 1302 * - packet rx timestamp 1303 * 1304 * Since only some of these items are available at this code 1305 * juncture, we will wait until after the packet has been 1306 * decrypted (if necessary) and parsed to call the log functions. 1307 * 1308 * Once the PACKET_RX event is finally logged, the timestamp 1309 * will come from packet_in->pi_received. For correct sequential 1310 * ordering of QLog events, be sure to process the QLogs downstream. 1311 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1312 */ 1313 packet_in_data = packet_in->pi_data; 1314 packet_in_size = packet_in->pi_data_sz; 1315 conn->cn_if->ci_packet_in(conn, packet_in); 1316 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1317 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1318 return 0; 1319} 1320 1321 1322void 1323lsquic_engine_destroy (lsquic_engine_t *engine) 1324{ 1325 struct lsquic_hash_elem *el; 1326 lsquic_conn_t *conn; 1327 1328 LSQ_DEBUG("destroying engine"); 1329#ifndef NDEBUG 1330 engine->flags |= ENG_DTOR; 1331#endif 1332 1333 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1334 { 1335 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1336 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1337 } 1338 1339 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1340 { 1341 assert(conn->cn_flags & LSCONN_TICKABLE); 1342 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1343 } 1344 1345 for (el = lsquic_hash_first(engine->conns_hash); el; 1346 el = lsquic_hash_next(engine->conns_hash)) 1347 { 1348 conn = lsquic_hashelem_getdata(el); 1349 force_close_conn(engine, conn); 1350 } 1351 lsquic_hash_destroy(engine->conns_hash); 1352 1353 assert(0 == engine->n_conns); 1354 assert(0 == engine->mini_conns_count); 1355 if (engine->pr_queue) 1356 prq_destroy(engine->pr_queue); 1357 if (engine->purga) 1358 lsquic_purga_destroy(engine->purga); 1359 attq_destroy(engine->attq); 1360 1361 assert(0 == lsquic_mh_count(&engine->conns_out)); 1362 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1363 if (engine->pub.enp_shi == &stock_shi) 1364 stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1365 lsquic_mm_cleanup(&engine->pub.enp_mm); 1366 free(engine->conns_tickable.mh_elems); 1367#ifndef NDEBUG 1368 if (engine->flags & ENG_LOSE_PACKETS) 1369 regfree(&engine->lose_packets_re); 1370#endif 1371 if (engine->pub.enp_tokgen) 1372 lsquic_tg_destroy(engine->pub.enp_tokgen); 1373#if LSQUIC_CONN_STATS 1374 if (engine->stats_fh) 1375 { 1376 const struct conn_stats *const stats = &engine->conn_stats_sum; 1377 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1378 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1379 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1380 fprintf(engine->stats_fh, "In:\n"); 1381 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1382 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1383 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1384 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1385 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1386 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1387 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1388 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1389 stats->in.headers_comp, stats->in.headers_uncomp, 1390 stats->in.headers_uncomp ? 1391 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1392 : 0); 1393 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1394 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1395 fprintf(engine->stats_fh, " ACK frames merged to new: %lu\n", stats->in.n_acks_merged[0]); 1396 fprintf(engine->stats_fh, " ACK frames merged to old: %lu\n", stats->in.n_acks_merged[1]); 1397 fprintf(engine->stats_fh, "Out:\n"); 1398 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1399 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1400 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1401 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1402 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1403 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1404 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1405 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1406 stats->out.headers_comp, stats->out.headers_uncomp, 1407 stats->out.headers_uncomp ? 1408 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1409 : 0); 1410 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1411 } 1412#endif 1413 if (engine->pub.enp_srst_hash) 1414 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1415#if LSQUIC_COUNT_ENGINE_CALLS 1416 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1417#endif 1418 free(engine); 1419} 1420 1421 1422static struct conn_cid_elem * 1423find_free_cce (struct lsquic_conn *conn) 1424{ 1425 struct conn_cid_elem *cce; 1426 1427 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1428 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1429 return cce; 1430 1431 return NULL; 1432} 1433 1434 1435static int 1436add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1437 const struct sockaddr *local_sa, void *peer_ctx) 1438{ 1439 struct conn_cid_elem *cce; 1440 1441 if (engine->flags & ENG_CONNS_BY_ADDR) 1442 { 1443 cce = find_free_cce(conn); 1444 if (!cce) 1445 { 1446 LSQ_ERROR("cannot find free CCE"); 1447 return -1; 1448 } 1449 cce->cce_port = sa2port(local_sa); 1450 cce->cce_flags = CCE_PORT; 1451 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1452 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1453 { 1454 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1455 return 0; 1456 } 1457 else 1458 return -1; 1459 1460 } 1461 else 1462 return insert_conn_into_hash(engine, conn, peer_ctx); 1463} 1464 1465 1466lsquic_conn_t * 1467lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1468 const struct sockaddr *local_sa, 1469 const struct sockaddr *peer_sa, 1470 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1471 const char *hostname, unsigned short max_packet_size, 1472 const unsigned char *zero_rtt, size_t zero_rtt_len, 1473 const unsigned char *token, size_t token_sz) 1474{ 1475 lsquic_conn_t *conn; 1476 unsigned flags, versions; 1477 int is_ipv4; 1478 1479 ENGINE_IN(engine); 1480 1481 if (engine->flags & ENG_SERVER) 1482 { 1483 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1484 goto err; 1485 } 1486 1487 if (engine->flags & ENG_CONNS_BY_ADDR 1488 && find_conn_by_addr(engine->conns_hash, local_sa)) 1489 { 1490 LSQ_ERROR("cannot have more than one connection on the same port"); 1491 goto err; 1492 } 1493 1494 if (0 != maybe_grow_conn_heaps(engine)) 1495 return NULL; 1496 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1497 is_ipv4 = peer_sa->sa_family == AF_INET; 1498 if (zero_rtt && zero_rtt_len) 1499 { 1500 version = lsquic_zero_rtt_version(zero_rtt, zero_rtt_len); 1501 if (version >= N_LSQVER) 1502 { 1503 LSQ_INFO("zero-rtt version is bad, won't use"); 1504 zero_rtt = NULL; 1505 zero_rtt_len = 0; 1506 } 1507 } 1508 if (version >= N_LSQVER) 1509 { 1510 if (version > N_LSQVER) 1511 LSQ_WARN("invalid version specified, engine will pick"); 1512 versions = engine->pub.enp_settings.es_versions; 1513 } 1514 else 1515 versions = 1u << version; 1516 if (versions & LSQUIC_IETF_VERSIONS) 1517 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1518 flags, hostname, max_packet_size, 1519 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1520 else 1521 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1522 flags, hostname, max_packet_size, is_ipv4, 1523 zero_rtt, zero_rtt_len); 1524 if (!conn) 1525 goto err; 1526 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1527 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1528 lsquic_ver2str[conn->cn_version]); 1529 ++engine->n_conns; 1530 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1531 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1532 { 1533 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1534 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1535 CID_BITS(cid)); 1536 destroy_conn(engine, conn, lsquic_time_now()); 1537 goto err; 1538 } 1539 assert(!(conn->cn_flags & 1540 (CONN_REF_FLAGS 1541 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1542 callbacks */ 1543 ))); 1544 conn->cn_flags |= LSCONN_HASHED; 1545 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1546 engine_incref_conn(conn, LSCONN_TICKABLE); 1547 lsquic_conn_set_ctx(conn, conn_ctx); 1548 conn->cn_if->ci_client_call_on_new(conn); 1549 end: 1550 ENGINE_OUT(engine); 1551 return conn; 1552 err: 1553 conn = NULL; 1554 goto end; 1555} 1556 1557 1558static void 1559remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1560{ 1561 remove_all_cces_from_hash(engine->conns_hash, conn); 1562 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1563} 1564 1565 1566static void 1567refflags2str (enum lsquic_conn_flags flags, char s[6]) 1568{ 1569 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1570 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1571 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1572 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1573 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1574 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1575 *s = '\0'; 1576} 1577 1578 1579static void 1580engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1581{ 1582 char str[2][7]; 1583 assert(flag & CONN_REF_FLAGS); 1584 assert(!(conn->cn_flags & flag)); 1585 conn->cn_flags |= flag; 1586 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1587 CID_BITS(lsquic_conn_log_cid(conn)), 1588 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1589 (refflags2str(conn->cn_flags, str[1]), str[1])); 1590} 1591 1592 1593static lsquic_conn_t * 1594engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1595 enum lsquic_conn_flags flags) 1596{ 1597 char str[2][7]; 1598 lsquic_time_t now; 1599 assert(flags & CONN_REF_FLAGS); 1600 assert(conn->cn_flags & flags); 1601#ifndef NDEBUG 1602 if (flags & LSCONN_CLOSING) 1603 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1604#endif 1605 conn->cn_flags &= ~flags; 1606 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1607 CID_BITS(lsquic_conn_log_cid(conn)), 1608 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1609 (refflags2str(conn->cn_flags, str[1]), str[1])); 1610 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1611 { 1612 now = lsquic_time_now(); 1613 if (conn->cn_flags & LSCONN_MINI) 1614 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1615 else 1616 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1617 destroy_conn(engine, conn, now); 1618 return NULL; 1619 } 1620 else 1621 return conn; 1622} 1623 1624 1625/* This is not a general-purpose function. Only call from engine dtor. */ 1626static void 1627force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1628{ 1629 assert(engine->flags & ENG_DTOR); 1630 const enum lsquic_conn_flags flags = conn->cn_flags; 1631 assert(conn->cn_flags & CONN_REF_FLAGS); 1632 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1633 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1634 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1635 if (flags & LSCONN_ATTQ) 1636 { 1637 attq_remove(engine->attq, conn); 1638 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1639 } 1640 if (flags & LSCONN_HASHED) 1641 remove_conn_from_hash(engine, conn); 1642} 1643 1644 1645/* Iterator for tickable connections (those on the Tickable Queue). Before 1646 * a connection is returned, it is removed from the Advisory Tick Time queue 1647 * if necessary. 1648 */ 1649static lsquic_conn_t * 1650conn_iter_next_tickable (struct lsquic_engine *engine) 1651{ 1652 lsquic_conn_t *conn; 1653 1654 if (engine->flags & ENG_SERVER) 1655 while (1) 1656 { 1657 conn = lsquic_mh_pop(&engine->conns_tickable); 1658 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1659 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1660 else 1661 break; 1662 } 1663 else 1664 conn = lsquic_mh_pop(&engine->conns_tickable); 1665 1666 if (conn) 1667 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1668 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1669 { 1670 attq_remove(engine->attq, conn); 1671 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1672 } 1673 1674 return conn; 1675} 1676 1677 1678static void 1679cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1680 void *update_ctx) 1681{ 1682 cub->cub_update_cids = update; 1683 cub->cub_update_ctx = update_ctx; 1684 cub->cub_count = 0; 1685} 1686 1687 1688static void 1689cub_flush (struct cid_update_batch *cub) 1690{ 1691 if (cub->cub_count > 0 && cub->cub_update_cids) 1692 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1693 cub->cub_cids, cub->cub_count); 1694 cub->cub_count = 0; 1695} 1696 1697 1698static void 1699cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1700{ 1701 cub->cub_cids [ cub->cub_count ] = *cid; 1702 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1703 ++cub->cub_count; 1704 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1705 cub_flush(cub); 1706} 1707 1708 1709/* Process registered CIDs */ 1710static void 1711cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1712{ 1713 struct cce_cid_iter citer; 1714 struct conn_cid_elem *cce; 1715 void *peer_ctx; 1716 1717 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1718 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1719 if (cce->cce_flags & CCE_REG) 1720 cub_add(cub, &cce->cce_cid, peer_ctx); 1721} 1722 1723 1724static void 1725drop_all_mini_conns (lsquic_engine_t *engine) 1726{ 1727 struct lsquic_hash_elem *el; 1728 lsquic_conn_t *conn; 1729 struct cid_update_batch cub; 1730 1731 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1732 1733 for (el = lsquic_hash_first(engine->conns_hash); el; 1734 el = lsquic_hash_next(engine->conns_hash)) 1735 { 1736 conn = lsquic_hashelem_getdata(el); 1737 if (conn->cn_flags & LSCONN_MINI) 1738 { 1739 /* If promoted, why is it still in this hash? */ 1740 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1741 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1742 cub_add_cids_from_cces(&cub, conn); 1743 remove_conn_from_hash(engine, conn); 1744 } 1745 } 1746 1747 cub_flush(&cub); 1748} 1749 1750 1751void 1752lsquic_engine_process_conns (lsquic_engine_t *engine) 1753{ 1754 lsquic_conn_t *conn; 1755 lsquic_time_t now; 1756 1757 ENGINE_IN(engine); 1758 1759 now = lsquic_time_now(); 1760 while ((conn = attq_pop(engine->attq, now))) 1761 { 1762 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1763 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 1764 { 1765 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1766 engine_incref_conn(conn, LSCONN_TICKABLE); 1767 } 1768 } 1769 1770 process_connections(engine, conn_iter_next_tickable, now); 1771 ENGINE_OUT(engine); 1772} 1773 1774 1775static void 1776release_or_return_enc_data (struct lsquic_engine *engine, 1777 void (*pmi_rel_or_ret) (void *, void *, void *, char), 1778 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 1779{ 1780 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1781 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 1782 packet_out->po_flags &= ~PO_ENCRYPTED; 1783 packet_out->po_enc_data = NULL; 1784} 1785 1786 1787static void 1788release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1789 struct lsquic_packet_out *packet_out) 1790{ 1791 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 1792 conn, packet_out); 1793} 1794 1795 1796static void 1797return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1798 struct lsquic_packet_out *packet_out) 1799{ 1800 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 1801 conn, packet_out); 1802} 1803 1804 1805static int 1806copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 1807 struct lsquic_packet_out *packet_out) 1808{ 1809 int ipv6; 1810 1811 ipv6 = NP_IS_IPv6(packet_out->po_path); 1812 if (packet_out->po_flags & PO_ENCRYPTED) 1813 { 1814 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1815 && packet_out->po_data_sz == packet_out->po_enc_data_sz 1816 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 1817 packet_out->po_data_sz)) 1818 return 0; 1819 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1820 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 1821 goto copy; 1822 return_enc_data(engine, conn, packet_out); 1823 } 1824 1825 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 1826 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1827 packet_out->po_data_sz, ipv6); 1828 if (!packet_out->po_enc_data) 1829 { 1830 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 1831 "of size %hu", packet_out->po_data_sz); 1832 return -1; 1833 } 1834 1835 copy: 1836 memcpy(packet_out->po_enc_data, packet_out->po_data, 1837 packet_out->po_data_sz); 1838 packet_out->po_enc_data_sz = packet_out->po_data_sz; 1839 packet_out->po_sent_sz = packet_out->po_data_sz; 1840 packet_out->po_flags &= ~PO_IPv6; 1841 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 1842 1843 return 0; 1844} 1845 1846 1847STAILQ_HEAD(conns_stailq, lsquic_conn); 1848TAILQ_HEAD(conns_tailq, lsquic_conn); 1849 1850 1851struct conns_out_iter 1852{ 1853 struct min_heap *coi_heap; 1854 struct pr_queue *coi_prq; 1855 TAILQ_HEAD(, lsquic_conn) coi_active_list, 1856 coi_inactive_list; 1857 lsquic_conn_t *coi_next; 1858#ifndef NDEBUG 1859 lsquic_time_t coi_last_sent; 1860#endif 1861}; 1862 1863 1864static void 1865coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 1866{ 1867 iter->coi_heap = &engine->conns_out; 1868 iter->coi_prq = engine->pr_queue; 1869 iter->coi_next = NULL; 1870 TAILQ_INIT(&iter->coi_active_list); 1871 TAILQ_INIT(&iter->coi_inactive_list); 1872#ifndef NDEBUG 1873 iter->coi_last_sent = 0; 1874#endif 1875} 1876 1877 1878static lsquic_conn_t * 1879coi_next (struct conns_out_iter *iter) 1880{ 1881 lsquic_conn_t *conn; 1882 1883 if (lsquic_mh_count(iter->coi_heap) > 0) 1884 { 1885 conn = lsquic_mh_pop(iter->coi_heap); 1886 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1887 conn->cn_flags |= LSCONN_COI_ACTIVE; 1888#ifndef NDEBUG 1889 if (iter->coi_last_sent) 1890 assert(iter->coi_last_sent <= conn->cn_last_sent); 1891 iter->coi_last_sent = conn->cn_last_sent; 1892#endif 1893 return conn; 1894 } 1895 else if (iter->coi_prq && (conn = prq_next_conn(iter->coi_prq))) 1896 { 1897 return conn; 1898 } 1899 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 1900 { 1901 iter->coi_prq = NULL; /* Save function call in previous conditional */ 1902 conn = iter->coi_next; 1903 if (!conn) 1904 conn = TAILQ_FIRST(&iter->coi_active_list); 1905 if (conn) 1906 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 1907 return conn; 1908 } 1909 else 1910 return NULL; 1911} 1912 1913 1914static void 1915coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1916{ 1917 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 1918 { 1919 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 1920 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1921 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1922 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 1923 conn->cn_flags |= LSCONN_COI_INACTIVE; 1924 } 1925} 1926 1927 1928static void 1929coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1930{ 1931 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 1932 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 1933 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 1934 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1935 conn->cn_flags |= LSCONN_COI_ACTIVE; 1936} 1937 1938 1939static void 1940coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 1941{ 1942 lsquic_conn_t *conn; 1943 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 1944 { 1945 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1946 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1947 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 1948 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 1949 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 1950 else /* Closed connection gets one shot at sending packets */ 1951 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1952 } 1953 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 1954 { 1955 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 1956 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 1957 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1958 } 1959} 1960 1961 1962#ifndef NDEBUG 1963static void 1964lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 1965 unsigned n) 1966{ 1967 const lsquic_cid_t *cid; 1968 struct iovec *iov; 1969 unsigned i; 1970 char packno_str[22]; 1971 1972 for (i = 0; i < n; ++i) 1973 { 1974 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 1975 batch->packets[i]->po_packno); 1976 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 1977 { 1978 for (iov = batch->outs[i].iov; iov < 1979 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 1980 batch->outs[i].iov->iov_len -= 1; 1981 cid = lsquic_conn_log_cid(batch->conns[i]); 1982 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 1983 CID_BITS(cid)); 1984 } 1985 } 1986} 1987 1988 1989#endif 1990 1991 1992#ifdef NDEBUG 1993#define CONST_BATCH const 1994#else 1995#define CONST_BATCH 1996#endif 1997 1998 1999static void 2000sockaddr2str (const struct sockaddr *addr, char *buf, size_t sz) 2001{ 2002 unsigned short port; 2003 int len; 2004 2005 switch (addr->sa_family) 2006 { 2007 case AF_INET: 2008 port = ((struct sockaddr_in *) addr)->sin_port; 2009 if (!inet_ntop(AF_INET, &((struct sockaddr_in *) addr)->sin_addr, 2010 buf, sz)) 2011 buf[0] = '\0'; 2012 break; 2013 case AF_INET6: 2014 port = ((struct sockaddr_in6 *) addr)->sin6_port; 2015 if (!inet_ntop(AF_INET6, &((struct sockaddr_in6 *) addr)->sin6_addr, 2016 buf, sz)) 2017 buf[0] = '\0'; 2018 break; 2019 default: 2020 port = 0; 2021 (void) snprintf(buf, sz, "<invalid family %d>", addr->sa_family); 2022 break; 2023 } 2024 2025 len = strlen(buf); 2026 if (len < (int) sz) 2027 snprintf(buf + len, sz - (size_t) len, ":%hu", port); 2028} 2029 2030 2031struct send_batch_ctx { 2032 struct conns_stailq *closed_conns; 2033 struct conns_tailq *ticked_conns; 2034 struct conns_out_iter *conns_iter; 2035 CONST_BATCH struct out_batch *batch; 2036}; 2037 2038 2039static void 2040close_conn_immediately (struct lsquic_engine *engine, 2041 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2042{ 2043 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2044 if (!(conn->cn_flags & LSCONN_CLOSING)) 2045 { 2046 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2047 engine_incref_conn(conn, LSCONN_CLOSING); 2048 if (conn->cn_flags & LSCONN_HASHED) 2049 remove_conn_from_hash(engine, conn); 2050 } 2051 if (conn->cn_flags & LSCONN_TICKED) 2052 { 2053 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2054 engine_decref_conn(engine, conn, LSCONN_TICKED); 2055 } 2056} 2057 2058 2059static void 2060close_conn_on_send_error (struct lsquic_engine *engine, 2061 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2062{ 2063 const struct out_batch *batch = sb_ctx->batch; 2064 struct lsquic_conn *const conn = batch->conns[n]; 2065 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2066 2067 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2068 "src: %s; dst: %s; errno: %d", 2069 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2070 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2071 CID_BITS(lsquic_conn_log_cid(conn)), 2072 (sockaddr2str(batch->outs[n].local_sa, buf[0], sizeof(buf[0])), buf[0]), 2073 (sockaddr2str(batch->outs[n].dest_sa, buf[1], sizeof(buf[1])), buf[1]), 2074 e_val); 2075 if (conn->cn_flags & LSCONN_EVANESCENT) 2076 lsquic_prq_drop(conn); 2077 else 2078 close_conn_immediately(engine, sb_ctx, conn); 2079} 2080 2081 2082static unsigned 2083send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2084 unsigned n_to_send) 2085{ 2086 int n_sent, i, e_val; 2087 lsquic_time_t now; 2088 unsigned off; 2089 size_t count; 2090 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2091 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2092 2093#ifndef NDEBUG 2094 if (engine->flags & ENG_LOSE_PACKETS) 2095 lose_matching_packets(engine, batch, n_to_send); 2096#endif 2097 /* Set sent time before the write to avoid underestimating RTT */ 2098 now = lsquic_time_now(); 2099 for (i = 0; i < (int) n_to_send; ++i) 2100 { 2101 off = batch->pack_off[i]; 2102 count = batch->outs[i].iovlen; 2103 assert(count > 0); 2104 packet_out = &batch->packets[off]; 2105 end = packet_out + count; 2106 do 2107 (*packet_out)->po_sent = now; 2108 while (++packet_out < end); 2109 } 2110 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs, 2111 n_to_send); 2112 e_val = errno; 2113 if (n_sent < (int) n_to_send) 2114 { 2115 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2116 engine->resume_sending_at = now + 1000000; 2117 LSQ_DEBUG("cannot send packets"); 2118 EV_LOG_GENERIC_EVENT("cannot send packets"); 2119 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2120 close_conn_on_send_error(engine, sb_ctx, 2121 n_sent < 0 ? 0 : n_sent, e_val); 2122 } 2123 if (n_sent >= 0) 2124 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, n_to_send); 2125 else 2126 { 2127 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2128 n_sent = 0; 2129 } 2130 if (n_sent > 0) 2131 engine->last_sent = now + n_sent; 2132 for (i = 0; i < n_sent; ++i) 2133 { 2134 eng_hist_inc(&engine->history, now, sl_packets_out); 2135 /* `i' is added to maintain relative order */ 2136 batch->conns[i]->cn_last_sent = now + i; 2137 2138 off = batch->pack_off[i]; 2139 count = batch->outs[i].iovlen; 2140 assert(count > 0); 2141 packet_out = &batch->packets[off]; 2142 end = packet_out + count; 2143 do 2144 { 2145#if LOG_PACKET_CHECKSUM 2146 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2147 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2148 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2149#endif 2150 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2151 *packet_out); 2152 /* Release packet out buffer as soon as the packet is sent 2153 * successfully. If not successfully sent, we hold on to 2154 * this buffer until the packet sending is attempted again 2155 * or until it times out and regenerated. 2156 */ 2157 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2158 release_enc_data(engine, batch->conns[i], *packet_out); 2159 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2160 *packet_out); 2161 } 2162 while (++packet_out < end); 2163 } 2164 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2165 for ( ; i < (int) n_to_send; ++i) 2166 { 2167 off = batch->pack_off[i]; 2168 count = batch->outs[i].iovlen; 2169 assert(count > 0); 2170 packet_out = &batch->packets[off]; 2171 end = packet_out + count; 2172 do 2173 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2174 *packet_out); 2175 while (++packet_out < end); 2176 } 2177 /* Return packets to the connection in reverse order so that the packet 2178 * ordering is maintained. 2179 */ 2180 for (i = (int) n_to_send - 1; i >= n_sent; --i) 2181 { 2182 off = batch->pack_off[i]; 2183 count = batch->outs[i].iovlen; 2184 assert(count > 0); 2185 packet_out = &batch->packets[off + count - 1]; 2186 end = &batch->packets[off - 1]; 2187 do 2188 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2189 *packet_out); 2190 while (--packet_out > end); 2191 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2192 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2193 } 2194 return n_sent; 2195} 2196 2197 2198/* Return 1 if went past deadline, 0 otherwise */ 2199static int 2200check_deadline (lsquic_engine_t *engine) 2201{ 2202 if (engine->pub.enp_settings.es_proc_time_thresh && 2203 lsquic_time_now() > engine->deadline) 2204 { 2205 LSQ_INFO("went past threshold of %u usec, stop sending", 2206 engine->pub.enp_settings.es_proc_time_thresh); 2207 engine->flags |= ENG_PAST_DEADLINE; 2208 return 1; 2209 } 2210 else 2211 return 0; 2212} 2213 2214 2215static size_t 2216iov_size (const struct iovec *iov, const struct iovec *const end) 2217{ 2218 size_t size; 2219 2220 assert(iov < end); 2221 2222 size = 0; 2223 do 2224 size += iov->iov_len; 2225 while (++iov < end); 2226 2227 return size; 2228} 2229 2230 2231/* XXX A lot of extra setup -- two extra arguments to this function, two extra 2232 * connection ref flags and queues -- is just to handle the ENCPA_BADCRYPT case, 2233 * which never really happens. 2234 */ 2235static void 2236send_packets_out (struct lsquic_engine *engine, 2237 struct conns_tailq *ticked_conns, 2238 struct conns_stailq *closed_conns) 2239{ 2240 unsigned n, w, n_sent, n_batches_sent; 2241 lsquic_packet_out_t *packet_out; 2242 struct lsquic_packet_out **packet; 2243 lsquic_conn_t *conn; 2244 struct out_batch *const batch = &engine->out_batch; 2245 struct iovec *iov, *packet_iov; 2246 struct conns_out_iter conns_iter; 2247 int shrink, deadline_exceeded; 2248 const struct send_batch_ctx sb_ctx = { 2249 closed_conns, 2250 ticked_conns, 2251 &conns_iter, 2252 &engine->out_batch, 2253 }; 2254 2255 coi_init(&conns_iter, engine); 2256 n_batches_sent = 0; 2257 n_sent = 0, n = 0; 2258 shrink = 0; 2259 deadline_exceeded = 0; 2260 iov = batch->iov; 2261 packet = batch->packets; 2262 2263 while ((conn = coi_next(&conns_iter))) 2264 { 2265 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2266 if (!packet_out) { 2267 /* Evanescent connection always has a packet to send: */ 2268 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2269 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2270 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2271 CID_BITS(lsquic_conn_log_cid(conn))); 2272 coi_deactivate(&conns_iter, conn); 2273 continue; 2274 } 2275 batch->outs[n].iov = packet_iov = iov; 2276 next_coa: 2277 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2278 { 2279 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2280 &engine->pub, conn, packet_out)) 2281 { 2282 case ENCPA_NOMEM: 2283 /* Send what we have and wait for a more opportune moment */ 2284 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2285 goto end_for; 2286 case ENCPA_BADCRYPT: 2287 /* This is pretty bad: close connection immediately */ 2288 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2289 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2290 CID_BITS(lsquic_conn_log_cid(conn))); 2291 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2292 { 2293 close_conn_immediately(engine, &sb_ctx, conn); 2294 coi_deactivate(&conns_iter, conn); 2295 } 2296 continue; 2297 case ENCPA_OK: 2298 break; 2299 } 2300 } 2301 else if ((packet_out->po_flags & PO_NOENCRYPT) 2302 && engine->pub.enp_pmi != &stock_pmi) 2303 { 2304 if (0 != copy_packet(engine, conn, packet_out)) 2305 { 2306 /* Copy can only fail if packet could not be allocated */ 2307 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2308 goto end_for; 2309 } 2310 } 2311 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2312 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2313 if (packet_out->po_flags & PO_ENCRYPTED) 2314 { 2315 iov->iov_base = packet_out->po_enc_data; 2316 iov->iov_len = packet_out->po_enc_data_sz; 2317 } 2318 else 2319 { 2320 iov->iov_base = packet_out->po_data; 2321 iov->iov_len = packet_out->po_data_sz; 2322 } 2323 if (packet_iov == iov) 2324 { 2325 batch->pack_off[n] = packet - batch->packets; 2326 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2327 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2328 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2329 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2330 batch->conns [n] = conn; 2331 } 2332 *packet = packet_out; 2333 ++packet; 2334 ++iov; 2335 if ((conn->cn_flags & LSCONN_IETF) 2336 && ((1 << packet_out->po_header_type) 2337 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2338#ifndef NDEBUG 2339 && (engine->flags & ENG_COALESCE) 2340#endif 2341 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2342 { 2343 const size_t size = iov_size(packet_iov, iov); 2344 packet_out = conn->cn_if->ci_next_packet_to_send(conn, size); 2345 if (packet_out) 2346 goto next_coa; 2347 } 2348 batch->outs [n].iovlen = iov - packet_iov; 2349 ++n; 2350 if (n == engine->batch_size 2351 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2352 { 2353 w = send_batch(engine, &sb_ctx, n); 2354 n = 0; 2355 iov = batch->iov; 2356 packet = batch->packets; 2357 ++n_batches_sent; 2358 n_sent += w; 2359 if (w < engine->batch_size) 2360 { 2361 shrink = 1; 2362 break; 2363 } 2364 deadline_exceeded = check_deadline(engine); 2365 if (deadline_exceeded) 2366 break; 2367 grow_batch_size(engine); 2368 } 2369 } 2370 end_for: 2371 2372 if (n > 0) { 2373 w = send_batch(engine, &sb_ctx, n); 2374 n_sent += w; 2375 shrink = w < n; 2376 ++n_batches_sent; 2377 } 2378 2379 if (shrink) 2380 shrink_batch_size(engine); 2381 else if (n_batches_sent > 1) 2382 { 2383 deadline_exceeded = check_deadline(engine); 2384 if (!deadline_exceeded) 2385 grow_batch_size(engine); 2386 } 2387 2388 coi_reheap(&conns_iter, engine); 2389 2390 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2391} 2392 2393 2394int 2395lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2396{ 2397 return lsquic_mh_count(&engine->conns_out) > 0 2398 || (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2399 ; 2400} 2401 2402 2403static void 2404reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2405{ 2406 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2407 engine->flags &= ~ENG_PAST_DEADLINE; 2408} 2409 2410 2411/* TODO: this is a user-facing function, account for load */ 2412void 2413lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2414{ 2415 lsquic_conn_t *conn; 2416 struct conns_stailq closed_conns; 2417 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2418 struct cid_update_batch cub; 2419 2420 ENGINE_IN(engine); 2421 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2422 STAILQ_INIT(&closed_conns); 2423 reset_deadline(engine, lsquic_time_now()); 2424 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2425 { 2426 LSQ_DEBUG("can send again"); 2427 EV_LOG_GENERIC_EVENT("can send again"); 2428 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2429 } 2430 2431 send_packets_out(engine, &ticked_conns, &closed_conns); 2432 2433 while ((conn = STAILQ_FIRST(&closed_conns))) { 2434 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2435 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2436 cub_add_cids_from_cces(&cub, conn); 2437 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2438 } 2439 2440 cub_flush(&cub); 2441 ENGINE_OUT(engine); 2442} 2443 2444 2445static lsquic_conn_t * 2446next_new_full_conn (struct conns_stailq *new_full_conns) 2447{ 2448 lsquic_conn_t *conn; 2449 2450 conn = STAILQ_FIRST(new_full_conns); 2451 if (conn) 2452 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2453 return conn; 2454} 2455 2456 2457static void 2458process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2459 lsquic_time_t now) 2460{ 2461 lsquic_conn_t *conn; 2462 enum tick_st tick_st; 2463 unsigned i, why; 2464 lsquic_time_t next_tick_time; 2465 struct conns_stailq closed_conns; 2466 struct conns_tailq ticked_conns; 2467 struct conns_stailq new_full_conns; 2468 struct cid_update_batch cub_old, cub_live; 2469 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2470 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2471 2472 eng_hist_tick(&engine->history, now); 2473 2474 STAILQ_INIT(&closed_conns); 2475 TAILQ_INIT(&ticked_conns); 2476 reset_deadline(engine, now); 2477 STAILQ_INIT(&new_full_conns); 2478 2479 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2480 && now > engine->resume_sending_at) 2481 { 2482 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2483 "timeout"); 2484 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2485 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2486 } 2487 2488 i = 0; 2489 while ((conn = next_conn(engine)) 2490 || (conn = next_new_full_conn(&new_full_conns))) 2491 { 2492 tick_st = conn->cn_if->ci_tick(conn, now); 2493 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2494 if (tick_st & TICK_PROMOTE) 2495 { 2496 lsquic_conn_t *new_conn; 2497 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2498 "scheduled for promotion"); 2499 assert(conn->cn_flags & LSCONN_MINI); 2500 new_conn = new_full_conn_server(engine, conn, now); 2501 if (new_conn) 2502 { 2503 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2504 new_conn->cn_last_sent = engine->last_sent; 2505 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2506 } 2507 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2508 conn->cn_flags |= LSCONN_PROMOTED; 2509 } 2510 if (tick_st & TICK_SEND) 2511 { 2512 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2513 { 2514 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2515 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2516 } 2517 } 2518 if (tick_st & TICK_CLOSE) 2519 { 2520 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2521 engine_incref_conn(conn, LSCONN_CLOSING); 2522 if (conn->cn_flags & LSCONN_HASHED) 2523 remove_conn_from_hash(engine, conn); 2524 } 2525 else 2526 { 2527 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2528 engine_incref_conn(conn, LSCONN_TICKED); 2529 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2530 && conn->cn_if->ci_report_live(conn, now)) 2531 cub_add_cids_from_cces(&cub_live, conn); 2532 } 2533 } 2534 2535 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2536 && lsquic_engine_has_unsent_packets(engine)) 2537 send_packets_out(engine, &ticked_conns, &closed_conns); 2538 2539 while ((conn = STAILQ_FIRST(&closed_conns))) { 2540 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2541 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2542 cub_add_cids_from_cces(&cub_old, conn); 2543 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2544 } 2545 2546 /* TODO Heapification can be optimized by switching to the Floyd method: 2547 * https://en.wikipedia.org/wiki/Binary_heap#Building_a_heap 2548 */ 2549 while ((conn = TAILQ_FIRST(&ticked_conns))) 2550 { 2551 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2552 engine_decref_conn(engine, conn, LSCONN_TICKED); 2553 if (!(conn->cn_flags & LSCONN_TICKABLE) 2554 && conn->cn_if->ci_is_tickable(conn)) 2555 { 2556 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2557 engine_incref_conn(conn, LSCONN_TICKABLE); 2558 } 2559 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2560 { 2561 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2562 if (next_tick_time) 2563 { 2564 if (0 == attq_add(engine->attq, conn, next_tick_time, why)) 2565 engine_incref_conn(conn, LSCONN_ATTQ); 2566 } 2567 else 2568 assert(0); 2569 } 2570 } 2571 2572 cub_flush(&engine->new_scids); 2573 cub_flush(&cub_live); 2574 cub_flush(&cub_old); 2575} 2576 2577 2578/* Return 0 if packet is being processed by a real connection, 1 if the 2579 * packet was processed, but not by a connection, and -1 on error. 2580 */ 2581int 2582lsquic_engine_packet_in (lsquic_engine_t *engine, 2583 const unsigned char *packet_in_data, size_t packet_in_size, 2584 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2585 void *peer_ctx, int ecn) 2586{ 2587 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2588 struct packin_parse_state ppstate; 2589 lsquic_packet_in_t *packet_in; 2590 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2591 int is_server, unsigned cid_len, struct packin_parse_state *); 2592 unsigned n_zeroes; 2593 int s; 2594 2595 ENGINE_CALLS_INCR(engine); 2596 2597 if (engine->flags & ENG_SERVER) 2598 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2599 else 2600 if (engine->flags & ENG_CONNS_BY_ADDR) 2601 { 2602 struct lsquic_hash_elem *el; 2603 const struct lsquic_conn *conn; 2604 el = find_conn_by_addr(engine->conns_hash, sa_local); 2605 if (!el) 2606 return -1; 2607 conn = lsquic_hashelem_getdata(el); 2608 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2609 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2610 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2611 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2612 else 2613 { 2614 assert(conn->cn_version == LSQVER_046 2615#if LSQUIC_USE_Q098 2616 || conn->cn_version == LSQVER_098 2617#endif 2618 2619 ); 2620 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2621 } 2622 } 2623 else 2624 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2625 2626 n_zeroes = 0; 2627 do 2628 { 2629 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 2630 if (!packet_in) 2631 return -1; 2632 /* Library does not modify packet_in_data, it is not referenced after 2633 * this function returns and subsequent release of pi_data is guarded 2634 * by PI_OWN_DATA flag. 2635 */ 2636 packet_in->pi_data = (unsigned char *) packet_in_data; 2637 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 2638 engine->flags & ENG_SERVER, 2639 engine->pub.enp_settings.es_scid_len, &ppstate)) 2640 { 2641 LSQ_DEBUG("Cannot parse incoming packet's header"); 2642 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 2643 errno = EINVAL; 2644 return -1; 2645 } 2646 2647 packet_in_data += packet_in->pi_data_sz; 2648 packet_in->pi_received = lsquic_time_now(); 2649 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 2650 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 2651 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 2652 peer_ctx, packet_in->pi_data_sz == packet_in_size); 2653 n_zeroes += s == 0; 2654 } 2655 while (0 == s && packet_in_data < packet_end); 2656 2657 return n_zeroes > 0 ? 0 : s; 2658} 2659 2660 2661#if __GNUC__ && !defined(NDEBUG) 2662__attribute__((weak)) 2663#endif 2664unsigned 2665lsquic_engine_quic_versions (const lsquic_engine_t *engine) 2666{ 2667 return engine->pub.enp_settings.es_versions; 2668} 2669 2670 2671void 2672lsquic_engine_cooldown (lsquic_engine_t *engine) 2673{ 2674 struct lsquic_hash_elem *el; 2675 lsquic_conn_t *conn; 2676 2677 if (engine->flags & ENG_COOLDOWN) 2678 /* AFAICT, there is no harm in calling this function more than once, 2679 * but log it just in case, as it may indicate an error in the caller. 2680 */ 2681 LSQ_INFO("cooldown called again"); 2682 engine->flags |= ENG_COOLDOWN; 2683 LSQ_INFO("entering cooldown mode"); 2684 if (engine->flags & ENG_SERVER) 2685 drop_all_mini_conns(engine); 2686 for (el = lsquic_hash_first(engine->conns_hash); el; 2687 el = lsquic_hash_next(engine->conns_hash)) 2688 { 2689 conn = lsquic_hashelem_getdata(el); 2690 lsquic_conn_going_away(conn); 2691 } 2692} 2693 2694 2695int 2696lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 2697{ 2698 const struct attq_elem *next_attq; 2699 lsquic_time_t now, next_time; 2700#if LSQUIC_DEBUG_NEXT_ADV_TICK 2701 const struct lsquic_conn *conn; 2702 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 2703#endif 2704 2705 ENGINE_CALLS_INCR(engine); 2706 2707 if ((engine->flags & ENG_PAST_DEADLINE) 2708 && lsquic_mh_count(&engine->conns_out)) 2709 { 2710#if LSQUIC_DEBUG_NEXT_ADV_TICK 2711 conn = lsquic_mh_peek(&engine->conns_out); 2712 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 2713 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 2714 lsquic_mh_count(&engine->conns_out), 2715 lsquic_mh_count(&engine->conns_out) != 1, "s", 2716 CID_BITS(lsquic_conn_log_cid(conn))); 2717#endif 2718 *diff = 0; 2719 return 1; 2720 } 2721 2722 if (engine->pr_queue && prq_have_pending(engine->pr_queue)) 2723 { 2724#if LSQUIC_DEBUG_NEXT_ADV_TICK 2725 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 2726#endif 2727 *diff = 0; 2728 return 1; 2729 } 2730 2731 if (lsquic_mh_count(&engine->conns_tickable)) 2732 { 2733#if LSQUIC_DEBUG_NEXT_ADV_TICK 2734 conn = lsquic_mh_peek(&engine->conns_tickable); 2735 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 2736 "connection%.*s (%"CID_FMT" first)", 2737 lsquic_mh_count(&engine->conns_tickable), 2738 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 2739 CID_BITS(lsquic_conn_log_cid(conn))); 2740#endif 2741 *diff = 0; 2742 return 1; 2743 } 2744 2745 next_attq = attq_next(engine->attq); 2746 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 2747 { 2748 if (next_attq) 2749 next_time = next_attq->ae_adv_time; 2750 else 2751 return 0; 2752 } 2753 else 2754 { 2755 if (next_attq) 2756 { 2757 next_time = next_attq->ae_adv_time; 2758 if (engine->resume_sending_at < next_time) 2759 { 2760 next_time = engine->resume_sending_at; 2761 next_attq = NULL; 2762 } 2763 } 2764 else 2765 next_time = engine->resume_sending_at; 2766 } 2767 2768 now = lsquic_time_now(); 2769 *diff = (int) ((int64_t) next_time - (int64_t) now); 2770#if LSQUIC_DEBUG_NEXT_ADV_TICK 2771 if (next_attq) 2772 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 2773 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 2774 lsquic_attq_why2str(next_attq->ae_why)); 2775 else 2776 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 2777#endif 2778 return 1; 2779} 2780 2781 2782unsigned 2783lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 2784{ 2785 lsquic_time_t now; 2786 ENGINE_CALLS_INCR(engine); 2787 now = lsquic_time_now(); 2788 if (from_now < 0) 2789 now -= from_now; 2790 else 2791 now += from_now; 2792 return attq_count_before(engine->attq, now); 2793} 2794 2795 2796int 2797lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 2798 struct lsquic_conn *conn, unsigned cce_idx) 2799{ 2800 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2801 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2802 void *peer_ctx; 2803 2804 assert(cce_idx < conn->cn_n_cces); 2805 assert(conn->cn_cces_mask & (1 << cce_idx)); 2806 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 2807 2808 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 2809 cce->cce_cid.len, conn, &cce->cce_hash_el)) 2810 { 2811 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 2812 CID_BITS(&cce->cce_cid)); 2813 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2814 cce->cce_flags |= CCE_REG; 2815 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 2816 return 0; 2817 } 2818 else 2819 { 2820 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 2821 CID_BITS(&cce->cce_cid)); 2822 return -1; 2823 } 2824} 2825 2826 2827void 2828lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 2829 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now) 2830{ 2831 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2832 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2833 void *peer_ctx; 2834 2835 assert(cce_idx < conn->cn_n_cces); 2836 2837 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 2838 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 2839 2840 if (engine->purga) 2841 { 2842 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2843 lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 2844 PUTY_CID_RETIRED, now); 2845 } 2846 conn->cn_cces_mask &= ~(1u << cce_idx); 2847 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 2848} 2849 2850 2851