lsquic_engine.c revision 77a28812
1/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#ifndef WIN32 17#include <sys/time.h> 18#include <netinet/in.h> 19#include <sys/types.h> 20#include <sys/stat.h> 21#include <fcntl.h> 22#include <unistd.h> 23#include <netdb.h> 24#endif 25 26#ifndef NDEBUG 27#include <sys/types.h> 28#include <regex.h> /* For code that loses packets */ 29#endif 30 31#if LOG_PACKET_CHECKSUM 32#include <zlib.h> 33#endif 34 35#include <openssl/aead.h> 36 37#include "lsquic.h" 38#include "lsquic_types.h" 39#include "lsquic_int_types.h" 40#include "lsquic_sizes.h" 41#include "lsquic_parse_common.h" 42#include "lsquic_parse.h" 43#include "lsquic_packet_in.h" 44#include "lsquic_packet_out.h" 45#include "lsquic_senhist.h" 46#include "lsquic_rtt.h" 47#include "lsquic_cubic.h" 48#include "lsquic_pacer.h" 49#include "lsquic_bw_sampler.h" 50#include "lsquic_minmax.h" 51#include "lsquic_bbr.h" 52#include "lsquic_send_ctl.h" 53#include "lsquic_set.h" 54#include "lsquic_conn_flow.h" 55#include "lsquic_sfcw.h" 56#include "lsquic_hash.h" 57#include "lsquic_conn.h" 58#include "lsquic_full_conn.h" 59#include "lsquic_util.h" 60#include "lsquic_qtags.h" 61#include "lsquic_enc_sess.h" 62#include "lsquic_mm.h" 63#include "lsquic_engine_public.h" 64#include "lsquic_eng_hist.h" 65#include "lsquic_ev_log.h" 66#include "lsquic_version.h" 67#include "lsquic_pr_queue.h" 68#include "lsquic_mini_conn.h" 69#include "lsquic_mini_conn_ietf.h" 70#include "lsquic_stock_shi.h" 71#include "lsquic_purga.h" 72#include "lsquic_tokgen.h" 73#include "lsquic_attq.h" 74#include "lsquic_min_heap.h" 75#include "lsquic_http1x_if.h" 76#include "lsquic_handshake.h" 77#include "lsquic_crand.h" 78#include "lsquic_ietf.h" 79 80#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 81#include "lsquic_logger.h" 82 83#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 84#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 85#endif 86 87#if LSQUIC_DEBUG_NEXT_ADV_TICK 88#include "lsquic_alarmset.h" 89#endif 90 91#define MIN(a, b) ((a) < (b) ? (a) : (b)) 92 93/* The batch of outgoing packets grows and shrinks dynamically */ 94#define MAX_OUT_BATCH_SIZE 1024 95#define MIN_OUT_BATCH_SIZE 4 96#define INITIAL_OUT_BATCH_SIZE 32 97 98struct out_batch 99{ 100 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 101 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 102 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 103 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 104 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 105}; 106 107typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 108 109static void 110process_connections (struct lsquic_engine *engine, conn_iter_f iter, 111 lsquic_time_t now); 112 113static void 114engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 115 116static lsquic_conn_t * 117engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 118 enum lsquic_conn_flags flag); 119 120static void 121force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 122 123#if LSQUIC_COUNT_ENGINE_CALLS 124#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 125#else 126#define ENGINE_CALLS_INCR(e) 127#endif 128 129/* Nested calls to LSQUIC are not supported */ 130#define ENGINE_IN(e) do { \ 131 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 132 (e)->pub.enp_flags |= ENPUB_PROC; \ 133 ENGINE_CALLS_INCR(e); \ 134} while (0) 135 136#define ENGINE_OUT(e) do { \ 137 assert((e)->pub.enp_flags & ENPUB_PROC); \ 138 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 139} while (0) 140 141/* A connection can be referenced from one of six places: 142 * 143 * 1. A hash is used to find connections in order to dispatch an incoming 144 * packet. Connections can be hashed by CIDs or by address. In the 145 * former case, each connection has one or more mappings in the hash 146 * table. IETF QUIC connections have up to eight (in our implementation) 147 * source CIDs and each of those would have a mapping. In client mode, 148 * depending on QUIC versions and options selected, it is may be 149 * necessary to hash connections by address, in which case incoming 150 * packets are delivered to connections based on the address. 151 * 152 * 2. Outgoing queue. 153 * 154 * 3. Tickable queue 155 * 156 * 4. Advisory Tick Time queue. 157 * 158 * 5. Closing connections queue. This is a transient queue -- it only 159 * exists for the duration of process_connections() function call. 160 * 161 * 6. Ticked connections queue. Another transient queue, similar to (5). 162 * 163 * The idea is to destroy the connection when it is no longer referenced. 164 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 165 * that case, the connection is referenced from two places: (2) and (5). 166 * After its packets are sent, it is only referenced in (5), and at the 167 * end of the function call, when it is removed from (5), reference count 168 * goes to zero and the connection is destroyed. If not all packets can 169 * be sent, at the end of the function call, the connection is referenced 170 * by (2) and will only be removed once all outgoing packets have been 171 * sent. 172 */ 173#define CONN_REF_FLAGS (LSCONN_HASHED \ 174 |LSCONN_HAS_OUTGOING \ 175 |LSCONN_TICKABLE \ 176 |LSCONN_TICKED \ 177 |LSCONN_CLOSING \ 178 |LSCONN_ATTQ) 179 180 181 182 183struct cid_update_batch 184{ 185 lsquic_cids_update_f cub_update_cids; 186 void *cub_update_ctx; 187 unsigned cub_count; 188 lsquic_cid_t cub_cids[20]; 189 void *cub_peer_ctxs[20]; 190}; 191 192static void 193cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 194 195 196struct lsquic_engine 197{ 198 struct lsquic_engine_public pub; 199 enum { 200 ENG_SERVER = LSENG_SERVER, 201 ENG_HTTP = LSENG_HTTP, 202 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 203 ENG_PAST_DEADLINE 204 = (1 << 8), /* Previous call to a processing 205 * function went past time threshold. 206 */ 207 ENG_CONNS_BY_ADDR 208 = (1 << 9), /* Connections are hashed by address */ 209#ifndef NDEBUG 210 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 211 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 212 ENG_DTOR = (1 << 26), /* Engine destructor */ 213#endif 214 } flags; 215 lsquic_packets_out_f packets_out; 216 void *packets_out_ctx; 217 lsquic_cids_update_f report_new_scids; 218 lsquic_cids_update_f report_live_scids; 219 lsquic_cids_update_f report_old_scids; 220 void *scids_ctx; 221 struct lsquic_hash *conns_hash; 222 struct min_heap conns_tickable; 223 struct min_heap conns_out; 224 struct eng_hist history; 225 unsigned batch_size; 226 struct pr_queue *pr_queue; 227 struct attq *attq; 228 /* Track time last time a packet was sent to give new connections 229 * priority lower than that of existing connections. 230 */ 231 lsquic_time_t last_sent; 232#ifndef NDEBUG 233 regex_t lose_packets_re; 234 const char *lose_packets_str; 235#endif 236 unsigned n_conns; 237 lsquic_time_t deadline; 238 lsquic_time_t resume_sending_at; 239 unsigned mini_conns_count; 240 struct lsquic_purga *purga; 241#if LSQUIC_CONN_STATS 242 struct { 243 unsigned conns; 244 } stats; 245 struct conn_stats conn_stats_sum; 246 FILE *stats_fh; 247#endif 248 struct cid_update_batch new_scids; 249 struct out_batch out_batch; 250#if LSQUIC_COUNT_ENGINE_CALLS 251 unsigned long n_engine_calls; 252#endif 253#if LSQUIC_DEBUG_NEXT_ADV_TICK 254 uintptr_t last_logged_conn; 255 unsigned last_logged_ae_why; 256 int last_tick_diff; 257#endif 258 struct crand crand; 259 EVP_AEAD_CTX retry_aead_ctx; 260}; 261 262 263void 264lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 265 unsigned flags) 266{ 267 memset(settings, 0, sizeof(*settings)); 268 settings->es_versions = LSQUIC_DF_VERSIONS; 269 if (flags & ENG_SERVER) 270 { 271 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 272 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 273 settings->es_init_max_data 274 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 275 settings->es_init_max_stream_data_bidi_remote 276 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 277 settings->es_init_max_stream_data_bidi_local 278 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 279 settings->es_init_max_stream_data_uni 280 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 281 settings->es_init_max_streams_uni 282 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 283 settings->es_ping_period = 0; 284 } 285 else 286 { 287 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 288 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 289 settings->es_init_max_data 290 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 291 settings->es_init_max_stream_data_bidi_remote 292 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 293 settings->es_init_max_stream_data_bidi_local 294 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 295 settings->es_init_max_stream_data_uni 296 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 297 settings->es_init_max_streams_uni 298 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 299 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 300 } 301 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 302 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 303 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 304 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 305 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 306 settings->es_max_header_list_size 307 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 308 settings->es_ua = LSQUIC_DF_UA; 309 settings->es_ecn = LSQUIC_DF_ECN; 310 311 settings->es_pdmd = QTAG_X509; 312 settings->es_aead = QTAG_AESG; 313 settings->es_kexs = QTAG_C255; 314 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 315 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 316 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 317 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 318 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 319 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 320 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 321 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 322 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 323 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 324 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 325 settings->es_sttl = LSQUIC_DF_STTL; 326 settings->es_init_max_streams_bidi 327 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 328 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 329 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 330 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 331 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 332 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 333 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 334 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 335 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 336 settings->es_spin = LSQUIC_DF_SPIN; 337 settings->es_delayed_acks = LSQUIC_DF_DELAYED_ACKS; 338 settings->es_timestamps = LSQUIC_DF_TIMESTAMPS; 339} 340 341 342/* Note: if returning an error, err_buf must be valid if non-NULL */ 343int 344lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 345 unsigned flags, 346 char *err_buf, size_t err_buf_sz) 347{ 348 if (settings->es_cfcw < LSQUIC_MIN_FCW || 349 settings->es_sfcw < LSQUIC_MIN_FCW) 350 { 351 if (err_buf) 352 snprintf(err_buf, err_buf_sz, "%s", 353 "flow control window set too low"); 354 return -1; 355 } 356 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 357 { 358 if (err_buf) 359 snprintf(err_buf, err_buf_sz, "%s", 360 "No supported QUIC versions specified"); 361 return -1; 362 } 363 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 364 { 365 if (err_buf) 366 snprintf(err_buf, err_buf_sz, "%s", 367 "one or more unsupported QUIC version is specified"); 368 return -1; 369 } 370 if (flags & ENG_SERVER) 371 { 372 if (settings->es_handshake_to > 373 MAX_MINI_CONN_LIFESPAN_IN_USEC) 374 { 375 if (err_buf) 376 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 377 " usec is too large. The maximum for server is %u usec", 378 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 379 return -1; 380 } 381 } 382 if (settings->es_idle_timeout > 600) 383 { 384 if (err_buf) 385 snprintf(err_buf, err_buf_sz, "%s", 386 "The maximum value of idle timeout is 600 seconds"); 387 return -1; 388 } 389 if (settings->es_scid_len > MAX_CID_LEN) 390 { 391 if (err_buf) 392 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 393 "bytes long; it must be between 0 and %u.", 394 settings->es_scid_len, MAX_CID_LEN); 395 return -1; 396 } 397 398 if (settings->es_cc_algo > 2) 399 { 400 if (err_buf) 401 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 402 "algorithm value %u", settings->es_cc_algo); 403 return -1; 404 } 405 406 if (!(settings->es_ql_bits >= 0 && settings->es_ql_bits <= 2)) 407 { 408 if (err_buf) 409 snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ", 410 settings->es_ql_bits); 411 return -1; 412 } 413 414 if (!(settings->es_spin == 0 || settings->es_spin == 1)) 415 { 416 if (err_buf) 417 snprintf(err_buf, err_buf_sz, "Invalid spin value %d", 418 settings->es_spin); 419 return -1; 420 } 421 422 return 0; 423} 424 425 426static void 427free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 428{ 429 free(packet_data); 430} 431 432 433static void * 434malloc_buf (void *ctx, void *conn_ctx, unsigned short size, char is_ipv6) 435{ 436 return malloc(size); 437} 438 439 440static const struct lsquic_packout_mem_if stock_pmi = 441{ 442 malloc_buf, free_packet, free_packet, 443}; 444 445 446static int 447hash_conns_by_addr (const struct lsquic_engine *engine) 448{ 449 if (engine->flags & ENG_SERVER) 450 return 0; 451 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 452 return 1; 453 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 454 && engine->pub.enp_settings.es_support_tcid0) 455 return 1; 456 if (engine->pub.enp_settings.es_scid_len == 0) 457 return 1; 458 return 0; 459} 460 461 462lsquic_engine_t * 463lsquic_engine_new (unsigned flags, 464 const struct lsquic_engine_api *api) 465{ 466 lsquic_engine_t *engine; 467 size_t alpn_len; 468 char err_buf[100]; 469 470 if (!api->ea_packets_out) 471 { 472 LSQ_ERROR("packets_out callback is not specified"); 473 return NULL; 474 } 475 476 if (!api->ea_stream_if) 477 { 478 LSQ_ERROR("stream interface is not specified"); 479 return NULL; 480 } 481 482 if (!(flags & LSENG_HTTP) && api->ea_alpn) 483 { 484 alpn_len = strlen(api->ea_alpn); 485 if (alpn_len < 1 || alpn_len > 255) 486 { 487 LSQ_ERROR("ALPN string length invalid: %zd bytes", alpn_len); 488 return NULL; 489 } 490 } 491 else 492 alpn_len = 0; 493 494 if (api->ea_settings && 495 0 != lsquic_engine_check_settings(api->ea_settings, flags, 496 err_buf, sizeof(err_buf))) 497 { 498 LSQ_ERROR("cannot create engine: %s", err_buf); 499 return NULL; 500 } 501 502 engine = calloc(1, sizeof(*engine)); 503 if (!engine) 504 return NULL; 505 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 506 { 507 free(engine); 508 return NULL; 509 } 510 if (api->ea_settings) 511 engine->pub.enp_settings = *api->ea_settings; 512 else 513 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 514 int tag_buf_len; 515 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 516 sizeof(engine->pub.enp_ver_tags_buf), 517 engine->pub.enp_settings.es_versions); 518 if (tag_buf_len <= 0) 519 { 520 LSQ_ERROR("cannot generate version tags buffer"); 521 free(engine); 522 return NULL; 523 } 524 engine->pub.enp_ver_tags_len = tag_buf_len; 525 engine->pub.enp_flags = ENPUB_CAN_SEND; 526 engine->pub.enp_stream_if = api->ea_stream_if; 527 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 528 529 engine->flags = flags; 530#ifndef NDEBUG 531 engine->flags |= ENG_COALESCE; 532#endif 533 engine->packets_out = api->ea_packets_out; 534 engine->packets_out_ctx = api->ea_packets_out_ctx; 535 engine->report_new_scids = api->ea_new_scids; 536 engine->report_live_scids = api->ea_live_scids; 537 engine->report_old_scids = api->ea_old_scids; 538 engine->scids_ctx = api->ea_cids_update_ctx; 539 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 540 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 541 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 542 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 543 if (api->ea_shi) 544 { 545 engine->pub.enp_shi = api->ea_shi; 546 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 547 } 548 else 549 { 550 engine->pub.enp_shi = &stock_shi; 551 engine->pub.enp_shi_ctx = lsquic_stock_shared_hash_new(); 552 if (!engine->pub.enp_shi_ctx) 553 { 554 free(engine); 555 return NULL; 556 } 557 } 558 if (api->ea_hsi_if) 559 { 560 engine->pub.enp_hsi_if = api->ea_hsi_if; 561 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 562 } 563 else 564 { 565 engine->pub.enp_hsi_if = lsquic_http1x_if; 566 engine->pub.enp_hsi_ctx = NULL; 567 } 568 if (api->ea_pmi) 569 { 570 engine->pub.enp_pmi = api->ea_pmi; 571 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 572 } 573 else 574 { 575 engine->pub.enp_pmi = &stock_pmi; 576 engine->pub.enp_pmi_ctx = NULL; 577 } 578 engine->pub.enp_verify_cert = api->ea_verify_cert; 579 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 580 engine->pub.enp_kli = api->ea_keylog_if; 581 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 582 engine->pub.enp_engine = engine; 583 if (hash_conns_by_addr(engine)) 584 engine->flags |= ENG_CONNS_BY_ADDR; 585 engine->conns_hash = lsquic_hash_create(); 586 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 587 if (!engine->pub.enp_tokgen) 588 return NULL; 589 engine->pub.enp_crand = &engine->crand; 590 if (flags & ENG_SERVER) 591 { 592 engine->pr_queue = lsquic_prq_create( 593 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 594 &engine->pub); 595 if (!engine->pr_queue) 596 { 597 lsquic_tg_destroy(engine->pub.enp_tokgen); 598 return NULL; 599 } 600 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 601 engine->report_old_scids, engine->scids_ctx); 602 if (!engine->purga) 603 { 604 lsquic_tg_destroy(engine->pub.enp_tokgen); 605 lsquic_prq_destroy(engine->pr_queue); 606 return NULL; 607 } 608 } 609 engine->attq = lsquic_attq_create(); 610 eng_hist_init(&engine->history); 611 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 612 if (engine->pub.enp_settings.es_honor_prst) 613 { 614 engine->pub.enp_srst_hash = lsquic_hash_create(); 615 if (!engine->pub.enp_srst_hash) 616 { 617 lsquic_engine_destroy(engine); 618 return NULL; 619 } 620 } 621 622 if (alpn_len) 623 { 624 engine->pub.enp_alpn = malloc(alpn_len + 1); 625 if (!engine->pub.enp_alpn) 626 { 627 lsquic_engine_destroy(engine); 628 return NULL; 629 } 630 engine->pub.enp_alpn[0] = alpn_len; 631 memcpy(engine->pub.enp_alpn + 1, api->ea_alpn, alpn_len); 632 } 633 634 if (flags & LSENG_HTTP) 635 engine->pub.enp_flags |= ENPUB_HTTP; 636 637#ifndef NDEBUG 638 { 639 const char *env; 640 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 641 if (env) 642 { 643 if (0 != regcomp(&engine->lose_packets_re, env, 644 REG_EXTENDED|REG_NOSUB)) 645 { 646 LSQ_ERROR("could not compile lost packet regex `%s'", env); 647 return NULL; 648 } 649 engine->flags |= ENG_LOSE_PACKETS; 650 engine->lose_packets_str = env; 651 LSQ_WARN("will lose packets that match the following regex: %s", 652 env); 653 } 654 env = getenv("LSQUIC_COALESCE"); 655 if (env) 656 { 657 engine->flags &= ~ENG_COALESCE; 658 if (atoi(env)) 659 { 660 engine->flags |= ENG_COALESCE; 661 LSQ_NOTICE("will coalesce packets"); 662 } 663 else 664 LSQ_NOTICE("will not coalesce packets"); 665 } 666 } 667#endif 668#if LSQUIC_CONN_STATS 669 engine->stats_fh = api->ea_stats_fh; 670#endif 671 if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx, EVP_aead_aes_128_gcm(), 672 IETF_RETRY_KEY_BUF, IETF_RETRY_KEY_SZ, 16, NULL)) 673 { 674 LSQ_ERROR("could not initialize retry AEAD ctx"); 675 lsquic_engine_destroy(engine); 676 return NULL; 677 } 678 engine->pub.enp_retry_aead_ctx = &engine->retry_aead_ctx; 679 680 LSQ_INFO("instantiated engine"); 681 return engine; 682} 683 684 685#if LOG_PACKET_CHECKSUM 686static void 687log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 688 const unsigned char *buf, size_t bufsz) 689{ 690 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 691 (uint32_t) crc32(0, buf, bufsz)); 692} 693 694 695#endif 696 697 698static void 699grow_batch_size (struct lsquic_engine *engine) 700{ 701 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 702} 703 704 705static void 706shrink_batch_size (struct lsquic_engine *engine) 707{ 708 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 709} 710 711 712struct cce_cid_iter 713{ 714 const struct lsquic_conn *conn; 715 unsigned todo, n; 716}; 717 718 719static struct conn_cid_elem * 720cce_iter_next (struct cce_cid_iter *citer) 721{ 722 struct conn_cid_elem *cce; 723 724 while (citer->todo) 725 if (citer->todo & (1 << citer->n)) 726 { 727 citer->todo &= ~(1 << citer->n); 728 cce = &citer->conn->cn_cces[ citer->n++ ]; 729 if (!(cce->cce_flags & CCE_PORT)) 730 return cce; 731 } 732 else 733 ++citer->n; 734 735 return NULL; 736} 737 738 739static struct conn_cid_elem * 740cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 741{ 742 citer->conn = conn; 743 citer->todo = conn->cn_cces_mask; 744 citer->n = 0; 745 return cce_iter_next(citer); 746} 747 748 749#if LSQUIC_CONN_STATS 750void 751update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 752{ 753 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 754 const unsigned long *src; 755 const struct conn_stats *stats; 756 unsigned i; 757 758 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 759 { 760 ++engine->stats.conns; 761 src = (unsigned long *) stats; 762 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 763 dst[i] += src[i]; 764 } 765} 766 767 768#endif 769 770 771/* Wrapper to make sure important things occur before the connection is 772 * really destroyed. 773 */ 774static void 775destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 776 lsquic_time_t now) 777{ 778 struct cce_cid_iter citer; 779 const struct conn_cid_elem *cce; 780 lsquic_time_t drain_time; 781 struct purga_el *puel; 782 783 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 784 if (engine->purga 785 /* Blacklist all CIDs except for promoted mini connections */ 786 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 787 != (LSCONN_MINI|LSCONN_PROMOTED)) 788 { 789 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 790 && conn->cn_if->ci_drain_time && 791 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 792 { 793 for (cce = cce_iter_first(&citer, conn); cce; 794 cce = cce_iter_next(&citer)) 795 { 796 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 797 lsquic_conn_get_peer_ctx(conn, NULL), 798 PUTY_CONN_DRAIN, now); 799 if (puel) 800 puel->puel_time = now + drain_time; 801 } 802 } 803 else 804 { 805 for (cce = cce_iter_first(&citer, conn); cce; 806 cce = cce_iter_next(&citer)) 807 { 808 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 809 lsquic_conn_get_peer_ctx(conn, NULL), 810 PUTY_CONN_DELETED, now); 811 if (puel) 812 { 813 puel->puel_time = now; 814 puel->puel_count = 0; 815 } 816 } 817 } 818 } 819#if LSQUIC_CONN_STATS 820 update_stats_sum(engine, conn); 821#endif 822 --engine->n_conns; 823 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 824 conn->cn_if->ci_destroy(conn); 825} 826 827 828static int 829maybe_grow_conn_heaps (struct lsquic_engine *engine) 830{ 831 struct min_heap_elem *els; 832 unsigned count; 833 834 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 835 return 0; /* Nothing to do */ 836 837 if (lsquic_mh_nalloc(&engine->conns_tickable)) 838 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 839 else 840 count = 8; 841 842 els = malloc(sizeof(els[0]) * count); 843 if (!els) 844 { 845 LSQ_ERROR("%s: malloc failed", __func__); 846 return -1; 847 } 848 849 LSQ_DEBUG("grew heaps to %u elements", count / 2); 850 memcpy(&els[0], engine->conns_tickable.mh_elems, 851 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 852 memcpy(&els[count / 2], engine->conns_out.mh_elems, 853 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 854 free(engine->conns_tickable.mh_elems); 855 engine->conns_tickable.mh_elems = els; 856 engine->conns_out.mh_elems = &els[count / 2]; 857 engine->conns_tickable.mh_nalloc = count / 2; 858 engine->conns_out.mh_nalloc = count / 2; 859 return 0; 860} 861 862 863static void 864remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 865 unsigned todo) 866{ 867 unsigned n; 868 869 for (n = 0; todo; todo &= ~(1 << n++)) 870 if ((todo & (1 << n)) && 871 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 872 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 873} 874 875 876static void 877remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 878{ 879 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 880} 881 882 883static void 884cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 885 886 887static int 888insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 889 void *peer_ctx) 890{ 891 struct conn_cid_elem *cce; 892 unsigned todo, done, n; 893 894 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 895 if (todo & (1 << n)) 896 { 897 cce = &conn->cn_cces[n]; 898 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 899 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 900 cce->cce_cid.len, conn, &cce->cce_hash_el)) 901 done |= 1 << n; 902 else 903 goto err; 904 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 905 { 906 cce->cce_flags |= CCE_REG; 907 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 908 } 909 } 910 911 return 0; 912 913 err: 914 remove_cces_from_hash(engine->conns_hash, conn, done); 915 return -1; 916} 917 918 919static lsquic_conn_t * 920new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 921 lsquic_time_t now) 922{ 923 const lsquic_cid_t *cid; 924 server_conn_ctor_f ctor; 925 lsquic_conn_t *conn; 926 unsigned flags; 927 if (0 != maybe_grow_conn_heaps(engine)) 928 return NULL; 929 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 930 931 if (mini_conn->cn_flags & LSCONN_IETF) 932 ctor = lsquic_ietf_full_conn_server_new; 933 else 934 ctor = lsquic_gquic_full_conn_server_new; 935 936 conn = ctor(&engine->pub, flags, mini_conn); 937 if (!conn) 938 { 939 /* Otherwise, full_conn_server_new prints its own warnings */ 940 if (ENOMEM == errno) 941 { 942 cid = lsquic_conn_log_cid(mini_conn); 943 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 944 CID_BITS(cid), strerror(errno)); 945 } 946 return NULL; 947 } 948 ++engine->n_conns; 949 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 950 { 951 cid = lsquic_conn_log_cid(conn); 952 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 953 CID_BITS(cid)); 954 destroy_conn(engine, conn, now); 955 return NULL; 956 } 957 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 958 conn->cn_flags |= LSCONN_HASHED; 959 return conn; 960} 961 962 963static enum 964{ 965 VER_NOT_SPECIFIED, 966 VER_SUPPORTED, 967 VER_UNSUPPORTED, 968} 969 970 971version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 972 enum lsquic_version *pversion) 973{ 974 lsquic_ver_tag_t ver_tag; 975 enum lsquic_version version; 976 977 if (!packet_in->pi_quic_ver) 978 { 979 LSQ_DEBUG("packet does not specify version"); 980 return VER_NOT_SPECIFIED; 981 } 982 983 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 984 version = lsquic_tag2ver(ver_tag); 985 if (version < N_LSQVER) 986 { 987 if (engine->pub.enp_settings.es_versions & (1 << version)) 988 { 989 LSQ_DEBUG("client-supplied version %s is supported", 990 lsquic_ver2str[version]); 991 *pversion = version; 992 return VER_SUPPORTED; 993 } 994 else 995 LSQ_DEBUG("client-supplied version %s is not supported", 996 lsquic_ver2str[version]); 997 } 998 else 999 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 1000 ver_tag); 1001 1002 return VER_UNSUPPORTED; 1003} 1004 1005 1006static void 1007schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 1008 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 1009 const struct sockaddr *sa_peer, void *peer_ctx) 1010{ 1011 assert(engine->pr_queue); 1012 if (0 == lsquic_prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 1013 sa_local, sa_peer)) 1014 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 1015 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 1016 else 1017 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 1018} 1019 1020 1021static unsigned short 1022sa2port (const struct sockaddr *sa) 1023{ 1024 if (sa->sa_family == AF_INET) 1025 { 1026 struct sockaddr_in *const sa4 = (void *) sa; 1027 return sa4->sin_port; 1028 } 1029 else 1030 { 1031 struct sockaddr_in6 *const sa6 = (void *) sa; 1032 return sa6->sin6_port; 1033 } 1034} 1035 1036 1037static struct lsquic_hash_elem * 1038find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 1039{ 1040 unsigned short port; 1041 1042 port = sa2port(sa); 1043 return lsquic_hash_find(hash, &port, sizeof(port)); 1044} 1045 1046 1047static lsquic_conn_t * 1048find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1049 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 1050{ 1051 struct lsquic_hash_elem *el; 1052 lsquic_conn_t *conn; 1053 1054 if (engine->flags & ENG_CONNS_BY_ADDR) 1055 el = find_conn_by_addr(engine->conns_hash, sa_local); 1056 else if (packet_in->pi_flags & PI_CONN_ID) 1057 el = lsquic_hash_find(engine->conns_hash, 1058 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1059 else 1060 { 1061 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1062 return NULL; 1063 } 1064 1065 if (!el) 1066 return NULL; 1067 1068 conn = lsquic_hashelem_getdata(el); 1069 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1070 if ((engine->flags & ENG_CONNS_BY_ADDR) 1071 && !(conn->cn_flags & LSCONN_IETF) 1072 && (packet_in->pi_flags & PI_CONN_ID) 1073 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1074 { 1075 LSQ_DEBUG("connection IDs do not match"); 1076 return NULL; 1077 } 1078 1079 return conn; 1080} 1081 1082 1083static lsquic_conn_t * 1084find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1085 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1086 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1087{ 1088 struct lsquic_hash_elem *el; 1089 struct purga_el *puel; 1090 lsquic_conn_t *conn; 1091 1092 if (!(packet_in->pi_flags & PI_CONN_ID)) 1093 { 1094 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1095 return NULL; 1096 } 1097 el = lsquic_hash_find(engine->conns_hash, 1098 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1099 1100 if (el) 1101 { 1102 conn = lsquic_hashelem_getdata(el); 1103 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1104 return conn; 1105 } 1106 1107 if (engine->flags & ENG_COOLDOWN) 1108 { /* Do not create incoming connections during cooldown */ 1109 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1110 return NULL; 1111 } 1112 1113 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1114 { 1115 LSQ_DEBUG("reached limit of %u inchoate connections", 1116 engine->pub.enp_settings.es_max_inchoate); 1117 return NULL; 1118 } 1119 1120 1121 if (engine->purga 1122 && (puel = lsquic_purga_contains(engine->purga, 1123 &packet_in->pi_conn_id), puel)) 1124 { 1125 switch (puel->puel_type) 1126 { 1127 case PUTY_CID_RETIRED: 1128 LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet", 1129 CID_BITS(&packet_in->pi_conn_id)); 1130 return NULL; 1131 case PUTY_CONN_DRAIN: 1132 LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64, 1133 puel->puel_time, packet_in->pi_received); 1134 if (puel->puel_time > packet_in->pi_received) 1135 { 1136 LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet", 1137 CID_BITS(&packet_in->pi_conn_id)); 1138 return NULL; 1139 } 1140 LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted", 1141 CID_BITS(&packet_in->pi_conn_id)); 1142 puel->puel_type = PUTY_CONN_DELETED; 1143 puel->puel_count = 0; 1144 puel->puel_time = 0; 1145 /* fall-through */ 1146 case PUTY_CONN_DELETED: 1147 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1148 CID_BITS(&packet_in->pi_conn_id)); 1149 if (puel->puel_time < packet_in->pi_received) 1150 { 1151 puel->puel_time = packet_in->pi_received 1152 /* Exponential back-off */ 1153 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1154 ++puel->puel_count; 1155 goto maybe_send_prst; 1156 } 1157 return NULL; 1158 default: 1159 assert(0); 1160 return NULL; 1161 } 1162 } 1163 1164 if (engine->pub.enp_settings.es_send_prst 1165 && !(packet_in->pi_flags & PI_GQUIC) 1166 && HETY_NOT_SET == packet_in->pi_header_type) 1167 goto maybe_send_prst; 1168 1169 if (0 != maybe_grow_conn_heaps(engine)) 1170 return NULL; 1171 1172 const struct parse_funcs *pf; 1173 enum lsquic_version version; 1174 switch (version_matches(engine, packet_in, &version)) 1175 { 1176 case VER_UNSUPPORTED: 1177 if (engine->flags & ENG_SERVER) 1178 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1179 sa_local, sa_peer, peer_ctx); 1180 return NULL; 1181 case VER_NOT_SPECIFIED: 1182 maybe_send_prst: 1183 if ((engine->flags & ENG_SERVER) && 1184 engine->pub.enp_settings.es_send_prst) 1185 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1186 sa_local, sa_peer, peer_ctx); 1187 return NULL; 1188 case VER_SUPPORTED: 1189 pf = select_pf_by_ver(version); 1190 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1191 break; 1192 } 1193 1194 1195 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1196 { 1197 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1198 sa_peer->sa_family == AF_INET, NULL, packet_in_size); 1199 } 1200 else 1201 { 1202 conn = lsquic_mini_conn_new(&engine->pub, packet_in, version); 1203 } 1204 if (!conn) 1205 return NULL; 1206 ++engine->mini_conns_count; 1207 ++engine->n_conns; 1208 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1209 { 1210 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1211 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1212 CID_BITS(cid)); 1213 destroy_conn(engine, conn, packet_in->pi_received); 1214 return NULL; 1215 } 1216 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1217 conn->cn_flags |= LSCONN_HASHED; 1218 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1219 conn->cn_last_sent = engine->last_sent; 1220 return conn; 1221} 1222 1223 1224lsquic_conn_t * 1225lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1226 const lsquic_cid_t *cid) 1227{ 1228 struct lsquic_hash_elem *el; 1229 lsquic_conn_t *conn = NULL; 1230 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1231 1232 if (el) 1233 conn = lsquic_hashelem_getdata(el); 1234 return conn; 1235} 1236 1237 1238#if !defined(NDEBUG) && __GNUC__ 1239__attribute__((weak)) 1240#endif 1241void 1242lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1243 lsquic_conn_t *conn) 1244{ 1245 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1246 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1247 { 1248 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1249 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1250 engine_incref_conn(conn, LSCONN_TICKABLE); 1251 } 1252} 1253 1254 1255void 1256lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1257 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1258{ 1259 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1260 if (conn->cn_flags & LSCONN_TICKABLE) 1261 { 1262 /* Optimization: no need to add the connection to the Advisory Tick 1263 * Time Queue: it is about to be ticked, after which it its next tick 1264 * time may be queried again. 1265 */; 1266 } 1267 else if (conn->cn_flags & LSCONN_ATTQ) 1268 { 1269 if (lsquic_conn_adv_time(conn) != tick_time) 1270 { 1271 lsquic_attq_remove(engine->attq, conn); 1272 if (0 != lsquic_attq_add(engine->attq, conn, tick_time, why)) 1273 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1274 } 1275 } 1276 else if (0 == lsquic_attq_add(engine->attq, conn, tick_time, why)) 1277 engine_incref_conn(conn, LSCONN_ATTQ); 1278} 1279 1280 1281static struct lsquic_conn * 1282find_conn_by_srst (struct lsquic_engine *engine, 1283 const struct lsquic_packet_in *packet_in) 1284{ 1285 struct lsquic_hash_elem *el; 1286 struct lsquic_conn *conn; 1287 1288 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1289 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1290 return NULL; 1291 1292 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1293 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1294 IQUIC_SRESET_TOKEN_SZ); 1295 if (!el) 1296 return NULL; 1297 1298 conn = lsquic_hashelem_getdata(el); 1299 return conn; 1300} 1301 1302 1303/* Return 0 if packet is being processed by a real connection (mini or full), 1304 * otherwise return 1. 1305 */ 1306static int 1307process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1308 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1309 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1310{ 1311 lsquic_conn_t *conn; 1312 const unsigned char *packet_in_data; 1313 1314 if (lsquic_packet_in_is_gquic_prst(packet_in) 1315 && !engine->pub.enp_settings.es_honor_prst) 1316 { 1317 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1318 LSQ_DEBUG("public reset packet: discarding"); 1319 return 1; 1320 } 1321 1322 if (engine->flags & ENG_SERVER) 1323 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1324 sa_peer, peer_ctx, packet_in_size); 1325 else 1326 conn = find_conn(engine, packet_in, ppstate, sa_local); 1327 1328 if (!conn) 1329 { 1330 if (engine->pub.enp_settings.es_honor_prst 1331 && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */ 1332 && !(packet_in->pi_flags & PI_GQUIC) 1333 && engine->pub.enp_srst_hash 1334 && (conn = find_conn_by_srst(engine, packet_in))) 1335 { 1336 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1337 CID_BITS(lsquic_conn_log_cid(conn))); 1338 conn->cn_if->ci_stateless_reset(conn); 1339 if (!(conn->cn_flags & LSCONN_TICKABLE) 1340 && conn->cn_if->ci_is_tickable(conn)) 1341 { 1342 lsquic_mh_insert(&engine->conns_tickable, conn, 1343 conn->cn_last_ticked); 1344 engine_incref_conn(conn, LSCONN_TICKABLE); 1345 } 1346 /* Even though the connection processes this packet, we return 1347 * 1 so that the caller does not add reset packet's random 1348 * bytes to the list of valid CIDs. 1349 */ 1350 } 1351 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1352 return 1; 1353 } 1354 1355 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1356 { 1357 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1358 engine_incref_conn(conn, LSCONN_TICKABLE); 1359 } 1360 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1361 sa_local, sa_peer); 1362 lsquic_packet_in_upref(packet_in); 1363#if LOG_PACKET_CHECKSUM 1364 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1365 packet_in->pi_data_sz); 1366#endif 1367 /* Note on QLog: 1368 * For the PACKET_RX QLog event, we are interested in logging these things: 1369 * - raw packet (however it comes in, encrypted or not) 1370 * - frames (list of frame names) 1371 * - packet type and number 1372 * - packet rx timestamp 1373 * 1374 * Since only some of these items are available at this code 1375 * juncture, we will wait until after the packet has been 1376 * decrypted (if necessary) and parsed to call the log functions. 1377 * 1378 * Once the PACKET_RX event is finally logged, the timestamp 1379 * will come from packet_in->pi_received. For correct sequential 1380 * ordering of QLog events, be sure to process the QLogs downstream. 1381 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1382 */ 1383 packet_in_data = packet_in->pi_data; 1384 packet_in_size = packet_in->pi_data_sz; 1385 conn->cn_if->ci_packet_in(conn, packet_in); 1386 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1387 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1388 return 0; 1389} 1390 1391 1392void 1393lsquic_engine_destroy (lsquic_engine_t *engine) 1394{ 1395 struct lsquic_hash_elem *el; 1396 lsquic_conn_t *conn; 1397 1398 LSQ_DEBUG("destroying engine"); 1399#ifndef NDEBUG 1400 engine->flags |= ENG_DTOR; 1401#endif 1402 1403 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1404 { 1405 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1406 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1407 } 1408 1409 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1410 { 1411 assert(conn->cn_flags & LSCONN_TICKABLE); 1412 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1413 } 1414 1415 for (el = lsquic_hash_first(engine->conns_hash); el; 1416 el = lsquic_hash_next(engine->conns_hash)) 1417 { 1418 conn = lsquic_hashelem_getdata(el); 1419 force_close_conn(engine, conn); 1420 } 1421 lsquic_hash_destroy(engine->conns_hash); 1422 1423 assert(0 == engine->n_conns); 1424 assert(0 == engine->mini_conns_count); 1425 if (engine->pr_queue) 1426 lsquic_prq_destroy(engine->pr_queue); 1427 if (engine->purga) 1428 lsquic_purga_destroy(engine->purga); 1429 lsquic_attq_destroy(engine->attq); 1430 1431 assert(0 == lsquic_mh_count(&engine->conns_out)); 1432 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1433 if (engine->pub.enp_shi == &stock_shi) 1434 lsquic_stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1435 lsquic_mm_cleanup(&engine->pub.enp_mm); 1436 free(engine->conns_tickable.mh_elems); 1437#ifndef NDEBUG 1438 if (engine->flags & ENG_LOSE_PACKETS) 1439 regfree(&engine->lose_packets_re); 1440#endif 1441 if (engine->pub.enp_tokgen) 1442 lsquic_tg_destroy(engine->pub.enp_tokgen); 1443#if LSQUIC_CONN_STATS 1444 if (engine->stats_fh) 1445 { 1446 const struct conn_stats *const stats = &engine->conn_stats_sum; 1447 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1448 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1449 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1450 fprintf(engine->stats_fh, "In:\n"); 1451 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1452 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1453 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1454 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1455 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1456 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1457 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1458 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1459 stats->in.headers_comp, stats->in.headers_uncomp, 1460 stats->in.headers_uncomp ? 1461 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1462 : 0); 1463 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1464 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1465 fprintf(engine->stats_fh, " ACK frames merged to new: %lu\n", stats->in.n_acks_merged[0]); 1466 fprintf(engine->stats_fh, " ACK frames merged to old: %lu\n", stats->in.n_acks_merged[1]); 1467 fprintf(engine->stats_fh, "Out:\n"); 1468 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1469 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1470 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1471 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1472 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1473 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1474 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1475 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1476 stats->out.headers_comp, stats->out.headers_uncomp, 1477 stats->out.headers_uncomp ? 1478 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1479 : 0); 1480 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1481 } 1482#endif 1483 if (engine->pub.enp_srst_hash) 1484 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1485#if LSQUIC_COUNT_ENGINE_CALLS 1486 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1487#endif 1488 if (engine->pub.enp_retry_aead_ctx) 1489 EVP_AEAD_CTX_cleanup(engine->pub.enp_retry_aead_ctx); 1490 free(engine->pub.enp_alpn); 1491 free(engine); 1492} 1493 1494 1495static struct conn_cid_elem * 1496find_free_cce (struct lsquic_conn *conn) 1497{ 1498 struct conn_cid_elem *cce; 1499 1500 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1501 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1502 return cce; 1503 1504 return NULL; 1505} 1506 1507 1508static int 1509add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1510 const struct sockaddr *local_sa, void *peer_ctx) 1511{ 1512 struct conn_cid_elem *cce; 1513 1514 if (engine->flags & ENG_CONNS_BY_ADDR) 1515 { 1516 cce = find_free_cce(conn); 1517 if (!cce) 1518 { 1519 LSQ_ERROR("cannot find free CCE"); 1520 return -1; 1521 } 1522 cce->cce_port = sa2port(local_sa); 1523 cce->cce_flags = CCE_PORT; 1524 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1525 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1526 { 1527 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1528 return 0; 1529 } 1530 else 1531 return -1; 1532 1533 } 1534 else 1535 return insert_conn_into_hash(engine, conn, peer_ctx); 1536} 1537 1538 1539lsquic_conn_t * 1540lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1541 const struct sockaddr *local_sa, 1542 const struct sockaddr *peer_sa, 1543 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1544 const char *hostname, unsigned short max_packet_size, 1545 const unsigned char *zero_rtt, size_t zero_rtt_len, 1546 const unsigned char *token, size_t token_sz) 1547{ 1548 lsquic_conn_t *conn; 1549 unsigned flags, versions; 1550 int is_ipv4; 1551 1552 ENGINE_IN(engine); 1553 1554 if (engine->flags & ENG_SERVER) 1555 { 1556 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1557 goto err; 1558 } 1559 1560 if (engine->flags & ENG_CONNS_BY_ADDR 1561 && find_conn_by_addr(engine->conns_hash, local_sa)) 1562 { 1563 LSQ_ERROR("cannot have more than one connection on the same port"); 1564 goto err; 1565 } 1566 1567 if (0 != maybe_grow_conn_heaps(engine)) 1568 return NULL; 1569 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1570 is_ipv4 = peer_sa->sa_family == AF_INET; 1571 if (zero_rtt && zero_rtt_len) 1572 { 1573 version = lsquic_zero_rtt_version(zero_rtt, zero_rtt_len); 1574 if (version >= N_LSQVER) 1575 { 1576 LSQ_INFO("zero-rtt version is bad, won't use"); 1577 zero_rtt = NULL; 1578 zero_rtt_len = 0; 1579 } 1580 } 1581 if (version >= N_LSQVER) 1582 { 1583 if (version > N_LSQVER) 1584 LSQ_WARN("invalid version specified, engine will pick"); 1585 versions = engine->pub.enp_settings.es_versions; 1586 } 1587 else 1588 versions = 1u << version; 1589 if (versions & LSQUIC_IETF_VERSIONS) 1590 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1591 flags, hostname, max_packet_size, 1592 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1593 else 1594 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1595 flags, hostname, max_packet_size, is_ipv4, 1596 zero_rtt, zero_rtt_len); 1597 if (!conn) 1598 goto err; 1599 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1600 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1601 lsquic_ver2str[conn->cn_version]); 1602 ++engine->n_conns; 1603 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1604 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1605 { 1606 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1607 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1608 CID_BITS(cid)); 1609 destroy_conn(engine, conn, lsquic_time_now()); 1610 goto err; 1611 } 1612 assert(!(conn->cn_flags & 1613 (CONN_REF_FLAGS 1614 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1615 callbacks */ 1616 ))); 1617 conn->cn_flags |= LSCONN_HASHED; 1618 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1619 engine_incref_conn(conn, LSCONN_TICKABLE); 1620 lsquic_conn_set_ctx(conn, conn_ctx); 1621 conn->cn_if->ci_client_call_on_new(conn); 1622 end: 1623 ENGINE_OUT(engine); 1624 return conn; 1625 err: 1626 conn = NULL; 1627 goto end; 1628} 1629 1630 1631static void 1632remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1633{ 1634 remove_all_cces_from_hash(engine->conns_hash, conn); 1635 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1636} 1637 1638 1639static void 1640refflags2str (enum lsquic_conn_flags flags, char s[7]) 1641{ 1642 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1643 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1644 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1645 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1646 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1647 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1648 *s = '\0'; 1649} 1650 1651 1652static void 1653engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1654{ 1655 char str[2][7]; 1656 assert(flag & CONN_REF_FLAGS); 1657 assert(!(conn->cn_flags & flag)); 1658 conn->cn_flags |= flag; 1659 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1660 CID_BITS(lsquic_conn_log_cid(conn)), 1661 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1662 (refflags2str(conn->cn_flags, str[1]), str[1])); 1663} 1664 1665 1666static lsquic_conn_t * 1667engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1668 enum lsquic_conn_flags flags) 1669{ 1670 char str[2][7]; 1671 lsquic_time_t now; 1672 assert(flags & CONN_REF_FLAGS); 1673 assert(conn->cn_flags & flags); 1674#ifndef NDEBUG 1675 if (flags & LSCONN_CLOSING) 1676 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1677#endif 1678 conn->cn_flags &= ~flags; 1679 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1680 CID_BITS(lsquic_conn_log_cid(conn)), 1681 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1682 (refflags2str(conn->cn_flags, str[1]), str[1])); 1683 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1684 { 1685 now = lsquic_time_now(); 1686 if (conn->cn_flags & LSCONN_MINI) 1687 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1688 else 1689 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1690 destroy_conn(engine, conn, now); 1691 return NULL; 1692 } 1693 else 1694 return conn; 1695} 1696 1697 1698/* This is not a general-purpose function. Only call from engine dtor. */ 1699static void 1700force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1701{ 1702 assert(engine->flags & ENG_DTOR); 1703 const enum lsquic_conn_flags flags = conn->cn_flags; 1704 assert(conn->cn_flags & CONN_REF_FLAGS); 1705 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1706 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1707 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1708 if (flags & LSCONN_ATTQ) 1709 { 1710 lsquic_attq_remove(engine->attq, conn); 1711 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1712 } 1713 if (flags & LSCONN_HASHED) 1714 remove_conn_from_hash(engine, conn); 1715} 1716 1717 1718/* Iterator for tickable connections (those on the Tickable Queue). Before 1719 * a connection is returned, it is removed from the Advisory Tick Time queue 1720 * if necessary. 1721 */ 1722static lsquic_conn_t * 1723conn_iter_next_tickable (struct lsquic_engine *engine) 1724{ 1725 lsquic_conn_t *conn; 1726 1727 if (engine->flags & ENG_SERVER) 1728 while (1) 1729 { 1730 conn = lsquic_mh_pop(&engine->conns_tickable); 1731 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1732 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1733 else 1734 break; 1735 } 1736 else 1737 conn = lsquic_mh_pop(&engine->conns_tickable); 1738 1739 if (conn) 1740 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1741 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1742 { 1743 lsquic_attq_remove(engine->attq, conn); 1744 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1745 } 1746 1747 return conn; 1748} 1749 1750 1751static void 1752cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1753 void *update_ctx) 1754{ 1755 cub->cub_update_cids = update; 1756 cub->cub_update_ctx = update_ctx; 1757 cub->cub_count = 0; 1758} 1759 1760 1761static void 1762cub_flush (struct cid_update_batch *cub) 1763{ 1764 if (cub->cub_count > 0 && cub->cub_update_cids) 1765 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1766 cub->cub_cids, cub->cub_count); 1767 cub->cub_count = 0; 1768} 1769 1770 1771static void 1772cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1773{ 1774 cub->cub_cids [ cub->cub_count ] = *cid; 1775 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1776 ++cub->cub_count; 1777 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1778 cub_flush(cub); 1779} 1780 1781 1782/* Process registered CIDs */ 1783static void 1784cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1785{ 1786 struct cce_cid_iter citer; 1787 struct conn_cid_elem *cce; 1788 void *peer_ctx; 1789 1790 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1791 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1792 if (cce->cce_flags & CCE_REG) 1793 cub_add(cub, &cce->cce_cid, peer_ctx); 1794} 1795 1796 1797static void 1798drop_all_mini_conns (lsquic_engine_t *engine) 1799{ 1800 struct lsquic_hash_elem *el; 1801 lsquic_conn_t *conn; 1802 struct cid_update_batch cub; 1803 1804 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1805 1806 for (el = lsquic_hash_first(engine->conns_hash); el; 1807 el = lsquic_hash_next(engine->conns_hash)) 1808 { 1809 conn = lsquic_hashelem_getdata(el); 1810 if (conn->cn_flags & LSCONN_MINI) 1811 { 1812 /* If promoted, why is it still in this hash? */ 1813 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1814 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1815 cub_add_cids_from_cces(&cub, conn); 1816 remove_conn_from_hash(engine, conn); 1817 } 1818 } 1819 1820 cub_flush(&cub); 1821} 1822 1823 1824void 1825lsquic_engine_process_conns (lsquic_engine_t *engine) 1826{ 1827 lsquic_conn_t *conn; 1828 lsquic_time_t now; 1829 1830 ENGINE_IN(engine); 1831 1832 now = lsquic_time_now(); 1833 while ((conn = lsquic_attq_pop(engine->attq, now))) 1834 { 1835 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1836 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 1837 { 1838 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1839 engine_incref_conn(conn, LSCONN_TICKABLE); 1840 } 1841 } 1842 1843 process_connections(engine, conn_iter_next_tickable, now); 1844 ENGINE_OUT(engine); 1845} 1846 1847 1848static void 1849release_or_return_enc_data (struct lsquic_engine *engine, 1850 void (*pmi_rel_or_ret) (void *, void *, void *, char), 1851 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 1852{ 1853 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1854 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 1855 packet_out->po_flags &= ~PO_ENCRYPTED; 1856 packet_out->po_enc_data = NULL; 1857} 1858 1859 1860static void 1861release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1862 struct lsquic_packet_out *packet_out) 1863{ 1864 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 1865 conn, packet_out); 1866} 1867 1868 1869static void 1870return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1871 struct lsquic_packet_out *packet_out) 1872{ 1873 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 1874 conn, packet_out); 1875} 1876 1877 1878static int 1879copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 1880 struct lsquic_packet_out *packet_out) 1881{ 1882 int ipv6; 1883 1884 ipv6 = NP_IS_IPv6(packet_out->po_path); 1885 if (packet_out->po_flags & PO_ENCRYPTED) 1886 { 1887 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1888 && packet_out->po_data_sz == packet_out->po_enc_data_sz 1889 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 1890 packet_out->po_data_sz)) 1891 return 0; 1892 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1893 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 1894 goto copy; 1895 return_enc_data(engine, conn, packet_out); 1896 } 1897 1898 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 1899 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1900 packet_out->po_data_sz, ipv6); 1901 if (!packet_out->po_enc_data) 1902 { 1903 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 1904 "of size %hu", packet_out->po_data_sz); 1905 return -1; 1906 } 1907 1908 copy: 1909 memcpy(packet_out->po_enc_data, packet_out->po_data, 1910 packet_out->po_data_sz); 1911 packet_out->po_enc_data_sz = packet_out->po_data_sz; 1912 packet_out->po_sent_sz = packet_out->po_data_sz; 1913 packet_out->po_flags &= ~PO_IPv6; 1914 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 1915 1916 return 0; 1917} 1918 1919 1920STAILQ_HEAD(conns_stailq, lsquic_conn); 1921TAILQ_HEAD(conns_tailq, lsquic_conn); 1922 1923 1924struct conns_out_iter 1925{ 1926 struct min_heap *coi_heap; 1927 struct pr_queue *coi_prq; 1928 TAILQ_HEAD(, lsquic_conn) coi_active_list, 1929 coi_inactive_list; 1930 lsquic_conn_t *coi_next; 1931#ifndef NDEBUG 1932 lsquic_time_t coi_last_sent; 1933#endif 1934}; 1935 1936 1937static void 1938coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 1939{ 1940 iter->coi_heap = &engine->conns_out; 1941 iter->coi_prq = engine->pr_queue; 1942 iter->coi_next = NULL; 1943 TAILQ_INIT(&iter->coi_active_list); 1944 TAILQ_INIT(&iter->coi_inactive_list); 1945#ifndef NDEBUG 1946 iter->coi_last_sent = 0; 1947#endif 1948} 1949 1950 1951static lsquic_conn_t * 1952coi_next (struct conns_out_iter *iter) 1953{ 1954 lsquic_conn_t *conn; 1955 1956 if (lsquic_mh_count(iter->coi_heap) > 0) 1957 { 1958 conn = lsquic_mh_pop(iter->coi_heap); 1959 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1960 conn->cn_flags |= LSCONN_COI_ACTIVE; 1961#ifndef NDEBUG 1962 if (iter->coi_last_sent) 1963 assert(iter->coi_last_sent <= conn->cn_last_sent); 1964 iter->coi_last_sent = conn->cn_last_sent; 1965#endif 1966 return conn; 1967 } 1968 else if (iter->coi_prq && (conn = lsquic_prq_next_conn(iter->coi_prq))) 1969 { 1970 return conn; 1971 } 1972 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 1973 { 1974 iter->coi_prq = NULL; /* Save function call in previous conditional */ 1975 conn = iter->coi_next; 1976 if (!conn) 1977 conn = TAILQ_FIRST(&iter->coi_active_list); 1978 if (conn) 1979 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 1980 return conn; 1981 } 1982 else 1983 return NULL; 1984} 1985 1986 1987static void 1988coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1989{ 1990 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 1991 { 1992 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 1993 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1994 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1995 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 1996 conn->cn_flags |= LSCONN_COI_INACTIVE; 1997 } 1998} 1999 2000 2001static void 2002coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2003{ 2004 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 2005 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2006 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2007 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2008 conn->cn_flags |= LSCONN_COI_ACTIVE; 2009} 2010 2011 2012static void 2013coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 2014{ 2015 lsquic_conn_t *conn; 2016 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 2017 { 2018 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2019 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2020 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 2021 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 2022 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 2023 else /* Closed connection gets one shot at sending packets */ 2024 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2025 } 2026 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 2027 { 2028 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2029 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2030 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2031 } 2032} 2033 2034 2035#ifndef NDEBUG 2036static void 2037lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 2038 unsigned n) 2039{ 2040 const lsquic_cid_t *cid; 2041 struct iovec *iov; 2042 unsigned i; 2043 char packno_str[22]; 2044 2045 for (i = 0; i < n; ++i) 2046 { 2047 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 2048 batch->packets[i]->po_packno); 2049 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 2050 { 2051 for (iov = batch->outs[i].iov; iov < 2052 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 2053 batch->outs[i].iov->iov_len -= 1; 2054 cid = lsquic_conn_log_cid(batch->conns[i]); 2055 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 2056 CID_BITS(cid)); 2057 } 2058 } 2059} 2060 2061 2062#endif 2063 2064 2065#ifdef NDEBUG 2066#define CONST_BATCH const 2067#else 2068#define CONST_BATCH 2069#endif 2070 2071 2072struct send_batch_ctx { 2073 struct conns_stailq *closed_conns; 2074 struct conns_tailq *ticked_conns; 2075 struct conns_out_iter *conns_iter; 2076 CONST_BATCH struct out_batch *batch; 2077}; 2078 2079 2080static void 2081close_conn_immediately (struct lsquic_engine *engine, 2082 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2083{ 2084 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2085 if (!(conn->cn_flags & LSCONN_CLOSING)) 2086 { 2087 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2088 engine_incref_conn(conn, LSCONN_CLOSING); 2089 if (conn->cn_flags & LSCONN_HASHED) 2090 remove_conn_from_hash(engine, conn); 2091 } 2092 if (conn->cn_flags & LSCONN_TICKED) 2093 { 2094 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2095 engine_decref_conn(engine, conn, LSCONN_TICKED); 2096 } 2097} 2098 2099 2100static void 2101close_conn_on_send_error (struct lsquic_engine *engine, 2102 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2103{ 2104 const struct out_batch *batch = sb_ctx->batch; 2105 struct lsquic_conn *const conn = batch->conns[n]; 2106 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2107 2108 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2109 "src: %s; dst: %s; errno: %d", 2110 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2111 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2112 CID_BITS(lsquic_conn_log_cid(conn)), 2113 SA2STR(batch->outs[n].local_sa, buf[0]), 2114 SA2STR(batch->outs[n].dest_sa, buf[1]), 2115 e_val); 2116 if (conn->cn_flags & LSCONN_EVANESCENT) 2117 lsquic_prq_drop(conn); 2118 else 2119 close_conn_immediately(engine, sb_ctx, conn); 2120} 2121 2122 2123static unsigned 2124send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2125 unsigned n_to_send) 2126{ 2127 int n_sent, i, e_val; 2128 lsquic_time_t now; 2129 unsigned off; 2130 size_t count; 2131 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2132 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2133 2134#ifndef NDEBUG 2135 if (engine->flags & ENG_LOSE_PACKETS) 2136 lose_matching_packets(engine, batch, n_to_send); 2137#endif 2138 /* Set sent time before the write to avoid underestimating RTT */ 2139 now = lsquic_time_now(); 2140 for (i = 0; i < (int) n_to_send; ++i) 2141 { 2142 off = batch->pack_off[i]; 2143 count = batch->outs[i].iovlen; 2144 assert(count > 0); 2145 packet_out = &batch->packets[off]; 2146 end = packet_out + count; 2147 do 2148 (*packet_out)->po_sent = now; 2149 while (++packet_out < end); 2150 } 2151 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs, 2152 n_to_send); 2153 e_val = errno; 2154 if (n_sent < (int) n_to_send) 2155 { 2156 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2157 engine->resume_sending_at = now + 1000000; 2158 LSQ_DEBUG("cannot send packets"); 2159 EV_LOG_GENERIC_EVENT("cannot send packets"); 2160 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2161 close_conn_on_send_error(engine, sb_ctx, 2162 n_sent < 0 ? 0 : n_sent, e_val); 2163 } 2164 if (n_sent >= 0) 2165 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, n_to_send); 2166 else 2167 { 2168 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2169 n_sent = 0; 2170 } 2171 if (n_sent > 0) 2172 engine->last_sent = now + n_sent; 2173 for (i = 0; i < n_sent; ++i) 2174 { 2175 eng_hist_inc(&engine->history, now, sl_packets_out); 2176 /* `i' is added to maintain relative order */ 2177 batch->conns[i]->cn_last_sent = now + i; 2178 2179 off = batch->pack_off[i]; 2180 count = batch->outs[i].iovlen; 2181 assert(count > 0); 2182 packet_out = &batch->packets[off]; 2183 end = packet_out + count; 2184 do 2185 { 2186#if LOG_PACKET_CHECKSUM 2187 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2188 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2189 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2190#endif 2191 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2192 *packet_out); 2193 /* Release packet out buffer as soon as the packet is sent 2194 * successfully. If not successfully sent, we hold on to 2195 * this buffer until the packet sending is attempted again 2196 * or until it times out and regenerated. 2197 */ 2198 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2199 release_enc_data(engine, batch->conns[i], *packet_out); 2200 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2201 *packet_out); 2202 } 2203 while (++packet_out < end); 2204 } 2205 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2206 for ( ; i < (int) n_to_send; ++i) 2207 { 2208 off = batch->pack_off[i]; 2209 count = batch->outs[i].iovlen; 2210 assert(count > 0); 2211 packet_out = &batch->packets[off]; 2212 end = packet_out + count; 2213 do 2214 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2215 *packet_out); 2216 while (++packet_out < end); 2217 } 2218 /* Return packets to the connection in reverse order so that the packet 2219 * ordering is maintained. 2220 */ 2221 for (i = (int) n_to_send - 1; i >= n_sent; --i) 2222 { 2223 off = batch->pack_off[i]; 2224 count = batch->outs[i].iovlen; 2225 assert(count > 0); 2226 packet_out = &batch->packets[off + count - 1]; 2227 end = &batch->packets[off - 1]; 2228 do 2229 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2230 *packet_out); 2231 while (--packet_out > end); 2232 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2233 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2234 } 2235 return n_sent; 2236} 2237 2238 2239/* Return 1 if went past deadline, 0 otherwise */ 2240static int 2241check_deadline (lsquic_engine_t *engine) 2242{ 2243 if (engine->pub.enp_settings.es_proc_time_thresh && 2244 lsquic_time_now() > engine->deadline) 2245 { 2246 LSQ_INFO("went past threshold of %u usec, stop sending", 2247 engine->pub.enp_settings.es_proc_time_thresh); 2248 engine->flags |= ENG_PAST_DEADLINE; 2249 return 1; 2250 } 2251 else 2252 return 0; 2253} 2254 2255 2256static size_t 2257iov_size (const struct iovec *iov, const struct iovec *const end) 2258{ 2259 size_t size; 2260 2261 assert(iov < end); 2262 2263 size = 0; 2264 do 2265 size += iov->iov_len; 2266 while (++iov < end); 2267 2268 return size; 2269} 2270 2271 2272static void 2273send_packets_out (struct lsquic_engine *engine, 2274 struct conns_tailq *ticked_conns, 2275 struct conns_stailq *closed_conns) 2276{ 2277 unsigned n, w, n_sent, n_batches_sent; 2278 lsquic_packet_out_t *packet_out; 2279 struct lsquic_packet_out **packet; 2280 lsquic_conn_t *conn; 2281 struct out_batch *const batch = &engine->out_batch; 2282 struct iovec *iov, *packet_iov; 2283 struct conns_out_iter conns_iter; 2284 int shrink, deadline_exceeded; 2285 const struct send_batch_ctx sb_ctx = { 2286 closed_conns, 2287 ticked_conns, 2288 &conns_iter, 2289 &engine->out_batch, 2290 }; 2291 2292 coi_init(&conns_iter, engine); 2293 n_batches_sent = 0; 2294 n_sent = 0, n = 0; 2295 shrink = 0; 2296 deadline_exceeded = 0; 2297 iov = batch->iov; 2298 packet = batch->packets; 2299 2300 while ((conn = coi_next(&conns_iter))) 2301 { 2302 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2303 if (!packet_out) { 2304 /* Evanescent connection always has a packet to send: */ 2305 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2306 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2307 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2308 CID_BITS(lsquic_conn_log_cid(conn))); 2309 coi_deactivate(&conns_iter, conn); 2310 continue; 2311 } 2312 batch->outs[n].iov = packet_iov = iov; 2313 next_coa: 2314 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2315 { 2316 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2317 &engine->pub, conn, packet_out)) 2318 { 2319 case ENCPA_NOMEM: 2320 /* Send what we have and wait for a more opportune moment */ 2321 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2322 goto end_for; 2323 case ENCPA_BADCRYPT: 2324 /* This is pretty bad: close connection immediately */ 2325 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2326 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2327 CID_BITS(lsquic_conn_log_cid(conn))); 2328 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2329 { 2330 close_conn_immediately(engine, &sb_ctx, conn); 2331 coi_deactivate(&conns_iter, conn); 2332 } 2333 continue; 2334 case ENCPA_OK: 2335 break; 2336 } 2337 } 2338 else if ((packet_out->po_flags & PO_NOENCRYPT) 2339 && engine->pub.enp_pmi != &stock_pmi) 2340 { 2341 if (0 != copy_packet(engine, conn, packet_out)) 2342 { 2343 /* Copy can only fail if packet could not be allocated */ 2344 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2345 goto end_for; 2346 } 2347 } 2348 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2349 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2350 if (packet_out->po_flags & PO_ENCRYPTED) 2351 { 2352 iov->iov_base = packet_out->po_enc_data; 2353 iov->iov_len = packet_out->po_enc_data_sz; 2354 } 2355 else 2356 { 2357 iov->iov_base = packet_out->po_data; 2358 iov->iov_len = packet_out->po_data_sz; 2359 } 2360 if (packet_iov == iov) 2361 { 2362 batch->pack_off[n] = packet - batch->packets; 2363 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2364 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2365 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2366 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2367 batch->conns [n] = conn; 2368 } 2369 *packet = packet_out; 2370 ++packet; 2371 ++iov; 2372 if ((conn->cn_flags & LSCONN_IETF) 2373 && ((1 << packet_out->po_header_type) 2374 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2375#ifndef NDEBUG 2376 && (engine->flags & ENG_COALESCE) 2377#endif 2378 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2379 { 2380 const size_t size = iov_size(packet_iov, iov); 2381 packet_out = conn->cn_if->ci_next_packet_to_send(conn, size); 2382 if (packet_out) 2383 goto next_coa; 2384 } 2385 batch->outs [n].iovlen = iov - packet_iov; 2386 ++n; 2387 if (n == engine->batch_size 2388 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2389 { 2390 w = send_batch(engine, &sb_ctx, n); 2391 n = 0; 2392 iov = batch->iov; 2393 packet = batch->packets; 2394 ++n_batches_sent; 2395 n_sent += w; 2396 if (w < engine->batch_size) 2397 { 2398 shrink = 1; 2399 break; 2400 } 2401 deadline_exceeded = check_deadline(engine); 2402 if (deadline_exceeded) 2403 break; 2404 grow_batch_size(engine); 2405 } 2406 } 2407 end_for: 2408 2409 if (n > 0) { 2410 w = send_batch(engine, &sb_ctx, n); 2411 n_sent += w; 2412 shrink = w < n; 2413 ++n_batches_sent; 2414 } 2415 2416 if (shrink) 2417 shrink_batch_size(engine); 2418 else if (n_batches_sent > 1) 2419 { 2420 deadline_exceeded = check_deadline(engine); 2421 if (!deadline_exceeded) 2422 grow_batch_size(engine); 2423 } 2424 2425 coi_reheap(&conns_iter, engine); 2426 2427 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2428} 2429 2430 2431int 2432lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2433{ 2434 return lsquic_mh_count(&engine->conns_out) > 0 2435 || (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 2436 ; 2437} 2438 2439 2440static void 2441reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2442{ 2443 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2444 engine->flags &= ~ENG_PAST_DEADLINE; 2445} 2446 2447 2448static void 2449check_tickable_conns_again (struct lsquic_engine *engine) 2450{ 2451 struct lsquic_hash_elem *el; 2452 struct lsquic_conn *conn; 2453 unsigned count; 2454 2455 count = 0; 2456 for (el = lsquic_hash_first(engine->conns_hash); el; 2457 el = lsquic_hash_next(engine->conns_hash)) 2458 { 2459 conn = lsquic_hashelem_getdata(el); 2460 if (!(conn->cn_flags & LSCONN_TICKABLE) 2461 && conn->cn_if->ci_is_tickable(conn)) 2462 { 2463 lsquic_mh_insert(&engine->conns_tickable, conn, 2464 conn->cn_last_ticked); 2465 engine_incref_conn(conn, LSCONN_TICKABLE); 2466 ++count; 2467 } 2468 } 2469 LSQ_DEBUG("%u connection%s tickable again after sending has been " 2470 "re-enabled", count, count == 1 ? " is" : "s are"); 2471} 2472 2473 2474void 2475lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2476{ 2477 lsquic_conn_t *conn; 2478 struct conns_stailq closed_conns; 2479 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2480 struct cid_update_batch cub; 2481 2482 ENGINE_IN(engine); 2483 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2484 STAILQ_INIT(&closed_conns); 2485 reset_deadline(engine, lsquic_time_now()); 2486 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2487 { 2488 LSQ_DEBUG("can send again"); 2489 EV_LOG_GENERIC_EVENT("can send again"); 2490 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2491 check_tickable_conns_again(engine); 2492 } 2493 2494 send_packets_out(engine, &ticked_conns, &closed_conns); 2495 2496 while ((conn = STAILQ_FIRST(&closed_conns))) { 2497 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2498 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2499 cub_add_cids_from_cces(&cub, conn); 2500 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2501 } 2502 2503 cub_flush(&cub); 2504 ENGINE_OUT(engine); 2505} 2506 2507 2508static lsquic_conn_t * 2509next_new_full_conn (struct conns_stailq *new_full_conns) 2510{ 2511 lsquic_conn_t *conn; 2512 2513 conn = STAILQ_FIRST(new_full_conns); 2514 if (conn) 2515 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2516 return conn; 2517} 2518 2519 2520static void 2521process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2522 lsquic_time_t now) 2523{ 2524 lsquic_conn_t *conn; 2525 enum tick_st tick_st; 2526 unsigned i, why; 2527 lsquic_time_t next_tick_time; 2528 struct conns_stailq closed_conns; 2529 struct conns_tailq ticked_conns; 2530 struct conns_stailq new_full_conns; 2531 struct cid_update_batch cub_old, cub_live; 2532 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2533 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2534 2535 eng_hist_tick(&engine->history, now); 2536 2537 STAILQ_INIT(&closed_conns); 2538 TAILQ_INIT(&ticked_conns); 2539 reset_deadline(engine, now); 2540 STAILQ_INIT(&new_full_conns); 2541 2542 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2543 && now > engine->resume_sending_at) 2544 { 2545 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2546 "timeout"); 2547 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2548 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2549 } 2550 2551 i = 0; 2552 while ((conn = next_conn(engine)) 2553 || (conn = next_new_full_conn(&new_full_conns))) 2554 { 2555 tick_st = conn->cn_if->ci_tick(conn, now); 2556 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2557 if (tick_st & TICK_PROMOTE) 2558 { 2559 lsquic_conn_t *new_conn; 2560 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2561 "scheduled for promotion"); 2562 assert(conn->cn_flags & LSCONN_MINI); 2563 new_conn = new_full_conn_server(engine, conn, now); 2564 if (new_conn) 2565 { 2566 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2567 new_conn->cn_last_sent = engine->last_sent; 2568 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2569 conn->cn_flags |= LSCONN_PROMOTED; 2570 } 2571 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2572 } 2573 if (tick_st & TICK_SEND) 2574 { 2575 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2576 { 2577 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2578 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2579 } 2580 } 2581 if (tick_st & TICK_CLOSE) 2582 { 2583 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2584 engine_incref_conn(conn, LSCONN_CLOSING); 2585 if (conn->cn_flags & LSCONN_HASHED) 2586 remove_conn_from_hash(engine, conn); 2587 } 2588 else 2589 { 2590 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2591 engine_incref_conn(conn, LSCONN_TICKED); 2592 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2593 && conn->cn_if->ci_report_live(conn, now)) 2594 cub_add_cids_from_cces(&cub_live, conn); 2595 } 2596 } 2597 2598 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2599 && lsquic_engine_has_unsent_packets(engine)) 2600 send_packets_out(engine, &ticked_conns, &closed_conns); 2601 2602 while ((conn = STAILQ_FIRST(&closed_conns))) { 2603 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2604 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2605 cub_add_cids_from_cces(&cub_old, conn); 2606 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2607 } 2608 2609 while ((conn = TAILQ_FIRST(&ticked_conns))) 2610 { 2611 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2612 engine_decref_conn(engine, conn, LSCONN_TICKED); 2613 if (!(conn->cn_flags & LSCONN_TICKABLE) 2614 && conn->cn_if->ci_is_tickable(conn)) 2615 { 2616 /* Floyd heapification is not faster, don't bother. */ 2617 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2618 engine_incref_conn(conn, LSCONN_TICKABLE); 2619 } 2620 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2621 { 2622 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2623 if (next_tick_time) 2624 { 2625 if (0 == lsquic_attq_add(engine->attq, conn, next_tick_time, 2626 why)) 2627 engine_incref_conn(conn, LSCONN_ATTQ); 2628 } 2629 else 2630 /* In all other cases, the idle timeout would make the next 2631 * tick time non-zero: 2632 */ 2633 assert((conn->cn_flags & LSCONN_IETF) 2634 && engine->pub.enp_settings.es_idle_timeout == 0); 2635 } 2636 } 2637 2638 cub_flush(&engine->new_scids); 2639 cub_flush(&cub_live); 2640 cub_flush(&cub_old); 2641} 2642 2643 2644/* Return 0 if packet is being processed by a real connection, 1 if the 2645 * packet was processed, but not by a connection, and -1 on error. 2646 */ 2647int 2648lsquic_engine_packet_in (lsquic_engine_t *engine, 2649 const unsigned char *packet_in_data, size_t packet_in_size, 2650 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2651 void *peer_ctx, int ecn) 2652{ 2653 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2654 struct packin_parse_state ppstate; 2655 lsquic_packet_in_t *packet_in; 2656 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2657 int is_server, unsigned cid_len, struct packin_parse_state *); 2658 unsigned n_zeroes; 2659 int s; 2660 2661 ENGINE_CALLS_INCR(engine); 2662 2663 if (engine->flags & ENG_SERVER) 2664 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2665 else if (engine->flags & ENG_CONNS_BY_ADDR) 2666 { 2667 struct lsquic_hash_elem *el; 2668 const struct lsquic_conn *conn; 2669 el = find_conn_by_addr(engine->conns_hash, sa_local); 2670 if (!el) 2671 return -1; 2672 conn = lsquic_hashelem_getdata(el); 2673 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2674 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2675 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2676 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2677 else if (conn->cn_version == LSQVER_050) 2678 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 2679 else 2680 { 2681 assert(conn->cn_version == LSQVER_046 2682#if LSQUIC_USE_Q098 2683 || conn->cn_version == LSQVER_098 2684#endif 2685 2686 ); 2687 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2688 } 2689 } 2690 else 2691 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2692 2693 n_zeroes = 0; 2694 do 2695 { 2696 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 2697 if (!packet_in) 2698 return -1; 2699 /* Library does not modify packet_in_data, it is not referenced after 2700 * this function returns and subsequent release of pi_data is guarded 2701 * by PI_OWN_DATA flag. 2702 */ 2703 packet_in->pi_data = (unsigned char *) packet_in_data; 2704 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 2705 engine->flags & ENG_SERVER, 2706 engine->pub.enp_settings.es_scid_len, &ppstate)) 2707 { 2708 LSQ_DEBUG("Cannot parse incoming packet's header"); 2709 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 2710 s = 1; 2711 break; 2712 } 2713 2714 packet_in_data += packet_in->pi_data_sz; 2715 packet_in->pi_received = lsquic_time_now(); 2716 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 2717 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 2718 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 2719 peer_ctx, packet_in_size); 2720 n_zeroes += s == 0; 2721 } 2722 while (0 == s && packet_in_data < packet_end); 2723 2724 return n_zeroes > 0 ? 0 : s; 2725} 2726 2727 2728#if __GNUC__ && !defined(NDEBUG) 2729__attribute__((weak)) 2730#endif 2731unsigned 2732lsquic_engine_quic_versions (const lsquic_engine_t *engine) 2733{ 2734 return engine->pub.enp_settings.es_versions; 2735} 2736 2737 2738void 2739lsquic_engine_cooldown (lsquic_engine_t *engine) 2740{ 2741 struct lsquic_hash_elem *el; 2742 lsquic_conn_t *conn; 2743 2744 if (engine->flags & ENG_COOLDOWN) 2745 /* AFAICT, there is no harm in calling this function more than once, 2746 * but log it just in case, as it may indicate an error in the caller. 2747 */ 2748 LSQ_INFO("cooldown called again"); 2749 engine->flags |= ENG_COOLDOWN; 2750 LSQ_INFO("entering cooldown mode"); 2751 if (engine->flags & ENG_SERVER) 2752 drop_all_mini_conns(engine); 2753 for (el = lsquic_hash_first(engine->conns_hash); el; 2754 el = lsquic_hash_next(engine->conns_hash)) 2755 { 2756 conn = lsquic_hashelem_getdata(el); 2757 lsquic_conn_going_away(conn); 2758 } 2759} 2760 2761 2762int 2763lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 2764{ 2765 const struct attq_elem *next_attq; 2766 lsquic_time_t now, next_time; 2767#if LSQUIC_DEBUG_NEXT_ADV_TICK 2768 const struct lsquic_conn *conn; 2769 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 2770#endif 2771 2772 ENGINE_CALLS_INCR(engine); 2773 2774 if ((engine->flags & ENG_PAST_DEADLINE) 2775 && lsquic_mh_count(&engine->conns_out)) 2776 { 2777#if LSQUIC_DEBUG_NEXT_ADV_TICK 2778 conn = lsquic_mh_peek(&engine->conns_out); 2779 engine->last_logged_conn = 0; 2780 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 2781 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 2782 lsquic_mh_count(&engine->conns_out), 2783 lsquic_mh_count(&engine->conns_out) != 1, "s", 2784 CID_BITS(lsquic_conn_log_cid(conn))); 2785#endif 2786 *diff = 0; 2787 return 1; 2788 } 2789 2790 if (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 2791 { 2792#if LSQUIC_DEBUG_NEXT_ADV_TICK 2793 engine->last_logged_conn = 0; 2794 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 2795#endif 2796 *diff = 0; 2797 return 1; 2798 } 2799 2800 if (lsquic_mh_count(&engine->conns_tickable)) 2801 { 2802#if LSQUIC_DEBUG_NEXT_ADV_TICK 2803 conn = lsquic_mh_peek(&engine->conns_tickable); 2804 engine->last_logged_conn = 0; 2805 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 2806 "connection%.*s (%"CID_FMT" first)", 2807 lsquic_mh_count(&engine->conns_tickable), 2808 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 2809 CID_BITS(lsquic_conn_log_cid(conn))); 2810#endif 2811 *diff = 0; 2812 return 1; 2813 } 2814 2815 next_attq = lsquic_attq_next(engine->attq); 2816 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 2817 { 2818 if (next_attq) 2819 next_time = next_attq->ae_adv_time; 2820 else 2821 return 0; 2822 } 2823 else 2824 { 2825 if (next_attq) 2826 { 2827 next_time = next_attq->ae_adv_time; 2828 if (engine->resume_sending_at < next_time) 2829 { 2830 next_time = engine->resume_sending_at; 2831 next_attq = NULL; 2832 } 2833 } 2834 else 2835 next_time = engine->resume_sending_at; 2836 } 2837 2838 now = lsquic_time_now(); 2839 *diff = (int) ((int64_t) next_time - (int64_t) now); 2840#if LSQUIC_DEBUG_NEXT_ADV_TICK 2841 if (next_attq) 2842 { 2843 /* Deduplicate consecutive log messages about the same reason for the 2844 * same connection. 2845 * If diff is always zero or diff reset to a higher value, event is 2846 * still logged. 2847 */ 2848 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 2849 && (uintptr_t) next_attq->ae_conn 2850 == engine->last_logged_conn 2851 && *diff < engine->last_tick_diff)) 2852 { 2853 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 2854 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 2855 engine->last_tick_diff = *diff; 2856 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 2857 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 2858 lsquic_attq_why2str(next_attq->ae_why)); 2859 } 2860 } 2861 else 2862 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 2863#endif 2864 return 1; 2865} 2866 2867 2868unsigned 2869lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 2870{ 2871 lsquic_time_t now; 2872 ENGINE_CALLS_INCR(engine); 2873 now = lsquic_time_now(); 2874 if (from_now < 0) 2875 now -= from_now; 2876 else 2877 now += from_now; 2878 return lsquic_attq_count_before(engine->attq, now); 2879} 2880 2881 2882int 2883lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 2884 struct lsquic_conn *conn, unsigned cce_idx) 2885{ 2886 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2887 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2888 void *peer_ctx; 2889 2890 assert(cce_idx < conn->cn_n_cces); 2891 assert(conn->cn_cces_mask & (1 << cce_idx)); 2892 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 2893 2894 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 2895 cce->cce_cid.len, conn, &cce->cce_hash_el)) 2896 { 2897 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 2898 CID_BITS(&cce->cce_cid)); 2899 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2900 cce->cce_flags |= CCE_REG; 2901 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 2902 return 0; 2903 } 2904 else 2905 { 2906 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 2907 CID_BITS(&cce->cce_cid)); 2908 return -1; 2909 } 2910} 2911 2912 2913void 2914lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 2915 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now) 2916{ 2917 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2918 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2919 void *peer_ctx; 2920 2921 assert(cce_idx < conn->cn_n_cces); 2922 2923 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 2924 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 2925 2926 if (engine->purga) 2927 { 2928 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2929 lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 2930 PUTY_CID_RETIRED, now); 2931 } 2932 conn->cn_cces_mask &= ~(1u << cce_idx); 2933 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 2934} 2935 2936 2937