lsquic_engine.c revision ee4d3930
1/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#ifndef WIN32 17#include <sys/time.h> 18#include <netinet/in.h> 19#include <sys/types.h> 20#include <sys/stat.h> 21#include <fcntl.h> 22#include <unistd.h> 23#include <netdb.h> 24#endif 25 26#ifndef NDEBUG 27#include <sys/types.h> 28#endif 29 30#if defined(WIN32) || defined(NDEBUG) 31#define CAN_LOSE_PACKETS 0 32#else 33#define CAN_LOSE_PACKETS 1 34#endif 35 36#if CAN_LOSE_PACKETS 37#include <regex.h> /* For code that loses packets */ 38#endif 39 40#if LOG_PACKET_CHECKSUM 41#include <zlib.h> 42#endif 43 44#include <openssl/aead.h> 45 46#include "lsquic.h" 47#include "lsquic_types.h" 48#include "lsquic_int_types.h" 49#include "lsquic_sizes.h" 50#include "lsquic_parse_common.h" 51#include "lsquic_parse.h" 52#include "lsquic_packet_in.h" 53#include "lsquic_packet_out.h" 54#include "lsquic_senhist.h" 55#include "lsquic_rtt.h" 56#include "lsquic_cubic.h" 57#include "lsquic_pacer.h" 58#include "lsquic_bw_sampler.h" 59#include "lsquic_minmax.h" 60#include "lsquic_bbr.h" 61#include "lsquic_adaptive_cc.h" 62#include "lsquic_set.h" 63#include "lsquic_conn_flow.h" 64#include "lsquic_sfcw.h" 65#include "lsquic_hash.h" 66#include "lsquic_conn.h" 67#include "lsquic_send_ctl.h" 68#include "lsquic_full_conn.h" 69#include "lsquic_util.h" 70#include "lsquic_qtags.h" 71#include "lsquic_enc_sess.h" 72#include "lsquic_mm.h" 73#include "lsquic_engine_public.h" 74#include "lsquic_eng_hist.h" 75#include "lsquic_ev_log.h" 76#include "lsquic_version.h" 77#include "lsquic_pr_queue.h" 78#include "lsquic_mini_conn.h" 79#include "lsquic_trechist.h" 80#include "lsquic_mini_conn_ietf.h" 81#include "lsquic_stock_shi.h" 82#include "lsquic_purga.h" 83#include "lsquic_tokgen.h" 84#include "lsquic_attq.h" 85#include "lsquic_min_heap.h" 86#include "lsquic_http1x_if.h" 87#include "lsquic_handshake.h" 88#include "lsquic_crand.h" 89#include "lsquic_ietf.h" 90#include "lsquic_handshake.h" 91 92#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 93#include "lsquic_logger.h" 94 95#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 96#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 97#endif 98 99#if LSQUIC_DEBUG_NEXT_ADV_TICK 100#include "lsquic_alarmset.h" 101#endif 102 103#define MIN(a, b) ((a) < (b) ? (a) : (b)) 104 105/* The batch of outgoing packets grows and shrinks dynamically */ 106/* Batch sizes must be powers of two */ 107#define MAX_OUT_BATCH_SIZE 1024 108#define MIN_OUT_BATCH_SIZE 4 109#define INITIAL_OUT_BATCH_SIZE 32 110 111struct out_batch 112{ 113 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 114 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 115 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 116 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 117 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 118}; 119 120typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 121 122static void 123process_connections (struct lsquic_engine *engine, conn_iter_f iter, 124 lsquic_time_t now); 125 126static void 127engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 128 129static lsquic_conn_t * 130engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 131 enum lsquic_conn_flags flag); 132 133static void 134force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 135 136#if LSQUIC_CONN_STATS 137static void 138update_busy_detector (struct lsquic_engine *, struct lsquic_conn *, int); 139#endif 140 141#if LSQUIC_COUNT_ENGINE_CALLS 142#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 143#else 144#define ENGINE_CALLS_INCR(e) 145#endif 146 147/* Nested calls to some LSQUIC functions are not supported. Functions that 148 * iterate over connections cannot be nested. 149 */ 150#define ENGINE_IN(e) do { \ 151 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 152 (e)->pub.enp_flags |= ENPUB_PROC; \ 153 ENGINE_CALLS_INCR(e); \ 154} while (0) 155 156#define ENGINE_OUT(e) do { \ 157 assert((e)->pub.enp_flags & ENPUB_PROC); \ 158 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 159} while (0) 160 161/* A connection can be referenced from one of six places: 162 * 163 * 1. A hash is used to find connections in order to dispatch an incoming 164 * packet. Connections can be hashed by CIDs or by address. In the 165 * former case, each connection has one or more mappings in the hash 166 * table. IETF QUIC connections have up to eight (in our implementation) 167 * source CIDs and each of those would have a mapping. In client mode, 168 * depending on QUIC versions and options selected, it is may be 169 * necessary to hash connections by address, in which case incoming 170 * packets are delivered to connections based on the address. 171 * 172 * 2. Outgoing queue. 173 * 174 * 3. Tickable queue 175 * 176 * 4. Advisory Tick Time queue. 177 * 178 * 5. Closing connections queue. This is a transient queue -- it only 179 * exists for the duration of process_connections() function call. 180 * 181 * 6. Ticked connections queue. Another transient queue, similar to (5). 182 * 183 * The idea is to destroy the connection when it is no longer referenced. 184 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 185 * that case, the connection is referenced from two places: (2) and (5). 186 * After its packets are sent, it is only referenced in (5), and at the 187 * end of the function call, when it is removed from (5), reference count 188 * goes to zero and the connection is destroyed. If not all packets can 189 * be sent, at the end of the function call, the connection is referenced 190 * by (2) and will only be removed once all outgoing packets have been 191 * sent. 192 */ 193#define CONN_REF_FLAGS (LSCONN_HASHED \ 194 |LSCONN_HAS_OUTGOING \ 195 |LSCONN_TICKABLE \ 196 |LSCONN_TICKED \ 197 |LSCONN_CLOSING \ 198 |LSCONN_ATTQ) 199 200 201 202 203struct cid_update_batch 204{ 205 lsquic_cids_update_f cub_update_cids; 206 void *cub_update_ctx; 207 unsigned cub_count; 208 lsquic_cid_t cub_cids[20]; 209 void *cub_peer_ctxs[20]; 210}; 211 212static void 213cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 214 215 216struct lsquic_engine 217{ 218 struct lsquic_engine_public pub; 219 enum { 220 ENG_SERVER = LSENG_SERVER, 221 ENG_HTTP = LSENG_HTTP, 222 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 223 ENG_PAST_DEADLINE 224 = (1 << 8), /* Previous call to a processing 225 * function went past time threshold. 226 */ 227 ENG_CONNS_BY_ADDR 228 = (1 << 9), /* Connections are hashed by address */ 229#ifndef NDEBUG 230 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 231#endif 232#if CAN_LOSE_PACKETS 233 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 234#endif 235#ifndef NDEBUG 236 ENG_DTOR = (1 << 26), /* Engine destructor */ 237#endif 238 } flags; 239 lsquic_packets_out_f packets_out; 240 void *packets_out_ctx; 241 lsquic_cids_update_f report_new_scids; 242 lsquic_cids_update_f report_live_scids; 243 lsquic_cids_update_f report_old_scids; 244 void *scids_ctx; 245 struct lsquic_hash *conns_hash; 246 struct min_heap conns_tickable; 247 struct min_heap conns_out; 248 struct eng_hist history; 249 unsigned batch_size; 250 struct lsquic_conn *curr_conn; 251 struct pr_queue *pr_queue; 252 struct attq *attq; 253 /* Track time last time a packet was sent to give new connections 254 * priority lower than that of existing connections. 255 */ 256 lsquic_time_t last_sent; 257#if CAN_LOSE_PACKETS 258 regex_t lose_packets_re; 259 const char *lose_packets_str; 260#endif 261 unsigned n_conns; 262 lsquic_time_t deadline; 263 lsquic_time_t resume_sending_at; 264 unsigned mini_conns_count; 265 struct lsquic_purga *purga; 266#if LSQUIC_CONN_STATS 267 struct { 268 unsigned conns; 269 } stats; 270 struct conn_stats conn_stats_sum; 271 FILE *stats_fh; 272#endif 273 struct cid_update_batch new_scids; 274 struct out_batch out_batch; 275#if LSQUIC_COUNT_ENGINE_CALLS 276 unsigned long n_engine_calls; 277#endif 278#if LSQUIC_DEBUG_NEXT_ADV_TICK 279 uintptr_t last_logged_conn; 280 unsigned last_logged_ae_why; 281 int last_tick_diff; 282#endif 283 struct crand crand; 284 EVP_AEAD_CTX retry_aead_ctx[N_IETF_RETRY_VERSIONS]; 285#if LSQUIC_CONN_STATS 286 struct { 287 uint16_t immed_ticks; /* bitmask */ 288#define MAX_IMMED_TICKS UINT16_MAX 289 struct lsquic_conn *last_conn, /* from last call */ 290 *pin_conn, /* last connection with packet in */ 291 *current; /* currently busy connection */ 292 lsquic_time_t last_log; 293 } busy; 294#endif 295}; 296 297 298void 299lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 300 unsigned flags) 301{ 302 memset(settings, 0, sizeof(*settings)); 303 settings->es_versions = LSQUIC_DF_VERSIONS; 304 if (flags & ENG_SERVER) 305 { 306 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 307 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 308 settings->es_init_max_data 309 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 310 settings->es_init_max_stream_data_bidi_remote 311 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 312 settings->es_init_max_stream_data_bidi_local 313 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 314 settings->es_init_max_stream_data_uni 315 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 316 settings->es_init_max_streams_uni 317 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 318 settings->es_ping_period = 0; 319 settings->es_noprogress_timeout 320 = LSQUIC_DF_NOPROGRESS_TIMEOUT_SERVER; 321 } 322 else 323 { 324 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 325 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 326 settings->es_init_max_data 327 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 328 settings->es_init_max_stream_data_bidi_remote 329 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 330 settings->es_init_max_stream_data_bidi_local 331 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 332 settings->es_init_max_stream_data_uni 333 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 334 settings->es_init_max_streams_uni 335 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 336 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 337 settings->es_noprogress_timeout 338 = LSQUIC_DF_NOPROGRESS_TIMEOUT_CLIENT; 339 } 340 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 341 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 342 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 343 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 344 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 345 settings->es_max_header_list_size 346 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 347 settings->es_ua = LSQUIC_DF_UA; 348 settings->es_ecn = LSQUIC_DF_ECN; 349 350 settings->es_pdmd = QTAG_X509; 351 settings->es_aead = QTAG_AESG; 352 settings->es_kexs = QTAG_C255; 353 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 354 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 355 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 356 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 357 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 358 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 359 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 360 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 361 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 362 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 363 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 364 settings->es_sttl = LSQUIC_DF_STTL; 365 settings->es_init_max_streams_bidi 366 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 367 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 368 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 369 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 370 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 371 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 372 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 373 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 374 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 375 settings->es_spin = LSQUIC_DF_SPIN; 376 settings->es_delayed_acks = LSQUIC_DF_DELAYED_ACKS; 377 settings->es_timestamps = LSQUIC_DF_TIMESTAMPS; 378 settings->es_grease_quic_bit = LSQUIC_DF_GREASE_QUIC_BIT; 379 settings->es_mtu_probe_timer = LSQUIC_DF_MTU_PROBE_TIMER; 380 settings->es_dplpmtud = LSQUIC_DF_DPLPMTUD; 381 settings->es_cc_algo = LSQUIC_DF_CC_ALGO; 382 settings->es_cc_rtt_thresh = LSQUIC_DF_CC_RTT_THRESH; 383 settings->es_optimistic_nat = LSQUIC_DF_OPTIMISTIC_NAT; 384 settings->es_ext_http_prio = LSQUIC_DF_EXT_HTTP_PRIO; 385} 386 387 388/* Note: if returning an error, err_buf must be valid if non-NULL */ 389int 390lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 391 unsigned flags, 392 char *err_buf, size_t err_buf_sz) 393{ 394 if (settings->es_cfcw < LSQUIC_MIN_FCW || 395 settings->es_sfcw < LSQUIC_MIN_FCW) 396 { 397 if (err_buf) 398 snprintf(err_buf, err_buf_sz, "%s", 399 "flow control window set too low"); 400 return -1; 401 } 402 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 403 { 404 if (err_buf) 405 snprintf(err_buf, err_buf_sz, "%s", 406 "No supported QUIC versions specified"); 407 return -1; 408 } 409 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 410 { 411 if (err_buf) 412 snprintf(err_buf, err_buf_sz, "%s", 413 "one or more unsupported QUIC version is specified"); 414 return -1; 415 } 416 if (flags & ENG_SERVER) 417 { 418 if (settings->es_handshake_to > 419 MAX_MINI_CONN_LIFESPAN_IN_USEC) 420 { 421 if (err_buf) 422 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 423 " usec is too large. The maximum for server is %u usec", 424 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 425 return -1; 426 } 427 } 428 if (settings->es_idle_timeout > 600) 429 { 430 if (err_buf) 431 snprintf(err_buf, err_buf_sz, "%s", 432 "The maximum value of idle timeout is 600 seconds"); 433 return -1; 434 } 435 if (settings->es_scid_len > MAX_CID_LEN) 436 { 437 if (err_buf) 438 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 439 "bytes long; it must be between 0 and %u.", 440 settings->es_scid_len, MAX_CID_LEN); 441 return -1; 442 } 443 444 if (settings->es_cc_algo > 3) 445 { 446 if (err_buf) 447 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 448 "algorithm value %u", settings->es_cc_algo); 449 return -1; 450 } 451 452 if (!(settings->es_ql_bits >= 0 && settings->es_ql_bits <= 2)) 453 { 454 if (err_buf) 455 snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ", 456 settings->es_ql_bits); 457 return -1; 458 } 459 460 if (!(settings->es_spin == 0 || settings->es_spin == 1)) 461 { 462 if (err_buf) 463 snprintf(err_buf, err_buf_sz, "Invalid spin value %d", 464 settings->es_spin); 465 return -1; 466 } 467 468 if (settings->es_mtu_probe_timer && settings->es_mtu_probe_timer < 1000) 469 { 470 if (err_buf) 471 snprintf(err_buf, err_buf_sz, "mtu probe timer is too small: " 472 "%u ms", settings->es_mtu_probe_timer); 473 return -1; 474 } 475 476 return 0; 477} 478 479 480static void 481free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 482{ 483 free(packet_data); 484} 485 486 487static void * 488malloc_buf (void *ctx, void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, unsigned short size, char is_ipv6) 489{ 490 return malloc(size); 491} 492 493 494static const struct lsquic_packout_mem_if stock_pmi = 495{ 496 malloc_buf, free_packet, free_packet, 497}; 498 499 500static int 501hash_conns_by_addr (const struct lsquic_engine *engine) 502{ 503 if (engine->flags & ENG_SERVER) 504 return 0; 505 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 506 return 1; 507 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 508 && engine->pub.enp_settings.es_support_tcid0) 509 return 1; 510 if (engine->pub.enp_settings.es_scid_len == 0) 511 return 1; 512 return 0; 513} 514 515 516lsquic_engine_t * 517lsquic_engine_new (unsigned flags, 518 const struct lsquic_engine_api *api) 519{ 520 lsquic_engine_t *engine; 521 size_t alpn_len; 522 unsigned i; 523 char err_buf[100]; 524 525 if (!api->ea_packets_out) 526 { 527 LSQ_ERROR("packets_out callback is not specified"); 528 return NULL; 529 } 530 531 if (!api->ea_stream_if) 532 { 533 LSQ_ERROR("stream interface is not specified"); 534 return NULL; 535 } 536 537 if (!(flags & LSENG_HTTP) && api->ea_alpn) 538 { 539 alpn_len = strlen(api->ea_alpn); 540 if (alpn_len < 1 || alpn_len > 255) 541 { 542 LSQ_ERROR("ALPN string length invalid: %zd bytes", alpn_len); 543 return NULL; 544 } 545 } 546 else 547 alpn_len = 0; 548 549 if (api->ea_settings && 550 0 != lsquic_engine_check_settings(api->ea_settings, flags, 551 err_buf, sizeof(err_buf))) 552 { 553 LSQ_ERROR("cannot create engine: %s", err_buf); 554 return NULL; 555 } 556 557 engine = calloc(1, sizeof(*engine)); 558 if (!engine) 559 return NULL; 560 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 561 { 562 free(engine); 563 return NULL; 564 } 565 if (api->ea_settings) 566 engine->pub.enp_settings = *api->ea_settings; 567 else 568 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 569 int tag_buf_len; 570 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 571 sizeof(engine->pub.enp_ver_tags_buf), 572 engine->pub.enp_settings.es_versions); 573 if (tag_buf_len <= 0) 574 { 575 LSQ_ERROR("cannot generate version tags buffer"); 576 free(engine); 577 return NULL; 578 } 579 engine->pub.enp_ver_tags_len = tag_buf_len; 580 engine->pub.enp_flags = ENPUB_CAN_SEND; 581 engine->pub.enp_stream_if = api->ea_stream_if; 582 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 583 584 engine->flags = flags; 585#ifndef NDEBUG 586 engine->flags |= ENG_COALESCE; 587#endif 588 engine->packets_out = api->ea_packets_out; 589 engine->packets_out_ctx = api->ea_packets_out_ctx; 590 engine->report_new_scids = api->ea_new_scids; 591 engine->report_live_scids = api->ea_live_scids; 592 engine->report_old_scids = api->ea_old_scids; 593 engine->scids_ctx = api->ea_cids_update_ctx; 594 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 595 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 596 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 597 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 598 599 if (api->ea_generate_scid) 600 engine->pub.enp_generate_scid = api->ea_generate_scid; 601 else 602 engine->pub.enp_generate_scid = lsquic_generate_scid; 603 604 if (api->ea_shi) 605 { 606 engine->pub.enp_shi = api->ea_shi; 607 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 608 } 609 else 610 { 611 engine->pub.enp_shi = &stock_shi; 612 engine->pub.enp_shi_ctx = lsquic_stock_shared_hash_new(); 613 if (!engine->pub.enp_shi_ctx) 614 { 615 free(engine); 616 return NULL; 617 } 618 } 619 if (api->ea_hsi_if) 620 { 621 engine->pub.enp_hsi_if = api->ea_hsi_if; 622 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 623 } 624 else 625 { 626 engine->pub.enp_hsi_if = lsquic_http1x_if; 627 engine->pub.enp_hsi_ctx = NULL; 628 } 629 if (api->ea_pmi) 630 { 631 engine->pub.enp_pmi = api->ea_pmi; 632 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 633 } 634 else 635 { 636 engine->pub.enp_pmi = &stock_pmi; 637 engine->pub.enp_pmi_ctx = NULL; 638 } 639 engine->pub.enp_verify_cert = api->ea_verify_cert; 640 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 641 engine->pub.enp_kli = api->ea_keylog_if; 642 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 643 engine->pub.enp_engine = engine; 644 if (hash_conns_by_addr(engine)) 645 engine->flags |= ENG_CONNS_BY_ADDR; 646 engine->conns_hash = lsquic_hash_create(); 647 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 648 if (!engine->pub.enp_tokgen) 649 return NULL; 650 if (engine->flags & ENG_SERVER) 651 for (i = 0; i < sizeof(engine->pub.enp_quic_ctx_sz) 652 / sizeof(engine->pub.enp_quic_ctx_sz[0]); ++i) 653 { 654 int sz = lsquic_enc_sess_ietf_gen_quic_ctx( 655 &engine->pub.enp_settings, 656 i == 0 ? LSQVER_ID27 : LSQVER_ID28, 657 engine->pub.enp_quic_ctx_buf[i], 658 sizeof(engine->pub.enp_quic_ctx_buf)); 659 if (sz < 0) 660 { 661 free(engine); 662 return NULL; 663 } 664 engine->pub.enp_quic_ctx_sz[i] = (unsigned) sz; 665 } 666 engine->pub.enp_crand = &engine->crand; 667 if (engine->pub.enp_settings.es_noprogress_timeout) 668 engine->pub.enp_noprog_timeout 669 = engine->pub.enp_settings.es_noprogress_timeout * 1000000; 670 engine->pub.enp_mtu_probe_timer = 1000 671 * (engine->pub.enp_settings.es_mtu_probe_timer 672 ? engine->pub.enp_settings.es_mtu_probe_timer 673 : LSQUIC_DF_MTU_PROBE_TIMER); 674 if (flags & ENG_SERVER) 675 { 676 engine->pr_queue = lsquic_prq_create( 677 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 678 &engine->pub); 679 if (!engine->pr_queue) 680 { 681 lsquic_tg_destroy(engine->pub.enp_tokgen); 682 return NULL; 683 } 684 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 685 engine->report_old_scids, engine->scids_ctx); 686 if (!engine->purga) 687 { 688 lsquic_tg_destroy(engine->pub.enp_tokgen); 689 lsquic_prq_destroy(engine->pr_queue); 690 return NULL; 691 } 692 } 693 engine->attq = lsquic_attq_create(); 694 eng_hist_init(&engine->history); 695 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 696 if (engine->pub.enp_settings.es_honor_prst) 697 { 698 engine->pub.enp_srst_hash = lsquic_hash_create(); 699 if (!engine->pub.enp_srst_hash) 700 { 701 lsquic_engine_destroy(engine); 702 return NULL; 703 } 704 } 705 if ((flags & LSENG_SERVER) && 0 != lsquic_init_gquic_crypto(&engine->pub)) 706 { 707 lsquic_engine_destroy(engine); 708 return NULL; 709 } 710 711 if (alpn_len) 712 { 713 engine->pub.enp_alpn = malloc(alpn_len + 1); 714 if (!engine->pub.enp_alpn) 715 { 716 lsquic_engine_destroy(engine); 717 return NULL; 718 } 719 engine->pub.enp_alpn[0] = alpn_len; 720 memcpy(engine->pub.enp_alpn + 1, api->ea_alpn, alpn_len); 721 } 722 723 if (flags & LSENG_HTTP) 724 engine->pub.enp_flags |= ENPUB_HTTP; 725 726#ifndef NDEBUG 727 { 728 const char *env; 729 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 730#if CAN_LOSE_PACKETS 731 if (env) 732 { 733 if (0 != regcomp(&engine->lose_packets_re, env, 734 REG_EXTENDED|REG_NOSUB)) 735 { 736 LSQ_ERROR("could not compile lost packet regex `%s'", env); 737 return NULL; 738 } 739 engine->flags |= ENG_LOSE_PACKETS; 740 engine->lose_packets_str = env; 741 LSQ_WARN("will lose packets that match the following regex: %s", 742 env); 743 } 744#endif 745 env = getenv("LSQUIC_COALESCE"); 746 if (env) 747 { 748 engine->flags &= ~ENG_COALESCE; 749 if (atoi(env)) 750 { 751 engine->flags |= ENG_COALESCE; 752 LSQ_NOTICE("will coalesce packets"); 753 } 754 else 755 LSQ_NOTICE("will not coalesce packets"); 756 } 757 } 758#endif 759#if LSQUIC_CONN_STATS 760 engine->stats_fh = api->ea_stats_fh; 761#endif 762 for (i = 0; i < sizeof(engine->retry_aead_ctx) 763 / sizeof(engine->retry_aead_ctx[0]); ++i) 764 if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx[i], 765 EVP_aead_aes_128_gcm(), lsquic_retry_key_buf[i], 766 IETF_RETRY_KEY_SZ, 16, NULL)) 767 { 768 LSQ_ERROR("could not initialize retry AEAD ctx #%u", i); 769 lsquic_engine_destroy(engine); 770 return NULL; 771 } 772 engine->pub.enp_retry_aead_ctx = engine->retry_aead_ctx; 773 774 LSQ_INFO("instantiated engine"); 775 return engine; 776} 777 778 779#if LOG_PACKET_CHECKSUM 780static void 781log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 782 const unsigned char *buf, size_t bufsz) 783{ 784 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 785 (uint32_t) crc32(0, buf, bufsz)); 786} 787 788 789#endif 790 791 792static void 793grow_batch_size (struct lsquic_engine *engine) 794{ 795 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 796} 797 798 799static void 800shrink_batch_size (struct lsquic_engine *engine) 801{ 802 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 803} 804 805 806struct cce_cid_iter 807{ 808 const struct lsquic_conn *conn; 809 unsigned todo, n; 810}; 811 812 813static struct conn_cid_elem * 814cce_iter_next (struct cce_cid_iter *citer) 815{ 816 struct conn_cid_elem *cce; 817 818 while (citer->todo) 819 if (citer->todo & (1 << citer->n)) 820 { 821 citer->todo &= ~(1 << citer->n); 822 cce = &citer->conn->cn_cces[ citer->n++ ]; 823 if (!(cce->cce_flags & CCE_PORT)) 824 return cce; 825 } 826 else 827 ++citer->n; 828 829 return NULL; 830} 831 832 833static struct conn_cid_elem * 834cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 835{ 836 citer->conn = conn; 837 citer->todo = conn->cn_cces_mask; 838 citer->n = 0; 839 return cce_iter_next(citer); 840} 841 842 843#if LSQUIC_CONN_STATS 844void 845update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 846{ 847 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 848 const unsigned long *src; 849 const struct conn_stats *stats; 850 unsigned i; 851 852 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 853 { 854 ++engine->stats.conns; 855 src = (unsigned long *) stats; 856 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 857 dst[i] += src[i]; 858 } 859} 860 861 862#endif 863 864 865/* Wrapper to make sure important things occur before the connection is 866 * really destroyed. 867 */ 868static void 869destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 870 lsquic_time_t now) 871{ 872 struct cce_cid_iter citer; 873 const struct conn_cid_elem *cce; 874 lsquic_time_t drain_time; 875 struct purga_el *puel; 876 877 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 878 if (engine->purga 879 /* Blacklist all CIDs except for promoted mini connections */ 880 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 881 != (LSCONN_MINI|LSCONN_PROMOTED)) 882 { 883 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 884 && conn->cn_if->ci_drain_time && 885 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 886 { 887 for (cce = cce_iter_first(&citer, conn); cce; 888 cce = cce_iter_next(&citer)) 889 { 890 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 891 lsquic_conn_get_peer_ctx(conn, NULL), 892 PUTY_CONN_DRAIN, now); 893 if (puel) 894 puel->puel_time = now + drain_time; 895 } 896 } 897 else 898 { 899 for (cce = cce_iter_first(&citer, conn); cce; 900 cce = cce_iter_next(&citer)) 901 { 902 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 903 lsquic_conn_get_peer_ctx(conn, NULL), 904 PUTY_CONN_DELETED, now); 905 if (puel) 906 { 907 puel->puel_time = now; 908 puel->puel_count = 0; 909 } 910 } 911 } 912 } 913#if LSQUIC_CONN_STATS 914 update_stats_sum(engine, conn); 915 if (engine->busy.last_conn == conn) 916 engine->busy.last_conn = NULL; 917 if (engine->busy.current == conn) 918 { 919 char cidstr[MAX_CID_LEN * 2 + 1]; 920 lsquic_logger_log1(LSQ_LOG_NOTICE, LSQLM_CONN_STATS, 921 "busy connection %s is destroyed", 922 (lsquic_cid2str(lsquic_conn_log_cid(conn), cidstr), cidstr)); 923 engine->busy.current = NULL; 924 engine->busy.last_log = 0; 925 } 926 if (engine->busy.pin_conn == conn) 927 engine->busy.pin_conn = NULL; 928#endif 929 --engine->n_conns; 930 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 931 conn->cn_if->ci_destroy(conn); 932} 933 934 935static int 936maybe_grow_conn_heaps (struct lsquic_engine *engine) 937{ 938 struct min_heap_elem *els; 939 unsigned count; 940 941 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 942 return 0; /* Nothing to do */ 943 944 if (lsquic_mh_nalloc(&engine->conns_tickable)) 945 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 946 else 947 count = 8; 948 949 els = malloc(sizeof(els[0]) * count); 950 if (!els) 951 { 952 LSQ_ERROR("%s: malloc failed", __func__); 953 return -1; 954 } 955 956 LSQ_DEBUG("grew heaps to %u elements", count / 2); 957 memcpy(&els[0], engine->conns_tickable.mh_elems, 958 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 959 memcpy(&els[count / 2], engine->conns_out.mh_elems, 960 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 961 free(engine->conns_tickable.mh_elems); 962 engine->conns_tickable.mh_elems = els; 963 engine->conns_out.mh_elems = &els[count / 2]; 964 engine->conns_tickable.mh_nalloc = count / 2; 965 engine->conns_out.mh_nalloc = count / 2; 966 return 0; 967} 968 969 970static void 971remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 972 unsigned todo) 973{ 974 unsigned n; 975 976 for (n = 0; todo; todo &= ~(1 << n++)) 977 if ((todo & (1 << n)) && 978 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 979 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 980} 981 982 983static void 984remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 985{ 986 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 987} 988 989 990static void 991cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 992 993 994static int 995insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 996 void *peer_ctx) 997{ 998 struct conn_cid_elem *cce; 999 unsigned todo, done, n; 1000 1001 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 1002 if (todo & (1 << n)) 1003 { 1004 cce = &conn->cn_cces[n]; 1005 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 1006 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 1007 cce->cce_cid.len, conn, &cce->cce_hash_el)) 1008 done |= 1 << n; 1009 else 1010 goto err; 1011 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 1012 { 1013 cce->cce_flags |= CCE_REG; 1014 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 1015 } 1016 } 1017 1018 return 0; 1019 1020 err: 1021 remove_cces_from_hash(engine->conns_hash, conn, done); 1022 return -1; 1023} 1024 1025 1026static lsquic_conn_t * 1027new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 1028 lsquic_time_t now) 1029{ 1030 const lsquic_cid_t *cid; 1031 server_conn_ctor_f ctor; 1032 lsquic_conn_t *conn; 1033 unsigned flags; 1034 if (0 != maybe_grow_conn_heaps(engine)) 1035 return NULL; 1036 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1037 1038 if (mini_conn->cn_flags & LSCONN_IETF) 1039 ctor = lsquic_ietf_full_conn_server_new; 1040 else 1041 ctor = lsquic_gquic_full_conn_server_new; 1042 1043 conn = ctor(&engine->pub, flags, mini_conn); 1044 if (!conn) 1045 { 1046 /* Otherwise, full_conn_server_new prints its own warnings */ 1047 if (ENOMEM == errno) 1048 { 1049 cid = lsquic_conn_log_cid(mini_conn); 1050 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 1051 CID_BITS(cid), strerror(errno)); 1052 } 1053 return NULL; 1054 } 1055 ++engine->n_conns; 1056 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 1057 { 1058 cid = lsquic_conn_log_cid(conn); 1059 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1060 CID_BITS(cid)); 1061 destroy_conn(engine, conn, now); 1062 return NULL; 1063 } 1064 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1065 conn->cn_flags |= LSCONN_HASHED; 1066 return conn; 1067} 1068 1069 1070static enum 1071{ 1072 VER_NOT_SPECIFIED, 1073 VER_SUPPORTED, 1074 VER_UNSUPPORTED, 1075} 1076 1077 1078version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 1079 enum lsquic_version *pversion) 1080{ 1081 lsquic_ver_tag_t ver_tag; 1082 enum lsquic_version version; 1083 1084 if (!packet_in->pi_quic_ver) 1085 { 1086 LSQ_DEBUG("packet does not specify version"); 1087 return VER_NOT_SPECIFIED; 1088 } 1089 1090 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 1091 version = lsquic_tag2ver(ver_tag); 1092 if (version < N_LSQVER) 1093 { 1094 if (engine->pub.enp_settings.es_versions & (1 << version)) 1095 { 1096 LSQ_DEBUG("client-supplied version %s is supported", 1097 lsquic_ver2str[version]); 1098 *pversion = version; 1099 return VER_SUPPORTED; 1100 } 1101 else 1102 LSQ_DEBUG("client-supplied version %s is not supported", 1103 lsquic_ver2str[version]); 1104 } 1105 else 1106 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 1107 ver_tag); 1108 1109 return VER_UNSUPPORTED; 1110} 1111 1112 1113static void 1114schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 1115 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 1116 const struct sockaddr *sa_peer, void *peer_ctx) 1117{ 1118 assert(engine->pr_queue); 1119 if (0 == lsquic_prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 1120 sa_local, sa_peer)) 1121 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 1122 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 1123 else 1124 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 1125} 1126 1127 1128static unsigned short 1129sa2port (const struct sockaddr *sa) 1130{ 1131 if (sa->sa_family == AF_INET) 1132 { 1133 struct sockaddr_in *const sa4 = (void *) sa; 1134 return sa4->sin_port; 1135 } 1136 else 1137 { 1138 struct sockaddr_in6 *const sa6 = (void *) sa; 1139 return sa6->sin6_port; 1140 } 1141} 1142 1143 1144static struct lsquic_hash_elem * 1145find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 1146{ 1147 unsigned short port; 1148 1149 port = sa2port(sa); 1150 return lsquic_hash_find(hash, &port, sizeof(port)); 1151} 1152 1153 1154/* When connections are identified by the local address, we need to drop 1155 * packets that use DCIDs that do not correspond to any of SCIDs. This 1156 * can happen when peer retires a SCID. This mimics the normal behavior, 1157 * when connections are looked up in engine->conns_hash by ID: when there 1158 * is no match, the packet is dropped. 1159 */ 1160static int 1161dcid_checks_out (const struct lsquic_conn *conn, const lsquic_cid_t *dcid) 1162{ 1163 const struct conn_cid_elem *cce; 1164 1165 if (LSQUIC_CIDS_EQ(CN_SCID(conn), dcid)) 1166 return 1; 1167 1168 /* Slow check for those rare cases */ 1169 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1170 if ((conn->cn_cces_mask & (1 << (cce - conn->cn_cces))) 1171 && !(cce->cce_flags & CCE_PORT) 1172 && LSQUIC_CIDS_EQ(&cce->cce_cid, dcid)) 1173 { 1174 LSQ_DEBUG("connection checks out"); 1175 return 1; 1176 } 1177 1178 return 0; 1179} 1180 1181 1182static lsquic_conn_t * 1183find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1184 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 1185{ 1186 struct lsquic_hash_elem *el; 1187 lsquic_conn_t *conn; 1188 1189 if (engine->flags & ENG_CONNS_BY_ADDR) 1190 { 1191 el = find_conn_by_addr(engine->conns_hash, sa_local); 1192 if ((packet_in->pi_flags & PI_CONN_ID) 1193 && !dcid_checks_out(lsquic_hashelem_getdata(el), 1194 &packet_in->pi_conn_id)) 1195 { 1196 LSQ_DEBUGC("DCID matches no SCID in connection %"CID_FMT": drop it", 1197 CID_BITS(&packet_in->pi_conn_id)); 1198 return NULL; 1199 } 1200 } 1201 else if (packet_in->pi_flags & PI_CONN_ID) 1202 el = lsquic_hash_find(engine->conns_hash, 1203 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1204 else 1205 { 1206 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1207 return NULL; 1208 } 1209 1210 if (!el) 1211 return NULL; 1212 1213 conn = lsquic_hashelem_getdata(el); 1214 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1215 if ((engine->flags & ENG_CONNS_BY_ADDR) 1216 && !(conn->cn_flags & LSCONN_IETF) 1217 && (packet_in->pi_flags & PI_CONN_ID) 1218 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1219 { 1220 LSQ_DEBUG("connection IDs do not match"); 1221 return NULL; 1222 } 1223 1224 return conn; 1225} 1226 1227 1228static lsquic_conn_t * 1229find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1230 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1231 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1232{ 1233 struct lsquic_hash_elem *el; 1234 struct purga_el *puel; 1235 lsquic_conn_t *conn; 1236 1237 if (!(packet_in->pi_flags & PI_CONN_ID)) 1238 { 1239 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1240 return NULL; 1241 } 1242 el = lsquic_hash_find(engine->conns_hash, 1243 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1244 1245 if (el) 1246 { 1247 conn = lsquic_hashelem_getdata(el); 1248 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1249 return conn; 1250 } 1251 1252 if (engine->flags & ENG_COOLDOWN) 1253 { /* Do not create incoming connections during cooldown */ 1254 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1255 return NULL; 1256 } 1257 1258 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1259 { 1260 LSQ_DEBUG("reached limit of %u inchoate connections", 1261 engine->pub.enp_settings.es_max_inchoate); 1262 return NULL; 1263 } 1264 1265 1266 if (engine->purga 1267 && (puel = lsquic_purga_contains(engine->purga, 1268 &packet_in->pi_conn_id), puel)) 1269 { 1270 switch (puel->puel_type) 1271 { 1272 case PUTY_CID_RETIRED: 1273 LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet", 1274 CID_BITS(&packet_in->pi_conn_id)); 1275 return NULL; 1276 case PUTY_CONN_DRAIN: 1277 LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64, 1278 puel->puel_time, packet_in->pi_received); 1279 if (puel->puel_time > packet_in->pi_received) 1280 { 1281 LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet", 1282 CID_BITS(&packet_in->pi_conn_id)); 1283 return NULL; 1284 } 1285 LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted", 1286 CID_BITS(&packet_in->pi_conn_id)); 1287 puel->puel_type = PUTY_CONN_DELETED; 1288 puel->puel_count = 0; 1289 puel->puel_time = 0; 1290 /* fall-through */ 1291 case PUTY_CONN_DELETED: 1292 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1293 CID_BITS(&packet_in->pi_conn_id)); 1294 if (puel->puel_time < packet_in->pi_received) 1295 { 1296 puel->puel_time = packet_in->pi_received 1297 /* Exponential back-off */ 1298 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1299 ++puel->puel_count; 1300 goto maybe_send_prst; 1301 } 1302 return NULL; 1303 default: 1304 assert(0); 1305 return NULL; 1306 } 1307 } 1308 1309 if (engine->pub.enp_settings.es_send_prst 1310 && !(packet_in->pi_flags & PI_GQUIC) 1311 && HETY_NOT_SET == packet_in->pi_header_type) 1312 goto maybe_send_prst; 1313 1314 if (0 != maybe_grow_conn_heaps(engine)) 1315 return NULL; 1316 1317 const struct parse_funcs *pf; 1318 enum lsquic_version version; 1319 switch (version_matches(engine, packet_in, &version)) 1320 { 1321 case VER_UNSUPPORTED: 1322 if (engine->flags & ENG_SERVER) 1323 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1324 sa_local, sa_peer, peer_ctx); 1325 return NULL; 1326 case VER_NOT_SPECIFIED: 1327 maybe_send_prst: 1328 if ((engine->flags & ENG_SERVER) && 1329 engine->pub.enp_settings.es_send_prst) 1330 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1331 sa_local, sa_peer, peer_ctx); 1332 return NULL; 1333 case VER_SUPPORTED: 1334 pf = select_pf_by_ver(version); 1335 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1336 break; 1337 } 1338 1339 1340 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1341 { 1342 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1343 sa_peer->sa_family == AF_INET, NULL, packet_in_size); 1344 } 1345 else 1346 { 1347 conn = lsquic_mini_conn_new(&engine->pub, packet_in, version); 1348 } 1349 if (!conn) 1350 return NULL; 1351 ++engine->mini_conns_count; 1352 ++engine->n_conns; 1353 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1354 { 1355 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1356 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1357 CID_BITS(cid)); 1358 destroy_conn(engine, conn, packet_in->pi_received); 1359 return NULL; 1360 } 1361 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1362 conn->cn_flags |= LSCONN_HASHED; 1363 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1364 conn->cn_last_sent = engine->last_sent; 1365 return conn; 1366} 1367 1368 1369lsquic_conn_t * 1370lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1371 const lsquic_cid_t *cid) 1372{ 1373 struct lsquic_hash_elem *el; 1374 lsquic_conn_t *conn = NULL; 1375 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1376 1377 if (el) 1378 conn = lsquic_hashelem_getdata(el); 1379 return conn; 1380} 1381 1382 1383#if !defined(NDEBUG) && __GNUC__ 1384__attribute__((weak)) 1385#endif 1386void 1387lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1388 lsquic_conn_t *conn) 1389{ 1390 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1391 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1392 { 1393 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1394 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1395 engine_incref_conn(conn, LSCONN_TICKABLE); 1396 } 1397} 1398 1399 1400void 1401lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1402 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1403{ 1404 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1405 if (conn->cn_flags & LSCONN_TICKABLE) 1406 { 1407 /* Optimization: no need to add the connection to the Advisory Tick 1408 * Time Queue: it is about to be ticked, after which it its next tick 1409 * time may be queried again. 1410 */; 1411 } 1412 else if (conn->cn_flags & LSCONN_ATTQ) 1413 { 1414 if (lsquic_conn_adv_time(conn) != tick_time) 1415 { 1416 lsquic_attq_remove(engine->attq, conn); 1417 if (0 != lsquic_attq_add(engine->attq, conn, tick_time, why)) 1418 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1419 } 1420 } 1421 else if (0 == lsquic_attq_add(engine->attq, conn, tick_time, why)) 1422 engine_incref_conn(conn, LSCONN_ATTQ); 1423} 1424 1425 1426static struct lsquic_conn * 1427find_conn_by_srst (struct lsquic_engine *engine, 1428 const struct lsquic_packet_in *packet_in) 1429{ 1430 struct lsquic_hash_elem *el; 1431 struct lsquic_conn *conn; 1432 1433 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1434 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1435 return NULL; 1436 1437 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1438 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1439 IQUIC_SRESET_TOKEN_SZ); 1440 if (!el) 1441 return NULL; 1442 1443 conn = lsquic_hashelem_getdata(el); 1444 return conn; 1445} 1446 1447 1448/* Return 0 if packet is being processed by a real connection (mini or full), 1449 * otherwise return 1. 1450 */ 1451static int 1452process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1453 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1454 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1455{ 1456 lsquic_conn_t *conn; 1457 const unsigned char *packet_in_data; 1458 1459 if (lsquic_packet_in_is_gquic_prst(packet_in) 1460 && !engine->pub.enp_settings.es_honor_prst) 1461 { 1462 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1463 LSQ_DEBUG("public reset packet: discarding"); 1464 return 1; 1465 } 1466 1467 if (engine->flags & ENG_SERVER) 1468 { 1469 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1470 sa_peer, peer_ctx, packet_in_size); 1471 if (!engine->curr_conn) 1472 engine->curr_conn = conn; 1473 } 1474 else 1475 conn = find_conn(engine, packet_in, ppstate, sa_local); 1476 1477 if (!conn) 1478 { 1479 if (engine->pub.enp_settings.es_honor_prst 1480 && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */ 1481 && !(packet_in->pi_flags & PI_GQUIC) 1482 && engine->pub.enp_srst_hash 1483 && (conn = find_conn_by_srst(engine, packet_in))) 1484 { 1485 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1486 CID_BITS(lsquic_conn_log_cid(conn))); 1487 conn->cn_if->ci_stateless_reset(conn); 1488 if (!(conn->cn_flags & LSCONN_TICKABLE) 1489 && conn->cn_if->ci_is_tickable(conn)) 1490 { 1491 lsquic_mh_insert(&engine->conns_tickable, conn, 1492 conn->cn_last_ticked); 1493 engine_incref_conn(conn, LSCONN_TICKABLE); 1494 } 1495 /* Even though the connection processes this packet, we return 1496 * 1 so that the caller does not add reset packet's random 1497 * bytes to the list of valid CIDs. 1498 */ 1499 } 1500 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1501 return 1; 1502 } 1503 1504 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1505 { 1506 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1507 engine_incref_conn(conn, LSCONN_TICKABLE); 1508 } 1509 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1510 sa_local, sa_peer); 1511 lsquic_packet_in_upref(packet_in); 1512#if LOG_PACKET_CHECKSUM 1513 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1514 packet_in->pi_data_sz); 1515#endif 1516 /* Note on QLog: 1517 * For the PACKET_RX QLog event, we are interested in logging these things: 1518 * - raw packet (however it comes in, encrypted or not) 1519 * - frames (list of frame names) 1520 * - packet type and number 1521 * - packet rx timestamp 1522 * 1523 * Since only some of these items are available at this code 1524 * juncture, we will wait until after the packet has been 1525 * decrypted (if necessary) and parsed to call the log functions. 1526 * 1527 * Once the PACKET_RX event is finally logged, the timestamp 1528 * will come from packet_in->pi_received. For correct sequential 1529 * ordering of QLog events, be sure to process the QLogs downstream. 1530 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1531 */ 1532 packet_in_data = packet_in->pi_data; 1533 packet_in_size = packet_in->pi_data_sz; 1534 conn->cn_if->ci_packet_in(conn, packet_in); 1535#if LSQUIC_CONN_STATS 1536 engine->busy.pin_conn = conn; 1537#endif 1538 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1539 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1540 return 0; 1541} 1542 1543 1544void 1545lsquic_engine_destroy (lsquic_engine_t *engine) 1546{ 1547 struct lsquic_hash_elem *el; 1548 lsquic_conn_t *conn; 1549 unsigned i; 1550 1551 LSQ_DEBUG("destroying engine"); 1552#ifndef NDEBUG 1553 engine->flags |= ENG_DTOR; 1554#endif 1555 1556 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1557 { 1558 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1559 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1560 } 1561 1562 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1563 { 1564 assert(conn->cn_flags & LSCONN_TICKABLE); 1565 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1566 } 1567 1568 for (el = lsquic_hash_first(engine->conns_hash); el; 1569 el = lsquic_hash_next(engine->conns_hash)) 1570 { 1571 conn = lsquic_hashelem_getdata(el); 1572 force_close_conn(engine, conn); 1573 } 1574 lsquic_hash_destroy(engine->conns_hash); 1575 1576 while ((conn = lsquic_attq_pop(engine->attq, UINT64_MAX))) 1577 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1578 1579 assert(0 == engine->n_conns); 1580 assert(0 == engine->mini_conns_count); 1581 if (engine->pr_queue) 1582 lsquic_prq_destroy(engine->pr_queue); 1583 if (engine->purga) 1584 lsquic_purga_destroy(engine->purga); 1585 lsquic_attq_destroy(engine->attq); 1586 1587 assert(0 == lsquic_mh_count(&engine->conns_out)); 1588 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1589 if (engine->pub.enp_shi == &stock_shi) 1590 lsquic_stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1591 lsquic_mm_cleanup(&engine->pub.enp_mm); 1592 free(engine->conns_tickable.mh_elems); 1593#if CAN_LOSE_PACKETS 1594 if (engine->flags & ENG_LOSE_PACKETS) 1595 regfree(&engine->lose_packets_re); 1596#endif 1597 if (engine->pub.enp_tokgen) 1598 lsquic_tg_destroy(engine->pub.enp_tokgen); 1599 if (engine->flags & LSENG_SERVER) 1600 lsquic_cleanup_gquic_crypto(&engine->pub); 1601#if LSQUIC_CONN_STATS 1602 if (engine->stats_fh) 1603 { 1604 const struct conn_stats *const stats = &engine->conn_stats_sum; 1605 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1606 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1607 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1608 fprintf(engine->stats_fh, "In:\n"); 1609 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1610 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1611 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1612 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1613 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1614 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1615 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1616 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1617 stats->in.headers_comp, stats->in.headers_uncomp, 1618 stats->in.headers_uncomp ? 1619 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1620 : 0); 1621 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1622 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1623 fprintf(engine->stats_fh, " ACK frames merged: %lu\n", stats->in.n_acks_merged); 1624 fprintf(engine->stats_fh, "Out:\n"); 1625 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1626 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1627 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1628 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1629 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1630 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1631 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1632 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1633 stats->out.headers_comp, stats->out.headers_uncomp, 1634 stats->out.headers_uncomp ? 1635 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1636 : 0); 1637 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1638 } 1639#endif 1640 if (engine->pub.enp_srst_hash) 1641 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1642#if LSQUIC_COUNT_ENGINE_CALLS 1643 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1644#endif 1645 for (i = 0; i < sizeof(engine->retry_aead_ctx) 1646 / sizeof(engine->retry_aead_ctx[0]); ++i) 1647 EVP_AEAD_CTX_cleanup(&engine->pub.enp_retry_aead_ctx[i]); 1648 free(engine->pub.enp_alpn); 1649 free(engine); 1650} 1651 1652 1653static struct conn_cid_elem * 1654find_free_cce (struct lsquic_conn *conn) 1655{ 1656 struct conn_cid_elem *cce; 1657 1658 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1659 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1660 return cce; 1661 1662 return NULL; 1663} 1664 1665 1666static int 1667add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1668 const struct sockaddr *local_sa, void *peer_ctx) 1669{ 1670 struct conn_cid_elem *cce; 1671 1672 if (engine->flags & ENG_CONNS_BY_ADDR) 1673 { 1674 cce = find_free_cce(conn); 1675 if (!cce) 1676 { 1677 LSQ_ERROR("cannot find free CCE"); 1678 return -1; 1679 } 1680 cce->cce_port = sa2port(local_sa); 1681 cce->cce_flags = CCE_PORT; 1682 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1683 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1684 { 1685 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1686 return 0; 1687 } 1688 else 1689 return -1; 1690 1691 } 1692 else 1693 return insert_conn_into_hash(engine, conn, peer_ctx); 1694} 1695 1696 1697lsquic_conn_t * 1698lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1699 const struct sockaddr *local_sa, 1700 const struct sockaddr *peer_sa, 1701 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1702 const char *hostname, unsigned short base_plpmtu, 1703 const unsigned char *sess_resume, size_t sess_resume_len, 1704 const unsigned char *token, size_t token_sz) 1705{ 1706 lsquic_conn_t *conn; 1707 unsigned flags, versions; 1708 int is_ipv4; 1709 1710 ENGINE_CALLS_INCR(engine); 1711 1712 if (engine->flags & ENG_SERVER) 1713 { 1714 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1715 goto err; 1716 } 1717 1718 if (engine->flags & ENG_CONNS_BY_ADDR 1719 && find_conn_by_addr(engine->conns_hash, local_sa)) 1720 { 1721 LSQ_ERROR("cannot have more than one connection on the same port"); 1722 goto err; 1723 } 1724 1725 if (0 != maybe_grow_conn_heaps(engine)) 1726 return NULL; 1727 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1728 is_ipv4 = peer_sa->sa_family == AF_INET; 1729 if (sess_resume && sess_resume_len) 1730 { 1731 version = lsquic_sess_resume_version(sess_resume, sess_resume_len); 1732 if (version >= N_LSQVER) 1733 { 1734 LSQ_INFO("session resumption version is bad, won't use"); 1735 sess_resume = NULL; 1736 sess_resume_len = 0; 1737 } 1738 } 1739 if (version >= N_LSQVER) 1740 { 1741 if (version > N_LSQVER) 1742 LSQ_WARN("invalid version specified, engine will pick"); 1743 versions = engine->pub.enp_settings.es_versions; 1744 } 1745 else 1746 versions = 1u << version; 1747 if (versions & LSQUIC_IETF_VERSIONS) 1748 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1749 flags, hostname, base_plpmtu, 1750 is_ipv4, sess_resume, sess_resume_len, token, token_sz, peer_ctx); 1751 else 1752 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1753 flags, hostname, base_plpmtu, is_ipv4, 1754 sess_resume, sess_resume_len); 1755 if (!conn) 1756 goto err; 1757 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1758 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1759 lsquic_ver2str[conn->cn_version]); 1760 ++engine->n_conns; 1761 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1762 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1763 { 1764 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1765 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1766 CID_BITS(cid)); 1767 destroy_conn(engine, conn, lsquic_time_now()); 1768 goto err; 1769 } 1770 assert(!(conn->cn_flags & 1771 (CONN_REF_FLAGS 1772 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1773 callbacks */ 1774 ))); 1775 conn->cn_flags |= LSCONN_HASHED; 1776 if (!(conn->cn_flags & LSCONN_TICKABLE)) 1777 { 1778 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1779 engine_incref_conn(conn, LSCONN_TICKABLE); 1780 } 1781 lsquic_conn_set_ctx(conn, conn_ctx); 1782 conn->cn_if->ci_client_call_on_new(conn); 1783 end: 1784 return conn; 1785 err: 1786 conn = NULL; 1787 goto end; 1788} 1789 1790 1791static void 1792remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1793{ 1794 remove_all_cces_from_hash(engine->conns_hash, conn); 1795 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1796} 1797 1798 1799static void 1800refflags2str (enum lsquic_conn_flags flags, char s[7]) 1801{ 1802 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1803 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1804 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1805 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1806 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1807 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1808 *s = '\0'; 1809} 1810 1811 1812static void 1813engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1814{ 1815 char str[2][7]; 1816 assert(flag & CONN_REF_FLAGS); 1817 assert(!(conn->cn_flags & flag)); 1818 conn->cn_flags |= flag; 1819 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1820 CID_BITS(lsquic_conn_log_cid(conn)), 1821 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1822 (refflags2str(conn->cn_flags, str[1]), str[1])); 1823} 1824 1825 1826static lsquic_conn_t * 1827engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1828 enum lsquic_conn_flags flags) 1829{ 1830 char str[2][7]; 1831 lsquic_time_t now; 1832 assert(flags & CONN_REF_FLAGS); 1833 assert(conn->cn_flags & flags); 1834#ifndef NDEBUG 1835 if (flags & LSCONN_CLOSING) 1836 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1837#endif 1838 conn->cn_flags &= ~flags; 1839 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1840 CID_BITS(lsquic_conn_log_cid(conn)), 1841 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1842 (refflags2str(conn->cn_flags, str[1]), str[1])); 1843 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1844 { 1845 now = lsquic_time_now(); 1846 if (conn->cn_flags & LSCONN_MINI) 1847 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1848 else 1849 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1850 destroy_conn(engine, conn, now); 1851 return NULL; 1852 } 1853 else 1854 return conn; 1855} 1856 1857 1858/* This is not a general-purpose function. Only call from engine dtor. */ 1859static void 1860force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1861{ 1862 assert(engine->flags & ENG_DTOR); 1863 const enum lsquic_conn_flags flags = conn->cn_flags; 1864 assert(conn->cn_flags & CONN_REF_FLAGS); 1865 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1866 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1867 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1868 if (flags & LSCONN_ATTQ) 1869 { 1870 lsquic_attq_remove(engine->attq, conn); 1871 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1872 } 1873 if (flags & LSCONN_HASHED) 1874 remove_conn_from_hash(engine, conn); 1875} 1876 1877 1878/* Iterator for tickable connections (those on the Tickable Queue). Before 1879 * a connection is returned, it is removed from the Advisory Tick Time queue 1880 * if necessary. 1881 */ 1882static lsquic_conn_t * 1883conn_iter_next_tickable (struct lsquic_engine *engine) 1884{ 1885 lsquic_conn_t *conn; 1886 1887 if (engine->flags & ENG_SERVER) 1888 while (1) 1889 { 1890 conn = lsquic_mh_pop(&engine->conns_tickable); 1891 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1892 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1893 else 1894 break; 1895 } 1896 else 1897 conn = lsquic_mh_pop(&engine->conns_tickable); 1898 1899 if (conn) 1900 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1901 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1902 { 1903 lsquic_attq_remove(engine->attq, conn); 1904 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1905 } 1906 1907 return conn; 1908} 1909 1910 1911static void 1912cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1913 void *update_ctx) 1914{ 1915 cub->cub_update_cids = update; 1916 cub->cub_update_ctx = update_ctx; 1917 cub->cub_count = 0; 1918} 1919 1920 1921static void 1922cub_flush (struct cid_update_batch *cub) 1923{ 1924 if (cub->cub_count > 0 && cub->cub_update_cids) 1925 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1926 cub->cub_cids, cub->cub_count); 1927 cub->cub_count = 0; 1928} 1929 1930 1931static void 1932cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1933{ 1934 cub->cub_cids [ cub->cub_count ] = *cid; 1935 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1936 ++cub->cub_count; 1937 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1938 cub_flush(cub); 1939} 1940 1941 1942/* Process registered CIDs */ 1943static void 1944cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1945{ 1946 struct cce_cid_iter citer; 1947 struct conn_cid_elem *cce; 1948 void *peer_ctx; 1949 1950 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1951 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1952 if (cce->cce_flags & CCE_REG) 1953 cub_add(cub, &cce->cce_cid, peer_ctx); 1954} 1955 1956 1957static void 1958drop_all_mini_conns (lsquic_engine_t *engine) 1959{ 1960 struct lsquic_hash_elem *el; 1961 lsquic_conn_t *conn; 1962 struct cid_update_batch cub; 1963 1964 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1965 1966 for (el = lsquic_hash_first(engine->conns_hash); el; 1967 el = lsquic_hash_next(engine->conns_hash)) 1968 { 1969 conn = lsquic_hashelem_getdata(el); 1970 if (conn->cn_flags & LSCONN_MINI) 1971 { 1972 /* If promoted, why is it still in this hash? */ 1973 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1974 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1975 cub_add_cids_from_cces(&cub, conn); 1976 remove_conn_from_hash(engine, conn); 1977 } 1978 } 1979 1980 cub_flush(&cub); 1981} 1982 1983 1984void 1985lsquic_engine_process_conns (lsquic_engine_t *engine) 1986{ 1987 lsquic_conn_t *conn; 1988 lsquic_time_t now; 1989 1990#if LSQUIC_CONN_STATS 1991 if (engine->busy.pin_conn) 1992 { 1993 update_busy_detector(engine, engine->busy.pin_conn, 1); 1994 engine->busy.pin_conn = NULL; 1995 } 1996#endif 1997 1998 ENGINE_IN(engine); 1999 2000 now = lsquic_time_now(); 2001 while ((conn = lsquic_attq_pop(engine->attq, now))) 2002 { 2003 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 2004 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 2005 { 2006 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2007 engine_incref_conn(conn, LSCONN_TICKABLE); 2008 } 2009 } 2010 2011 process_connections(engine, conn_iter_next_tickable, now); 2012 ENGINE_OUT(engine); 2013} 2014 2015 2016static void 2017release_or_return_enc_data (struct lsquic_engine *engine, 2018 void (*pmi_rel_or_ret) (void *, void *, void *, char), 2019 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 2020{ 2021 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 2022 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 2023 packet_out->po_flags &= ~PO_ENCRYPTED; 2024 packet_out->po_enc_data = NULL; 2025} 2026 2027 2028static void 2029release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 2030 struct lsquic_packet_out *packet_out) 2031{ 2032 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 2033 conn, packet_out); 2034} 2035 2036 2037static void 2038return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 2039 struct lsquic_packet_out *packet_out) 2040{ 2041 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 2042 conn, packet_out); 2043} 2044 2045 2046static int 2047copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 2048 struct lsquic_packet_out *packet_out) 2049{ 2050 int ipv6; 2051 2052 ipv6 = NP_IS_IPv6(packet_out->po_path); 2053 if (packet_out->po_flags & PO_ENCRYPTED) 2054 { 2055 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 2056 && packet_out->po_data_sz == packet_out->po_enc_data_sz 2057 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 2058 packet_out->po_data_sz)) 2059 return 0; 2060 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 2061 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 2062 goto copy; 2063 return_enc_data(engine, conn, packet_out); 2064 } 2065 2066 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 2067 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 2068 conn->cn_conn_ctx, packet_out->po_data_sz, ipv6); 2069 if (!packet_out->po_enc_data) 2070 { 2071 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 2072 "of size %hu", packet_out->po_data_sz); 2073 return -1; 2074 } 2075 2076 copy: 2077 memcpy(packet_out->po_enc_data, packet_out->po_data, 2078 packet_out->po_data_sz); 2079 packet_out->po_enc_data_sz = packet_out->po_data_sz; 2080 packet_out->po_sent_sz = packet_out->po_data_sz; 2081 packet_out->po_flags &= ~PO_IPv6; 2082 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 2083 2084 return 0; 2085} 2086 2087 2088STAILQ_HEAD(conns_stailq, lsquic_conn); 2089TAILQ_HEAD(conns_tailq, lsquic_conn); 2090 2091 2092struct conns_out_iter 2093{ 2094 struct min_heap *coi_heap; 2095 struct pr_queue *coi_prq; 2096 TAILQ_HEAD(, lsquic_conn) coi_active_list, 2097 coi_inactive_list; 2098 lsquic_conn_t *coi_next; 2099#ifndef NDEBUG 2100 lsquic_time_t coi_last_sent; 2101#endif 2102}; 2103 2104 2105static void 2106coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 2107{ 2108 iter->coi_heap = &engine->conns_out; 2109 iter->coi_prq = engine->pr_queue; 2110 iter->coi_next = NULL; 2111 TAILQ_INIT(&iter->coi_active_list); 2112 TAILQ_INIT(&iter->coi_inactive_list); 2113#ifndef NDEBUG 2114 iter->coi_last_sent = 0; 2115#endif 2116} 2117 2118 2119static lsquic_conn_t * 2120coi_next (struct conns_out_iter *iter) 2121{ 2122 lsquic_conn_t *conn; 2123 2124 if (lsquic_mh_count(iter->coi_heap) > 0) 2125 { 2126 conn = lsquic_mh_pop(iter->coi_heap); 2127 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2128 conn->cn_flags |= LSCONN_COI_ACTIVE; 2129#ifndef NDEBUG 2130 if (iter->coi_last_sent) 2131 assert(iter->coi_last_sent <= conn->cn_last_sent); 2132 iter->coi_last_sent = conn->cn_last_sent; 2133#endif 2134 return conn; 2135 } 2136 else if (iter->coi_prq && (conn = lsquic_prq_next_conn(iter->coi_prq))) 2137 { 2138 return conn; 2139 } 2140 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 2141 { 2142 iter->coi_prq = NULL; /* Save function call in previous conditional */ 2143 conn = iter->coi_next; 2144 if (!conn) 2145 conn = TAILQ_FIRST(&iter->coi_active_list); 2146 if (conn) 2147 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 2148 return conn; 2149 } 2150 else 2151 return NULL; 2152} 2153 2154 2155static void 2156coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2157{ 2158 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2159 { 2160 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 2161 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2162 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2163 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 2164 conn->cn_flags |= LSCONN_COI_INACTIVE; 2165 } 2166} 2167 2168 2169static void 2170coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2171{ 2172 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 2173 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2174 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2175 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2176 conn->cn_flags |= LSCONN_COI_ACTIVE; 2177} 2178 2179 2180static void 2181coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 2182{ 2183 lsquic_conn_t *conn; 2184 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 2185 { 2186 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2187 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2188 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 2189 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 2190 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 2191 else /* Closed connection gets one shot at sending packets */ 2192 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2193 } 2194 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 2195 { 2196 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2197 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2198 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2199 } 2200} 2201 2202 2203#if CAN_LOSE_PACKETS 2204static void 2205lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 2206 unsigned n) 2207{ 2208 const lsquic_cid_t *cid; 2209 struct iovec *iov; 2210 unsigned i; 2211 char packno_str[22]; 2212 2213 for (i = 0; i < n; ++i) 2214 { 2215 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 2216 batch->packets[i]->po_packno); 2217 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 2218 { 2219 for (iov = batch->outs[i].iov; iov < 2220 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 2221 batch->outs[i].iov->iov_len -= 1; 2222 cid = lsquic_conn_log_cid(batch->conns[i]); 2223 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 2224 CID_BITS(cid)); 2225 } 2226 } 2227} 2228 2229 2230#endif 2231 2232 2233#ifdef NDEBUG 2234#define CONST_BATCH const 2235#else 2236#define CONST_BATCH 2237#endif 2238 2239 2240struct send_batch_ctx { 2241 struct conns_stailq *closed_conns; 2242 struct conns_tailq *ticked_conns; 2243 struct conns_out_iter *conns_iter; 2244 CONST_BATCH struct out_batch *batch; 2245}; 2246 2247 2248static void 2249close_conn_immediately (struct lsquic_engine *engine, 2250 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2251{ 2252 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2253 if (!(conn->cn_flags & LSCONN_CLOSING)) 2254 { 2255 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2256 engine_incref_conn(conn, LSCONN_CLOSING); 2257 if (conn->cn_flags & LSCONN_HASHED) 2258 remove_conn_from_hash(engine, conn); 2259 } 2260 if (conn->cn_flags & LSCONN_TICKED) 2261 { 2262 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2263 engine_decref_conn(engine, conn, LSCONN_TICKED); 2264 } 2265} 2266 2267 2268static void 2269close_conn_on_send_error (struct lsquic_engine *engine, 2270 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2271{ 2272 const struct out_batch *batch = sb_ctx->batch; 2273 struct lsquic_conn *const conn = batch->conns[n]; 2274 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2275 2276 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2277 "src: %s; dst: %s; errno: %d", 2278 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2279 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2280 CID_BITS(lsquic_conn_log_cid(conn)), 2281 SA2STR(batch->outs[n].local_sa, buf[0]), 2282 SA2STR(batch->outs[n].dest_sa, buf[1]), 2283 e_val); 2284 if (conn->cn_flags & LSCONN_EVANESCENT) 2285 lsquic_prq_drop(conn); 2286 else 2287 close_conn_immediately(engine, sb_ctx, conn); 2288} 2289 2290 2291static void 2292apply_hp (struct conns_out_iter *iter) 2293{ 2294 struct lsquic_conn *conn; 2295 2296 TAILQ_FOREACH(conn, &iter->coi_active_list, cn_next_out) 2297 if (conn->cn_esf_c->esf_flush_encryption && conn->cn_enc_session) 2298 conn->cn_esf_c->esf_flush_encryption(conn->cn_enc_session); 2299 TAILQ_FOREACH(conn, &iter->coi_inactive_list, cn_next_out) 2300 if (conn->cn_esf_c->esf_flush_encryption && conn->cn_enc_session) 2301 conn->cn_esf_c->esf_flush_encryption(conn->cn_enc_session); 2302} 2303 2304 2305#if LSQUIC_CONN_STATS 2306static void 2307update_batch_size_stats (struct lsquic_engine *engine, unsigned batch_size) 2308{ 2309 struct batch_size_stats *const stats = &engine->pub.enp_batch_size_stats; 2310 2311 ++stats->count; 2312 if (batch_size > stats->max) 2313 stats->max = batch_size; 2314 if (batch_size < stats->min || 0 == stats->min) 2315 stats->min = batch_size; 2316 if (stats->avg) 2317 stats->avg = ((float) batch_size - stats->avg) * 0.4 + stats->avg; 2318 else 2319 stats->avg = (float) batch_size; 2320} 2321 2322 2323#endif 2324 2325 2326static unsigned 2327send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2328 unsigned n_to_send) 2329{ 2330 int n_sent, i, e_val; 2331 lsquic_time_t now; 2332 unsigned off, skip; 2333 size_t count; 2334 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2335 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2336 2337#if LSQUIC_CONN_STATS 2338 update_batch_size_stats(engine, n_to_send); 2339#endif 2340 2341 apply_hp(sb_ctx->conns_iter); 2342#if CAN_LOSE_PACKETS 2343 if (engine->flags & ENG_LOSE_PACKETS) 2344 lose_matching_packets(engine, batch, n_to_send); 2345#endif 2346 skip = 0; 2347 restart_batch: 2348 /* Set sent time before the write to avoid underestimating RTT */ 2349 now = lsquic_time_now(); 2350 for (i = skip; i < (int) (n_to_send - skip); ++i) 2351 { 2352 off = batch->pack_off[i]; 2353 count = batch->outs[i].iovlen; 2354 assert(count > 0); 2355 packet_out = &batch->packets[off]; 2356 end = packet_out + count; 2357 do 2358 (*packet_out)->po_sent = now; 2359 while (++packet_out < end); 2360 } 2361 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs + skip, 2362 n_to_send - skip); 2363 e_val = errno; 2364 if (n_sent < (int) (n_to_send - skip) && e_val != EMSGSIZE) 2365 { 2366 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2367 engine->resume_sending_at = now + 1000000; 2368 LSQ_DEBUG("cannot send packets"); 2369 EV_LOG_GENERIC_EVENT("cannot send packets"); 2370 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2371 close_conn_on_send_error(engine, sb_ctx, 2372 n_sent < 0 ? 0 : n_sent, e_val); 2373 } 2374 if (n_sent >= 0) 2375 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, 2376 n_to_send - skip); 2377 else 2378 { 2379 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2380 n_sent = 0; 2381 } 2382 if (n_sent > 0) 2383 engine->last_sent = now + n_sent; 2384 for (i = skip; i < (int) (skip + n_sent); ++i) 2385 { 2386 eng_hist_inc(&engine->history, now, sl_packets_out); 2387 /* `i' is added to maintain relative order */ 2388 batch->conns[i]->cn_last_sent = now + i; 2389 2390 off = batch->pack_off[i]; 2391 count = batch->outs[i].iovlen; 2392 assert(count > 0); 2393 packet_out = &batch->packets[off]; 2394 end = packet_out + count; 2395 do 2396 { 2397#if LOG_PACKET_CHECKSUM 2398 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2399 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2400 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2401#endif 2402 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2403 *packet_out); 2404 /* Release packet out buffer as soon as the packet is sent 2405 * successfully. If not successfully sent, we hold on to 2406 * this buffer until the packet sending is attempted again 2407 * or until it times out and regenerated. 2408 */ 2409 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2410 release_enc_data(engine, batch->conns[i], *packet_out); 2411 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2412 *packet_out); 2413 } 2414 while (++packet_out < end); 2415 } 2416 if (i < (int) n_to_send && e_val == EMSGSIZE) 2417 { 2418 LSQ_DEBUG("packet #%d could not be sent out for being too large", i); 2419 if (batch->conns[i]->cn_if->ci_packet_too_large 2420 && batch->outs[i].iovlen == 1) 2421 { 2422 off = batch->pack_off[i]; 2423 packet_out = &batch->packets[off]; 2424 batch->conns[i]->cn_if->ci_packet_too_large(batch->conns[i], 2425 *packet_out); 2426 ++i; 2427 if (i < (int) n_to_send) 2428 { 2429 skip = i; 2430 LSQ_DEBUG("restart batch starting at packet #%u", skip); 2431 goto restart_batch; 2432 } 2433 n_sent = n_to_send; 2434 } 2435 else 2436 close_conn_on_send_error(engine, sb_ctx, i, e_val); 2437 } 2438 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2439 for ( ; i < (int) n_to_send; ++i) 2440 { 2441 off = batch->pack_off[i]; 2442 count = batch->outs[i].iovlen; 2443 assert(count > 0); 2444 packet_out = &batch->packets[off]; 2445 end = packet_out + count; 2446 do 2447 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2448 *packet_out); 2449 while (++packet_out < end); 2450 } 2451 /* Return packets to the connection in reverse order so that the packet 2452 * ordering is maintained. 2453 */ 2454 for (i = (int) n_to_send - 1; i >= (int) (skip + n_sent); --i) 2455 { 2456 off = batch->pack_off[i]; 2457 count = batch->outs[i].iovlen; 2458 assert(count > 0); 2459 packet_out = &batch->packets[off + count - 1]; 2460 end = &batch->packets[off - 1]; 2461 do 2462 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2463 *packet_out); 2464 while (--packet_out > end); 2465 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2466 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2467 } 2468 return skip + n_sent; 2469} 2470 2471 2472/* Return 1 if went past deadline, 0 otherwise */ 2473static int 2474check_deadline (lsquic_engine_t *engine) 2475{ 2476 if (engine->pub.enp_settings.es_proc_time_thresh && 2477 lsquic_time_now() > engine->deadline) 2478 { 2479 LSQ_INFO("went past threshold of %u usec, stop sending", 2480 engine->pub.enp_settings.es_proc_time_thresh); 2481 engine->flags |= ENG_PAST_DEADLINE; 2482 return 1; 2483 } 2484 else 2485 return 0; 2486} 2487 2488 2489static size_t 2490iov_size (const struct iovec *iov, const struct iovec *const end) 2491{ 2492 size_t size; 2493 2494 assert(iov < end); 2495 2496 size = 0; 2497 do 2498 size += iov->iov_len; 2499 while (++iov < end); 2500 2501 return size; 2502} 2503 2504 2505static void 2506send_packets_out (struct lsquic_engine *engine, 2507 struct conns_tailq *ticked_conns, 2508 struct conns_stailq *closed_conns) 2509{ 2510 unsigned n, w, n_sent, n_batches_sent; 2511 lsquic_packet_out_t *packet_out; 2512 struct lsquic_packet_out **packet; 2513 lsquic_conn_t *conn; 2514 struct out_batch *const batch = &engine->out_batch; 2515 struct iovec *iov, *packet_iov; 2516 struct conns_out_iter conns_iter; 2517 int shrink, deadline_exceeded; 2518 const struct send_batch_ctx sb_ctx = { 2519 closed_conns, 2520 ticked_conns, 2521 &conns_iter, 2522 &engine->out_batch, 2523 }; 2524 2525 coi_init(&conns_iter, engine); 2526 n_batches_sent = 0; 2527 n_sent = 0, n = 0; 2528 shrink = 0; 2529 deadline_exceeded = 0; 2530 iov = batch->iov; 2531 packet = batch->packets; 2532 2533 while ((conn = coi_next(&conns_iter))) 2534 { 2535 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2536 if (!packet_out) { 2537 /* Evanescent connection always has a packet to send: */ 2538 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2539 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2540 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2541 CID_BITS(lsquic_conn_log_cid(conn))); 2542 coi_deactivate(&conns_iter, conn); 2543 continue; 2544 } 2545 batch->outs[n].iov = packet_iov = iov; 2546 next_coa: 2547 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2548 { 2549 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2550 &engine->pub, conn, packet_out)) 2551 { 2552 case ENCPA_NOMEM: 2553 /* Send what we have and wait for a more opportune moment */ 2554 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2555 goto end_for; 2556 case ENCPA_BADCRYPT: 2557 /* This is pretty bad: close connection immediately */ 2558 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2559 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2560 CID_BITS(lsquic_conn_log_cid(conn))); 2561 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2562 { 2563 close_conn_immediately(engine, &sb_ctx, conn); 2564 coi_deactivate(&conns_iter, conn); 2565 } 2566 continue; 2567 case ENCPA_OK: 2568 break; 2569 } 2570 } 2571 else if ((packet_out->po_flags & PO_NOENCRYPT) 2572 && engine->pub.enp_pmi != &stock_pmi) 2573 { 2574 if (0 != copy_packet(engine, conn, packet_out)) 2575 { 2576 /* Copy can only fail if packet could not be allocated */ 2577 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2578 goto end_for; 2579 } 2580 } 2581 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2582 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2583 if (packet_out->po_flags & PO_ENCRYPTED) 2584 { 2585 iov->iov_base = packet_out->po_enc_data; 2586 iov->iov_len = packet_out->po_enc_data_sz; 2587 } 2588 else 2589 { 2590 iov->iov_base = packet_out->po_data; 2591 iov->iov_len = packet_out->po_data_sz; 2592 } 2593 if (packet_iov == iov) 2594 { 2595 batch->pack_off[n] = packet - batch->packets; 2596 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2597 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2598 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2599 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2600 batch->outs [n].conn_ctx = conn->cn_conn_ctx; 2601 batch->conns [n] = conn; 2602 } 2603 *packet = packet_out; 2604 ++packet; 2605 ++iov; 2606 if ((conn->cn_flags & LSCONN_IETF) 2607 && ((1 << packet_out->po_header_type) 2608 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2609#ifndef NDEBUG 2610 && (engine->flags & ENG_COALESCE) 2611#endif 2612 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2613 { 2614 const struct to_coal to_coal = { 2615 .prev_packet = packet_out, 2616 .prev_sz_sum = iov_size(packet_iov, iov), 2617 }; 2618 packet_out = conn->cn_if->ci_next_packet_to_send(conn, &to_coal); 2619 if (packet_out) 2620 goto next_coa; 2621 } 2622 batch->outs [n].iovlen = iov - packet_iov; 2623 ++n; 2624 if (n == engine->batch_size 2625 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2626 { 2627 w = send_batch(engine, &sb_ctx, n); 2628 n = 0; 2629 iov = batch->iov; 2630 packet = batch->packets; 2631 ++n_batches_sent; 2632 n_sent += w; 2633 if (w < engine->batch_size) 2634 { 2635 shrink = 1; 2636 break; 2637 } 2638 deadline_exceeded = check_deadline(engine); 2639 if (deadline_exceeded) 2640 break; 2641 grow_batch_size(engine); 2642 } 2643 } 2644 end_for: 2645 2646 if (n > 0) { 2647 w = send_batch(engine, &sb_ctx, n); 2648 n_sent += w; 2649 shrink = w < n; 2650 ++n_batches_sent; 2651 } 2652 2653 if (shrink) 2654 shrink_batch_size(engine); 2655 else if (n_batches_sent > 1) 2656 { 2657 deadline_exceeded = check_deadline(engine); 2658 if (!deadline_exceeded) 2659 grow_batch_size(engine); 2660 } 2661 2662 coi_reheap(&conns_iter, engine); 2663 2664 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2665} 2666 2667 2668int 2669lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2670{ 2671 return lsquic_mh_count(&engine->conns_out) > 0 2672 || (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 2673 ; 2674} 2675 2676 2677static void 2678reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2679{ 2680 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2681 engine->flags &= ~ENG_PAST_DEADLINE; 2682} 2683 2684 2685static void 2686check_tickable_conns_again (struct lsquic_engine *engine) 2687{ 2688 struct lsquic_hash_elem *el; 2689 struct lsquic_conn *conn; 2690 unsigned count; 2691 2692 count = 0; 2693 for (el = lsquic_hash_first(engine->conns_hash); el; 2694 el = lsquic_hash_next(engine->conns_hash)) 2695 { 2696 conn = lsquic_hashelem_getdata(el); 2697 if (!(conn->cn_flags & LSCONN_TICKABLE) 2698 && conn->cn_if->ci_is_tickable(conn)) 2699 { 2700 lsquic_mh_insert(&engine->conns_tickable, conn, 2701 conn->cn_last_ticked); 2702 engine_incref_conn(conn, LSCONN_TICKABLE); 2703 ++count; 2704 } 2705 } 2706 LSQ_DEBUG("%u connection%s tickable again after sending has been " 2707 "re-enabled", count, count == 1 ? " is" : "s are"); 2708} 2709 2710 2711void 2712lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2713{ 2714 lsquic_conn_t *conn; 2715 struct conns_stailq closed_conns; 2716 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2717 struct cid_update_batch cub; 2718 2719 ENGINE_IN(engine); 2720 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2721 STAILQ_INIT(&closed_conns); 2722 reset_deadline(engine, lsquic_time_now()); 2723 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2724 { 2725 LSQ_DEBUG("can send again"); 2726 EV_LOG_GENERIC_EVENT("can send again"); 2727 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2728 check_tickable_conns_again(engine); 2729 } 2730 2731 send_packets_out(engine, &ticked_conns, &closed_conns); 2732 2733 while ((conn = STAILQ_FIRST(&closed_conns))) { 2734 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2735 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2736 cub_add_cids_from_cces(&cub, conn); 2737 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2738 } 2739 2740 cub_flush(&cub); 2741 ENGINE_OUT(engine); 2742} 2743 2744 2745static lsquic_conn_t * 2746next_new_full_conn (struct conns_stailq *new_full_conns) 2747{ 2748 lsquic_conn_t *conn; 2749 2750 conn = STAILQ_FIRST(new_full_conns); 2751 if (conn) 2752 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2753 return conn; 2754} 2755 2756 2757#if LSQUIC_CONN_STATS 2758static void 2759maybe_log_conn_stats (struct lsquic_engine *engine, struct lsquic_conn *conn, 2760 lsquic_time_t now) 2761{ 2762 char cidstr[MAX_CID_LEN * 2 + 1]; 2763 2764 if (!LSQ_LOG_ENABLED_EXT(LSQ_LOG_NOTICE, LSQLM_CONN_STATS)) 2765 return; 2766 2767 if (conn->cn_last_ticked + 1000000 >= now) 2768 { 2769 if (0 == engine->busy.last_log 2770 || engine->busy.last_log + 1000000 - 1000 < now) 2771 { 2772 engine->busy.last_log = now; 2773 conn->cn_if->ci_log_stats(conn); 2774 } 2775 } 2776 else 2777 { 2778 lsquic_logger_log1(LSQ_LOG_NOTICE, LSQLM_CONN_STATS, 2779 "stop logging status for connection %s: no longer busy", 2780 (lsquic_cid2str(lsquic_conn_log_cid(conn), cidstr), cidstr)); 2781 engine->busy.current = NULL; 2782 engine->busy.last_log = 0; 2783 } 2784} 2785 2786 2787#endif 2788 2789 2790static void 2791process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2792 lsquic_time_t now) 2793{ 2794 lsquic_conn_t *conn; 2795 enum tick_st tick_st; 2796 unsigned i, why; 2797 lsquic_time_t next_tick_time; 2798 struct conns_stailq closed_conns; 2799 struct conns_tailq ticked_conns; 2800 struct conns_stailq new_full_conns; 2801 struct cid_update_batch cub_old, cub_live; 2802 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2803 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2804 2805 eng_hist_tick(&engine->history, now); 2806 2807 STAILQ_INIT(&closed_conns); 2808 TAILQ_INIT(&ticked_conns); 2809 reset_deadline(engine, now); 2810 STAILQ_INIT(&new_full_conns); 2811 2812 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2813 && now > engine->resume_sending_at) 2814 { 2815 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2816 "timeout"); 2817 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2818 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2819 } 2820 2821 i = 0; 2822 while ((conn = next_conn(engine)) 2823 || (conn = next_new_full_conn(&new_full_conns))) 2824 { 2825 tick_st = conn->cn_if->ci_tick(conn, now); 2826#if LSQUIC_CONN_STATS 2827 if (conn == engine->busy.current) 2828 maybe_log_conn_stats(engine, conn, now); 2829#endif 2830 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2831 if (tick_st & TICK_PROMOTE) 2832 { 2833 lsquic_conn_t *new_conn; 2834 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2835 "scheduled for promotion"); 2836 assert(conn->cn_flags & LSCONN_MINI); 2837 new_conn = new_full_conn_server(engine, conn, now); 2838 if (new_conn) 2839 { 2840 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2841 new_conn->cn_last_sent = engine->last_sent; 2842 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2843 conn->cn_flags |= LSCONN_PROMOTED; 2844 } 2845 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2846 } 2847 if (tick_st & TICK_SEND) 2848 { 2849 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2850 { 2851 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2852 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2853 } 2854 } 2855 if (tick_st & TICK_CLOSE) 2856 { 2857 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2858 engine_incref_conn(conn, LSCONN_CLOSING); 2859 if (conn->cn_flags & LSCONN_HASHED) 2860 remove_conn_from_hash(engine, conn); 2861 } 2862 else 2863 { 2864 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2865 engine_incref_conn(conn, LSCONN_TICKED); 2866 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2867 && conn->cn_if->ci_report_live(conn, now)) 2868 cub_add_cids_from_cces(&cub_live, conn); 2869 } 2870 } 2871 2872 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2873 && lsquic_engine_has_unsent_packets(engine)) 2874 send_packets_out(engine, &ticked_conns, &closed_conns); 2875 2876 while ((conn = STAILQ_FIRST(&closed_conns))) { 2877 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2878 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2879 cub_add_cids_from_cces(&cub_old, conn); 2880 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2881 } 2882 2883 while ((conn = TAILQ_FIRST(&ticked_conns))) 2884 { 2885 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2886 engine_decref_conn(engine, conn, LSCONN_TICKED); 2887 if (!(conn->cn_flags & LSCONN_TICKABLE) 2888 && conn->cn_if->ci_is_tickable(conn)) 2889 { 2890 /* Floyd heapification is not faster, don't bother. */ 2891 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2892 engine_incref_conn(conn, LSCONN_TICKABLE); 2893 } 2894 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2895 { 2896 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2897 if (next_tick_time) 2898 { 2899 if (0 == lsquic_attq_add(engine->attq, conn, next_tick_time, 2900 why)) 2901 engine_incref_conn(conn, LSCONN_ATTQ); 2902 } 2903 else 2904 /* In all other cases, the idle timeout would make the next 2905 * tick time non-zero: 2906 */ 2907 assert((conn->cn_flags & LSCONN_IETF) 2908 && engine->pub.enp_settings.es_idle_timeout == 0); 2909 } 2910 } 2911 2912 cub_flush(&engine->new_scids); 2913 cub_flush(&cub_live); 2914 cub_flush(&cub_old); 2915} 2916 2917 2918static void 2919maybe_count_garbage (struct lsquic_engine *engine, size_t garbage_sz) 2920{ 2921 /* This is not very pretty (action at a distance via engine->curr_conn), 2922 * but it's the cheapest I can come up with to handle the "count garbage 2923 * toward amplification limit" requirement in 2924 * [draft-ietf-quic-transport-28] Section 8.1. 2925 */ 2926 if (engine->curr_conn && engine->curr_conn->cn_if->ci_count_garbage) 2927 engine->curr_conn->cn_if->ci_count_garbage(engine->curr_conn, 2928 garbage_sz); 2929} 2930 2931 2932/* Return 0 if packet is being processed by a real connection, 1 if the 2933 * packet was processed, but not by a connection, and -1 on error. 2934 */ 2935int 2936lsquic_engine_packet_in (lsquic_engine_t *engine, 2937 const unsigned char *packet_in_data, size_t packet_in_size, 2938 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2939 void *peer_ctx, int ecn) 2940{ 2941 const unsigned char *const packet_begin = packet_in_data; 2942 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2943 struct packin_parse_state ppstate; 2944 lsquic_packet_in_t *packet_in; 2945 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2946 int is_server, unsigned cid_len, struct packin_parse_state *); 2947 unsigned n_zeroes; 2948 int s, is_ietf; 2949 lsquic_cid_t cid; 2950 2951 ENGINE_CALLS_INCR(engine); 2952 2953 if (engine->flags & ENG_SERVER) 2954 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2955 else if (engine->flags & ENG_CONNS_BY_ADDR) 2956 { 2957 struct lsquic_hash_elem *el; 2958 const struct lsquic_conn *conn; 2959 el = find_conn_by_addr(engine->conns_hash, sa_local); 2960 if (!el) 2961 return -1; 2962 conn = lsquic_hashelem_getdata(el); 2963 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2964 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2965 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2966 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2967 else if (conn->cn_version == LSQVER_050) 2968 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 2969 else 2970 { 2971#if LSQUIC_USE_Q098 2972 assert(conn->cn_version == LSQVER_046 || conn->cn_version == LSQVER_098); 2973#else 2974 assert(conn->cn_version == LSQVER_046); 2975#endif 2976 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2977 } 2978 } 2979 else 2980 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2981 2982 engine->curr_conn = NULL; 2983 n_zeroes = 0; 2984 is_ietf = 0; 2985#ifdef _MSC_VER 2986 s = 0; 2987 cid.len = 0; 2988 cid.idbuf[0] = 0; 2989#endif 2990 do 2991 { 2992 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 2993 if (!packet_in) 2994 return -1; 2995 /* Library does not modify packet_in_data, it is not referenced after 2996 * this function returns and subsequent release of pi_data is guarded 2997 * by PI_OWN_DATA flag. 2998 */ 2999 packet_in->pi_data = (unsigned char *) packet_in_data; 3000 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 3001 engine->flags & ENG_SERVER, 3002 engine->pub.enp_settings.es_scid_len, &ppstate)) 3003 { 3004 LSQ_DEBUG("Cannot parse incoming packet's header"); 3005 maybe_count_garbage(engine, packet_end - packet_in_data); 3006 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 3007 s = 1; 3008 break; 3009 } 3010 3011 /* [draft-ietf-quic-transport-30] Section 12.2: 3012 * " Receivers SHOULD ignore any subsequent packets with a different 3013 * " Destination Connection ID than the first packet in the datagram. 3014 */ 3015 if (is_ietf && packet_in_data > packet_begin) 3016 { 3017 if (!((packet_in->pi_flags & (PI_GQUIC|PI_CONN_ID)) == PI_CONN_ID 3018 && LSQUIC_CIDS_EQ(&packet_in->pi_dcid, &cid))) 3019 { 3020 packet_in_data += packet_in->pi_data_sz; 3021 maybe_count_garbage(engine, packet_in->pi_data_sz); 3022 continue; 3023 } 3024 } 3025 3026 is_ietf = 0 == (packet_in->pi_flags & PI_GQUIC); 3027 packet_in_data += packet_in->pi_data_sz; 3028 if (is_ietf && packet_in_data < packet_end) 3029 { 3030 cid = packet_in->pi_dcid; 3031 if (packet_begin == packet_in->pi_data) /* Only log once: */ 3032 LSQ_DEBUGC("received coalesced datagram of %zd bytes for " 3033 "connection %"CID_FMT, packet_in_size, CID_BITS(&cid)); 3034 } 3035 packet_in->pi_received = lsquic_time_now(); 3036 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 3037 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 3038 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 3039 peer_ctx, packet_in_size); 3040 n_zeroes += s == 0; 3041 } 3042 while (0 == s && packet_in_data < packet_end); 3043 3044 return n_zeroes > 0 ? 0 : s; 3045} 3046 3047 3048#if __GNUC__ && !defined(NDEBUG) 3049__attribute__((weak)) 3050#endif 3051unsigned 3052lsquic_engine_quic_versions (const lsquic_engine_t *engine) 3053{ 3054 return engine->pub.enp_settings.es_versions; 3055} 3056 3057 3058void 3059lsquic_engine_cooldown (lsquic_engine_t *engine) 3060{ 3061 struct lsquic_hash_elem *el; 3062 lsquic_conn_t *conn; 3063 3064 if (engine->flags & ENG_COOLDOWN) 3065 /* AFAICT, there is no harm in calling this function more than once, 3066 * but log it just in case, as it may indicate an error in the caller. 3067 */ 3068 LSQ_INFO("cooldown called again"); 3069 engine->flags |= ENG_COOLDOWN; 3070 LSQ_INFO("entering cooldown mode"); 3071 if (engine->flags & ENG_SERVER) 3072 drop_all_mini_conns(engine); 3073 for (el = lsquic_hash_first(engine->conns_hash); el; 3074 el = lsquic_hash_next(engine->conns_hash)) 3075 { 3076 conn = lsquic_hashelem_getdata(el); 3077 lsquic_conn_going_away(conn); 3078 } 3079} 3080 3081 3082#if LSQUIC_CONN_STATS 3083static void 3084update_busy_detector (struct lsquic_engine *engine, struct lsquic_conn *conn, 3085 int immed) 3086{ 3087 char cidstr[MAX_CID_LEN * 2 + 1]; 3088 3089 if (!(LSQ_LOG_ENABLED_EXT(LSQ_LOG_NOTICE, LSQLM_CONN_STATS) 3090 && conn->cn_if->ci_log_stats)) 3091 return; 3092 3093 if (conn == engine->busy.last_conn) 3094 { 3095 engine->busy.immed_ticks <<= 1u; 3096 engine->busy.immed_ticks |= immed; 3097 if (MAX_IMMED_TICKS == engine->busy.immed_ticks) 3098 { 3099 if (engine->busy.current != conn) 3100 lsquic_logger_log1(LSQ_LOG_NOTICE, LSQLM_CONN_STATS, 3101 "connection %s marked busy: it's had %u immediate ticks " 3102 "in a row", 3103 (lsquic_cid2str(lsquic_conn_log_cid(conn), cidstr), cidstr), 3104 (unsigned) (sizeof(engine->busy.immed_ticks) * 8)); 3105 engine->busy.current = conn; 3106 } 3107 } 3108 else 3109 engine->busy.immed_ticks <<= 1; 3110 3111 engine->busy.last_conn = conn; 3112} 3113 3114 3115#endif 3116 3117 3118int 3119lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 3120{ 3121 const struct attq_elem *next_attq; 3122 lsquic_time_t now, next_time; 3123#if LSQUIC_DEBUG_NEXT_ADV_TICK || LSQUIC_CONN_STATS 3124 struct lsquic_conn *conn; 3125 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 3126#endif 3127 3128 ENGINE_CALLS_INCR(engine); 3129 3130 if ((engine->flags & ENG_PAST_DEADLINE) 3131 && lsquic_mh_count(&engine->conns_out)) 3132 { 3133#if LSQUIC_DEBUG_NEXT_ADV_TICK 3134 conn = lsquic_mh_peek(&engine->conns_out); 3135 engine->last_logged_conn = 0; 3136 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 3137 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 3138 lsquic_mh_count(&engine->conns_out), 3139 lsquic_mh_count(&engine->conns_out) != 1, "s", 3140 CID_BITS(lsquic_conn_log_cid(conn))); 3141#endif 3142#if LSQUIC_CONN_STATS 3143 conn = lsquic_mh_peek(&engine->conns_out); 3144 update_busy_detector(engine, conn, 1); 3145#endif 3146 *diff = 0; 3147 return 1; 3148 } 3149 3150 if (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 3151 { 3152#if LSQUIC_DEBUG_NEXT_ADV_TICK 3153 engine->last_logged_conn = 0; 3154 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 3155#endif 3156 *diff = 0; 3157 return 1; 3158 } 3159 3160 if (lsquic_mh_count(&engine->conns_tickable)) 3161 { 3162#if LSQUIC_DEBUG_NEXT_ADV_TICK 3163 conn = lsquic_mh_peek(&engine->conns_tickable); 3164 engine->last_logged_conn = 0; 3165 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 3166 "connection%.*s (%"CID_FMT" first)", 3167 lsquic_mh_count(&engine->conns_tickable), 3168 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 3169 CID_BITS(lsquic_conn_log_cid(conn))); 3170#endif 3171#if LSQUIC_CONN_STATS 3172 conn = lsquic_mh_peek(&engine->conns_tickable); 3173 update_busy_detector(engine, conn, 1); 3174#endif 3175 *diff = 0; 3176 return 1; 3177 } 3178 3179 next_attq = lsquic_attq_next(engine->attq); 3180 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 3181 { 3182 if (next_attq) 3183 next_time = next_attq->ae_adv_time; 3184 else 3185 return 0; 3186 } 3187 else 3188 { 3189 if (next_attq) 3190 { 3191 next_time = next_attq->ae_adv_time; 3192 if (engine->resume_sending_at < next_time) 3193 { 3194 next_time = engine->resume_sending_at; 3195 next_attq = NULL; 3196 } 3197 } 3198 else 3199 next_time = engine->resume_sending_at; 3200 } 3201 3202 now = lsquic_time_now(); 3203 *diff = (int) ((int64_t) next_time - (int64_t) now); 3204#if LSQUIC_DEBUG_NEXT_ADV_TICK 3205 if (next_attq) 3206 { 3207 /* Deduplicate consecutive log messages about the same reason for the 3208 * same connection. 3209 * If diff is always zero or diff reset to a higher value, event is 3210 * still logged. 3211 */ 3212 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 3213 && (uintptr_t) next_attq->ae_conn 3214 == engine->last_logged_conn 3215 && *diff < engine->last_tick_diff)) 3216 { 3217 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 3218 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 3219 engine->last_tick_diff = *diff; 3220 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 3221 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 3222 lsquic_attq_why2str(next_attq->ae_why)); 3223 } 3224 } 3225 else 3226 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 3227#endif 3228 3229#if LSQUIC_CONN_STATS 3230 if (next_attq) 3231 update_busy_detector(engine, next_attq->ae_conn, 3232 /* Immediate if: a) time is now or in the past */ 3233 *diff <= 0 3234 /* b) next event is pacer, which means that the 3235 * connection wants to send, but is prevented 3236 * by the pacer from doing so. 3237 */ 3238 || next_attq->ae_why == AEW_PACER 3239 /* c) next event is to retransmit data (meaning 3240 * that there is data in flight) and the 3241 * time is small, which implies a small RTT. 3242 */ 3243 || (next_attq->ae_why == N_AEWS + AL_RETX_APP 3244 && *diff < 5000)); 3245#endif 3246 3247 return 1; 3248} 3249 3250 3251unsigned 3252lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 3253{ 3254 lsquic_time_t now; 3255 ENGINE_CALLS_INCR(engine); 3256 now = lsquic_time_now(); 3257 if (from_now < 0) 3258 now -= from_now; 3259 else 3260 now += from_now; 3261 return lsquic_attq_count_before(engine->attq, now); 3262} 3263 3264 3265int 3266lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 3267 struct lsquic_conn *conn, unsigned cce_idx) 3268{ 3269 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 3270 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 3271 void *peer_ctx; 3272 3273 assert(cce_idx < conn->cn_n_cces); 3274 assert(conn->cn_cces_mask & (1 << cce_idx)); 3275 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 3276 3277 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 3278 cce->cce_cid.len, conn, &cce->cce_hash_el)) 3279 { 3280 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 3281 CID_BITS(&cce->cce_cid)); 3282 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 3283 cce->cce_flags |= CCE_REG; 3284 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 3285 return 0; 3286 } 3287 else 3288 { 3289 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 3290 CID_BITS(&cce->cce_cid)); 3291 return -1; 3292 } 3293} 3294 3295 3296void 3297lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 3298 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now) 3299{ 3300 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 3301 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 3302 void *peer_ctx; 3303 3304 assert(cce_idx < conn->cn_n_cces); 3305 3306 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 3307 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 3308 3309 if (engine->purga) 3310 { 3311 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 3312 lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 3313 PUTY_CID_RETIRED, now); 3314 } 3315 conn->cn_cces_mask &= ~(1u << cce_idx); 3316 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 3317} 3318 3319 3320