lsquic_engine.c revision f38b395a
1/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#ifndef WIN32 17#include <sys/time.h> 18#include <netinet/in.h> 19#include <sys/types.h> 20#include <sys/stat.h> 21#include <fcntl.h> 22#include <unistd.h> 23#include <netdb.h> 24#endif 25 26#ifndef NDEBUG 27#include <sys/types.h> 28#endif 29 30#if defined(WIN32) || defined(NDEBUG) 31#define CAN_LOSE_PACKETS 0 32#else 33#define CAN_LOSE_PACKETS 1 34#endif 35 36#if CAN_LOSE_PACKETS 37#include <regex.h> /* For code that loses packets */ 38#endif 39 40#if LOG_PACKET_CHECKSUM 41#include <zlib.h> 42#endif 43 44#include <openssl/aead.h> 45 46#include "lsquic.h" 47#include "lsquic_types.h" 48#include "lsquic_int_types.h" 49#include "lsquic_sizes.h" 50#include "lsquic_parse_common.h" 51#include "lsquic_parse.h" 52#include "lsquic_packet_in.h" 53#include "lsquic_packet_out.h" 54#include "lsquic_senhist.h" 55#include "lsquic_rtt.h" 56#include "lsquic_cubic.h" 57#include "lsquic_pacer.h" 58#include "lsquic_bw_sampler.h" 59#include "lsquic_minmax.h" 60#include "lsquic_bbr.h" 61#include "lsquic_adaptive_cc.h" 62#include "lsquic_set.h" 63#include "lsquic_conn_flow.h" 64#include "lsquic_sfcw.h" 65#include "lsquic_hash.h" 66#include "lsquic_conn.h" 67#include "lsquic_send_ctl.h" 68#include "lsquic_full_conn.h" 69#include "lsquic_util.h" 70#include "lsquic_qtags.h" 71#include "lsquic_enc_sess.h" 72#include "lsquic_mm.h" 73#include "lsquic_engine_public.h" 74#include "lsquic_eng_hist.h" 75#include "lsquic_ev_log.h" 76#include "lsquic_version.h" 77#include "lsquic_pr_queue.h" 78#include "lsquic_mini_conn.h" 79#include "lsquic_trechist.h" 80#include "lsquic_mini_conn_ietf.h" 81#include "lsquic_stock_shi.h" 82#include "lsquic_purga.h" 83#include "lsquic_tokgen.h" 84#include "lsquic_attq.h" 85#include "lsquic_min_heap.h" 86#include "lsquic_http1x_if.h" 87#include "lsquic_handshake.h" 88#include "lsquic_crand.h" 89#include "lsquic_ietf.h" 90#include "lsquic_handshake.h" 91 92#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 93#include "lsquic_logger.h" 94 95#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 96#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 97#endif 98 99#if LSQUIC_DEBUG_NEXT_ADV_TICK 100#include "lsquic_alarmset.h" 101#endif 102 103#define MIN(a, b) ((a) < (b) ? (a) : (b)) 104 105/* The batch of outgoing packets grows and shrinks dynamically */ 106/* Batch sizes must be powers of two */ 107#define MAX_OUT_BATCH_SIZE 1024 108#define MIN_OUT_BATCH_SIZE 4 109#define INITIAL_OUT_BATCH_SIZE 32 110 111struct out_batch 112{ 113 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 114 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 115 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 116 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 117 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 118}; 119 120typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 121 122static void 123process_connections (struct lsquic_engine *engine, conn_iter_f iter, 124 lsquic_time_t now); 125 126static void 127engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 128 129static lsquic_conn_t * 130engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 131 enum lsquic_conn_flags flag); 132 133static void 134force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 135 136#if LSQUIC_CONN_STATS 137static void 138update_busy_detector (struct lsquic_engine *, struct lsquic_conn *, int); 139#endif 140 141#if LSQUIC_COUNT_ENGINE_CALLS 142#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 143#else 144#define ENGINE_CALLS_INCR(e) 145#endif 146 147/* Nested calls to some LSQUIC functions are not supported. Functions that 148 * iterate over connections cannot be nested. 149 */ 150#define ENGINE_IN(e) do { \ 151 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 152 (e)->pub.enp_flags |= ENPUB_PROC; \ 153 ENGINE_CALLS_INCR(e); \ 154} while (0) 155 156#define ENGINE_OUT(e) do { \ 157 assert((e)->pub.enp_flags & ENPUB_PROC); \ 158 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 159} while (0) 160 161/* A connection can be referenced from one of six places: 162 * 163 * 1. A hash is used to find connections in order to dispatch an incoming 164 * packet. Connections can be hashed by CIDs or by address. In the 165 * former case, each connection has one or more mappings in the hash 166 * table. IETF QUIC connections have up to eight (in our implementation) 167 * source CIDs and each of those would have a mapping. In client mode, 168 * depending on QUIC versions and options selected, it is may be 169 * necessary to hash connections by address, in which case incoming 170 * packets are delivered to connections based on the address. 171 * 172 * 2. Outgoing queue. 173 * 174 * 3. Tickable queue 175 * 176 * 4. Advisory Tick Time queue. 177 * 178 * 5. Closing connections queue. This is a transient queue -- it only 179 * exists for the duration of process_connections() function call. 180 * 181 * 6. Ticked connections queue. Another transient queue, similar to (5). 182 * 183 * The idea is to destroy the connection when it is no longer referenced. 184 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 185 * that case, the connection is referenced from two places: (2) and (5). 186 * After its packets are sent, it is only referenced in (5), and at the 187 * end of the function call, when it is removed from (5), reference count 188 * goes to zero and the connection is destroyed. If not all packets can 189 * be sent, at the end of the function call, the connection is referenced 190 * by (2) and will only be removed once all outgoing packets have been 191 * sent. 192 */ 193#define CONN_REF_FLAGS (LSCONN_HASHED \ 194 |LSCONN_HAS_OUTGOING \ 195 |LSCONN_TICKABLE \ 196 |LSCONN_TICKED \ 197 |LSCONN_CLOSING \ 198 |LSCONN_ATTQ) 199 200 201 202 203struct cid_update_batch 204{ 205 lsquic_cids_update_f cub_update_cids; 206 void *cub_update_ctx; 207 unsigned cub_count; 208 lsquic_cid_t cub_cids[20]; 209 void *cub_peer_ctxs[20]; 210}; 211 212static void 213cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 214 215 216struct lsquic_engine 217{ 218 struct lsquic_engine_public pub; 219 enum { 220 ENG_SERVER = LSENG_SERVER, 221 ENG_HTTP = LSENG_HTTP, 222 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 223 ENG_PAST_DEADLINE 224 = (1 << 8), /* Previous call to a processing 225 * function went past time threshold. 226 */ 227 ENG_CONNS_BY_ADDR 228 = (1 << 9), /* Connections are hashed by address */ 229#ifndef NDEBUG 230 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 231#endif 232#if CAN_LOSE_PACKETS 233 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 234#endif 235#ifndef NDEBUG 236 ENG_DTOR = (1 << 26), /* Engine destructor */ 237#endif 238 } flags; 239 lsquic_packets_out_f packets_out; 240 void *packets_out_ctx; 241 lsquic_cids_update_f report_new_scids; 242 lsquic_cids_update_f report_live_scids; 243 lsquic_cids_update_f report_old_scids; 244 void *scids_ctx; 245 struct lsquic_hash *conns_hash; 246 struct min_heap conns_tickable; 247 struct min_heap conns_out; 248 struct eng_hist history; 249 unsigned batch_size; 250 struct lsquic_conn *curr_conn; 251 struct pr_queue *pr_queue; 252 struct attq *attq; 253 /* Track time last time a packet was sent to give new connections 254 * priority lower than that of existing connections. 255 */ 256 lsquic_time_t last_sent; 257#if CAN_LOSE_PACKETS 258 regex_t lose_packets_re; 259 const char *lose_packets_str; 260#endif 261 unsigned n_conns; 262 lsquic_time_t deadline; 263 lsquic_time_t resume_sending_at; 264 unsigned mini_conns_count; 265 struct lsquic_purga *purga; 266#if LSQUIC_CONN_STATS 267 struct { 268 unsigned conns; 269 } stats; 270 struct conn_stats conn_stats_sum; 271 FILE *stats_fh; 272#endif 273 struct cid_update_batch new_scids; 274 struct out_batch out_batch; 275#if LSQUIC_COUNT_ENGINE_CALLS 276 unsigned long n_engine_calls; 277#endif 278#if LSQUIC_DEBUG_NEXT_ADV_TICK 279 uintptr_t last_logged_conn; 280 unsigned last_logged_ae_why; 281 int last_tick_diff; 282#endif 283 struct crand crand; 284 EVP_AEAD_CTX retry_aead_ctx[N_IETF_RETRY_VERSIONS]; 285#if LSQUIC_CONN_STATS 286 struct { 287 uint16_t immed_ticks; /* bitmask */ 288#define MAX_IMMED_TICKS UINT16_MAX 289 struct lsquic_conn *last_conn, /* from last call */ 290 *pin_conn, /* last connection with packet in */ 291 *current; /* currently busy connection */ 292 lsquic_time_t last_log; 293 } busy; 294#endif 295}; 296 297 298void 299lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 300 unsigned flags) 301{ 302 memset(settings, 0, sizeof(*settings)); 303 settings->es_versions = LSQUIC_DF_VERSIONS; 304 if (flags & ENG_SERVER) 305 { 306 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 307 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 308 settings->es_init_max_data 309 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 310 settings->es_init_max_stream_data_bidi_remote 311 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 312 settings->es_init_max_stream_data_bidi_local 313 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 314 settings->es_init_max_stream_data_uni 315 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 316 settings->es_init_max_streams_uni 317 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 318 settings->es_ping_period = 0; 319 settings->es_noprogress_timeout 320 = LSQUIC_DF_NOPROGRESS_TIMEOUT_SERVER; 321 } 322 else 323 { 324 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 325 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 326 settings->es_init_max_data 327 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 328 settings->es_init_max_stream_data_bidi_remote 329 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 330 settings->es_init_max_stream_data_bidi_local 331 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 332 settings->es_init_max_stream_data_uni 333 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 334 settings->es_init_max_streams_uni 335 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 336 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 337 settings->es_noprogress_timeout 338 = LSQUIC_DF_NOPROGRESS_TIMEOUT_CLIENT; 339 } 340 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 341 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 342 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 343 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 344 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 345 settings->es_max_header_list_size 346 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 347 settings->es_ua = LSQUIC_DF_UA; 348 settings->es_ecn = LSQUIC_DF_ECN; 349 350 settings->es_pdmd = QTAG_X509; 351 settings->es_aead = QTAG_AESG; 352 settings->es_kexs = QTAG_C255; 353 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 354 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 355 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 356 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 357 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 358 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 359 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 360 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 361 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 362 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 363 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 364 settings->es_sttl = LSQUIC_DF_STTL; 365 settings->es_init_max_streams_bidi 366 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 367 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 368 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 369 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 370 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 371 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 372 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 373 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 374 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 375 settings->es_spin = LSQUIC_DF_SPIN; 376 settings->es_delayed_acks = LSQUIC_DF_DELAYED_ACKS; 377 settings->es_timestamps = LSQUIC_DF_TIMESTAMPS; 378 settings->es_grease_quic_bit = LSQUIC_DF_GREASE_QUIC_BIT; 379 settings->es_mtu_probe_timer = LSQUIC_DF_MTU_PROBE_TIMER; 380 settings->es_dplpmtud = LSQUIC_DF_DPLPMTUD; 381 settings->es_cc_algo = LSQUIC_DF_CC_ALGO; 382 settings->es_cc_rtt_thresh = LSQUIC_DF_CC_RTT_THRESH; 383 settings->es_optimistic_nat = LSQUIC_DF_OPTIMISTIC_NAT; 384 settings->es_ext_http_prio = LSQUIC_DF_EXT_HTTP_PRIO; 385 settings->es_ptpc_periodicity= LSQUIC_DF_PTPC_PERIODICITY; 386 settings->es_ptpc_max_packtol= LSQUIC_DF_PTPC_MAX_PACKTOL; 387 settings->es_ptpc_dyn_target = LSQUIC_DF_PTPC_DYN_TARGET; 388 settings->es_ptpc_target = LSQUIC_DF_PTPC_TARGET; 389 settings->es_ptpc_prop_gain = LSQUIC_DF_PTPC_PROP_GAIN; 390 settings->es_ptpc_int_gain = LSQUIC_DF_PTPC_INT_GAIN; 391 settings->es_ptpc_err_thresh = LSQUIC_DF_PTPC_ERR_THRESH; 392 settings->es_ptpc_err_divisor= LSQUIC_DF_PTPC_ERR_DIVISOR; 393} 394 395 396/* Note: if returning an error, err_buf must be valid if non-NULL */ 397int 398lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 399 unsigned flags, 400 char *err_buf, size_t err_buf_sz) 401{ 402 if (settings->es_cfcw < LSQUIC_MIN_FCW || 403 settings->es_sfcw < LSQUIC_MIN_FCW) 404 { 405 if (err_buf) 406 snprintf(err_buf, err_buf_sz, "%s", 407 "flow control window set too low"); 408 return -1; 409 } 410 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 411 { 412 if (err_buf) 413 snprintf(err_buf, err_buf_sz, "%s", 414 "No supported QUIC versions specified"); 415 return -1; 416 } 417 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 418 { 419 if (err_buf) 420 snprintf(err_buf, err_buf_sz, "%s", 421 "one or more unsupported QUIC version is specified"); 422 return -1; 423 } 424 if (flags & ENG_SERVER) 425 { 426 if (settings->es_handshake_to > 427 MAX_MINI_CONN_LIFESPAN_IN_USEC) 428 { 429 if (err_buf) 430 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 431 " usec is too large. The maximum for server is %u usec", 432 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 433 return -1; 434 } 435 } 436 if (settings->es_idle_timeout > 600) 437 { 438 if (err_buf) 439 snprintf(err_buf, err_buf_sz, "%s", 440 "The maximum value of idle timeout is 600 seconds"); 441 return -1; 442 } 443 if (settings->es_scid_len > MAX_CID_LEN) 444 { 445 if (err_buf) 446 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 447 "bytes long; it must be between 0 and %u.", 448 settings->es_scid_len, MAX_CID_LEN); 449 return -1; 450 } 451 452 if (settings->es_cc_algo > 3) 453 { 454 if (err_buf) 455 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 456 "algorithm value %u", settings->es_cc_algo); 457 return -1; 458 } 459 460 if (!(settings->es_ql_bits >= 0 && settings->es_ql_bits <= 2)) 461 { 462 if (err_buf) 463 snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ", 464 settings->es_ql_bits); 465 return -1; 466 } 467 468 if (!(settings->es_spin == 0 || settings->es_spin == 1)) 469 { 470 if (err_buf) 471 snprintf(err_buf, err_buf_sz, "Invalid spin value %d", 472 settings->es_spin); 473 return -1; 474 } 475 476 if (settings->es_mtu_probe_timer && settings->es_mtu_probe_timer < 1000) 477 { 478 if (err_buf) 479 snprintf(err_buf, err_buf_sz, "mtu probe timer is too small: " 480 "%u ms", settings->es_mtu_probe_timer); 481 return -1; 482 } 483 484 return 0; 485} 486 487 488static void 489free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 490{ 491 free(packet_data); 492} 493 494 495static void * 496malloc_buf (void *ctx, void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, unsigned short size, char is_ipv6) 497{ 498 return malloc(size); 499} 500 501 502static const struct lsquic_packout_mem_if stock_pmi = 503{ 504 malloc_buf, free_packet, free_packet, 505}; 506 507 508static int 509hash_conns_by_addr (const struct lsquic_engine *engine) 510{ 511 if (engine->flags & ENG_SERVER) 512 return 0; 513 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 514 return 1; 515 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 516 && engine->pub.enp_settings.es_support_tcid0) 517 return 1; 518 if (engine->pub.enp_settings.es_scid_len == 0) 519 return 1; 520 return 0; 521} 522 523 524lsquic_engine_t * 525lsquic_engine_new (unsigned flags, 526 const struct lsquic_engine_api *api) 527{ 528 lsquic_engine_t *engine; 529 size_t alpn_len; 530 unsigned i; 531 char err_buf[100]; 532 533 if (!api->ea_packets_out) 534 { 535 LSQ_ERROR("packets_out callback is not specified"); 536 return NULL; 537 } 538 539 if (!api->ea_stream_if) 540 { 541 LSQ_ERROR("stream interface is not specified"); 542 return NULL; 543 } 544 545 if (!(flags & LSENG_HTTP) && api->ea_alpn) 546 { 547 alpn_len = strlen(api->ea_alpn); 548 if (alpn_len < 1 || alpn_len > 255) 549 { 550 LSQ_ERROR("ALPN string length invalid: %zd bytes", alpn_len); 551 return NULL; 552 } 553 } 554 else 555 alpn_len = 0; 556 557 if (api->ea_settings && 558 0 != lsquic_engine_check_settings(api->ea_settings, flags, 559 err_buf, sizeof(err_buf))) 560 { 561 LSQ_ERROR("cannot create engine: %s", err_buf); 562 return NULL; 563 } 564 565 engine = calloc(1, sizeof(*engine)); 566 if (!engine) 567 return NULL; 568 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 569 { 570 free(engine); 571 return NULL; 572 } 573 if (api->ea_settings) 574 engine->pub.enp_settings = *api->ea_settings; 575 else 576 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 577 int tag_buf_len; 578 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 579 sizeof(engine->pub.enp_ver_tags_buf), 580 engine->pub.enp_settings.es_versions); 581 if (tag_buf_len <= 0) 582 { 583 LSQ_ERROR("cannot generate version tags buffer"); 584 free(engine); 585 return NULL; 586 } 587 engine->pub.enp_ver_tags_len = tag_buf_len; 588 engine->pub.enp_flags = ENPUB_CAN_SEND; 589 engine->pub.enp_stream_if = api->ea_stream_if; 590 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 591 592 engine->flags = flags; 593#ifndef NDEBUG 594 engine->flags |= ENG_COALESCE; 595#endif 596 engine->packets_out = api->ea_packets_out; 597 engine->packets_out_ctx = api->ea_packets_out_ctx; 598 engine->report_new_scids = api->ea_new_scids; 599 engine->report_live_scids = api->ea_live_scids; 600 engine->report_old_scids = api->ea_old_scids; 601 engine->scids_ctx = api->ea_cids_update_ctx; 602 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 603 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 604 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 605 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 606 607 if (api->ea_generate_scid) 608 engine->pub.enp_generate_scid = api->ea_generate_scid; 609 else 610 engine->pub.enp_generate_scid = lsquic_generate_scid; 611 612 if (api->ea_shi) 613 { 614 engine->pub.enp_shi = api->ea_shi; 615 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 616 } 617 else 618 { 619 engine->pub.enp_shi = &stock_shi; 620 engine->pub.enp_shi_ctx = lsquic_stock_shared_hash_new(); 621 if (!engine->pub.enp_shi_ctx) 622 { 623 free(engine); 624 return NULL; 625 } 626 } 627 if (api->ea_hsi_if) 628 { 629 engine->pub.enp_hsi_if = api->ea_hsi_if; 630 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 631 } 632 else 633 { 634 engine->pub.enp_hsi_if = lsquic_http1x_if; 635 engine->pub.enp_hsi_ctx = NULL; 636 } 637 if (api->ea_pmi) 638 { 639 engine->pub.enp_pmi = api->ea_pmi; 640 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 641 } 642 else 643 { 644 engine->pub.enp_pmi = &stock_pmi; 645 engine->pub.enp_pmi_ctx = NULL; 646 } 647 engine->pub.enp_verify_cert = api->ea_verify_cert; 648 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 649 engine->pub.enp_kli = api->ea_keylog_if; 650 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 651 engine->pub.enp_engine = engine; 652 if (hash_conns_by_addr(engine)) 653 engine->flags |= ENG_CONNS_BY_ADDR; 654 engine->conns_hash = lsquic_hash_create(); 655 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 656 if (!engine->pub.enp_tokgen) 657 return NULL; 658 if (engine->flags & ENG_SERVER) 659 for (i = 0; i < sizeof(engine->pub.enp_quic_ctx_sz) 660 / sizeof(engine->pub.enp_quic_ctx_sz[0]); ++i) 661 { 662 int sz = lsquic_enc_sess_ietf_gen_quic_ctx( 663 &engine->pub.enp_settings, 664 i == 0 ? LSQVER_ID27 : LSQVER_ID28, 665 engine->pub.enp_quic_ctx_buf[i], 666 sizeof(engine->pub.enp_quic_ctx_buf)); 667 if (sz < 0) 668 { 669 free(engine); 670 return NULL; 671 } 672 engine->pub.enp_quic_ctx_sz[i] = (unsigned) sz; 673 } 674 engine->pub.enp_crand = &engine->crand; 675 if (engine->pub.enp_settings.es_noprogress_timeout) 676 engine->pub.enp_noprog_timeout 677 = engine->pub.enp_settings.es_noprogress_timeout * 1000000; 678 engine->pub.enp_mtu_probe_timer = 1000 679 * (engine->pub.enp_settings.es_mtu_probe_timer 680 ? engine->pub.enp_settings.es_mtu_probe_timer 681 : LSQUIC_DF_MTU_PROBE_TIMER); 682 if (flags & ENG_SERVER) 683 { 684 engine->pr_queue = lsquic_prq_create( 685 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 686 &engine->pub); 687 if (!engine->pr_queue) 688 { 689 lsquic_tg_destroy(engine->pub.enp_tokgen); 690 return NULL; 691 } 692 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 693 engine->report_old_scids, engine->scids_ctx); 694 if (!engine->purga) 695 { 696 lsquic_tg_destroy(engine->pub.enp_tokgen); 697 lsquic_prq_destroy(engine->pr_queue); 698 return NULL; 699 } 700 } 701 engine->attq = lsquic_attq_create(); 702 eng_hist_init(&engine->history); 703 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 704 if (engine->pub.enp_settings.es_honor_prst) 705 { 706 engine->pub.enp_srst_hash = lsquic_hash_create(); 707 if (!engine->pub.enp_srst_hash) 708 { 709 lsquic_engine_destroy(engine); 710 return NULL; 711 } 712 } 713 if ((flags & LSENG_SERVER) && 0 != lsquic_init_gquic_crypto(&engine->pub)) 714 { 715 lsquic_engine_destroy(engine); 716 return NULL; 717 } 718 719 if (alpn_len) 720 { 721 engine->pub.enp_alpn = malloc(alpn_len + 1); 722 if (!engine->pub.enp_alpn) 723 { 724 lsquic_engine_destroy(engine); 725 return NULL; 726 } 727 engine->pub.enp_alpn[0] = alpn_len; 728 memcpy(engine->pub.enp_alpn + 1, api->ea_alpn, alpn_len); 729 } 730 731 if (flags & LSENG_HTTP) 732 engine->pub.enp_flags |= ENPUB_HTTP; 733 734#ifndef NDEBUG 735 { 736 const char *env; 737 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 738#if CAN_LOSE_PACKETS 739 if (env) 740 { 741 if (0 != regcomp(&engine->lose_packets_re, env, 742 REG_EXTENDED|REG_NOSUB)) 743 { 744 LSQ_ERROR("could not compile lost packet regex `%s'", env); 745 return NULL; 746 } 747 engine->flags |= ENG_LOSE_PACKETS; 748 engine->lose_packets_str = env; 749 LSQ_WARN("will lose packets that match the following regex: %s", 750 env); 751 } 752#endif 753 env = getenv("LSQUIC_COALESCE"); 754 if (env) 755 { 756 engine->flags &= ~ENG_COALESCE; 757 if (atoi(env)) 758 { 759 engine->flags |= ENG_COALESCE; 760 LSQ_NOTICE("will coalesce packets"); 761 } 762 else 763 LSQ_NOTICE("will not coalesce packets"); 764 } 765 } 766#endif 767#if LSQUIC_CONN_STATS 768 engine->stats_fh = api->ea_stats_fh; 769#endif 770 for (i = 0; i < sizeof(engine->retry_aead_ctx) 771 / sizeof(engine->retry_aead_ctx[0]); ++i) 772 if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx[i], 773 EVP_aead_aes_128_gcm(), lsquic_retry_key_buf[i], 774 IETF_RETRY_KEY_SZ, 16, NULL)) 775 { 776 LSQ_ERROR("could not initialize retry AEAD ctx #%u", i); 777 lsquic_engine_destroy(engine); 778 return NULL; 779 } 780 engine->pub.enp_retry_aead_ctx = engine->retry_aead_ctx; 781 782 LSQ_INFO("instantiated engine"); 783 return engine; 784} 785 786 787#if LOG_PACKET_CHECKSUM 788static void 789log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 790 const unsigned char *buf, size_t bufsz) 791{ 792 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 793 (uint32_t) crc32(0, buf, bufsz)); 794} 795 796 797#endif 798 799 800static void 801grow_batch_size (struct lsquic_engine *engine) 802{ 803 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 804} 805 806 807static void 808shrink_batch_size (struct lsquic_engine *engine) 809{ 810 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 811} 812 813 814struct cce_cid_iter 815{ 816 const struct lsquic_conn *conn; 817 unsigned todo, n; 818}; 819 820 821static struct conn_cid_elem * 822cce_iter_next (struct cce_cid_iter *citer) 823{ 824 struct conn_cid_elem *cce; 825 826 while (citer->todo) 827 if (citer->todo & (1 << citer->n)) 828 { 829 citer->todo &= ~(1 << citer->n); 830 cce = &citer->conn->cn_cces[ citer->n++ ]; 831 if (!(cce->cce_flags & CCE_PORT)) 832 return cce; 833 } 834 else 835 ++citer->n; 836 837 return NULL; 838} 839 840 841static struct conn_cid_elem * 842cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 843{ 844 citer->conn = conn; 845 citer->todo = conn->cn_cces_mask; 846 citer->n = 0; 847 return cce_iter_next(citer); 848} 849 850 851#if LSQUIC_CONN_STATS 852void 853update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 854{ 855 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 856 const unsigned long *src; 857 const struct conn_stats *stats; 858 unsigned i; 859 860 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 861 { 862 ++engine->stats.conns; 863 src = (unsigned long *) stats; 864 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 865 dst[i] += src[i]; 866 } 867} 868 869 870#endif 871 872 873/* Wrapper to make sure important things occur before the connection is 874 * really destroyed. 875 */ 876static void 877destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 878 lsquic_time_t now) 879{ 880 struct cce_cid_iter citer; 881 const struct conn_cid_elem *cce; 882 lsquic_time_t drain_time; 883 struct purga_el *puel; 884 885 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 886 if (engine->purga 887 /* Blacklist all CIDs except for promoted mini connections */ 888 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 889 != (LSCONN_MINI|LSCONN_PROMOTED)) 890 { 891 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 892 && conn->cn_if->ci_drain_time && 893 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 894 { 895 for (cce = cce_iter_first(&citer, conn); cce; 896 cce = cce_iter_next(&citer)) 897 { 898 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 899 lsquic_conn_get_peer_ctx(conn, NULL), 900 PUTY_CONN_DRAIN, now); 901 if (puel) 902 puel->puel_time = now + drain_time; 903 } 904 } 905 else 906 { 907 for (cce = cce_iter_first(&citer, conn); cce; 908 cce = cce_iter_next(&citer)) 909 { 910 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 911 lsquic_conn_get_peer_ctx(conn, NULL), 912 PUTY_CONN_DELETED, now); 913 if (puel) 914 { 915 puel->puel_time = now; 916 puel->puel_count = 0; 917 } 918 } 919 } 920 } 921#if LSQUIC_CONN_STATS 922 update_stats_sum(engine, conn); 923 if (engine->busy.last_conn == conn) 924 engine->busy.last_conn = NULL; 925 if (engine->busy.current == conn) 926 { 927 char cidstr[MAX_CID_LEN * 2 + 1]; 928 lsquic_logger_log1(LSQ_LOG_NOTICE, LSQLM_CONN_STATS, 929 "busy connection %s is destroyed", 930 (lsquic_cid2str(lsquic_conn_log_cid(conn), cidstr), cidstr)); 931 engine->busy.current = NULL; 932 engine->busy.last_log = 0; 933 } 934 if (engine->busy.pin_conn == conn) 935 engine->busy.pin_conn = NULL; 936#endif 937 --engine->n_conns; 938 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 939 conn->cn_if->ci_destroy(conn); 940} 941 942 943static int 944maybe_grow_conn_heaps (struct lsquic_engine *engine) 945{ 946 struct min_heap_elem *els; 947 unsigned count; 948 949 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 950 return 0; /* Nothing to do */ 951 952 if (lsquic_mh_nalloc(&engine->conns_tickable)) 953 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 954 else 955 count = 8; 956 957 els = malloc(sizeof(els[0]) * count); 958 if (!els) 959 { 960 LSQ_ERROR("%s: malloc failed", __func__); 961 return -1; 962 } 963 964 LSQ_DEBUG("grew heaps to %u elements", count / 2); 965 memcpy(&els[0], engine->conns_tickable.mh_elems, 966 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 967 memcpy(&els[count / 2], engine->conns_out.mh_elems, 968 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 969 free(engine->conns_tickable.mh_elems); 970 engine->conns_tickable.mh_elems = els; 971 engine->conns_out.mh_elems = &els[count / 2]; 972 engine->conns_tickable.mh_nalloc = count / 2; 973 engine->conns_out.mh_nalloc = count / 2; 974 return 0; 975} 976 977 978static void 979remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 980 unsigned todo) 981{ 982 unsigned n; 983 984 for (n = 0; todo; todo &= ~(1 << n++)) 985 if ((todo & (1 << n)) && 986 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 987 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 988} 989 990 991static void 992remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 993{ 994 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 995} 996 997 998static void 999cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 1000 1001 1002static int 1003insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1004 void *peer_ctx) 1005{ 1006 struct conn_cid_elem *cce; 1007 unsigned todo, done, n; 1008 1009 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 1010 if (todo & (1 << n)) 1011 { 1012 cce = &conn->cn_cces[n]; 1013 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 1014 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 1015 cce->cce_cid.len, conn, &cce->cce_hash_el)) 1016 done |= 1 << n; 1017 else 1018 goto err; 1019 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 1020 { 1021 cce->cce_flags |= CCE_REG; 1022 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 1023 } 1024 } 1025 1026 return 0; 1027 1028 err: 1029 remove_cces_from_hash(engine->conns_hash, conn, done); 1030 return -1; 1031} 1032 1033 1034static lsquic_conn_t * 1035new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 1036 lsquic_time_t now) 1037{ 1038 const lsquic_cid_t *cid; 1039 server_conn_ctor_f ctor; 1040 lsquic_conn_t *conn; 1041 unsigned flags; 1042 if (0 != maybe_grow_conn_heaps(engine)) 1043 return NULL; 1044 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1045 1046 if (mini_conn->cn_flags & LSCONN_IETF) 1047 ctor = lsquic_ietf_full_conn_server_new; 1048 else 1049 ctor = lsquic_gquic_full_conn_server_new; 1050 1051 conn = ctor(&engine->pub, flags, mini_conn); 1052 if (!conn) 1053 { 1054 /* Otherwise, full_conn_server_new prints its own warnings */ 1055 if (ENOMEM == errno) 1056 { 1057 cid = lsquic_conn_log_cid(mini_conn); 1058 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 1059 CID_BITS(cid), strerror(errno)); 1060 } 1061 return NULL; 1062 } 1063 ++engine->n_conns; 1064 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 1065 { 1066 cid = lsquic_conn_log_cid(conn); 1067 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1068 CID_BITS(cid)); 1069 destroy_conn(engine, conn, now); 1070 return NULL; 1071 } 1072 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1073 conn->cn_flags |= LSCONN_HASHED; 1074 return conn; 1075} 1076 1077 1078static enum 1079{ 1080 VER_NOT_SPECIFIED, 1081 VER_SUPPORTED, 1082 VER_UNSUPPORTED, 1083} 1084 1085 1086version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 1087 enum lsquic_version *pversion) 1088{ 1089 lsquic_ver_tag_t ver_tag; 1090 enum lsquic_version version; 1091 1092 if (!packet_in->pi_quic_ver) 1093 { 1094 LSQ_DEBUG("packet does not specify version"); 1095 return VER_NOT_SPECIFIED; 1096 } 1097 1098 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 1099 version = lsquic_tag2ver(ver_tag); 1100 if (version < N_LSQVER) 1101 { 1102 if (engine->pub.enp_settings.es_versions & (1 << version)) 1103 { 1104 LSQ_DEBUG("client-supplied version %s is supported", 1105 lsquic_ver2str[version]); 1106 *pversion = version; 1107 return VER_SUPPORTED; 1108 } 1109 else 1110 LSQ_DEBUG("client-supplied version %s is not supported", 1111 lsquic_ver2str[version]); 1112 } 1113 else 1114 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 1115 ver_tag); 1116 1117 return VER_UNSUPPORTED; 1118} 1119 1120 1121static void 1122schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 1123 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 1124 const struct sockaddr *sa_peer, void *peer_ctx) 1125{ 1126 assert(engine->pr_queue); 1127 if (0 == lsquic_prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 1128 sa_local, sa_peer)) 1129 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 1130 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 1131 else 1132 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 1133} 1134 1135 1136static unsigned short 1137sa2port (const struct sockaddr *sa) 1138{ 1139 if (sa->sa_family == AF_INET) 1140 { 1141 struct sockaddr_in *const sa4 = (void *) sa; 1142 return sa4->sin_port; 1143 } 1144 else 1145 { 1146 struct sockaddr_in6 *const sa6 = (void *) sa; 1147 return sa6->sin6_port; 1148 } 1149} 1150 1151 1152static struct lsquic_hash_elem * 1153find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 1154{ 1155 unsigned short port; 1156 1157 port = sa2port(sa); 1158 return lsquic_hash_find(hash, &port, sizeof(port)); 1159} 1160 1161 1162/* When connections are identified by the local address, we need to drop 1163 * packets that use DCIDs that do not correspond to any of SCIDs. This 1164 * can happen when peer retires a SCID. This mimics the normal behavior, 1165 * when connections are looked up in engine->conns_hash by ID: when there 1166 * is no match, the packet is dropped. 1167 */ 1168static int 1169dcid_checks_out (const struct lsquic_conn *conn, const lsquic_cid_t *dcid) 1170{ 1171 const struct conn_cid_elem *cce; 1172 1173 if (LSQUIC_CIDS_EQ(CN_SCID(conn), dcid)) 1174 return 1; 1175 1176 /* Slow check for those rare cases */ 1177 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1178 if ((conn->cn_cces_mask & (1 << (cce - conn->cn_cces))) 1179 && !(cce->cce_flags & CCE_PORT) 1180 && LSQUIC_CIDS_EQ(&cce->cce_cid, dcid)) 1181 { 1182 LSQ_DEBUG("connection checks out"); 1183 return 1; 1184 } 1185 1186 return 0; 1187} 1188 1189 1190static lsquic_conn_t * 1191find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1192 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 1193{ 1194 struct lsquic_hash_elem *el; 1195 lsquic_conn_t *conn; 1196 1197 if (engine->flags & ENG_CONNS_BY_ADDR) 1198 { 1199 el = find_conn_by_addr(engine->conns_hash, sa_local); 1200 if ((packet_in->pi_flags & PI_CONN_ID) 1201 && !dcid_checks_out(lsquic_hashelem_getdata(el), 1202 &packet_in->pi_conn_id)) 1203 { 1204 LSQ_DEBUGC("DCID matches no SCID in connection %"CID_FMT": drop it", 1205 CID_BITS(&packet_in->pi_conn_id)); 1206 return NULL; 1207 } 1208 } 1209 else if (packet_in->pi_flags & PI_CONN_ID) 1210 el = lsquic_hash_find(engine->conns_hash, 1211 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1212 else 1213 { 1214 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1215 return NULL; 1216 } 1217 1218 if (!el) 1219 return NULL; 1220 1221 conn = lsquic_hashelem_getdata(el); 1222 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1223 if ((engine->flags & ENG_CONNS_BY_ADDR) 1224 && !(conn->cn_flags & LSCONN_IETF) 1225 && (packet_in->pi_flags & PI_CONN_ID) 1226 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1227 { 1228 LSQ_DEBUG("connection IDs do not match"); 1229 return NULL; 1230 } 1231 1232 return conn; 1233} 1234 1235 1236static const char *const puty2str[] = { 1237 [PUTY_CONN_DELETED] = "deleted", 1238 [PUTY_CONN_DRAIN] = "being drained", 1239 [PUTY_CID_RETIRED] = "retired", 1240}; 1241 1242 1243static lsquic_conn_t * 1244find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1245 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1246 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1247{ 1248 struct lsquic_hash_elem *el; 1249 struct purga_el *puel; 1250 lsquic_conn_t *conn; 1251 1252 if (!(packet_in->pi_flags & PI_CONN_ID)) 1253 { 1254 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1255 return NULL; 1256 } 1257 el = lsquic_hash_find(engine->conns_hash, 1258 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1259 1260 if (el) 1261 { 1262 conn = lsquic_hashelem_getdata(el); 1263 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1264 return conn; 1265 } 1266 1267 if (engine->flags & ENG_COOLDOWN) 1268 { /* Do not create incoming connections during cooldown */ 1269 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1270 return NULL; 1271 } 1272 1273 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1274 { 1275 LSQ_DEBUG("reached limit of %u inchoate connections", 1276 engine->pub.enp_settings.es_max_inchoate); 1277 return NULL; 1278 } 1279 1280 1281 if (engine->purga 1282 && (puel = lsquic_purga_contains(engine->purga, 1283 &packet_in->pi_conn_id), puel)) 1284 { 1285 switch (puel->puel_type) 1286 { 1287 case PUTY_CID_RETIRED: 1288 case PUTY_CONN_DRAIN: 1289 if (puel->puel_time > packet_in->pi_received) 1290 { 1291 LSQ_DEBUGC("CID %"CID_FMT" is %s for another %"PRIu64 1292 "usec, ignore packet", CID_BITS(&packet_in->pi_conn_id), 1293 puty2str[puel->puel_type], 1294 puel->puel_time - packet_in->pi_received); 1295 return NULL; 1296 } 1297 LSQ_DEBUGC("CID %"CID_FMT" is no longer %s", 1298 CID_BITS(&packet_in->pi_conn_id), puty2str[puel->puel_type]); 1299 break; 1300 case PUTY_CONN_DELETED: 1301 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1302 CID_BITS(&packet_in->pi_conn_id)); 1303 if (puel->puel_time < packet_in->pi_received) 1304 { 1305 if (puel->puel_count < 4) 1306 { 1307 puel->puel_time = packet_in->pi_received 1308 /* Exponential back-off */ 1309 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1310 ++puel->puel_count; 1311 goto maybe_send_prst; 1312 } 1313 else 1314 break; 1315 } 1316 return NULL; 1317 default: 1318 assert(0); 1319 return NULL; 1320 } 1321 } 1322 1323 if (engine->pub.enp_settings.es_send_prst 1324 && !(packet_in->pi_flags & PI_GQUIC) 1325 && HETY_NOT_SET == packet_in->pi_header_type) 1326 goto maybe_send_prst; 1327 1328 if (0 != maybe_grow_conn_heaps(engine)) 1329 return NULL; 1330 1331 const struct parse_funcs *pf; 1332 enum lsquic_version version; 1333 switch (version_matches(engine, packet_in, &version)) 1334 { 1335 case VER_UNSUPPORTED: 1336 if (engine->flags & ENG_SERVER) 1337 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1338 sa_local, sa_peer, peer_ctx); 1339 return NULL; 1340 case VER_NOT_SPECIFIED: 1341 maybe_send_prst: 1342 if ((engine->flags & ENG_SERVER) && 1343 engine->pub.enp_settings.es_send_prst) 1344 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1345 sa_local, sa_peer, peer_ctx); 1346 return NULL; 1347 case VER_SUPPORTED: 1348 pf = select_pf_by_ver(version); 1349 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1350 break; 1351 } 1352 1353 1354 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1355 { 1356 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1357 sa_peer->sa_family == AF_INET, NULL, packet_in_size); 1358 } 1359 else 1360 { 1361 conn = lsquic_mini_conn_new(&engine->pub, packet_in, version); 1362 } 1363 if (!conn) 1364 return NULL; 1365 ++engine->mini_conns_count; 1366 ++engine->n_conns; 1367 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1368 { 1369 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1370 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1371 CID_BITS(cid)); 1372 destroy_conn(engine, conn, packet_in->pi_received); 1373 return NULL; 1374 } 1375 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1376 conn->cn_flags |= LSCONN_HASHED; 1377 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1378 conn->cn_last_sent = engine->last_sent; 1379 return conn; 1380} 1381 1382 1383lsquic_conn_t * 1384lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1385 const lsquic_cid_t *cid) 1386{ 1387 struct lsquic_hash_elem *el; 1388 lsquic_conn_t *conn = NULL; 1389 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1390 1391 if (el) 1392 conn = lsquic_hashelem_getdata(el); 1393 return conn; 1394} 1395 1396 1397#if !defined(NDEBUG) && __GNUC__ 1398__attribute__((weak)) 1399#endif 1400void 1401lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1402 lsquic_conn_t *conn) 1403{ 1404 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1405 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1406 { 1407 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1408 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1409 engine_incref_conn(conn, LSCONN_TICKABLE); 1410 } 1411} 1412 1413 1414void 1415lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1416 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1417{ 1418 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1419 if (conn->cn_flags & LSCONN_TICKABLE) 1420 { 1421 /* Optimization: no need to add the connection to the Advisory Tick 1422 * Time Queue: it is about to be ticked, after which it its next tick 1423 * time may be queried again. 1424 */; 1425 } 1426 else if (conn->cn_flags & LSCONN_ATTQ) 1427 { 1428 if (lsquic_conn_adv_time(conn) != tick_time) 1429 { 1430 lsquic_attq_remove(engine->attq, conn); 1431 if (0 != lsquic_attq_add(engine->attq, conn, tick_time, why)) 1432 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1433 } 1434 } 1435 else if (0 == lsquic_attq_add(engine->attq, conn, tick_time, why)) 1436 engine_incref_conn(conn, LSCONN_ATTQ); 1437} 1438 1439 1440static struct lsquic_conn * 1441find_conn_by_srst (struct lsquic_engine *engine, 1442 const struct lsquic_packet_in *packet_in) 1443{ 1444 struct lsquic_hash_elem *el; 1445 struct lsquic_conn *conn; 1446 1447 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1448 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1449 return NULL; 1450 1451 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1452 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1453 IQUIC_SRESET_TOKEN_SZ); 1454 if (!el) 1455 return NULL; 1456 1457 conn = lsquic_hashelem_getdata(el); 1458 return conn; 1459} 1460 1461 1462/* Return 0 if packet is being processed by a real connection (mini or full), 1463 * otherwise return 1. 1464 */ 1465static int 1466process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1467 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1468 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1469{ 1470 lsquic_conn_t *conn; 1471 const unsigned char *packet_in_data; 1472 1473 if (lsquic_packet_in_is_gquic_prst(packet_in) 1474 && !engine->pub.enp_settings.es_honor_prst) 1475 { 1476 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1477 LSQ_DEBUG("public reset packet: discarding"); 1478 return 1; 1479 } 1480 1481 if (engine->flags & ENG_SERVER) 1482 { 1483 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1484 sa_peer, peer_ctx, packet_in_size); 1485 if (!engine->curr_conn) 1486 engine->curr_conn = conn; 1487 } 1488 else 1489 conn = find_conn(engine, packet_in, ppstate, sa_local); 1490 1491 if (!conn) 1492 { 1493 if (engine->pub.enp_settings.es_honor_prst 1494 && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */ 1495 && !(packet_in->pi_flags & PI_GQUIC) 1496 && engine->pub.enp_srst_hash 1497 && (conn = find_conn_by_srst(engine, packet_in))) 1498 { 1499 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1500 CID_BITS(lsquic_conn_log_cid(conn))); 1501 conn->cn_if->ci_stateless_reset(conn); 1502 if (!(conn->cn_flags & LSCONN_TICKABLE) 1503 && conn->cn_if->ci_is_tickable(conn)) 1504 { 1505 lsquic_mh_insert(&engine->conns_tickable, conn, 1506 conn->cn_last_ticked); 1507 engine_incref_conn(conn, LSCONN_TICKABLE); 1508 } 1509 /* Even though the connection processes this packet, we return 1510 * 1 so that the caller does not add reset packet's random 1511 * bytes to the list of valid CIDs. 1512 */ 1513 } 1514 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1515 return 1; 1516 } 1517 1518 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1519 { 1520 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1521 engine_incref_conn(conn, LSCONN_TICKABLE); 1522 } 1523 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1524 sa_local, sa_peer); 1525 lsquic_packet_in_upref(packet_in); 1526#if LOG_PACKET_CHECKSUM 1527 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1528 packet_in->pi_data_sz); 1529#endif 1530 /* Note on QLog: 1531 * For the PACKET_RX QLog event, we are interested in logging these things: 1532 * - raw packet (however it comes in, encrypted or not) 1533 * - frames (list of frame names) 1534 * - packet type and number 1535 * - packet rx timestamp 1536 * 1537 * Since only some of these items are available at this code 1538 * juncture, we will wait until after the packet has been 1539 * decrypted (if necessary) and parsed to call the log functions. 1540 * 1541 * Once the PACKET_RX event is finally logged, the timestamp 1542 * will come from packet_in->pi_received. For correct sequential 1543 * ordering of QLog events, be sure to process the QLogs downstream. 1544 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1545 */ 1546 packet_in_data = packet_in->pi_data; 1547 packet_in_size = packet_in->pi_data_sz; 1548 conn->cn_if->ci_packet_in(conn, packet_in); 1549#if LSQUIC_CONN_STATS 1550 engine->busy.pin_conn = conn; 1551#endif 1552 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1553 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1554 return 0; 1555} 1556 1557 1558void 1559lsquic_engine_destroy (lsquic_engine_t *engine) 1560{ 1561 struct lsquic_hash_elem *el; 1562 lsquic_conn_t *conn; 1563 unsigned i; 1564 1565 LSQ_DEBUG("destroying engine"); 1566#ifndef NDEBUG 1567 engine->flags |= ENG_DTOR; 1568#endif 1569 1570 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1571 { 1572 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1573 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1574 } 1575 1576 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1577 { 1578 assert(conn->cn_flags & LSCONN_TICKABLE); 1579 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1580 } 1581 1582 for (el = lsquic_hash_first(engine->conns_hash); el; 1583 el = lsquic_hash_next(engine->conns_hash)) 1584 { 1585 conn = lsquic_hashelem_getdata(el); 1586 force_close_conn(engine, conn); 1587 } 1588 lsquic_hash_destroy(engine->conns_hash); 1589 1590 while ((conn = lsquic_attq_pop(engine->attq, UINT64_MAX))) 1591 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1592 1593 assert(0 == engine->n_conns); 1594 assert(0 == engine->mini_conns_count); 1595 if (engine->pr_queue) 1596 lsquic_prq_destroy(engine->pr_queue); 1597 if (engine->purga) 1598 lsquic_purga_destroy(engine->purga); 1599 lsquic_attq_destroy(engine->attq); 1600 1601 assert(0 == lsquic_mh_count(&engine->conns_out)); 1602 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1603 if (engine->pub.enp_shi == &stock_shi) 1604 lsquic_stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1605 lsquic_mm_cleanup(&engine->pub.enp_mm); 1606 free(engine->conns_tickable.mh_elems); 1607#if CAN_LOSE_PACKETS 1608 if (engine->flags & ENG_LOSE_PACKETS) 1609 regfree(&engine->lose_packets_re); 1610#endif 1611 if (engine->pub.enp_tokgen) 1612 lsquic_tg_destroy(engine->pub.enp_tokgen); 1613 if (engine->flags & LSENG_SERVER) 1614 lsquic_cleanup_gquic_crypto(&engine->pub); 1615#if LSQUIC_CONN_STATS 1616 if (engine->stats_fh) 1617 { 1618 const struct conn_stats *const stats = &engine->conn_stats_sum; 1619 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1620 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1621 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1622 fprintf(engine->stats_fh, "In:\n"); 1623 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1624 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1625 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1626 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1627 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1628 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1629 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1630 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1631 stats->in.headers_comp, stats->in.headers_uncomp, 1632 stats->in.headers_uncomp ? 1633 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1634 : 0); 1635 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1636 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1637 fprintf(engine->stats_fh, " ACK frames merged: %lu\n", stats->in.n_acks_merged); 1638 fprintf(engine->stats_fh, "Out:\n"); 1639 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1640 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1641 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1642 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1643 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1644 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1645 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1646 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1647 stats->out.headers_comp, stats->out.headers_uncomp, 1648 stats->out.headers_uncomp ? 1649 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1650 : 0); 1651 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1652 } 1653#endif 1654 if (engine->pub.enp_srst_hash) 1655 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1656#if LSQUIC_COUNT_ENGINE_CALLS 1657 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1658#endif 1659 for (i = 0; i < sizeof(engine->retry_aead_ctx) 1660 / sizeof(engine->retry_aead_ctx[0]); ++i) 1661 EVP_AEAD_CTX_cleanup(&engine->pub.enp_retry_aead_ctx[i]); 1662 free(engine->pub.enp_alpn); 1663 free(engine); 1664} 1665 1666 1667static struct conn_cid_elem * 1668find_free_cce (struct lsquic_conn *conn) 1669{ 1670 struct conn_cid_elem *cce; 1671 1672 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1673 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1674 return cce; 1675 1676 return NULL; 1677} 1678 1679 1680static int 1681add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1682 const struct sockaddr *local_sa, void *peer_ctx) 1683{ 1684 struct conn_cid_elem *cce; 1685 1686 if (engine->flags & ENG_CONNS_BY_ADDR) 1687 { 1688 cce = find_free_cce(conn); 1689 if (!cce) 1690 { 1691 LSQ_ERROR("cannot find free CCE"); 1692 return -1; 1693 } 1694 cce->cce_port = sa2port(local_sa); 1695 cce->cce_flags = CCE_PORT; 1696 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1697 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1698 { 1699 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1700 return 0; 1701 } 1702 else 1703 return -1; 1704 1705 } 1706 else 1707 return insert_conn_into_hash(engine, conn, peer_ctx); 1708} 1709 1710 1711lsquic_conn_t * 1712lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1713 const struct sockaddr *local_sa, 1714 const struct sockaddr *peer_sa, 1715 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1716 const char *hostname, unsigned short base_plpmtu, 1717 const unsigned char *sess_resume, size_t sess_resume_len, 1718 const unsigned char *token, size_t token_sz) 1719{ 1720 lsquic_conn_t *conn; 1721 unsigned flags, versions; 1722 int is_ipv4; 1723 1724 ENGINE_CALLS_INCR(engine); 1725 1726 if (engine->flags & ENG_SERVER) 1727 { 1728 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1729 goto err; 1730 } 1731 1732 if (engine->flags & ENG_CONNS_BY_ADDR 1733 && find_conn_by_addr(engine->conns_hash, local_sa)) 1734 { 1735 LSQ_ERROR("cannot have more than one connection on the same port"); 1736 goto err; 1737 } 1738 1739 if (0 != maybe_grow_conn_heaps(engine)) 1740 return NULL; 1741 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1742 is_ipv4 = peer_sa->sa_family == AF_INET; 1743 if (sess_resume && sess_resume_len) 1744 { 1745 version = lsquic_sess_resume_version(sess_resume, sess_resume_len); 1746 if (version >= N_LSQVER) 1747 { 1748 LSQ_INFO("session resumption version is bad, won't use"); 1749 sess_resume = NULL; 1750 sess_resume_len = 0; 1751 } 1752 } 1753 if (version >= N_LSQVER) 1754 { 1755 if (version > N_LSQVER) 1756 LSQ_WARN("invalid version specified, engine will pick"); 1757 versions = engine->pub.enp_settings.es_versions; 1758 } 1759 else 1760 versions = 1u << version; 1761 if (versions & LSQUIC_IETF_VERSIONS) 1762 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1763 flags, hostname, base_plpmtu, 1764 is_ipv4, sess_resume, sess_resume_len, token, token_sz, peer_ctx); 1765 else 1766 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1767 flags, hostname, base_plpmtu, is_ipv4, 1768 sess_resume, sess_resume_len); 1769 if (!conn) 1770 goto err; 1771 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1772 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1773 lsquic_ver2str[conn->cn_version]); 1774 ++engine->n_conns; 1775 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1776 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1777 { 1778 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1779 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1780 CID_BITS(cid)); 1781 destroy_conn(engine, conn, lsquic_time_now()); 1782 goto err; 1783 } 1784 assert(!(conn->cn_flags & 1785 (CONN_REF_FLAGS 1786 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1787 callbacks */ 1788 ))); 1789 conn->cn_flags |= LSCONN_HASHED; 1790 if (!(conn->cn_flags & LSCONN_TICKABLE)) 1791 { 1792 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1793 engine_incref_conn(conn, LSCONN_TICKABLE); 1794 } 1795 lsquic_conn_set_ctx(conn, conn_ctx); 1796 conn->cn_if->ci_client_call_on_new(conn); 1797 end: 1798 return conn; 1799 err: 1800 conn = NULL; 1801 goto end; 1802} 1803 1804 1805static void 1806remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1807{ 1808 remove_all_cces_from_hash(engine->conns_hash, conn); 1809 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1810} 1811 1812 1813static void 1814refflags2str (enum lsquic_conn_flags flags, char s[7]) 1815{ 1816 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1817 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1818 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1819 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1820 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1821 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1822 *s = '\0'; 1823} 1824 1825 1826static void 1827engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1828{ 1829 char str[2][7]; 1830 assert(flag & CONN_REF_FLAGS); 1831 assert(!(conn->cn_flags & flag)); 1832 conn->cn_flags |= flag; 1833 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1834 CID_BITS(lsquic_conn_log_cid(conn)), 1835 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1836 (refflags2str(conn->cn_flags, str[1]), str[1])); 1837} 1838 1839 1840static lsquic_conn_t * 1841engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1842 enum lsquic_conn_flags flags) 1843{ 1844 char str[2][7]; 1845 lsquic_time_t now; 1846 assert(flags & CONN_REF_FLAGS); 1847 assert(conn->cn_flags & flags); 1848#ifndef NDEBUG 1849 if (flags & LSCONN_CLOSING) 1850 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1851#endif 1852 conn->cn_flags &= ~flags; 1853 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1854 CID_BITS(lsquic_conn_log_cid(conn)), 1855 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1856 (refflags2str(conn->cn_flags, str[1]), str[1])); 1857 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1858 { 1859 now = lsquic_time_now(); 1860 if (conn->cn_flags & LSCONN_MINI) 1861 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1862 else 1863 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1864 destroy_conn(engine, conn, now); 1865 return NULL; 1866 } 1867 else 1868 return conn; 1869} 1870 1871 1872/* This is not a general-purpose function. Only call from engine dtor. */ 1873static void 1874force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1875{ 1876 assert(engine->flags & ENG_DTOR); 1877 const enum lsquic_conn_flags flags = conn->cn_flags; 1878 assert(conn->cn_flags & CONN_REF_FLAGS); 1879 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1880 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1881 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1882 if (flags & LSCONN_ATTQ) 1883 { 1884 lsquic_attq_remove(engine->attq, conn); 1885 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1886 } 1887 if (flags & LSCONN_HASHED) 1888 remove_conn_from_hash(engine, conn); 1889} 1890 1891 1892/* Iterator for tickable connections (those on the Tickable Queue). Before 1893 * a connection is returned, it is removed from the Advisory Tick Time queue 1894 * if necessary. 1895 */ 1896static lsquic_conn_t * 1897conn_iter_next_tickable (struct lsquic_engine *engine) 1898{ 1899 lsquic_conn_t *conn; 1900 1901 if (engine->flags & ENG_SERVER) 1902 while (1) 1903 { 1904 conn = lsquic_mh_pop(&engine->conns_tickable); 1905 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1906 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1907 else 1908 break; 1909 } 1910 else 1911 conn = lsquic_mh_pop(&engine->conns_tickable); 1912 1913 if (conn) 1914 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1915 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1916 { 1917 lsquic_attq_remove(engine->attq, conn); 1918 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1919 } 1920 1921 return conn; 1922} 1923 1924 1925static void 1926cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1927 void *update_ctx) 1928{ 1929 cub->cub_update_cids = update; 1930 cub->cub_update_ctx = update_ctx; 1931 cub->cub_count = 0; 1932} 1933 1934 1935static void 1936cub_flush (struct cid_update_batch *cub) 1937{ 1938 if (cub->cub_count > 0 && cub->cub_update_cids) 1939 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1940 cub->cub_cids, cub->cub_count); 1941 cub->cub_count = 0; 1942} 1943 1944 1945static void 1946cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1947{ 1948 cub->cub_cids [ cub->cub_count ] = *cid; 1949 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1950 ++cub->cub_count; 1951 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1952 cub_flush(cub); 1953} 1954 1955 1956/* Process registered CIDs */ 1957static void 1958cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1959{ 1960 struct cce_cid_iter citer; 1961 struct conn_cid_elem *cce; 1962 void *peer_ctx; 1963 1964 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1965 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1966 if (cce->cce_flags & CCE_REG) 1967 cub_add(cub, &cce->cce_cid, peer_ctx); 1968} 1969 1970 1971static void 1972drop_all_mini_conns (lsquic_engine_t *engine) 1973{ 1974 struct lsquic_hash_elem *el; 1975 lsquic_conn_t *conn; 1976 struct cid_update_batch cub; 1977 1978 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1979 1980 for (el = lsquic_hash_first(engine->conns_hash); el; 1981 el = lsquic_hash_next(engine->conns_hash)) 1982 { 1983 conn = lsquic_hashelem_getdata(el); 1984 if (conn->cn_flags & LSCONN_MINI) 1985 { 1986 /* If promoted, why is it still in this hash? */ 1987 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1988 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1989 cub_add_cids_from_cces(&cub, conn); 1990 remove_conn_from_hash(engine, conn); 1991 } 1992 } 1993 1994 cub_flush(&cub); 1995} 1996 1997 1998void 1999lsquic_engine_process_conns (lsquic_engine_t *engine) 2000{ 2001 lsquic_conn_t *conn; 2002 lsquic_time_t now; 2003 2004#if LSQUIC_CONN_STATS 2005 if (engine->busy.pin_conn) 2006 { 2007 update_busy_detector(engine, engine->busy.pin_conn, 1); 2008 engine->busy.pin_conn = NULL; 2009 } 2010#endif 2011 2012 ENGINE_IN(engine); 2013 2014 now = lsquic_time_now(); 2015 while ((conn = lsquic_attq_pop(engine->attq, now))) 2016 { 2017 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 2018 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 2019 { 2020 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2021 engine_incref_conn(conn, LSCONN_TICKABLE); 2022 } 2023 } 2024 2025 process_connections(engine, conn_iter_next_tickable, now); 2026 ENGINE_OUT(engine); 2027} 2028 2029 2030static void 2031release_or_return_enc_data (struct lsquic_engine *engine, 2032 void (*pmi_rel_or_ret) (void *, void *, void *, char), 2033 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 2034{ 2035 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 2036 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 2037 packet_out->po_flags &= ~PO_ENCRYPTED; 2038 packet_out->po_enc_data = NULL; 2039} 2040 2041 2042static void 2043release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 2044 struct lsquic_packet_out *packet_out) 2045{ 2046 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 2047 conn, packet_out); 2048} 2049 2050 2051static void 2052return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 2053 struct lsquic_packet_out *packet_out) 2054{ 2055 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 2056 conn, packet_out); 2057} 2058 2059 2060static int 2061copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 2062 struct lsquic_packet_out *packet_out) 2063{ 2064 int ipv6; 2065 2066 ipv6 = NP_IS_IPv6(packet_out->po_path); 2067 if (packet_out->po_flags & PO_ENCRYPTED) 2068 { 2069 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 2070 && packet_out->po_data_sz == packet_out->po_enc_data_sz 2071 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 2072 packet_out->po_data_sz)) 2073 return 0; 2074 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 2075 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 2076 goto copy; 2077 return_enc_data(engine, conn, packet_out); 2078 } 2079 2080 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 2081 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 2082 conn->cn_conn_ctx, packet_out->po_data_sz, ipv6); 2083 if (!packet_out->po_enc_data) 2084 { 2085 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 2086 "of size %hu", packet_out->po_data_sz); 2087 return -1; 2088 } 2089 2090 copy: 2091 memcpy(packet_out->po_enc_data, packet_out->po_data, 2092 packet_out->po_data_sz); 2093 packet_out->po_enc_data_sz = packet_out->po_data_sz; 2094 packet_out->po_sent_sz = packet_out->po_data_sz; 2095 packet_out->po_flags &= ~PO_IPv6; 2096 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 2097 2098 return 0; 2099} 2100 2101 2102STAILQ_HEAD(conns_stailq, lsquic_conn); 2103TAILQ_HEAD(conns_tailq, lsquic_conn); 2104 2105 2106struct conns_out_iter 2107{ 2108 struct min_heap *coi_heap; 2109 struct pr_queue *coi_prq; 2110 TAILQ_HEAD(, lsquic_conn) coi_active_list, 2111 coi_inactive_list; 2112 lsquic_conn_t *coi_next; 2113#ifndef NDEBUG 2114 lsquic_time_t coi_last_sent; 2115#endif 2116}; 2117 2118 2119static void 2120coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 2121{ 2122 iter->coi_heap = &engine->conns_out; 2123 iter->coi_prq = engine->pr_queue; 2124 iter->coi_next = NULL; 2125 TAILQ_INIT(&iter->coi_active_list); 2126 TAILQ_INIT(&iter->coi_inactive_list); 2127#ifndef NDEBUG 2128 iter->coi_last_sent = 0; 2129#endif 2130} 2131 2132 2133static lsquic_conn_t * 2134coi_next (struct conns_out_iter *iter) 2135{ 2136 lsquic_conn_t *conn; 2137 2138 if (lsquic_mh_count(iter->coi_heap) > 0) 2139 { 2140 conn = lsquic_mh_pop(iter->coi_heap); 2141 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2142 conn->cn_flags |= LSCONN_COI_ACTIVE; 2143#ifndef NDEBUG 2144 if (iter->coi_last_sent) 2145 assert(iter->coi_last_sent <= conn->cn_last_sent); 2146 iter->coi_last_sent = conn->cn_last_sent; 2147#endif 2148 return conn; 2149 } 2150 else if (iter->coi_prq && (conn = lsquic_prq_next_conn(iter->coi_prq))) 2151 { 2152 return conn; 2153 } 2154 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 2155 { 2156 iter->coi_prq = NULL; /* Save function call in previous conditional */ 2157 conn = iter->coi_next; 2158 if (!conn) 2159 conn = TAILQ_FIRST(&iter->coi_active_list); 2160 if (conn) 2161 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 2162 return conn; 2163 } 2164 else 2165 return NULL; 2166} 2167 2168 2169static void 2170coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2171{ 2172 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2173 { 2174 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 2175 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2176 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2177 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 2178 conn->cn_flags |= LSCONN_COI_INACTIVE; 2179 } 2180} 2181 2182 2183static void 2184coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2185{ 2186 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 2187 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2188 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2189 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2190 conn->cn_flags |= LSCONN_COI_ACTIVE; 2191} 2192 2193 2194static void 2195coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 2196{ 2197 lsquic_conn_t *conn; 2198 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 2199 { 2200 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2201 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2202 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 2203 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 2204 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 2205 else /* Closed connection gets one shot at sending packets */ 2206 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2207 } 2208 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 2209 { 2210 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2211 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2212 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2213 } 2214} 2215 2216 2217#if CAN_LOSE_PACKETS 2218static void 2219lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 2220 unsigned n) 2221{ 2222 const lsquic_cid_t *cid; 2223 struct iovec *iov; 2224 unsigned i; 2225 char packno_str[22]; 2226 2227 for (i = 0; i < n; ++i) 2228 { 2229 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 2230 batch->packets[i]->po_packno); 2231 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 2232 { 2233 for (iov = batch->outs[i].iov; iov < 2234 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 2235 batch->outs[i].iov->iov_len -= 1; 2236 cid = lsquic_conn_log_cid(batch->conns[i]); 2237 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 2238 CID_BITS(cid)); 2239 } 2240 } 2241} 2242 2243 2244#endif 2245 2246 2247#ifdef NDEBUG 2248#define CONST_BATCH const 2249#else 2250#define CONST_BATCH 2251#endif 2252 2253 2254struct send_batch_ctx { 2255 struct conns_stailq *closed_conns; 2256 struct conns_tailq *ticked_conns; 2257 struct conns_out_iter *conns_iter; 2258 CONST_BATCH struct out_batch *batch; 2259}; 2260 2261 2262static void 2263close_conn_immediately (struct lsquic_engine *engine, 2264 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2265{ 2266 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2267 if (!(conn->cn_flags & LSCONN_CLOSING)) 2268 { 2269 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2270 engine_incref_conn(conn, LSCONN_CLOSING); 2271 if (conn->cn_flags & LSCONN_HASHED) 2272 remove_conn_from_hash(engine, conn); 2273 } 2274 if (conn->cn_flags & LSCONN_TICKED) 2275 { 2276 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2277 engine_decref_conn(engine, conn, LSCONN_TICKED); 2278 } 2279} 2280 2281 2282static void 2283close_conn_on_send_error (struct lsquic_engine *engine, 2284 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2285{ 2286 const struct out_batch *batch = sb_ctx->batch; 2287 struct lsquic_conn *const conn = batch->conns[n]; 2288 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2289 2290 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2291 "src: %s; dst: %s; errno: %d", 2292 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2293 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2294 CID_BITS(lsquic_conn_log_cid(conn)), 2295 SA2STR(batch->outs[n].local_sa, buf[0]), 2296 SA2STR(batch->outs[n].dest_sa, buf[1]), 2297 e_val); 2298 if (conn->cn_flags & LSCONN_EVANESCENT) 2299 lsquic_prq_drop(conn); 2300 else 2301 close_conn_immediately(engine, sb_ctx, conn); 2302} 2303 2304 2305static void 2306apply_hp (struct conns_out_iter *iter) 2307{ 2308 struct lsquic_conn *conn; 2309 2310 TAILQ_FOREACH(conn, &iter->coi_active_list, cn_next_out) 2311 if (conn->cn_esf_c->esf_flush_encryption && conn->cn_enc_session) 2312 conn->cn_esf_c->esf_flush_encryption(conn->cn_enc_session); 2313 TAILQ_FOREACH(conn, &iter->coi_inactive_list, cn_next_out) 2314 if (conn->cn_esf_c->esf_flush_encryption && conn->cn_enc_session) 2315 conn->cn_esf_c->esf_flush_encryption(conn->cn_enc_session); 2316} 2317 2318 2319#if LSQUIC_CONN_STATS 2320static void 2321update_batch_size_stats (struct lsquic_engine *engine, unsigned batch_size) 2322{ 2323 struct batch_size_stats *const stats = &engine->pub.enp_batch_size_stats; 2324 2325 ++stats->count; 2326 if (batch_size > stats->max) 2327 stats->max = batch_size; 2328 if (batch_size < stats->min || 0 == stats->min) 2329 stats->min = batch_size; 2330 if (stats->avg) 2331 stats->avg = ((float) batch_size - stats->avg) * 0.4 + stats->avg; 2332 else 2333 stats->avg = (float) batch_size; 2334} 2335 2336 2337#endif 2338 2339 2340static unsigned 2341send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2342 unsigned n_to_send) 2343{ 2344 int n_sent, i, e_val; 2345 lsquic_time_t now; 2346 unsigned off, skip; 2347 size_t count; 2348 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2349 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2350 2351#if LSQUIC_CONN_STATS 2352 update_batch_size_stats(engine, n_to_send); 2353#endif 2354 2355 apply_hp(sb_ctx->conns_iter); 2356#if CAN_LOSE_PACKETS 2357 if (engine->flags & ENG_LOSE_PACKETS) 2358 lose_matching_packets(engine, batch, n_to_send); 2359#endif 2360 skip = 0; 2361 restart_batch: 2362 /* Set sent time before the write to avoid underestimating RTT */ 2363 now = lsquic_time_now(); 2364 for (i = skip; i < (int) (n_to_send - skip); ++i) 2365 { 2366 off = batch->pack_off[i]; 2367 count = batch->outs[i].iovlen; 2368 assert(count > 0); 2369 packet_out = &batch->packets[off]; 2370 end = packet_out + count; 2371 do 2372 (*packet_out)->po_sent = now; 2373 while (++packet_out < end); 2374 } 2375 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs + skip, 2376 n_to_send - skip); 2377 e_val = errno; 2378 if (n_sent < (int) (n_to_send - skip) && e_val != EMSGSIZE) 2379 { 2380 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2381 engine->resume_sending_at = now + 1000000; 2382 LSQ_DEBUG("cannot send packets"); 2383 EV_LOG_GENERIC_EVENT("cannot send packets"); 2384 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2385 close_conn_on_send_error(engine, sb_ctx, 2386 n_sent < 0 ? 0 : n_sent, e_val); 2387 } 2388 if (n_sent >= 0) 2389 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, 2390 n_to_send - skip); 2391 else 2392 { 2393 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2394 n_sent = 0; 2395 } 2396 if (n_sent > 0) 2397 engine->last_sent = now + n_sent; 2398 for (i = skip; i < (int) (skip + n_sent); ++i) 2399 { 2400 eng_hist_inc(&engine->history, now, sl_packets_out); 2401 /* `i' is added to maintain relative order */ 2402 batch->conns[i]->cn_last_sent = now + i; 2403 2404 off = batch->pack_off[i]; 2405 count = batch->outs[i].iovlen; 2406 assert(count > 0); 2407 packet_out = &batch->packets[off]; 2408 end = packet_out + count; 2409 do 2410 { 2411#if LOG_PACKET_CHECKSUM 2412 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2413 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2414 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2415#endif 2416 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2417 *packet_out); 2418 /* Release packet out buffer as soon as the packet is sent 2419 * successfully. If not successfully sent, we hold on to 2420 * this buffer until the packet sending is attempted again 2421 * or until it times out and regenerated. 2422 */ 2423 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2424 release_enc_data(engine, batch->conns[i], *packet_out); 2425 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2426 *packet_out); 2427 } 2428 while (++packet_out < end); 2429 } 2430 if (i < (int) n_to_send && e_val == EMSGSIZE) 2431 { 2432 LSQ_DEBUG("packet #%d could not be sent out for being too large", i); 2433 if (batch->conns[i]->cn_if->ci_packet_too_large 2434 && batch->outs[i].iovlen == 1) 2435 { 2436 off = batch->pack_off[i]; 2437 packet_out = &batch->packets[off]; 2438 batch->conns[i]->cn_if->ci_packet_too_large(batch->conns[i], 2439 *packet_out); 2440 ++i; 2441 if (i < (int) n_to_send) 2442 { 2443 skip = i; 2444 LSQ_DEBUG("restart batch starting at packet #%u", skip); 2445 goto restart_batch; 2446 } 2447 n_sent = n_to_send; 2448 } 2449 else 2450 close_conn_on_send_error(engine, sb_ctx, i, e_val); 2451 } 2452 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2453 for ( ; i < (int) n_to_send; ++i) 2454 { 2455 off = batch->pack_off[i]; 2456 count = batch->outs[i].iovlen; 2457 assert(count > 0); 2458 packet_out = &batch->packets[off]; 2459 end = packet_out + count; 2460 do 2461 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2462 *packet_out); 2463 while (++packet_out < end); 2464 } 2465 /* Return packets to the connection in reverse order so that the packet 2466 * ordering is maintained. 2467 */ 2468 for (i = (int) n_to_send - 1; i >= (int) (skip + n_sent); --i) 2469 { 2470 off = batch->pack_off[i]; 2471 count = batch->outs[i].iovlen; 2472 assert(count > 0); 2473 packet_out = &batch->packets[off + count - 1]; 2474 end = &batch->packets[off - 1]; 2475 do 2476 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2477 *packet_out); 2478 while (--packet_out > end); 2479 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2480 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2481 } 2482 return skip + n_sent; 2483} 2484 2485 2486/* Return 1 if went past deadline, 0 otherwise */ 2487static int 2488check_deadline (lsquic_engine_t *engine) 2489{ 2490 if (engine->pub.enp_settings.es_proc_time_thresh && 2491 lsquic_time_now() > engine->deadline) 2492 { 2493 LSQ_INFO("went past threshold of %u usec, stop sending", 2494 engine->pub.enp_settings.es_proc_time_thresh); 2495 engine->flags |= ENG_PAST_DEADLINE; 2496 return 1; 2497 } 2498 else 2499 return 0; 2500} 2501 2502 2503static size_t 2504iov_size (const struct iovec *iov, const struct iovec *const end) 2505{ 2506 size_t size; 2507 2508 assert(iov < end); 2509 2510 size = 0; 2511 do 2512 size += iov->iov_len; 2513 while (++iov < end); 2514 2515 return size; 2516} 2517 2518 2519static void 2520send_packets_out (struct lsquic_engine *engine, 2521 struct conns_tailq *ticked_conns, 2522 struct conns_stailq *closed_conns) 2523{ 2524 unsigned n, w, n_sent, n_batches_sent; 2525 lsquic_packet_out_t *packet_out; 2526 struct lsquic_packet_out **packet; 2527 lsquic_conn_t *conn; 2528 struct out_batch *const batch = &engine->out_batch; 2529 struct iovec *iov, *packet_iov; 2530 struct conns_out_iter conns_iter; 2531 int shrink, deadline_exceeded; 2532 const struct send_batch_ctx sb_ctx = { 2533 closed_conns, 2534 ticked_conns, 2535 &conns_iter, 2536 &engine->out_batch, 2537 }; 2538 2539 coi_init(&conns_iter, engine); 2540 n_batches_sent = 0; 2541 n_sent = 0, n = 0; 2542 shrink = 0; 2543 deadline_exceeded = 0; 2544 iov = batch->iov; 2545 packet = batch->packets; 2546 2547 while ((conn = coi_next(&conns_iter))) 2548 { 2549 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2550 if (!packet_out) { 2551 /* Evanescent connection always has a packet to send: */ 2552 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2553 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2554 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2555 CID_BITS(lsquic_conn_log_cid(conn))); 2556 coi_deactivate(&conns_iter, conn); 2557 continue; 2558 } 2559 batch->outs[n].iov = packet_iov = iov; 2560 next_coa: 2561 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2562 { 2563 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2564 &engine->pub, conn, packet_out)) 2565 { 2566 case ENCPA_NOMEM: 2567 /* Send what we have and wait for a more opportune moment */ 2568 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2569 goto end_for; 2570 case ENCPA_BADCRYPT: 2571 /* This is pretty bad: close connection immediately */ 2572 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2573 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2574 CID_BITS(lsquic_conn_log_cid(conn))); 2575 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2576 { 2577 close_conn_immediately(engine, &sb_ctx, conn); 2578 coi_deactivate(&conns_iter, conn); 2579 } 2580 continue; 2581 case ENCPA_OK: 2582 break; 2583 } 2584 } 2585 else if ((packet_out->po_flags & PO_NOENCRYPT) 2586 && engine->pub.enp_pmi != &stock_pmi) 2587 { 2588 if (0 != copy_packet(engine, conn, packet_out)) 2589 { 2590 /* Copy can only fail if packet could not be allocated */ 2591 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2592 goto end_for; 2593 } 2594 } 2595 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2596 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2597 if (packet_out->po_flags & PO_ENCRYPTED) 2598 { 2599 iov->iov_base = packet_out->po_enc_data; 2600 iov->iov_len = packet_out->po_enc_data_sz; 2601 } 2602 else 2603 { 2604 iov->iov_base = packet_out->po_data; 2605 iov->iov_len = packet_out->po_data_sz; 2606 } 2607 if (packet_iov == iov) 2608 { 2609 batch->pack_off[n] = packet - batch->packets; 2610 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2611 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2612 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2613 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2614 batch->outs [n].conn_ctx = conn->cn_conn_ctx; 2615 batch->conns [n] = conn; 2616 } 2617 *packet = packet_out; 2618 ++packet; 2619 ++iov; 2620 if ((conn->cn_flags & LSCONN_IETF) 2621 && ((1 << packet_out->po_header_type) 2622 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2623#ifndef NDEBUG 2624 && (engine->flags & ENG_COALESCE) 2625#endif 2626 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2627 { 2628 const struct to_coal to_coal = { 2629 .prev_packet = packet_out, 2630 .prev_sz_sum = iov_size(packet_iov, iov), 2631 }; 2632 packet_out = conn->cn_if->ci_next_packet_to_send(conn, &to_coal); 2633 if (packet_out) 2634 goto next_coa; 2635 } 2636 batch->outs [n].iovlen = iov - packet_iov; 2637 ++n; 2638 if (n == engine->batch_size 2639 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2640 { 2641 w = send_batch(engine, &sb_ctx, n); 2642 n = 0; 2643 iov = batch->iov; 2644 packet = batch->packets; 2645 ++n_batches_sent; 2646 n_sent += w; 2647 if (w < engine->batch_size) 2648 { 2649 shrink = 1; 2650 break; 2651 } 2652 deadline_exceeded = check_deadline(engine); 2653 if (deadline_exceeded) 2654 break; 2655 grow_batch_size(engine); 2656 } 2657 } 2658 end_for: 2659 2660 if (n > 0) { 2661 w = send_batch(engine, &sb_ctx, n); 2662 n_sent += w; 2663 shrink = w < n; 2664 ++n_batches_sent; 2665 } 2666 2667 if (shrink) 2668 shrink_batch_size(engine); 2669 else if (n_batches_sent > 1) 2670 { 2671 deadline_exceeded = check_deadline(engine); 2672 if (!deadline_exceeded) 2673 grow_batch_size(engine); 2674 } 2675 2676 coi_reheap(&conns_iter, engine); 2677 2678 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2679} 2680 2681 2682int 2683lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2684{ 2685 return lsquic_mh_count(&engine->conns_out) > 0 2686 || (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 2687 ; 2688} 2689 2690 2691static void 2692reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2693{ 2694 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2695 engine->flags &= ~ENG_PAST_DEADLINE; 2696} 2697 2698 2699static void 2700check_tickable_conns_again (struct lsquic_engine *engine) 2701{ 2702 struct lsquic_hash_elem *el; 2703 struct lsquic_conn *conn; 2704 unsigned count; 2705 2706 count = 0; 2707 for (el = lsquic_hash_first(engine->conns_hash); el; 2708 el = lsquic_hash_next(engine->conns_hash)) 2709 { 2710 conn = lsquic_hashelem_getdata(el); 2711 if (!(conn->cn_flags & LSCONN_TICKABLE) 2712 && conn->cn_if->ci_is_tickable(conn)) 2713 { 2714 lsquic_mh_insert(&engine->conns_tickable, conn, 2715 conn->cn_last_ticked); 2716 engine_incref_conn(conn, LSCONN_TICKABLE); 2717 ++count; 2718 } 2719 } 2720 LSQ_DEBUG("%u connection%s tickable again after sending has been " 2721 "re-enabled", count, count == 1 ? " is" : "s are"); 2722} 2723 2724 2725void 2726lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2727{ 2728 lsquic_conn_t *conn; 2729 struct conns_stailq closed_conns; 2730 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2731 struct cid_update_batch cub; 2732 2733 ENGINE_IN(engine); 2734 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2735 STAILQ_INIT(&closed_conns); 2736 reset_deadline(engine, lsquic_time_now()); 2737 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2738 { 2739 LSQ_DEBUG("can send again"); 2740 EV_LOG_GENERIC_EVENT("can send again"); 2741 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2742 check_tickable_conns_again(engine); 2743 } 2744 2745 send_packets_out(engine, &ticked_conns, &closed_conns); 2746 2747 while ((conn = STAILQ_FIRST(&closed_conns))) { 2748 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2749 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2750 cub_add_cids_from_cces(&cub, conn); 2751 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2752 } 2753 2754 cub_flush(&cub); 2755 ENGINE_OUT(engine); 2756} 2757 2758 2759static lsquic_conn_t * 2760next_new_full_conn (struct conns_stailq *new_full_conns) 2761{ 2762 lsquic_conn_t *conn; 2763 2764 conn = STAILQ_FIRST(new_full_conns); 2765 if (conn) 2766 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2767 return conn; 2768} 2769 2770 2771#if LSQUIC_CONN_STATS 2772static void 2773maybe_log_conn_stats (struct lsquic_engine *engine, struct lsquic_conn *conn, 2774 lsquic_time_t now) 2775{ 2776 char cidstr[MAX_CID_LEN * 2 + 1]; 2777 2778 if (!LSQ_LOG_ENABLED_EXT(LSQ_LOG_NOTICE, LSQLM_CONN_STATS)) 2779 return; 2780 2781 if (conn->cn_last_ticked + 1000000 >= now) 2782 { 2783 if (0 == engine->busy.last_log 2784 || engine->busy.last_log + 1000000 - 1000 < now) 2785 { 2786 engine->busy.last_log = now; 2787 conn->cn_if->ci_log_stats(conn); 2788 } 2789 } 2790 else 2791 { 2792 lsquic_logger_log1(LSQ_LOG_NOTICE, LSQLM_CONN_STATS, 2793 "stop logging status for connection %s: no longer busy", 2794 (lsquic_cid2str(lsquic_conn_log_cid(conn), cidstr), cidstr)); 2795 engine->busy.current = NULL; 2796 engine->busy.last_log = 0; 2797 } 2798} 2799 2800 2801#endif 2802 2803 2804static void 2805process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2806 lsquic_time_t now) 2807{ 2808 lsquic_conn_t *conn; 2809 enum tick_st tick_st; 2810 unsigned i, why; 2811 lsquic_time_t next_tick_time; 2812 struct conns_stailq closed_conns; 2813 struct conns_tailq ticked_conns; 2814 struct conns_stailq new_full_conns; 2815 struct cid_update_batch cub_old, cub_live; 2816 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2817 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2818 2819 eng_hist_tick(&engine->history, now); 2820 2821 STAILQ_INIT(&closed_conns); 2822 TAILQ_INIT(&ticked_conns); 2823 reset_deadline(engine, now); 2824 STAILQ_INIT(&new_full_conns); 2825 2826 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2827 && now > engine->resume_sending_at) 2828 { 2829 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2830 "timeout"); 2831 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2832 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2833 } 2834 2835 i = 0; 2836 while ((conn = next_conn(engine)) 2837 || (conn = next_new_full_conn(&new_full_conns))) 2838 { 2839 tick_st = conn->cn_if->ci_tick(conn, now); 2840#if LSQUIC_CONN_STATS 2841 if (conn == engine->busy.current) 2842 maybe_log_conn_stats(engine, conn, now); 2843#endif 2844 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2845 if (tick_st & TICK_PROMOTE) 2846 { 2847 lsquic_conn_t *new_conn; 2848 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2849 "scheduled for promotion"); 2850 assert(conn->cn_flags & LSCONN_MINI); 2851 new_conn = new_full_conn_server(engine, conn, now); 2852 if (new_conn) 2853 { 2854 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2855 new_conn->cn_last_sent = engine->last_sent; 2856 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2857 conn->cn_flags |= LSCONN_PROMOTED; 2858 } 2859 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2860 } 2861 if (tick_st & TICK_SEND) 2862 { 2863 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2864 { 2865 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2866 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2867 } 2868 } 2869 if (tick_st & TICK_CLOSE) 2870 { 2871 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2872 engine_incref_conn(conn, LSCONN_CLOSING); 2873 if (conn->cn_flags & LSCONN_HASHED) 2874 remove_conn_from_hash(engine, conn); 2875 } 2876 else 2877 { 2878 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2879 engine_incref_conn(conn, LSCONN_TICKED); 2880 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2881 && conn->cn_if->ci_report_live(conn, now)) 2882 cub_add_cids_from_cces(&cub_live, conn); 2883 } 2884 } 2885 2886 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2887 && lsquic_engine_has_unsent_packets(engine)) 2888 send_packets_out(engine, &ticked_conns, &closed_conns); 2889 2890 while ((conn = STAILQ_FIRST(&closed_conns))) { 2891 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2892 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2893 cub_add_cids_from_cces(&cub_old, conn); 2894 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2895 } 2896 2897 while ((conn = TAILQ_FIRST(&ticked_conns))) 2898 { 2899 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2900 engine_decref_conn(engine, conn, LSCONN_TICKED); 2901 if (!(conn->cn_flags & LSCONN_TICKABLE) 2902 && conn->cn_if->ci_is_tickable(conn)) 2903 { 2904 /* Floyd heapification is not faster, don't bother. */ 2905 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2906 engine_incref_conn(conn, LSCONN_TICKABLE); 2907 } 2908 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2909 { 2910 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2911 if (next_tick_time) 2912 { 2913 if (0 == lsquic_attq_add(engine->attq, conn, next_tick_time, 2914 why)) 2915 engine_incref_conn(conn, LSCONN_ATTQ); 2916 } 2917 else 2918 /* In all other cases, the idle timeout would make the next 2919 * tick time non-zero: 2920 */ 2921 assert((conn->cn_flags & LSCONN_IETF) 2922 && engine->pub.enp_settings.es_idle_timeout == 0); 2923 } 2924 } 2925 2926 cub_flush(&engine->new_scids); 2927 cub_flush(&cub_live); 2928 cub_flush(&cub_old); 2929} 2930 2931 2932static void 2933maybe_count_garbage (struct lsquic_engine *engine, size_t garbage_sz) 2934{ 2935 /* This is not very pretty (action at a distance via engine->curr_conn), 2936 * but it's the cheapest I can come up with to handle the "count garbage 2937 * toward amplification limit" requirement in 2938 * [draft-ietf-quic-transport-28] Section 8.1. 2939 */ 2940 if (engine->curr_conn && engine->curr_conn->cn_if->ci_count_garbage) 2941 engine->curr_conn->cn_if->ci_count_garbage(engine->curr_conn, 2942 garbage_sz); 2943} 2944 2945 2946/* Return 0 if packet is being processed by a real connection, 1 if the 2947 * packet was processed, but not by a connection, and -1 on error. 2948 */ 2949int 2950lsquic_engine_packet_in (lsquic_engine_t *engine, 2951 const unsigned char *packet_in_data, size_t packet_in_size, 2952 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2953 void *peer_ctx, int ecn) 2954{ 2955 const unsigned char *const packet_begin = packet_in_data; 2956 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2957 struct packin_parse_state ppstate; 2958 lsquic_packet_in_t *packet_in; 2959 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2960 int is_server, unsigned cid_len, struct packin_parse_state *); 2961 unsigned n_zeroes; 2962 int s, is_ietf; 2963 lsquic_cid_t cid; 2964 2965 ENGINE_CALLS_INCR(engine); 2966 2967 if (engine->flags & ENG_SERVER) 2968 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2969 else if (engine->flags & ENG_CONNS_BY_ADDR) 2970 { 2971 struct lsquic_hash_elem *el; 2972 const struct lsquic_conn *conn; 2973 el = find_conn_by_addr(engine->conns_hash, sa_local); 2974 if (!el) 2975 return -1; 2976 conn = lsquic_hashelem_getdata(el); 2977 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2978 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2979 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2980 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2981 else if (conn->cn_version == LSQVER_050) 2982 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 2983 else 2984 { 2985#if LSQUIC_USE_Q098 2986 assert(conn->cn_version == LSQVER_046 || conn->cn_version == LSQVER_098); 2987#else 2988 assert(conn->cn_version == LSQVER_046); 2989#endif 2990 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2991 } 2992 } 2993 else 2994 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2995 2996 engine->curr_conn = NULL; 2997 n_zeroes = 0; 2998 is_ietf = 0; 2999#ifdef _MSC_VER 3000 s = 0; 3001 cid.len = 0; 3002 cid.idbuf[0] = 0; 3003#endif 3004 do 3005 { 3006 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 3007 if (!packet_in) 3008 return -1; 3009 /* Library does not modify packet_in_data, it is not referenced after 3010 * this function returns and subsequent release of pi_data is guarded 3011 * by PI_OWN_DATA flag. 3012 */ 3013 packet_in->pi_data = (unsigned char *) packet_in_data; 3014 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 3015 engine->flags & ENG_SERVER, 3016 engine->pub.enp_settings.es_scid_len, &ppstate)) 3017 { 3018 LSQ_DEBUG("Cannot parse incoming packet's header"); 3019 maybe_count_garbage(engine, packet_end - packet_in_data); 3020 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 3021 s = 1; 3022 break; 3023 } 3024 3025 /* [draft-ietf-quic-transport-30] Section 12.2: 3026 * " Receivers SHOULD ignore any subsequent packets with a different 3027 * " Destination Connection ID than the first packet in the datagram. 3028 */ 3029 if (is_ietf && packet_in_data > packet_begin) 3030 { 3031 if (!((packet_in->pi_flags & (PI_GQUIC|PI_CONN_ID)) == PI_CONN_ID 3032 && LSQUIC_CIDS_EQ(&packet_in->pi_dcid, &cid))) 3033 { 3034 packet_in_data += packet_in->pi_data_sz; 3035 maybe_count_garbage(engine, packet_in->pi_data_sz); 3036 continue; 3037 } 3038 } 3039 3040 is_ietf = 0 == (packet_in->pi_flags & PI_GQUIC); 3041 packet_in_data += packet_in->pi_data_sz; 3042 if (is_ietf && packet_in_data < packet_end) 3043 { 3044 cid = packet_in->pi_dcid; 3045 if (packet_begin == packet_in->pi_data) /* Only log once: */ 3046 LSQ_DEBUGC("received coalesced datagram of %zd bytes for " 3047 "connection %"CID_FMT, packet_in_size, CID_BITS(&cid)); 3048 } 3049 packet_in->pi_received = lsquic_time_now(); 3050 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 3051 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 3052 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 3053 peer_ctx, packet_in_size); 3054 n_zeroes += s == 0; 3055 } 3056 while (0 == s && packet_in_data < packet_end); 3057 3058 return n_zeroes > 0 ? 0 : s; 3059} 3060 3061 3062#if __GNUC__ && !defined(NDEBUG) 3063__attribute__((weak)) 3064#endif 3065unsigned 3066lsquic_engine_quic_versions (const lsquic_engine_t *engine) 3067{ 3068 return engine->pub.enp_settings.es_versions; 3069} 3070 3071 3072void 3073lsquic_engine_cooldown (lsquic_engine_t *engine) 3074{ 3075 struct lsquic_hash_elem *el; 3076 lsquic_conn_t *conn; 3077 3078 if (engine->flags & ENG_COOLDOWN) 3079 /* AFAICT, there is no harm in calling this function more than once, 3080 * but log it just in case, as it may indicate an error in the caller. 3081 */ 3082 LSQ_INFO("cooldown called again"); 3083 engine->flags |= ENG_COOLDOWN; 3084 LSQ_INFO("entering cooldown mode"); 3085 if (engine->flags & ENG_SERVER) 3086 drop_all_mini_conns(engine); 3087 for (el = lsquic_hash_first(engine->conns_hash); el; 3088 el = lsquic_hash_next(engine->conns_hash)) 3089 { 3090 conn = lsquic_hashelem_getdata(el); 3091 lsquic_conn_going_away(conn); 3092 } 3093} 3094 3095 3096#if LSQUIC_CONN_STATS 3097static void 3098update_busy_detector (struct lsquic_engine *engine, struct lsquic_conn *conn, 3099 int immed) 3100{ 3101 char cidstr[MAX_CID_LEN * 2 + 1]; 3102 3103 if (!(LSQ_LOG_ENABLED_EXT(LSQ_LOG_NOTICE, LSQLM_CONN_STATS) 3104 && conn->cn_if->ci_log_stats)) 3105 return; 3106 3107 if (conn == engine->busy.last_conn) 3108 { 3109 engine->busy.immed_ticks <<= 1u; 3110 engine->busy.immed_ticks |= immed; 3111 if (MAX_IMMED_TICKS == engine->busy.immed_ticks) 3112 { 3113 if (engine->busy.current != conn) 3114 lsquic_logger_log1(LSQ_LOG_NOTICE, LSQLM_CONN_STATS, 3115 "connection %s marked busy: it's had %u immediate ticks " 3116 "in a row", 3117 (lsquic_cid2str(lsquic_conn_log_cid(conn), cidstr), cidstr), 3118 (unsigned) (sizeof(engine->busy.immed_ticks) * 8)); 3119 engine->busy.current = conn; 3120 } 3121 } 3122 else 3123 engine->busy.immed_ticks <<= 1; 3124 3125 engine->busy.last_conn = conn; 3126} 3127 3128 3129#endif 3130 3131 3132int 3133lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 3134{ 3135 const struct attq_elem *next_attq; 3136 lsquic_time_t now, next_time; 3137#if LSQUIC_DEBUG_NEXT_ADV_TICK || LSQUIC_CONN_STATS 3138 struct lsquic_conn *conn; 3139 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 3140#endif 3141 3142 ENGINE_CALLS_INCR(engine); 3143 3144 if ((engine->flags & ENG_PAST_DEADLINE) 3145 && lsquic_mh_count(&engine->conns_out)) 3146 { 3147#if LSQUIC_DEBUG_NEXT_ADV_TICK 3148 conn = lsquic_mh_peek(&engine->conns_out); 3149 engine->last_logged_conn = 0; 3150 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 3151 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 3152 lsquic_mh_count(&engine->conns_out), 3153 lsquic_mh_count(&engine->conns_out) != 1, "s", 3154 CID_BITS(lsquic_conn_log_cid(conn))); 3155#endif 3156#if LSQUIC_CONN_STATS 3157 conn = lsquic_mh_peek(&engine->conns_out); 3158 update_busy_detector(engine, conn, 1); 3159#endif 3160 *diff = 0; 3161 return 1; 3162 } 3163 3164 if (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 3165 { 3166#if LSQUIC_DEBUG_NEXT_ADV_TICK 3167 engine->last_logged_conn = 0; 3168 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 3169#endif 3170 *diff = 0; 3171 return 1; 3172 } 3173 3174 if (lsquic_mh_count(&engine->conns_tickable)) 3175 { 3176#if LSQUIC_DEBUG_NEXT_ADV_TICK 3177 conn = lsquic_mh_peek(&engine->conns_tickable); 3178 engine->last_logged_conn = 0; 3179 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 3180 "connection%.*s (%"CID_FMT" first)", 3181 lsquic_mh_count(&engine->conns_tickable), 3182 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 3183 CID_BITS(lsquic_conn_log_cid(conn))); 3184#endif 3185#if LSQUIC_CONN_STATS 3186 conn = lsquic_mh_peek(&engine->conns_tickable); 3187 update_busy_detector(engine, conn, 1); 3188#endif 3189 *diff = 0; 3190 return 1; 3191 } 3192 3193 next_attq = lsquic_attq_next(engine->attq); 3194 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 3195 { 3196 if (next_attq) 3197 next_time = next_attq->ae_adv_time; 3198 else 3199 return 0; 3200 } 3201 else 3202 { 3203 if (next_attq) 3204 { 3205 next_time = next_attq->ae_adv_time; 3206 if (engine->resume_sending_at < next_time) 3207 { 3208 next_time = engine->resume_sending_at; 3209 next_attq = NULL; 3210 } 3211 } 3212 else 3213 next_time = engine->resume_sending_at; 3214 } 3215 3216 now = lsquic_time_now(); 3217 *diff = (int) ((int64_t) next_time - (int64_t) now); 3218#if LSQUIC_DEBUG_NEXT_ADV_TICK 3219 if (next_attq) 3220 { 3221 /* Deduplicate consecutive log messages about the same reason for the 3222 * same connection. 3223 * If diff is always zero or diff reset to a higher value, event is 3224 * still logged. 3225 */ 3226 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 3227 && (uintptr_t) next_attq->ae_conn 3228 == engine->last_logged_conn 3229 && *diff < engine->last_tick_diff)) 3230 { 3231 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 3232 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 3233 engine->last_tick_diff = *diff; 3234 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 3235 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 3236 lsquic_attq_why2str(next_attq->ae_why)); 3237 } 3238 } 3239 else 3240 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 3241#endif 3242 3243#if LSQUIC_CONN_STATS 3244 if (next_attq) 3245 update_busy_detector(engine, next_attq->ae_conn, 3246 /* Immediate if: a) time is now or in the past */ 3247 *diff <= 0 3248 /* b) next event is pacer, which means that the 3249 * connection wants to send, but is prevented 3250 * by the pacer from doing so. 3251 */ 3252 || next_attq->ae_why == AEW_PACER 3253 /* c) next event is to retransmit data (meaning 3254 * that there is data in flight) and the 3255 * time is small, which implies a small RTT. 3256 */ 3257 || (next_attq->ae_why == N_AEWS + AL_RETX_APP 3258 && *diff < 5000)); 3259#endif 3260 3261 return 1; 3262} 3263 3264 3265unsigned 3266lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 3267{ 3268 lsquic_time_t now; 3269 ENGINE_CALLS_INCR(engine); 3270 now = lsquic_time_now(); 3271 if (from_now < 0) 3272 now -= from_now; 3273 else 3274 now += from_now; 3275 return lsquic_attq_count_before(engine->attq, now); 3276} 3277 3278 3279int 3280lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 3281 struct lsquic_conn *conn, unsigned cce_idx) 3282{ 3283 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 3284 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 3285 void *peer_ctx; 3286 3287 assert(cce_idx < conn->cn_n_cces); 3288 assert(conn->cn_cces_mask & (1 << cce_idx)); 3289 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 3290 3291 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 3292 cce->cce_cid.len, conn, &cce->cce_hash_el)) 3293 { 3294 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 3295 CID_BITS(&cce->cce_cid)); 3296 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 3297 cce->cce_flags |= CCE_REG; 3298 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 3299 return 0; 3300 } 3301 else 3302 { 3303 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 3304 CID_BITS(&cce->cce_cid)); 3305 return -1; 3306 } 3307} 3308 3309 3310void 3311lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 3312 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now, 3313 lsquic_time_t drain_time) 3314{ 3315 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 3316 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 3317 struct purga_el *puel; 3318 void *peer_ctx; 3319 3320 assert(cce_idx < conn->cn_n_cces); 3321 3322 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 3323 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 3324 3325 if (engine->purga) 3326 { 3327 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 3328 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 3329 PUTY_CID_RETIRED, now); 3330 if (puel) 3331 puel->puel_time = now + drain_time; 3332 } 3333 conn->cn_cces_mask &= ~(1u << cce_idx); 3334 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 3335} 3336 3337 3338