lsquic_engine.c revision 26e8f082
1/* Copyright (c) 2017 - 2021 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#ifndef WIN32 17#include <sys/time.h> 18#include <netinet/in.h> 19#include <sys/types.h> 20#include <sys/stat.h> 21#include <fcntl.h> 22#include <unistd.h> 23#include <netdb.h> 24#endif 25 26#ifndef NDEBUG 27#include <sys/types.h> 28#endif 29 30#if defined(WIN32) || defined(NDEBUG) 31#define CAN_LOSE_PACKETS 0 32#else 33#define CAN_LOSE_PACKETS 1 34#endif 35 36#if CAN_LOSE_PACKETS 37#include <regex.h> /* For code that loses packets */ 38#endif 39 40#if LOG_PACKET_CHECKSUM 41#include <zlib.h> 42#endif 43 44#include <openssl/aead.h> 45 46#include "lsquic.h" 47#include "lsquic_types.h" 48#include "lsquic_int_types.h" 49#include "lsquic_sizes.h" 50#include "lsquic_parse_common.h" 51#include "lsquic_parse.h" 52#include "lsquic_packet_in.h" 53#include "lsquic_packet_out.h" 54#include "lsquic_senhist.h" 55#include "lsquic_rtt.h" 56#include "lsquic_cubic.h" 57#include "lsquic_pacer.h" 58#include "lsquic_bw_sampler.h" 59#include "lsquic_minmax.h" 60#include "lsquic_bbr.h" 61#include "lsquic_adaptive_cc.h" 62#include "lsquic_set.h" 63#include "lsquic_conn_flow.h" 64#include "lsquic_sfcw.h" 65#include "lsquic_hash.h" 66#include "lsquic_conn.h" 67#include "lsquic_send_ctl.h" 68#include "lsquic_full_conn.h" 69#include "lsquic_util.h" 70#include "lsquic_qtags.h" 71#include "lsquic_enc_sess.h" 72#include "lsquic_mm.h" 73#include "lsquic_engine_public.h" 74#include "lsquic_eng_hist.h" 75#include "lsquic_ev_log.h" 76#include "lsquic_version.h" 77#include "lsquic_pr_queue.h" 78#include "lsquic_mini_conn.h" 79#include "lsquic_trechist.h" 80#include "lsquic_mini_conn_ietf.h" 81#include "lsquic_stock_shi.h" 82#include "lsquic_purga.h" 83#include "lsquic_tokgen.h" 84#include "lsquic_attq.h" 85#include "lsquic_min_heap.h" 86#include "lsquic_http1x_if.h" 87#include "lsquic_handshake.h" 88#include "lsquic_crand.h" 89#include "lsquic_ietf.h" 90#include "lsquic_handshake.h" 91 92#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 93#include "lsquic_logger.h" 94 95#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 96#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 97#endif 98 99#if LSQUIC_DEBUG_NEXT_ADV_TICK 100#include "lsquic_alarmset.h" 101#endif 102 103#define MIN(a, b) ((a) < (b) ? (a) : (b)) 104#define MAX(a, b) ((a) > (b) ? (a) : (b)) 105 106/* The batch of outgoing packets grows and shrinks dynamically */ 107/* Batch sizes do not have to be powers of two */ 108#define MAX_OUT_BATCH_SIZE 1024 109#define MIN_OUT_BATCH_SIZE 4 110#define INITIAL_OUT_BATCH_SIZE 32 111 112struct out_batch 113{ 114 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 115 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 116 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 117 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 118 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 119}; 120 121typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 122 123static void 124process_connections (struct lsquic_engine *engine, conn_iter_f iter, 125 lsquic_time_t now); 126 127static void 128engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 129 130static lsquic_conn_t * 131engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 132 enum lsquic_conn_flags flag); 133 134static void 135force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 136 137#if LSQUIC_CONN_STATS 138static void 139update_busy_detector (struct lsquic_engine *, struct lsquic_conn *, int); 140#endif 141 142#if LSQUIC_COUNT_ENGINE_CALLS 143#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 144#else 145#define ENGINE_CALLS_INCR(e) 146#endif 147 148/* Nested calls to some LSQUIC functions are not supported. Functions that 149 * iterate over connections cannot be nested. 150 */ 151#define ENGINE_IN(e) do { \ 152 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 153 (e)->pub.enp_flags |= ENPUB_PROC; \ 154 ENGINE_CALLS_INCR(e); \ 155} while (0) 156 157#define ENGINE_OUT(e) do { \ 158 assert((e)->pub.enp_flags & ENPUB_PROC); \ 159 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 160} while (0) 161 162/* A connection can be referenced from one of six places: 163 * 164 * 1. A hash is used to find connections in order to dispatch an incoming 165 * packet. Connections can be hashed by CIDs or by address. In the 166 * former case, each connection has one or more mappings in the hash 167 * table. IETF QUIC connections have up to eight (in our implementation) 168 * source CIDs and each of those would have a mapping. In client mode, 169 * depending on QUIC versions and options selected, it is may be 170 * necessary to hash connections by address, in which case incoming 171 * packets are delivered to connections based on the address. 172 * 173 * 2. Outgoing queue. 174 * 175 * 3. Tickable queue 176 * 177 * 4. Advisory Tick Time queue. 178 * 179 * 5. Closing connections queue. This is a transient queue -- it only 180 * exists for the duration of process_connections() function call. 181 * 182 * 6. Ticked connections queue. Another transient queue, similar to (5). 183 * 184 * The idea is to destroy the connection when it is no longer referenced. 185 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 186 * that case, the connection is referenced from two places: (2) and (5). 187 * After its packets are sent, it is only referenced in (5), and at the 188 * end of the function call, when it is removed from (5), reference count 189 * goes to zero and the connection is destroyed. If not all packets can 190 * be sent, at the end of the function call, the connection is referenced 191 * by (2) and will only be removed once all outgoing packets have been 192 * sent. 193 */ 194#define CONN_REF_FLAGS (LSCONN_HASHED \ 195 |LSCONN_HAS_OUTGOING \ 196 |LSCONN_TICKABLE \ 197 |LSCONN_TICKED \ 198 |LSCONN_CLOSING \ 199 |LSCONN_ATTQ) 200 201 202 203 204struct cid_update_batch 205{ 206 lsquic_cids_update_f cub_update_cids; 207 void *cub_update_ctx; 208 unsigned cub_count; 209 lsquic_cid_t cub_cids[20]; 210 void *cub_peer_ctxs[20]; 211}; 212 213static void 214cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 215 216 217struct lsquic_engine 218{ 219 struct lsquic_engine_public pub; 220 enum { 221 ENG_SERVER = LSENG_SERVER, 222 ENG_HTTP = LSENG_HTTP, 223 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 224 ENG_PAST_DEADLINE 225 = (1 << 8), /* Previous call to a processing 226 * function went past time threshold. 227 */ 228 ENG_CONNS_BY_ADDR 229 = (1 << 9), /* Connections are hashed by address */ 230#ifndef NDEBUG 231 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 232#endif 233#if CAN_LOSE_PACKETS 234 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 235#endif 236#ifndef NDEBUG 237 ENG_DTOR = (1 << 26), /* Engine destructor */ 238#endif 239 } flags; 240 lsquic_packets_out_f packets_out; 241 void *packets_out_ctx; 242 lsquic_cids_update_f report_new_scids; 243 lsquic_cids_update_f report_live_scids; 244 lsquic_cids_update_f report_old_scids; 245 void *scids_ctx; 246 struct lsquic_hash *conns_hash; 247 struct min_heap conns_tickable; 248 struct min_heap conns_out; 249 struct eng_hist history; 250 unsigned batch_size; 251 unsigned min_batch_size, max_batch_size; 252 struct lsquic_conn *curr_conn; 253 struct pr_queue *pr_queue; 254 struct attq *attq; 255 /* Track time last time a packet was sent to give new connections 256 * priority lower than that of existing connections. 257 */ 258 lsquic_time_t last_sent; 259#if CAN_LOSE_PACKETS 260 regex_t lose_packets_re; 261 const char *lose_packets_str; 262#endif 263 unsigned n_conns; 264 lsquic_time_t deadline; 265 lsquic_time_t resume_sending_at; 266 unsigned mini_conns_count; 267 struct lsquic_purga *purga; 268#if LSQUIC_CONN_STATS 269 struct { 270 unsigned conns; 271 } stats; 272 struct conn_stats conn_stats_sum; 273 FILE *stats_fh; 274#endif 275 struct cid_update_batch new_scids; 276 struct out_batch out_batch; 277#if LSQUIC_COUNT_ENGINE_CALLS 278 unsigned long n_engine_calls; 279#endif 280#if LSQUIC_DEBUG_NEXT_ADV_TICK 281 uintptr_t last_logged_conn; 282 unsigned last_logged_ae_why; 283 int last_tick_diff; 284#endif 285 struct crand crand; 286 EVP_AEAD_CTX retry_aead_ctx[N_IETF_RETRY_VERSIONS]; 287#if LSQUIC_CONN_STATS 288 struct { 289 uint16_t immed_ticks; /* bitmask */ 290#define MAX_IMMED_TICKS UINT16_MAX 291 struct lsquic_conn *last_conn, /* from last call */ 292 *pin_conn, /* last connection with packet in */ 293 *current; /* currently busy connection */ 294 lsquic_time_t last_log; 295 } busy; 296#endif 297}; 298 299 300void 301lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 302 unsigned flags) 303{ 304 memset(settings, 0, sizeof(*settings)); 305 settings->es_versions = LSQUIC_DF_VERSIONS; 306 if (flags & ENG_SERVER) 307 { 308 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 309 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 310 settings->es_init_max_data 311 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 312 settings->es_init_max_stream_data_bidi_remote 313 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 314 settings->es_init_max_stream_data_bidi_local 315 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 316 settings->es_init_max_stream_data_uni 317 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 318 settings->es_init_max_streams_uni 319 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 320 settings->es_ping_period = 0; 321 settings->es_noprogress_timeout 322 = LSQUIC_DF_NOPROGRESS_TIMEOUT_SERVER; 323 } 324 else 325 { 326 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 327 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 328 settings->es_init_max_data 329 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 330 settings->es_init_max_stream_data_bidi_remote 331 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 332 settings->es_init_max_stream_data_bidi_local 333 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 334 settings->es_init_max_stream_data_uni 335 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 336 settings->es_init_max_streams_uni 337 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 338 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 339 settings->es_noprogress_timeout 340 = LSQUIC_DF_NOPROGRESS_TIMEOUT_CLIENT; 341 } 342 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 343 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 344 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 345 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 346 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 347 settings->es_max_header_list_size 348 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 349 settings->es_ua = LSQUIC_DF_UA; 350 settings->es_ecn = LSQUIC_DF_ECN; 351 352 settings->es_pdmd = QTAG_X509; 353 settings->es_aead = QTAG_AESG; 354 settings->es_kexs = QTAG_C255; 355 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 356 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 357 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 358 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 359 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 360 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 361 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 362 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 363 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 364 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 365 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 366 settings->es_sttl = LSQUIC_DF_STTL; 367 settings->es_init_max_streams_bidi 368 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 369 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 370 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 371 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 372 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 373 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 374 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 375 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 376 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 377 settings->es_spin = LSQUIC_DF_SPIN; 378 settings->es_delayed_acks = LSQUIC_DF_DELAYED_ACKS; 379 settings->es_timestamps = LSQUIC_DF_TIMESTAMPS; 380 settings->es_grease_quic_bit = LSQUIC_DF_GREASE_QUIC_BIT; 381 settings->es_mtu_probe_timer = LSQUIC_DF_MTU_PROBE_TIMER; 382 settings->es_dplpmtud = LSQUIC_DF_DPLPMTUD; 383 settings->es_cc_algo = LSQUIC_DF_CC_ALGO; 384 settings->es_cc_rtt_thresh = LSQUIC_DF_CC_RTT_THRESH; 385 settings->es_optimistic_nat = LSQUIC_DF_OPTIMISTIC_NAT; 386 settings->es_ext_http_prio = LSQUIC_DF_EXT_HTTP_PRIO; 387 settings->es_ptpc_periodicity= LSQUIC_DF_PTPC_PERIODICITY; 388 settings->es_ptpc_max_packtol= LSQUIC_DF_PTPC_MAX_PACKTOL; 389 settings->es_ptpc_dyn_target = LSQUIC_DF_PTPC_DYN_TARGET; 390 settings->es_ptpc_target = LSQUIC_DF_PTPC_TARGET; 391 settings->es_ptpc_prop_gain = LSQUIC_DF_PTPC_PROP_GAIN; 392 settings->es_ptpc_int_gain = LSQUIC_DF_PTPC_INT_GAIN; 393 settings->es_ptpc_err_thresh = LSQUIC_DF_PTPC_ERR_THRESH; 394 settings->es_ptpc_err_divisor= LSQUIC_DF_PTPC_ERR_DIVISOR; 395 settings->es_delay_onclose = LSQUIC_DF_DELAY_ONCLOSE; 396 settings->es_check_tp_sanity = LSQUIC_DF_CHECK_TP_SANITY; 397} 398 399 400/* Note: if returning an error, err_buf must be valid if non-NULL */ 401int 402lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 403 unsigned flags, 404 char *err_buf, size_t err_buf_sz) 405{ 406 if (settings->es_cfcw < LSQUIC_MIN_FCW || 407 settings->es_sfcw < LSQUIC_MIN_FCW) 408 { 409 if (err_buf) 410 snprintf(err_buf, err_buf_sz, "%s", 411 "flow control window set too low"); 412 return -1; 413 } 414 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 415 { 416 if (err_buf) 417 snprintf(err_buf, err_buf_sz, "%s", 418 "No supported QUIC versions specified"); 419 return -1; 420 } 421 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 422 { 423 if (err_buf) 424 snprintf(err_buf, err_buf_sz, "%s", 425 "one or more unsupported QUIC version is specified"); 426 return -1; 427 } 428 if (flags & ENG_SERVER) 429 { 430 if (settings->es_handshake_to > 431 MAX_MINI_CONN_LIFESPAN_IN_USEC) 432 { 433 if (err_buf) 434 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 435 " usec is too large. The maximum for server is %u usec", 436 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 437 return -1; 438 } 439 } 440 if (settings->es_idle_timeout > 600) 441 { 442 if (err_buf) 443 snprintf(err_buf, err_buf_sz, "%s", 444 "The maximum value of idle timeout is 600 seconds"); 445 return -1; 446 } 447 if (settings->es_scid_len > MAX_CID_LEN) 448 { 449 if (err_buf) 450 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 451 "bytes long; it must be between 0 and %u.", 452 settings->es_scid_len, MAX_CID_LEN); 453 return -1; 454 } 455 456 if (settings->es_cc_algo > 3) 457 { 458 if (err_buf) 459 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 460 "algorithm value %u", settings->es_cc_algo); 461 return -1; 462 } 463 464 if (!(settings->es_ql_bits >= 0 && settings->es_ql_bits <= 2)) 465 { 466 if (err_buf) 467 snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ", 468 settings->es_ql_bits); 469 return -1; 470 } 471 472 if (!(settings->es_spin == 0 || settings->es_spin == 1)) 473 { 474 if (err_buf) 475 snprintf(err_buf, err_buf_sz, "Invalid spin value %d", 476 settings->es_spin); 477 return -1; 478 } 479 480 if (settings->es_mtu_probe_timer && settings->es_mtu_probe_timer < 1000) 481 { 482 if (err_buf) 483 snprintf(err_buf, err_buf_sz, "mtu probe timer is too small: " 484 "%u ms", settings->es_mtu_probe_timer); 485 return -1; 486 } 487 488 if (settings->es_max_batch_size > MAX_OUT_BATCH_SIZE) 489 { 490 if (err_buf) 491 snprintf(err_buf, err_buf_sz, "max batch size is greater than " 492 "the allowed maximum of %u", (unsigned) MAX_OUT_BATCH_SIZE); 493 return -1; 494 } 495 496 return 0; 497} 498 499 500static void 501free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 502{ 503 free(packet_data); 504} 505 506 507static void * 508malloc_buf (void *ctx, void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, unsigned short size, char is_ipv6) 509{ 510 return malloc(size); 511} 512 513 514static const struct lsquic_packout_mem_if stock_pmi = 515{ 516 malloc_buf, free_packet, free_packet, 517}; 518 519 520static int 521hash_conns_by_addr (const struct lsquic_engine *engine) 522{ 523 if (engine->flags & ENG_SERVER) 524 return 0; 525 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 526 return 1; 527 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 528 && engine->pub.enp_settings.es_support_tcid0) 529 return 1; 530 if (engine->pub.enp_settings.es_scid_len == 0) 531 return 1; 532 return 0; 533} 534 535 536lsquic_engine_t * 537lsquic_engine_new (unsigned flags, 538 const struct lsquic_engine_api *api) 539{ 540 lsquic_engine_t *engine; 541 size_t alpn_len; 542 unsigned i; 543 char err_buf[100]; 544 545 if (!api->ea_packets_out) 546 { 547 LSQ_ERROR("packets_out callback is not specified"); 548 return NULL; 549 } 550 551 if (!api->ea_stream_if) 552 { 553 LSQ_ERROR("stream interface is not specified"); 554 return NULL; 555 } 556 557 if (!(flags & LSENG_HTTP) && api->ea_alpn) 558 { 559 alpn_len = strlen(api->ea_alpn); 560 if (alpn_len < 1 || alpn_len > 255) 561 { 562 LSQ_ERROR("ALPN string length invalid: %zd bytes", alpn_len); 563 return NULL; 564 } 565 } 566 else 567 alpn_len = 0; 568 569 if (api->ea_settings && 570 0 != lsquic_engine_check_settings(api->ea_settings, flags, 571 err_buf, sizeof(err_buf))) 572 { 573 LSQ_ERROR("cannot create engine: %s", err_buf); 574 return NULL; 575 } 576 577 engine = calloc(1, sizeof(*engine)); 578 if (!engine) 579 return NULL; 580 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 581 { 582 free(engine); 583 return NULL; 584 } 585 if (api->ea_settings) 586 engine->pub.enp_settings = *api->ea_settings; 587 else 588 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 589 int tag_buf_len; 590 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 591 sizeof(engine->pub.enp_ver_tags_buf), 592 engine->pub.enp_settings.es_versions); 593 if (tag_buf_len <= 0) 594 { 595 LSQ_ERROR("cannot generate version tags buffer"); 596 free(engine); 597 return NULL; 598 } 599 engine->pub.enp_ver_tags_len = tag_buf_len; 600 engine->pub.enp_flags = ENPUB_CAN_SEND; 601 engine->pub.enp_stream_if = api->ea_stream_if; 602 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 603 604 engine->flags = flags; 605#ifndef NDEBUG 606 engine->flags |= ENG_COALESCE; 607#endif 608 engine->packets_out = api->ea_packets_out; 609 engine->packets_out_ctx = api->ea_packets_out_ctx; 610 engine->report_new_scids = api->ea_new_scids; 611 engine->report_live_scids = api->ea_live_scids; 612 engine->report_old_scids = api->ea_old_scids; 613 engine->scids_ctx = api->ea_cids_update_ctx; 614 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 615 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 616 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 617 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 618 619 if (api->ea_generate_scid) 620 { 621 engine->pub.enp_generate_scid = api->ea_generate_scid; 622 engine->pub.enp_gen_scid_ctx = api->ea_gen_scid_ctx; 623 } 624 else 625 engine->pub.enp_generate_scid = lsquic_generate_scid; 626 627 if (api->ea_shi) 628 { 629 engine->pub.enp_shi = api->ea_shi; 630 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 631 } 632 else 633 { 634 engine->pub.enp_shi = &stock_shi; 635 engine->pub.enp_shi_ctx = lsquic_stock_shared_hash_new(); 636 if (!engine->pub.enp_shi_ctx) 637 { 638 free(engine); 639 return NULL; 640 } 641 } 642 if (api->ea_hsi_if) 643 { 644 engine->pub.enp_hsi_if = api->ea_hsi_if; 645 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 646 } 647 else 648 { 649 engine->pub.enp_hsi_if = lsquic_http1x_if; 650 engine->pub.enp_hsi_ctx = NULL; 651 } 652 if (api->ea_pmi) 653 { 654 engine->pub.enp_pmi = api->ea_pmi; 655 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 656 } 657 else 658 { 659 engine->pub.enp_pmi = &stock_pmi; 660 engine->pub.enp_pmi_ctx = NULL; 661 } 662 engine->pub.enp_verify_cert = api->ea_verify_cert; 663 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 664 engine->pub.enp_engine = engine; 665 if (hash_conns_by_addr(engine)) 666 engine->flags |= ENG_CONNS_BY_ADDR; 667 engine->conns_hash = lsquic_hash_create(); 668 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 669 if (!engine->pub.enp_tokgen) 670 return NULL; 671 if (engine->flags & ENG_SERVER) 672 for (i = 0; i < sizeof(engine->pub.enp_quic_ctx_sz) 673 / sizeof(engine->pub.enp_quic_ctx_sz[0]); ++i) 674 { 675 int sz = lsquic_enc_sess_ietf_gen_quic_ctx( 676 &engine->pub.enp_settings, 677 i == 0 ? LSQVER_ID27 : LSQVER_ID29, 678 engine->pub.enp_quic_ctx_buf[i], 679 sizeof(engine->pub.enp_quic_ctx_buf)); 680 if (sz < 0) 681 { 682 free(engine); 683 return NULL; 684 } 685 engine->pub.enp_quic_ctx_sz[i] = (unsigned) sz; 686 } 687 engine->pub.enp_crand = &engine->crand; 688 if (engine->pub.enp_settings.es_noprogress_timeout) 689 engine->pub.enp_noprog_timeout 690 = engine->pub.enp_settings.es_noprogress_timeout * 1000000; 691 engine->pub.enp_mtu_probe_timer = 1000 692 * (engine->pub.enp_settings.es_mtu_probe_timer 693 ? engine->pub.enp_settings.es_mtu_probe_timer 694 : LSQUIC_DF_MTU_PROBE_TIMER); 695 if (flags & ENG_SERVER) 696 { 697 engine->pr_queue = lsquic_prq_create( 698 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 699 &engine->pub); 700 if (!engine->pr_queue) 701 { 702 lsquic_tg_destroy(engine->pub.enp_tokgen); 703 return NULL; 704 } 705 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 706 engine->report_old_scids, engine->scids_ctx); 707 if (!engine->purga) 708 { 709 lsquic_tg_destroy(engine->pub.enp_tokgen); 710 lsquic_prq_destroy(engine->pr_queue); 711 return NULL; 712 } 713 } 714 engine->attq = lsquic_attq_create(); 715 eng_hist_init(&engine->history); 716 if (engine->pub.enp_settings.es_max_batch_size) 717 { 718 engine->max_batch_size = engine->pub.enp_settings.es_max_batch_size; 719 engine->min_batch_size = MIN(4, engine->max_batch_size); 720 engine->batch_size = MAX(engine->max_batch_size / 4, 721 engine->min_batch_size); 722 } 723 else 724 { 725 engine->min_batch_size = MIN_OUT_BATCH_SIZE; 726 engine->max_batch_size = MAX_OUT_BATCH_SIZE; 727 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 728 } 729 if (engine->pub.enp_settings.es_honor_prst) 730 { 731 engine->pub.enp_srst_hash = lsquic_hash_create(); 732 if (!engine->pub.enp_srst_hash) 733 { 734 lsquic_engine_destroy(engine); 735 return NULL; 736 } 737 } 738 if ((flags & LSENG_SERVER) && 0 != lsquic_init_gquic_crypto(&engine->pub)) 739 { 740 lsquic_engine_destroy(engine); 741 return NULL; 742 } 743 744 if (alpn_len) 745 { 746 engine->pub.enp_alpn = malloc(alpn_len + 1); 747 if (!engine->pub.enp_alpn) 748 { 749 lsquic_engine_destroy(engine); 750 return NULL; 751 } 752 engine->pub.enp_alpn[0] = alpn_len; 753 memcpy(engine->pub.enp_alpn + 1, api->ea_alpn, alpn_len); 754 } 755 756 if (flags & LSENG_HTTP) 757 engine->pub.enp_flags |= ENPUB_HTTP; 758 759#ifndef NDEBUG 760 { 761 const char *env; 762 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 763#if CAN_LOSE_PACKETS 764 if (env) 765 { 766 if (0 != regcomp(&engine->lose_packets_re, env, 767 REG_EXTENDED|REG_NOSUB)) 768 { 769 LSQ_ERROR("could not compile lost packet regex `%s'", env); 770 return NULL; 771 } 772 engine->flags |= ENG_LOSE_PACKETS; 773 engine->lose_packets_str = env; 774 LSQ_WARN("will lose packets that match the following regex: %s", 775 env); 776 } 777#endif 778 env = getenv("LSQUIC_COALESCE"); 779 if (env) 780 { 781 engine->flags &= ~ENG_COALESCE; 782 if (atoi(env)) 783 { 784 engine->flags |= ENG_COALESCE; 785 LSQ_NOTICE("will coalesce packets"); 786 } 787 else 788 LSQ_NOTICE("will not coalesce packets"); 789 } 790 } 791#endif 792#if LSQUIC_CONN_STATS 793 engine->stats_fh = api->ea_stats_fh; 794#endif 795 for (i = 0; i < sizeof(engine->retry_aead_ctx) 796 / sizeof(engine->retry_aead_ctx[0]); ++i) 797 if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx[i], 798 EVP_aead_aes_128_gcm(), lsquic_retry_key_buf[i], 799 IETF_RETRY_KEY_SZ, 16, NULL)) 800 { 801 LSQ_ERROR("could not initialize retry AEAD ctx #%u", i); 802 lsquic_engine_destroy(engine); 803 return NULL; 804 } 805 engine->pub.enp_retry_aead_ctx = engine->retry_aead_ctx; 806 807 LSQ_INFO("instantiated engine"); 808 return engine; 809} 810 811 812#if LOG_PACKET_CHECKSUM 813static void 814log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 815 const unsigned char *buf, size_t bufsz) 816{ 817 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 818 (uint32_t) crc32(0, buf, bufsz)); 819} 820 821 822#endif 823 824 825static void 826grow_batch_size (struct lsquic_engine *engine) 827{ 828 engine->batch_size = MIN(engine->batch_size * 2, engine->max_batch_size); 829} 830 831 832static void 833shrink_batch_size (struct lsquic_engine *engine) 834{ 835 engine->batch_size = MAX(engine->batch_size / 2, engine->min_batch_size); 836} 837 838 839struct cce_cid_iter 840{ 841 const struct lsquic_conn *conn; 842 unsigned todo, n; 843}; 844 845 846static struct conn_cid_elem * 847cce_iter_next (struct cce_cid_iter *citer) 848{ 849 struct conn_cid_elem *cce; 850 851 while (citer->todo) 852 if (citer->todo & (1 << citer->n)) 853 { 854 citer->todo &= ~(1 << citer->n); 855 cce = &citer->conn->cn_cces[ citer->n++ ]; 856 if (!(cce->cce_flags & CCE_PORT)) 857 return cce; 858 } 859 else 860 ++citer->n; 861 862 return NULL; 863} 864 865 866static struct conn_cid_elem * 867cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 868{ 869 citer->conn = conn; 870 citer->todo = conn->cn_cces_mask; 871 citer->n = 0; 872 return cce_iter_next(citer); 873} 874 875 876#if LSQUIC_CONN_STATS 877void 878update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 879{ 880 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 881 const unsigned long *src; 882 const struct conn_stats *stats; 883 unsigned i; 884 885 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 886 { 887 ++engine->stats.conns; 888 src = (unsigned long *) stats; 889 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 890 dst[i] += src[i]; 891 } 892} 893 894 895#endif 896 897 898/* Wrapper to make sure important things occur before the connection is 899 * really destroyed. 900 */ 901static void 902destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 903 lsquic_time_t now) 904{ 905 struct cce_cid_iter citer; 906 const struct conn_cid_elem *cce; 907 lsquic_time_t drain_time; 908 struct purga_el *puel; 909 910 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 911 if (engine->purga 912 /* Blacklist all CIDs except for promoted mini connections */ 913 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 914 != (LSCONN_MINI|LSCONN_PROMOTED)) 915 { 916 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 917 && conn->cn_if->ci_drain_time && 918 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 919 { 920 for (cce = cce_iter_first(&citer, conn); cce; 921 cce = cce_iter_next(&citer)) 922 { 923 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 924 lsquic_conn_get_peer_ctx(conn, NULL), 925 PUTY_CONN_DRAIN, now); 926 if (puel) 927 puel->puel_time = now + drain_time; 928 } 929 } 930 else 931 { 932 for (cce = cce_iter_first(&citer, conn); cce; 933 cce = cce_iter_next(&citer)) 934 { 935 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 936 lsquic_conn_get_peer_ctx(conn, NULL), 937 PUTY_CONN_DELETED, now); 938 if (puel) 939 { 940 puel->puel_time = now; 941 puel->puel_count = 0; 942 } 943 } 944 } 945 } 946#if LSQUIC_CONN_STATS 947 update_stats_sum(engine, conn); 948 if (engine->busy.last_conn == conn) 949 engine->busy.last_conn = NULL; 950 if (engine->busy.current == conn) 951 { 952 char cidstr[MAX_CID_LEN * 2 + 1]; 953 lsquic_logger_log1(LSQ_LOG_NOTICE, LSQLM_CONN_STATS, 954 "busy connection %s is destroyed", 955 (lsquic_cid2str(lsquic_conn_log_cid(conn), cidstr), cidstr)); 956 engine->busy.current = NULL; 957 engine->busy.last_log = 0; 958 } 959 if (engine->busy.pin_conn == conn) 960 engine->busy.pin_conn = NULL; 961#endif 962 --engine->n_conns; 963 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 964 conn->cn_if->ci_destroy(conn); 965} 966 967 968static int 969maybe_grow_conn_heaps (struct lsquic_engine *engine) 970{ 971 struct min_heap_elem *els; 972 unsigned count; 973 974 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 975 return 0; /* Nothing to do */ 976 977 if (lsquic_mh_nalloc(&engine->conns_tickable)) 978 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 979 else 980 count = 8; 981 982 els = malloc(sizeof(els[0]) * count); 983 if (!els) 984 { 985 LSQ_ERROR("%s: malloc failed", __func__); 986 return -1; 987 } 988 989 LSQ_DEBUG("grew heaps to %u elements", count / 2); 990 memcpy(&els[0], engine->conns_tickable.mh_elems, 991 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 992 memcpy(&els[count / 2], engine->conns_out.mh_elems, 993 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 994 free(engine->conns_tickable.mh_elems); 995 engine->conns_tickable.mh_elems = els; 996 engine->conns_out.mh_elems = &els[count / 2]; 997 engine->conns_tickable.mh_nalloc = count / 2; 998 engine->conns_out.mh_nalloc = count / 2; 999 return 0; 1000} 1001 1002 1003static void 1004remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 1005 unsigned todo) 1006{ 1007 unsigned n; 1008 1009 for (n = 0; todo; todo &= ~(1 << n++)) 1010 if ((todo & (1 << n)) && 1011 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 1012 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 1013} 1014 1015 1016static void 1017remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 1018{ 1019 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 1020} 1021 1022 1023static void 1024cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 1025 1026 1027static int 1028insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1029 void *peer_ctx) 1030{ 1031 struct conn_cid_elem *cce; 1032 unsigned todo, done, n; 1033 1034 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 1035 if (todo & (1 << n)) 1036 { 1037 cce = &conn->cn_cces[n]; 1038 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 1039 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 1040 cce->cce_cid.len, conn, &cce->cce_hash_el)) 1041 done |= 1 << n; 1042 else 1043 goto err; 1044 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 1045 { 1046 cce->cce_flags |= CCE_REG; 1047 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 1048 } 1049 } 1050 1051 return 0; 1052 1053 err: 1054 remove_cces_from_hash(engine->conns_hash, conn, done); 1055 return -1; 1056} 1057 1058 1059static lsquic_conn_t * 1060new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 1061 lsquic_time_t now) 1062{ 1063 const lsquic_cid_t *cid; 1064 server_conn_ctor_f ctor; 1065 lsquic_conn_t *conn; 1066 unsigned flags; 1067 if (0 != maybe_grow_conn_heaps(engine)) 1068 return NULL; 1069 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1070 1071 if (mini_conn->cn_flags & LSCONN_IETF) 1072 ctor = lsquic_ietf_full_conn_server_new; 1073 else 1074 ctor = lsquic_gquic_full_conn_server_new; 1075 1076 conn = ctor(&engine->pub, flags, mini_conn); 1077 if (!conn) 1078 { 1079 /* Otherwise, full_conn_server_new prints its own warnings */ 1080 if (ENOMEM == errno) 1081 { 1082 cid = lsquic_conn_log_cid(mini_conn); 1083 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 1084 CID_BITS(cid), strerror(errno)); 1085 } 1086 return NULL; 1087 } 1088 ++engine->n_conns; 1089 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 1090 { 1091 cid = lsquic_conn_log_cid(conn); 1092 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1093 CID_BITS(cid)); 1094 destroy_conn(engine, conn, now); 1095 return NULL; 1096 } 1097 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1098 conn->cn_flags |= LSCONN_HASHED; 1099 return conn; 1100} 1101 1102 1103static enum 1104{ 1105 VER_NOT_SPECIFIED, 1106 VER_SUPPORTED, 1107 VER_UNSUPPORTED, 1108} 1109 1110 1111version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 1112 enum lsquic_version *pversion) 1113{ 1114 lsquic_ver_tag_t ver_tag; 1115 enum lsquic_version version; 1116 1117 if (!packet_in->pi_quic_ver) 1118 { 1119 LSQ_DEBUG("packet does not specify version"); 1120 return VER_NOT_SPECIFIED; 1121 } 1122 1123 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 1124 version = lsquic_tag2ver(ver_tag); 1125 if (version < N_LSQVER) 1126 { 1127 if (engine->pub.enp_settings.es_versions & (1 << version)) 1128 { 1129 LSQ_DEBUG("client-supplied version %s is supported", 1130 lsquic_ver2str[version]); 1131 *pversion = version; 1132 return VER_SUPPORTED; 1133 } 1134 else 1135 LSQ_DEBUG("client-supplied version %s is not supported", 1136 lsquic_ver2str[version]); 1137 } 1138 else 1139 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 1140 ver_tag); 1141 1142 return VER_UNSUPPORTED; 1143} 1144 1145 1146static void 1147schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 1148 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 1149 const struct sockaddr *sa_peer, void *peer_ctx) 1150{ 1151 assert(engine->pr_queue); 1152 if (0 == lsquic_prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 1153 sa_local, sa_peer)) 1154 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 1155 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 1156 else 1157 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 1158} 1159 1160 1161static unsigned short 1162sa2port (const struct sockaddr *sa) 1163{ 1164 if (sa->sa_family == AF_INET) 1165 { 1166 struct sockaddr_in *const sa4 = (void *) sa; 1167 return sa4->sin_port; 1168 } 1169 else 1170 { 1171 struct sockaddr_in6 *const sa6 = (void *) sa; 1172 return sa6->sin6_port; 1173 } 1174} 1175 1176 1177static struct lsquic_hash_elem * 1178find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 1179{ 1180 unsigned short port; 1181 1182 port = sa2port(sa); 1183 return lsquic_hash_find(hash, &port, sizeof(port)); 1184} 1185 1186 1187/* When connections are identified by the local address, we need to drop 1188 * packets that use DCIDs that do not correspond to any of SCIDs. This 1189 * can happen when peer retires a SCID. This mimics the normal behavior, 1190 * when connections are looked up in engine->conns_hash by ID: when there 1191 * is no match, the packet is dropped. 1192 */ 1193static int 1194dcid_checks_out (const struct lsquic_conn *conn, const lsquic_cid_t *dcid) 1195{ 1196 const struct conn_cid_elem *cce; 1197 1198 if (LSQUIC_CIDS_EQ(CN_SCID(conn), dcid)) 1199 return 1; 1200 1201 /* Slow check for those rare cases */ 1202 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1203 if ((conn->cn_cces_mask & (1 << (cce - conn->cn_cces))) 1204 && !(cce->cce_flags & CCE_PORT) 1205 && LSQUIC_CIDS_EQ(&cce->cce_cid, dcid)) 1206 { 1207 LSQ_DEBUG("connection checks out"); 1208 return 1; 1209 } 1210 1211 return 0; 1212} 1213 1214 1215static lsquic_conn_t * 1216find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1217 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 1218{ 1219 struct lsquic_hash_elem *el; 1220 lsquic_conn_t *conn; 1221 1222 if (engine->flags & ENG_CONNS_BY_ADDR) 1223 { 1224 el = find_conn_by_addr(engine->conns_hash, sa_local); 1225 if ((packet_in->pi_flags & PI_CONN_ID) 1226 && !dcid_checks_out(lsquic_hashelem_getdata(el), 1227 &packet_in->pi_conn_id)) 1228 { 1229 LSQ_DEBUGC("DCID matches no SCID in connection %"CID_FMT": drop it", 1230 CID_BITS(&packet_in->pi_conn_id)); 1231 return NULL; 1232 } 1233 } 1234 else if (packet_in->pi_flags & PI_CONN_ID) 1235 el = lsquic_hash_find(engine->conns_hash, 1236 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1237 else 1238 { 1239 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1240 return NULL; 1241 } 1242 1243 if (!el) 1244 return NULL; 1245 1246 conn = lsquic_hashelem_getdata(el); 1247 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1248 if ((engine->flags & ENG_CONNS_BY_ADDR) 1249 && !(conn->cn_flags & LSCONN_IETF) 1250 && (packet_in->pi_flags & PI_CONN_ID) 1251 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1252 { 1253 LSQ_DEBUG("connection IDs do not match"); 1254 return NULL; 1255 } 1256 1257 return conn; 1258} 1259 1260 1261static const char *const puty2str[] = { 1262 [PUTY_CONN_DELETED] = "deleted", 1263 [PUTY_CONN_DRAIN] = "being drained", 1264 [PUTY_CID_RETIRED] = "retired", 1265}; 1266 1267 1268static lsquic_conn_t * 1269find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1270 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1271 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1272{ 1273 struct lsquic_hash_elem *el; 1274 struct purga_el *puel; 1275 lsquic_conn_t *conn; 1276 1277 if (!(packet_in->pi_flags & PI_CONN_ID)) 1278 { 1279 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1280 return NULL; 1281 } 1282 el = lsquic_hash_find(engine->conns_hash, 1283 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1284 1285 if (el) 1286 { 1287 conn = lsquic_hashelem_getdata(el); 1288 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1289 return conn; 1290 } 1291 1292 if (engine->flags & ENG_COOLDOWN) 1293 { /* Do not create incoming connections during cooldown */ 1294 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1295 return NULL; 1296 } 1297 1298 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1299 { 1300 LSQ_DEBUG("reached limit of %u inchoate connections", 1301 engine->pub.enp_settings.es_max_inchoate); 1302 return NULL; 1303 } 1304 1305 1306 if (engine->purga 1307 && (puel = lsquic_purga_contains(engine->purga, 1308 &packet_in->pi_conn_id), puel)) 1309 { 1310 switch (puel->puel_type) 1311 { 1312 case PUTY_CID_RETIRED: 1313 case PUTY_CONN_DRAIN: 1314 if (puel->puel_time > packet_in->pi_received) 1315 { 1316 LSQ_DEBUGC("CID %"CID_FMT" is %s for another %"PRIu64 1317 "usec, ignore packet", CID_BITS(&packet_in->pi_conn_id), 1318 puty2str[puel->puel_type], 1319 puel->puel_time - packet_in->pi_received); 1320 return NULL; 1321 } 1322 LSQ_DEBUGC("CID %"CID_FMT" is no longer %s", 1323 CID_BITS(&packet_in->pi_conn_id), puty2str[puel->puel_type]); 1324 break; 1325 case PUTY_CONN_DELETED: 1326 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1327 CID_BITS(&packet_in->pi_conn_id)); 1328 if (puel->puel_time < packet_in->pi_received) 1329 { 1330 if (puel->puel_count < 4) 1331 { 1332 puel->puel_time = packet_in->pi_received 1333 /* Exponential back-off */ 1334 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1335 ++puel->puel_count; 1336 goto maybe_send_prst; 1337 } 1338 else 1339 break; 1340 } 1341 return NULL; 1342 default: 1343 assert(0); 1344 return NULL; 1345 } 1346 } 1347 1348 if (engine->pub.enp_settings.es_send_prst 1349 && !(packet_in->pi_flags & PI_GQUIC) 1350 && HETY_NOT_SET == packet_in->pi_header_type) 1351 goto maybe_send_prst; 1352 1353 if (0 != maybe_grow_conn_heaps(engine)) 1354 return NULL; 1355 1356 const struct parse_funcs *pf; 1357 enum lsquic_version version; 1358 switch (version_matches(engine, packet_in, &version)) 1359 { 1360 case VER_UNSUPPORTED: 1361 if (engine->flags & ENG_SERVER) 1362 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1363 sa_local, sa_peer, peer_ctx); 1364 return NULL; 1365 case VER_NOT_SPECIFIED: 1366 maybe_send_prst: 1367 if ((engine->flags & ENG_SERVER) && 1368 engine->pub.enp_settings.es_send_prst) 1369 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1370 sa_local, sa_peer, peer_ctx); 1371 return NULL; 1372 case VER_SUPPORTED: 1373 pf = select_pf_by_ver(version); 1374 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1375 break; 1376 } 1377 1378 1379 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1380 { 1381 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1382 sa_peer->sa_family == AF_INET, NULL, packet_in_size); 1383 } 1384 else 1385 { 1386 conn = lsquic_mini_conn_new(&engine->pub, packet_in, version); 1387 } 1388 if (!conn) 1389 return NULL; 1390 ++engine->mini_conns_count; 1391 ++engine->n_conns; 1392 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1393 { 1394 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1395 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1396 CID_BITS(cid)); 1397 destroy_conn(engine, conn, packet_in->pi_received); 1398 return NULL; 1399 } 1400 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1401 conn->cn_flags |= LSCONN_HASHED; 1402 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1403 conn->cn_last_sent = engine->last_sent; 1404 return conn; 1405} 1406 1407 1408lsquic_conn_t * 1409lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1410 const lsquic_cid_t *cid) 1411{ 1412 struct lsquic_hash_elem *el; 1413 lsquic_conn_t *conn = NULL; 1414 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1415 1416 if (el) 1417 conn = lsquic_hashelem_getdata(el); 1418 return conn; 1419} 1420 1421 1422#if !defined(NDEBUG) && __GNUC__ 1423__attribute__((weak)) 1424#endif 1425void 1426lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1427 lsquic_conn_t *conn) 1428{ 1429 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1430 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1431 { 1432 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1433 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1434 engine_incref_conn(conn, LSCONN_TICKABLE); 1435 } 1436} 1437 1438 1439void 1440lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1441 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1442{ 1443 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1444 if (conn->cn_flags & LSCONN_TICKABLE) 1445 { 1446 /* Optimization: no need to add the connection to the Advisory Tick 1447 * Time Queue: it is about to be ticked, after which it its next tick 1448 * time may be queried again. 1449 */; 1450 } 1451 else if (conn->cn_flags & LSCONN_ATTQ) 1452 { 1453 if (lsquic_conn_adv_time(conn) != tick_time) 1454 { 1455 lsquic_attq_remove(engine->attq, conn); 1456 if (0 != lsquic_attq_add(engine->attq, conn, tick_time, why)) 1457 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1458 } 1459 } 1460 else if (0 == lsquic_attq_add(engine->attq, conn, tick_time, why)) 1461 engine_incref_conn(conn, LSCONN_ATTQ); 1462} 1463 1464 1465static struct lsquic_conn * 1466find_conn_by_srst (struct lsquic_engine *engine, 1467 const struct lsquic_packet_in *packet_in) 1468{ 1469 struct lsquic_hash_elem *el; 1470 struct lsquic_conn *conn; 1471 1472 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1473 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1474 return NULL; 1475 1476 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1477 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1478 IQUIC_SRESET_TOKEN_SZ); 1479 if (!el) 1480 return NULL; 1481 1482 conn = lsquic_hashelem_getdata(el); 1483 return conn; 1484} 1485 1486 1487/* Return 0 if packet is being processed by a real connection (mini or full), 1488 * otherwise return 1. 1489 */ 1490static int 1491process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1492 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1493 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1494{ 1495 lsquic_conn_t *conn; 1496 const unsigned char *packet_in_data; 1497 1498 if (lsquic_packet_in_is_gquic_prst(packet_in) 1499 && !engine->pub.enp_settings.es_honor_prst) 1500 { 1501 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1502 LSQ_DEBUG("public reset packet: discarding"); 1503 return 1; 1504 } 1505 1506 if (engine->flags & ENG_SERVER) 1507 { 1508 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1509 sa_peer, peer_ctx, packet_in_size); 1510 if (!engine->curr_conn) 1511 engine->curr_conn = conn; 1512 } 1513 else 1514 conn = find_conn(engine, packet_in, ppstate, sa_local); 1515 1516 if (!conn) 1517 { 1518 if (engine->pub.enp_settings.es_honor_prst 1519 && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */ 1520 && !(packet_in->pi_flags & PI_GQUIC) 1521 && engine->pub.enp_srst_hash 1522 && (conn = find_conn_by_srst(engine, packet_in))) 1523 { 1524 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1525 CID_BITS(lsquic_conn_log_cid(conn))); 1526 conn->cn_if->ci_stateless_reset(conn); 1527 if (!(conn->cn_flags & LSCONN_TICKABLE) 1528 && conn->cn_if->ci_is_tickable(conn)) 1529 { 1530 lsquic_mh_insert(&engine->conns_tickable, conn, 1531 conn->cn_last_ticked); 1532 engine_incref_conn(conn, LSCONN_TICKABLE); 1533 } 1534 /* Even though the connection processes this packet, we return 1535 * 1 so that the caller does not add reset packet's random 1536 * bytes to the list of valid CIDs. 1537 */ 1538 } 1539 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1540 return 1; 1541 } 1542 1543 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1544 { 1545 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1546 engine_incref_conn(conn, LSCONN_TICKABLE); 1547 } 1548 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1549 sa_local, sa_peer); 1550 lsquic_packet_in_upref(packet_in); 1551#if LOG_PACKET_CHECKSUM 1552 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1553 packet_in->pi_data_sz); 1554#endif 1555 /* Note on QLog: 1556 * For the PACKET_RX QLog event, we are interested in logging these things: 1557 * - raw packet (however it comes in, encrypted or not) 1558 * - frames (list of frame names) 1559 * - packet type and number 1560 * - packet rx timestamp 1561 * 1562 * Since only some of these items are available at this code 1563 * juncture, we will wait until after the packet has been 1564 * decrypted (if necessary) and parsed to call the log functions. 1565 * 1566 * Once the PACKET_RX event is finally logged, the timestamp 1567 * will come from packet_in->pi_received. For correct sequential 1568 * ordering of QLog events, be sure to process the QLogs downstream. 1569 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1570 */ 1571 packet_in_data = packet_in->pi_data; 1572 packet_in_size = packet_in->pi_data_sz; 1573 conn->cn_if->ci_packet_in(conn, packet_in); 1574#if LSQUIC_CONN_STATS 1575 engine->busy.pin_conn = conn; 1576#endif 1577 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1578 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1579 return 0; 1580} 1581 1582 1583void 1584lsquic_engine_destroy (lsquic_engine_t *engine) 1585{ 1586 struct lsquic_hash_elem *el; 1587 lsquic_conn_t *conn; 1588 unsigned i; 1589 1590 LSQ_DEBUG("destroying engine"); 1591#ifndef NDEBUG 1592 engine->flags |= ENG_DTOR; 1593#endif 1594 1595 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1596 { 1597 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1598 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1599 } 1600 1601 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1602 { 1603 assert(conn->cn_flags & LSCONN_TICKABLE); 1604 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1605 } 1606 1607 for (el = lsquic_hash_first(engine->conns_hash); el; 1608 el = lsquic_hash_next(engine->conns_hash)) 1609 { 1610 conn = lsquic_hashelem_getdata(el); 1611 force_close_conn(engine, conn); 1612 } 1613 lsquic_hash_destroy(engine->conns_hash); 1614 1615 while ((conn = lsquic_attq_pop(engine->attq, UINT64_MAX))) 1616 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1617 1618 assert(0 == engine->n_conns); 1619 assert(0 == engine->mini_conns_count); 1620 if (engine->pr_queue) 1621 lsquic_prq_destroy(engine->pr_queue); 1622 if (engine->purga) 1623 lsquic_purga_destroy(engine->purga); 1624 lsquic_attq_destroy(engine->attq); 1625 1626 assert(0 == lsquic_mh_count(&engine->conns_out)); 1627 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1628 if (engine->pub.enp_shi == &stock_shi) 1629 lsquic_stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1630 lsquic_mm_cleanup(&engine->pub.enp_mm); 1631 free(engine->conns_tickable.mh_elems); 1632#if CAN_LOSE_PACKETS 1633 if (engine->flags & ENG_LOSE_PACKETS) 1634 regfree(&engine->lose_packets_re); 1635#endif 1636 if (engine->pub.enp_tokgen) 1637 lsquic_tg_destroy(engine->pub.enp_tokgen); 1638 if (engine->flags & LSENG_SERVER) 1639 lsquic_cleanup_gquic_crypto(&engine->pub); 1640#if LSQUIC_CONN_STATS 1641 if (engine->stats_fh) 1642 { 1643 const struct conn_stats *const stats = &engine->conn_stats_sum; 1644 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1645 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1646 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1647 fprintf(engine->stats_fh, "In:\n"); 1648 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1649 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1650 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1651 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1652 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1653 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1654 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1655 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1656 stats->in.headers_comp, stats->in.headers_uncomp, 1657 stats->in.headers_uncomp ? 1658 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1659 : 0); 1660 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1661 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1662 fprintf(engine->stats_fh, " ACK frames merged: %lu\n", stats->in.n_acks_merged); 1663 fprintf(engine->stats_fh, "Out:\n"); 1664 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1665 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1666 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1667 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1668 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1669 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1670 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1671 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1672 stats->out.headers_comp, stats->out.headers_uncomp, 1673 stats->out.headers_uncomp ? 1674 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1675 : 0); 1676 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1677 } 1678#endif 1679 if (engine->pub.enp_srst_hash) 1680 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1681#if LSQUIC_COUNT_ENGINE_CALLS 1682 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1683#endif 1684 for (i = 0; i < sizeof(engine->retry_aead_ctx) 1685 / sizeof(engine->retry_aead_ctx[0]); ++i) 1686 EVP_AEAD_CTX_cleanup(&engine->pub.enp_retry_aead_ctx[i]); 1687 free(engine->pub.enp_alpn); 1688 free(engine); 1689} 1690 1691 1692static struct conn_cid_elem * 1693find_free_cce (struct lsquic_conn *conn) 1694{ 1695 struct conn_cid_elem *cce; 1696 1697 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1698 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1699 return cce; 1700 1701 return NULL; 1702} 1703 1704 1705static int 1706add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1707 const struct sockaddr *local_sa, void *peer_ctx) 1708{ 1709 struct conn_cid_elem *cce; 1710 1711 if (engine->flags & ENG_CONNS_BY_ADDR) 1712 { 1713 cce = find_free_cce(conn); 1714 if (!cce) 1715 { 1716 LSQ_ERROR("cannot find free CCE"); 1717 return -1; 1718 } 1719 cce->cce_port = sa2port(local_sa); 1720 cce->cce_flags = CCE_PORT; 1721 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1722 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1723 { 1724 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1725 return 0; 1726 } 1727 else 1728 return -1; 1729 1730 } 1731 else 1732 return insert_conn_into_hash(engine, conn, peer_ctx); 1733} 1734 1735 1736lsquic_conn_t * 1737lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1738 const struct sockaddr *local_sa, 1739 const struct sockaddr *peer_sa, 1740 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1741 const char *hostname, unsigned short base_plpmtu, 1742 const unsigned char *sess_resume, size_t sess_resume_len, 1743 const unsigned char *token, size_t token_sz) 1744{ 1745 lsquic_conn_t *conn; 1746 unsigned flags, versions; 1747 int is_ipv4; 1748 1749 ENGINE_CALLS_INCR(engine); 1750 1751 if (engine->flags & ENG_SERVER) 1752 { 1753 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1754 goto err; 1755 } 1756 1757 if (engine->flags & ENG_CONNS_BY_ADDR 1758 && find_conn_by_addr(engine->conns_hash, local_sa)) 1759 { 1760 LSQ_ERROR("cannot have more than one connection on the same port"); 1761 goto err; 1762 } 1763 1764 if (0 != maybe_grow_conn_heaps(engine)) 1765 return NULL; 1766 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1767 is_ipv4 = peer_sa->sa_family == AF_INET; 1768 if (sess_resume && sess_resume_len) 1769 { 1770 version = lsquic_sess_resume_version(sess_resume, sess_resume_len); 1771 if (version >= N_LSQVER) 1772 { 1773 LSQ_INFO("session resumption version is bad, won't use"); 1774 sess_resume = NULL; 1775 sess_resume_len = 0; 1776 } 1777 } 1778 if (version >= N_LSQVER) 1779 { 1780 if (version > N_LSQVER) 1781 LSQ_WARN("invalid version specified, engine will pick"); 1782 versions = engine->pub.enp_settings.es_versions; 1783 } 1784 else 1785 versions = 1u << version; 1786 if (versions & LSQUIC_IETF_VERSIONS) 1787 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1788 flags, hostname, base_plpmtu, 1789 is_ipv4, sess_resume, sess_resume_len, token, token_sz, peer_ctx); 1790 else 1791 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1792 flags, hostname, base_plpmtu, is_ipv4, 1793 sess_resume, sess_resume_len); 1794 if (!conn) 1795 goto err; 1796 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1797 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1798 lsquic_ver2str[conn->cn_version]); 1799 ++engine->n_conns; 1800 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1801 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1802 { 1803 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1804 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1805 CID_BITS(cid)); 1806 destroy_conn(engine, conn, lsquic_time_now()); 1807 goto err; 1808 } 1809 assert(!(conn->cn_flags & 1810 (CONN_REF_FLAGS 1811 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1812 callbacks */ 1813 ))); 1814 conn->cn_flags |= LSCONN_HASHED; 1815 if (!(conn->cn_flags & LSCONN_TICKABLE)) 1816 { 1817 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1818 engine_incref_conn(conn, LSCONN_TICKABLE); 1819 } 1820 lsquic_conn_set_ctx(conn, conn_ctx); 1821 conn->cn_if->ci_client_call_on_new(conn); 1822 end: 1823 return conn; 1824 err: 1825 conn = NULL; 1826 goto end; 1827} 1828 1829 1830static void 1831remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1832{ 1833 remove_all_cces_from_hash(engine->conns_hash, conn); 1834 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1835} 1836 1837 1838static void 1839refflags2str (enum lsquic_conn_flags flags, char s[7]) 1840{ 1841 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1842 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1843 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1844 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1845 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1846 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1847 *s = '\0'; 1848} 1849 1850 1851static void 1852engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1853{ 1854 char str[2][7]; 1855 assert(flag & CONN_REF_FLAGS); 1856 assert(!(conn->cn_flags & flag)); 1857 conn->cn_flags |= flag; 1858 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1859 CID_BITS(lsquic_conn_log_cid(conn)), 1860 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1861 (refflags2str(conn->cn_flags, str[1]), str[1])); 1862} 1863 1864 1865static lsquic_conn_t * 1866engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1867 enum lsquic_conn_flags flags) 1868{ 1869 char str[2][7]; 1870 lsquic_time_t now; 1871 assert(flags & CONN_REF_FLAGS); 1872 assert(conn->cn_flags & flags); 1873#ifndef NDEBUG 1874 if (flags & LSCONN_CLOSING) 1875 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1876#endif 1877 conn->cn_flags &= ~flags; 1878 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1879 CID_BITS(lsquic_conn_log_cid(conn)), 1880 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1881 (refflags2str(conn->cn_flags, str[1]), str[1])); 1882 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1883 { 1884 now = lsquic_time_now(); 1885 if (conn->cn_flags & LSCONN_MINI) 1886 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1887 else 1888 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1889 destroy_conn(engine, conn, now); 1890 return NULL; 1891 } 1892 else 1893 return conn; 1894} 1895 1896 1897/* This is not a general-purpose function. Only call from engine dtor. */ 1898static void 1899force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1900{ 1901 assert(engine->flags & ENG_DTOR); 1902 const enum lsquic_conn_flags flags = conn->cn_flags; 1903 assert(conn->cn_flags & CONN_REF_FLAGS); 1904 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1905 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1906 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1907 if (flags & LSCONN_ATTQ) 1908 { 1909 lsquic_attq_remove(engine->attq, conn); 1910 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1911 } 1912 if (flags & LSCONN_HASHED) 1913 remove_conn_from_hash(engine, conn); 1914} 1915 1916 1917/* Iterator for tickable connections (those on the Tickable Queue). Before 1918 * a connection is returned, it is removed from the Advisory Tick Time queue 1919 * if necessary. 1920 */ 1921static lsquic_conn_t * 1922conn_iter_next_tickable (struct lsquic_engine *engine) 1923{ 1924 lsquic_conn_t *conn; 1925 1926 if (engine->flags & ENG_SERVER) 1927 while (1) 1928 { 1929 conn = lsquic_mh_pop(&engine->conns_tickable); 1930 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1931 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1932 else 1933 break; 1934 } 1935 else 1936 conn = lsquic_mh_pop(&engine->conns_tickable); 1937 1938 if (conn) 1939 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1940 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1941 { 1942 lsquic_attq_remove(engine->attq, conn); 1943 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1944 } 1945 1946 return conn; 1947} 1948 1949 1950static void 1951cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1952 void *update_ctx) 1953{ 1954 cub->cub_update_cids = update; 1955 cub->cub_update_ctx = update_ctx; 1956 cub->cub_count = 0; 1957} 1958 1959 1960static void 1961cub_flush (struct cid_update_batch *cub) 1962{ 1963 if (cub->cub_count > 0 && cub->cub_update_cids) 1964 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1965 cub->cub_cids, cub->cub_count); 1966 cub->cub_count = 0; 1967} 1968 1969 1970static void 1971cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1972{ 1973 cub->cub_cids [ cub->cub_count ] = *cid; 1974 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1975 ++cub->cub_count; 1976 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1977 cub_flush(cub); 1978} 1979 1980 1981/* Process registered CIDs */ 1982static void 1983cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1984{ 1985 struct cce_cid_iter citer; 1986 struct conn_cid_elem *cce; 1987 void *peer_ctx; 1988 1989 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1990 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1991 if (cce->cce_flags & CCE_REG) 1992 cub_add(cub, &cce->cce_cid, peer_ctx); 1993} 1994 1995 1996static void 1997drop_all_mini_conns (lsquic_engine_t *engine) 1998{ 1999 struct lsquic_hash_elem *el; 2000 lsquic_conn_t *conn; 2001 struct cid_update_batch cub; 2002 2003 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2004 2005 for (el = lsquic_hash_first(engine->conns_hash); el; 2006 el = lsquic_hash_next(engine->conns_hash)) 2007 { 2008 conn = lsquic_hashelem_getdata(el); 2009 if (conn->cn_flags & LSCONN_MINI) 2010 { 2011 /* If promoted, why is it still in this hash? */ 2012 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 2013 if (!(conn->cn_flags & LSCONN_PROMOTED)) 2014 cub_add_cids_from_cces(&cub, conn); 2015 remove_conn_from_hash(engine, conn); 2016 } 2017 } 2018 2019 cub_flush(&cub); 2020} 2021 2022 2023void 2024lsquic_engine_process_conns (lsquic_engine_t *engine) 2025{ 2026 lsquic_conn_t *conn; 2027 lsquic_time_t now; 2028 2029#if LSQUIC_CONN_STATS 2030 if (engine->busy.pin_conn) 2031 { 2032 update_busy_detector(engine, engine->busy.pin_conn, 1); 2033 engine->busy.pin_conn = NULL; 2034 } 2035#endif 2036 2037 ENGINE_IN(engine); 2038 2039 now = lsquic_time_now(); 2040 while ((conn = lsquic_attq_pop(engine->attq, now))) 2041 { 2042 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 2043 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 2044 { 2045 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2046 engine_incref_conn(conn, LSCONN_TICKABLE); 2047 } 2048 } 2049 2050 process_connections(engine, conn_iter_next_tickable, now); 2051 ENGINE_OUT(engine); 2052} 2053 2054 2055static void 2056release_or_return_enc_data (struct lsquic_engine *engine, 2057 void (*pmi_rel_or_ret) (void *, void *, void *, char), 2058 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 2059{ 2060 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 2061 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 2062 packet_out->po_flags &= ~PO_ENCRYPTED; 2063 packet_out->po_enc_data = NULL; 2064} 2065 2066 2067static void 2068release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 2069 struct lsquic_packet_out *packet_out) 2070{ 2071 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 2072 conn, packet_out); 2073} 2074 2075 2076static void 2077return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 2078 struct lsquic_packet_out *packet_out) 2079{ 2080 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 2081 conn, packet_out); 2082} 2083 2084 2085static int 2086copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 2087 struct lsquic_packet_out *packet_out) 2088{ 2089 int ipv6; 2090 2091 ipv6 = NP_IS_IPv6(packet_out->po_path); 2092 if (packet_out->po_flags & PO_ENCRYPTED) 2093 { 2094 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 2095 && packet_out->po_data_sz == packet_out->po_enc_data_sz 2096 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 2097 packet_out->po_data_sz)) 2098 return 0; 2099 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 2100 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 2101 goto copy; 2102 return_enc_data(engine, conn, packet_out); 2103 } 2104 2105 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 2106 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 2107 conn->cn_conn_ctx, packet_out->po_data_sz, ipv6); 2108 if (!packet_out->po_enc_data) 2109 { 2110 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 2111 "of size %hu", packet_out->po_data_sz); 2112 return -1; 2113 } 2114 2115 copy: 2116 memcpy(packet_out->po_enc_data, packet_out->po_data, 2117 packet_out->po_data_sz); 2118 packet_out->po_enc_data_sz = packet_out->po_data_sz; 2119 packet_out->po_sent_sz = packet_out->po_data_sz; 2120 packet_out->po_flags &= ~PO_IPv6; 2121 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 2122 2123 return 0; 2124} 2125 2126 2127STAILQ_HEAD(conns_stailq, lsquic_conn); 2128TAILQ_HEAD(conns_tailq, lsquic_conn); 2129 2130 2131struct conns_out_iter 2132{ 2133 struct min_heap *coi_heap; 2134 struct pr_queue *coi_prq; 2135 TAILQ_HEAD(, lsquic_conn) coi_active_list, 2136 coi_inactive_list; 2137 lsquic_conn_t *coi_next; 2138#ifndef NDEBUG 2139 lsquic_time_t coi_last_sent; 2140#endif 2141}; 2142 2143 2144static void 2145coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 2146{ 2147 iter->coi_heap = &engine->conns_out; 2148 iter->coi_prq = engine->pr_queue; 2149 iter->coi_next = NULL; 2150 TAILQ_INIT(&iter->coi_active_list); 2151 TAILQ_INIT(&iter->coi_inactive_list); 2152#ifndef NDEBUG 2153 iter->coi_last_sent = 0; 2154#endif 2155} 2156 2157 2158static lsquic_conn_t * 2159coi_next (struct conns_out_iter *iter) 2160{ 2161 lsquic_conn_t *conn; 2162 2163 if (lsquic_mh_count(iter->coi_heap) > 0) 2164 { 2165 conn = lsquic_mh_pop(iter->coi_heap); 2166 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2167 conn->cn_flags |= LSCONN_COI_ACTIVE; 2168#ifndef NDEBUG 2169 if (iter->coi_last_sent) 2170 assert(iter->coi_last_sent <= conn->cn_last_sent); 2171 iter->coi_last_sent = conn->cn_last_sent; 2172#endif 2173 return conn; 2174 } 2175 else if (iter->coi_prq && (conn = lsquic_prq_next_conn(iter->coi_prq))) 2176 { 2177 return conn; 2178 } 2179 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 2180 { 2181 iter->coi_prq = NULL; /* Save function call in previous conditional */ 2182 conn = iter->coi_next; 2183 if (!conn) 2184 conn = TAILQ_FIRST(&iter->coi_active_list); 2185 if (conn) 2186 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 2187 return conn; 2188 } 2189 else 2190 return NULL; 2191} 2192 2193 2194static void 2195coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2196{ 2197 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2198 { 2199 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 2200 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2201 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2202 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 2203 conn->cn_flags |= LSCONN_COI_INACTIVE; 2204 } 2205} 2206 2207 2208static void 2209coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2210{ 2211 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 2212 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2213 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2214 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2215 conn->cn_flags |= LSCONN_COI_ACTIVE; 2216} 2217 2218 2219static void 2220coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 2221{ 2222 lsquic_conn_t *conn; 2223 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 2224 { 2225 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2226 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2227 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 2228 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 2229 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 2230 else /* Closed connection gets one shot at sending packets */ 2231 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2232 } 2233 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 2234 { 2235 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2236 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2237 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2238 } 2239} 2240 2241 2242#if CAN_LOSE_PACKETS 2243static void 2244lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 2245 unsigned n) 2246{ 2247 const lsquic_cid_t *cid; 2248 struct iovec *iov; 2249 unsigned i; 2250 char packno_str[22]; 2251 2252 for (i = 0; i < n; ++i) 2253 { 2254 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 2255 batch->packets[i]->po_packno); 2256 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 2257 { 2258 for (iov = batch->outs[i].iov; iov < 2259 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 2260 batch->outs[i].iov->iov_len -= 1; 2261 cid = lsquic_conn_log_cid(batch->conns[i]); 2262 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 2263 CID_BITS(cid)); 2264 } 2265 } 2266} 2267 2268 2269#endif 2270 2271 2272#ifdef NDEBUG 2273#define CONST_BATCH const 2274#else 2275#define CONST_BATCH 2276#endif 2277 2278 2279struct send_batch_ctx { 2280 struct conns_stailq *closed_conns; 2281 struct conns_tailq *ticked_conns; 2282 struct conns_out_iter *conns_iter; 2283 CONST_BATCH struct out_batch *batch; 2284}; 2285 2286 2287static void 2288close_conn_immediately (struct lsquic_engine *engine, 2289 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2290{ 2291 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2292 if (!(conn->cn_flags & LSCONN_CLOSING)) 2293 { 2294 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2295 engine_incref_conn(conn, LSCONN_CLOSING); 2296 if (conn->cn_flags & LSCONN_HASHED) 2297 remove_conn_from_hash(engine, conn); 2298 } 2299 if (conn->cn_flags & LSCONN_TICKED) 2300 { 2301 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2302 engine_decref_conn(engine, conn, LSCONN_TICKED); 2303 } 2304} 2305 2306 2307static void 2308close_conn_on_send_error (struct lsquic_engine *engine, 2309 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2310{ 2311 const struct out_batch *batch = sb_ctx->batch; 2312 struct lsquic_conn *const conn = batch->conns[n]; 2313 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2314 2315 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2316 "src: %s; dst: %s; errno: %d", 2317 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2318 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2319 CID_BITS(lsquic_conn_log_cid(conn)), 2320 SA2STR(batch->outs[n].local_sa, buf[0]), 2321 SA2STR(batch->outs[n].dest_sa, buf[1]), 2322 e_val); 2323 if (conn->cn_flags & LSCONN_EVANESCENT) 2324 lsquic_prq_drop(conn); 2325 else 2326 close_conn_immediately(engine, sb_ctx, conn); 2327} 2328 2329 2330static void 2331apply_hp (struct conns_out_iter *iter) 2332{ 2333 struct lsquic_conn *conn; 2334 2335 TAILQ_FOREACH(conn, &iter->coi_active_list, cn_next_out) 2336 if (conn->cn_esf_c->esf_flush_encryption && conn->cn_enc_session) 2337 conn->cn_esf_c->esf_flush_encryption(conn->cn_enc_session); 2338 TAILQ_FOREACH(conn, &iter->coi_inactive_list, cn_next_out) 2339 if (conn->cn_esf_c->esf_flush_encryption && conn->cn_enc_session) 2340 conn->cn_esf_c->esf_flush_encryption(conn->cn_enc_session); 2341} 2342 2343 2344#if LSQUIC_CONN_STATS 2345static void 2346update_batch_size_stats (struct lsquic_engine *engine, unsigned batch_size) 2347{ 2348 struct batch_size_stats *const stats = &engine->pub.enp_batch_size_stats; 2349 2350 ++stats->count; 2351 if (batch_size > stats->max) 2352 stats->max = batch_size; 2353 if (batch_size < stats->min || 0 == stats->min) 2354 stats->min = batch_size; 2355 if (stats->avg) 2356 stats->avg = ((float) batch_size - stats->avg) * 0.4 + stats->avg; 2357 else 2358 stats->avg = (float) batch_size; 2359} 2360 2361 2362#endif 2363 2364 2365static unsigned 2366send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2367 unsigned n_to_send) 2368{ 2369 int n_sent, i, e_val; 2370 lsquic_time_t now; 2371 unsigned off, skip; 2372 size_t count; 2373 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2374 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2375 2376#if LSQUIC_CONN_STATS 2377 update_batch_size_stats(engine, n_to_send); 2378#endif 2379 2380 apply_hp(sb_ctx->conns_iter); 2381#if CAN_LOSE_PACKETS 2382 if (engine->flags & ENG_LOSE_PACKETS) 2383 lose_matching_packets(engine, batch, n_to_send); 2384#endif 2385 skip = 0; 2386 restart_batch: 2387 /* Set sent time before the write to avoid underestimating RTT */ 2388 now = lsquic_time_now(); 2389 for (i = skip; i < (int) (n_to_send - skip); ++i) 2390 { 2391 off = batch->pack_off[i]; 2392 count = batch->outs[i].iovlen; 2393 assert(count > 0); 2394 packet_out = &batch->packets[off]; 2395 end = packet_out + count; 2396 do 2397 (*packet_out)->po_sent = now; 2398 while (++packet_out < end); 2399 } 2400 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs + skip, 2401 n_to_send - skip); 2402 e_val = errno; 2403 if (n_sent < (int) (n_to_send - skip) && e_val != EMSGSIZE) 2404 { 2405 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2406 engine->resume_sending_at = now + 1000000; 2407 LSQ_DEBUG("cannot send packets"); 2408 EV_LOG_GENERIC_EVENT("cannot send packets"); 2409 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2410 close_conn_on_send_error(engine, sb_ctx, 2411 n_sent < 0 ? 0 : n_sent, e_val); 2412 } 2413 if (n_sent >= 0) 2414 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, 2415 n_to_send - skip); 2416 else 2417 { 2418 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2419 n_sent = 0; 2420 } 2421 if (n_sent > 0) 2422 engine->last_sent = now + n_sent; 2423 for (i = skip; i < (int) (skip + n_sent); ++i) 2424 { 2425 eng_hist_inc(&engine->history, now, sl_packets_out); 2426 /* `i' is added to maintain relative order */ 2427 batch->conns[i]->cn_last_sent = now + i; 2428 2429 off = batch->pack_off[i]; 2430 count = batch->outs[i].iovlen; 2431 assert(count > 0); 2432 packet_out = &batch->packets[off]; 2433 end = packet_out + count; 2434 do 2435 { 2436#if LOG_PACKET_CHECKSUM 2437 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2438 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2439 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2440#endif 2441 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2442 *packet_out); 2443 /* Release packet out buffer as soon as the packet is sent 2444 * successfully. If not successfully sent, we hold on to 2445 * this buffer until the packet sending is attempted again 2446 * or until it times out and regenerated. 2447 */ 2448 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2449 release_enc_data(engine, batch->conns[i], *packet_out); 2450 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2451 *packet_out); 2452 } 2453 while (++packet_out < end); 2454 } 2455 if (i < (int) n_to_send && e_val == EMSGSIZE) 2456 { 2457 LSQ_DEBUG("packet #%d could not be sent out for being too large", i); 2458 if (batch->conns[i]->cn_if->ci_packet_too_large 2459 && batch->outs[i].iovlen == 1) 2460 { 2461 off = batch->pack_off[i]; 2462 packet_out = &batch->packets[off]; 2463 batch->conns[i]->cn_if->ci_packet_too_large(batch->conns[i], 2464 *packet_out); 2465 ++i; 2466 if (i < (int) n_to_send) 2467 { 2468 skip = i; 2469 LSQ_DEBUG("restart batch starting at packet #%u", skip); 2470 goto restart_batch; 2471 } 2472 n_sent = n_to_send; 2473 } 2474 else 2475 close_conn_on_send_error(engine, sb_ctx, i, e_val); 2476 } 2477 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2478 for ( ; i < (int) n_to_send; ++i) 2479 { 2480 off = batch->pack_off[i]; 2481 count = batch->outs[i].iovlen; 2482 assert(count > 0); 2483 packet_out = &batch->packets[off]; 2484 end = packet_out + count; 2485 do 2486 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2487 *packet_out); 2488 while (++packet_out < end); 2489 } 2490 /* Return packets to the connection in reverse order so that the packet 2491 * ordering is maintained. 2492 */ 2493 for (i = (int) n_to_send - 1; i >= (int) (skip + n_sent); --i) 2494 { 2495 off = batch->pack_off[i]; 2496 count = batch->outs[i].iovlen; 2497 assert(count > 0); 2498 packet_out = &batch->packets[off + count - 1]; 2499 end = &batch->packets[off - 1]; 2500 do 2501 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2502 *packet_out); 2503 while (--packet_out > end); 2504 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2505 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2506 } 2507 return skip + n_sent; 2508} 2509 2510 2511/* Return 1 if went past deadline, 0 otherwise */ 2512static int 2513check_deadline (lsquic_engine_t *engine) 2514{ 2515 if (engine->pub.enp_settings.es_proc_time_thresh && 2516 lsquic_time_now() > engine->deadline) 2517 { 2518 LSQ_INFO("went past threshold of %u usec, stop sending", 2519 engine->pub.enp_settings.es_proc_time_thresh); 2520 engine->flags |= ENG_PAST_DEADLINE; 2521 return 1; 2522 } 2523 else 2524 return 0; 2525} 2526 2527 2528static size_t 2529iov_size (const struct iovec *iov, const struct iovec *const end) 2530{ 2531 size_t size; 2532 2533 assert(iov < end); 2534 2535 size = 0; 2536 do 2537 size += iov->iov_len; 2538 while (++iov < end); 2539 2540 return size; 2541} 2542 2543 2544static void 2545send_packets_out (struct lsquic_engine *engine, 2546 struct conns_tailq *ticked_conns, 2547 struct conns_stailq *closed_conns) 2548{ 2549 unsigned n, w, n_sent, n_batches_sent; 2550 lsquic_packet_out_t *packet_out; 2551 struct lsquic_packet_out **packet; 2552 lsquic_conn_t *conn; 2553 struct out_batch *const batch = &engine->out_batch; 2554 struct iovec *iov, *packet_iov; 2555 struct conns_out_iter conns_iter; 2556 int shrink, deadline_exceeded; 2557 const struct send_batch_ctx sb_ctx = { 2558 closed_conns, 2559 ticked_conns, 2560 &conns_iter, 2561 &engine->out_batch, 2562 }; 2563 2564 coi_init(&conns_iter, engine); 2565 n_batches_sent = 0; 2566 n_sent = 0, n = 0; 2567 shrink = 0; 2568 deadline_exceeded = 0; 2569 iov = batch->iov; 2570 packet = batch->packets; 2571 2572 while ((conn = coi_next(&conns_iter))) 2573 { 2574 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2575 if (!packet_out) { 2576 /* Evanescent connection always has a packet to send: */ 2577 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2578 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2579 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2580 CID_BITS(lsquic_conn_log_cid(conn))); 2581 coi_deactivate(&conns_iter, conn); 2582 continue; 2583 } 2584 batch->outs[n].iov = packet_iov = iov; 2585 next_coa: 2586 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2587 { 2588 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2589 &engine->pub, conn, packet_out)) 2590 { 2591 case ENCPA_NOMEM: 2592 /* Send what we have and wait for a more opportune moment */ 2593 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2594 goto end_for; 2595 case ENCPA_BADCRYPT: 2596 /* This is pretty bad: close connection immediately */ 2597 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2598 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2599 CID_BITS(lsquic_conn_log_cid(conn))); 2600 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2601 { 2602 close_conn_immediately(engine, &sb_ctx, conn); 2603 coi_deactivate(&conns_iter, conn); 2604 } 2605 continue; 2606 case ENCPA_OK: 2607 break; 2608 } 2609 } 2610 else if ((packet_out->po_flags & PO_NOENCRYPT) 2611 && engine->pub.enp_pmi != &stock_pmi) 2612 { 2613 if (0 != copy_packet(engine, conn, packet_out)) 2614 { 2615 /* Copy can only fail if packet could not be allocated */ 2616 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2617 goto end_for; 2618 } 2619 } 2620 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2621 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2622 if (packet_out->po_flags & PO_ENCRYPTED) 2623 { 2624 iov->iov_base = packet_out->po_enc_data; 2625 iov->iov_len = packet_out->po_enc_data_sz; 2626 } 2627 else 2628 { 2629 iov->iov_base = packet_out->po_data; 2630 iov->iov_len = packet_out->po_data_sz; 2631 } 2632 if (packet_iov == iov) 2633 { 2634 batch->pack_off[n] = packet - batch->packets; 2635 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2636 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2637 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2638 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2639 batch->outs [n].conn_ctx = conn->cn_conn_ctx; 2640 batch->conns [n] = conn; 2641 } 2642 *packet = packet_out; 2643 ++packet; 2644 ++iov; 2645 if ((conn->cn_flags & LSCONN_IETF) 2646 && ((1 << packet_out->po_header_type) 2647 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2648#ifndef NDEBUG 2649 && (engine->flags & ENG_COALESCE) 2650#endif 2651 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2652 { 2653 const struct to_coal to_coal = { 2654 .prev_packet = packet_out, 2655 .prev_sz_sum = iov_size(packet_iov, iov), 2656 }; 2657 packet_out = conn->cn_if->ci_next_packet_to_send(conn, &to_coal); 2658 if (packet_out) 2659 goto next_coa; 2660 } 2661 batch->outs [n].iovlen = iov - packet_iov; 2662 ++n; 2663 if (n == engine->batch_size 2664 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2665 { 2666 w = send_batch(engine, &sb_ctx, n); 2667 n = 0; 2668 iov = batch->iov; 2669 packet = batch->packets; 2670 ++n_batches_sent; 2671 n_sent += w; 2672 if (w < engine->batch_size) 2673 { 2674 shrink = 1; 2675 break; 2676 } 2677 deadline_exceeded = check_deadline(engine); 2678 if (deadline_exceeded) 2679 break; 2680 grow_batch_size(engine); 2681 } 2682 } 2683 end_for: 2684 2685 if (n > 0) { 2686 w = send_batch(engine, &sb_ctx, n); 2687 n_sent += w; 2688 shrink = w < n; 2689 ++n_batches_sent; 2690 } 2691 2692 if (shrink) 2693 shrink_batch_size(engine); 2694 else if (n_batches_sent > 1) 2695 { 2696 deadline_exceeded = check_deadline(engine); 2697 if (!deadline_exceeded) 2698 grow_batch_size(engine); 2699 } 2700 2701 coi_reheap(&conns_iter, engine); 2702 2703 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2704} 2705 2706 2707int 2708lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2709{ 2710 return lsquic_mh_count(&engine->conns_out) > 0 2711 || (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 2712 ; 2713} 2714 2715 2716static void 2717reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2718{ 2719 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2720 engine->flags &= ~ENG_PAST_DEADLINE; 2721} 2722 2723 2724static void 2725check_tickable_conns_again (struct lsquic_engine *engine) 2726{ 2727 struct lsquic_hash_elem *el; 2728 struct lsquic_conn *conn; 2729 unsigned count; 2730 2731 count = 0; 2732 for (el = lsquic_hash_first(engine->conns_hash); el; 2733 el = lsquic_hash_next(engine->conns_hash)) 2734 { 2735 conn = lsquic_hashelem_getdata(el); 2736 if (!(conn->cn_flags & LSCONN_TICKABLE) 2737 && conn->cn_if->ci_is_tickable(conn)) 2738 { 2739 lsquic_mh_insert(&engine->conns_tickable, conn, 2740 conn->cn_last_ticked); 2741 engine_incref_conn(conn, LSCONN_TICKABLE); 2742 ++count; 2743 } 2744 } 2745 LSQ_DEBUG("%u connection%s tickable again after sending has been " 2746 "re-enabled", count, count == 1 ? " is" : "s are"); 2747} 2748 2749 2750void 2751lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2752{ 2753 lsquic_conn_t *conn; 2754 struct conns_stailq closed_conns; 2755 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2756 struct cid_update_batch cub; 2757 2758 ENGINE_IN(engine); 2759 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2760 STAILQ_INIT(&closed_conns); 2761 reset_deadline(engine, lsquic_time_now()); 2762 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2763 { 2764 LSQ_DEBUG("can send again"); 2765 EV_LOG_GENERIC_EVENT("can send again"); 2766 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2767 check_tickable_conns_again(engine); 2768 } 2769 2770 send_packets_out(engine, &ticked_conns, &closed_conns); 2771 2772 while ((conn = STAILQ_FIRST(&closed_conns))) { 2773 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2774 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2775 cub_add_cids_from_cces(&cub, conn); 2776 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2777 } 2778 2779 cub_flush(&cub); 2780 ENGINE_OUT(engine); 2781} 2782 2783 2784static lsquic_conn_t * 2785next_new_full_conn (struct conns_stailq *new_full_conns) 2786{ 2787 lsquic_conn_t *conn; 2788 2789 conn = STAILQ_FIRST(new_full_conns); 2790 if (conn) 2791 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2792 return conn; 2793} 2794 2795 2796#if LSQUIC_CONN_STATS 2797static void 2798maybe_log_conn_stats (struct lsquic_engine *engine, struct lsquic_conn *conn, 2799 lsquic_time_t now) 2800{ 2801 char cidstr[MAX_CID_LEN * 2 + 1]; 2802 2803 if (!LSQ_LOG_ENABLED_EXT(LSQ_LOG_NOTICE, LSQLM_CONN_STATS)) 2804 return; 2805 2806 if (conn->cn_last_ticked + 1000000 >= now) 2807 { 2808 if (0 == engine->busy.last_log 2809 || engine->busy.last_log + 1000000 - 1000 < now) 2810 { 2811 engine->busy.last_log = now; 2812 conn->cn_if->ci_log_stats(conn); 2813 } 2814 } 2815 else 2816 { 2817 lsquic_logger_log1(LSQ_LOG_NOTICE, LSQLM_CONN_STATS, 2818 "stop logging status for connection %s: no longer busy", 2819 (lsquic_cid2str(lsquic_conn_log_cid(conn), cidstr), cidstr)); 2820 engine->busy.current = NULL; 2821 engine->busy.last_log = 0; 2822 } 2823} 2824 2825 2826#endif 2827 2828 2829static void 2830process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2831 lsquic_time_t now) 2832{ 2833 lsquic_conn_t *conn; 2834 enum tick_st tick_st; 2835 unsigned i, why; 2836 lsquic_time_t next_tick_time; 2837 struct conns_stailq closed_conns; 2838 struct conns_tailq ticked_conns; 2839 struct conns_stailq new_full_conns; 2840 struct cid_update_batch cub_old, cub_live; 2841 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2842 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2843 2844 eng_hist_tick(&engine->history, now); 2845 2846 STAILQ_INIT(&closed_conns); 2847 TAILQ_INIT(&ticked_conns); 2848 reset_deadline(engine, now); 2849 STAILQ_INIT(&new_full_conns); 2850 2851 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2852 && now > engine->resume_sending_at) 2853 { 2854 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2855 "timeout"); 2856 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2857 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2858 } 2859 2860 i = 0; 2861 while ((conn = next_conn(engine)) 2862 || (conn = next_new_full_conn(&new_full_conns))) 2863 { 2864 tick_st = conn->cn_if->ci_tick(conn, now); 2865#if LSQUIC_CONN_STATS 2866 if (conn == engine->busy.current) 2867 maybe_log_conn_stats(engine, conn, now); 2868#endif 2869 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2870 if (tick_st & TICK_PROMOTE) 2871 { 2872 lsquic_conn_t *new_conn; 2873 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2874 "scheduled for promotion"); 2875 assert(conn->cn_flags & LSCONN_MINI); 2876 new_conn = new_full_conn_server(engine, conn, now); 2877 if (new_conn) 2878 { 2879 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2880 new_conn->cn_last_sent = engine->last_sent; 2881 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2882 conn->cn_flags |= LSCONN_PROMOTED; 2883 } 2884 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2885 } 2886 if (tick_st & TICK_SEND) 2887 { 2888 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2889 { 2890 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2891 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2892 } 2893 } 2894 if (tick_st & TICK_CLOSE) 2895 { 2896 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2897 engine_incref_conn(conn, LSCONN_CLOSING); 2898 if (conn->cn_flags & LSCONN_HASHED) 2899 remove_conn_from_hash(engine, conn); 2900 } 2901 else 2902 { 2903 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2904 engine_incref_conn(conn, LSCONN_TICKED); 2905 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2906 && conn->cn_if->ci_report_live(conn, now)) 2907 cub_add_cids_from_cces(&cub_live, conn); 2908 } 2909 } 2910 2911 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2912 && lsquic_engine_has_unsent_packets(engine)) 2913 send_packets_out(engine, &ticked_conns, &closed_conns); 2914 2915 while ((conn = STAILQ_FIRST(&closed_conns))) { 2916 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2917 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2918 cub_add_cids_from_cces(&cub_old, conn); 2919 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2920 } 2921 2922 while ((conn = TAILQ_FIRST(&ticked_conns))) 2923 { 2924 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2925 engine_decref_conn(engine, conn, LSCONN_TICKED); 2926 if (!(conn->cn_flags & LSCONN_TICKABLE) 2927 && conn->cn_if->ci_is_tickable(conn)) 2928 { 2929 /* Floyd heapification is not faster, don't bother. */ 2930 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2931 engine_incref_conn(conn, LSCONN_TICKABLE); 2932 } 2933 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2934 { 2935 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2936 if (next_tick_time) 2937 { 2938 if (0 == lsquic_attq_add(engine->attq, conn, next_tick_time, 2939 why)) 2940 engine_incref_conn(conn, LSCONN_ATTQ); 2941 } 2942 else 2943 /* In all other cases, the idle timeout would make the next 2944 * tick time non-zero: 2945 */ 2946 assert((conn->cn_flags & LSCONN_IETF) 2947 && engine->pub.enp_settings.es_idle_timeout == 0); 2948 } 2949 } 2950 2951 cub_flush(&engine->new_scids); 2952 cub_flush(&cub_live); 2953 cub_flush(&cub_old); 2954} 2955 2956 2957static void 2958maybe_count_garbage (struct lsquic_engine *engine, size_t garbage_sz) 2959{ 2960 /* This is not very pretty (action at a distance via engine->curr_conn), 2961 * but it's the cheapest I can come up with to handle the "count garbage 2962 * toward amplification limit" requirement in 2963 * [draft-ietf-quic-transport-28] Section 8.1. 2964 */ 2965 if (engine->curr_conn && engine->curr_conn->cn_if->ci_count_garbage) 2966 engine->curr_conn->cn_if->ci_count_garbage(engine->curr_conn, 2967 garbage_sz); 2968} 2969 2970 2971/* Return 0 if packet is being processed by a real connection, 1 if the 2972 * packet was processed, but not by a connection, and -1 on error. 2973 */ 2974int 2975lsquic_engine_packet_in (lsquic_engine_t *engine, 2976 const unsigned char *packet_in_data, size_t packet_in_size, 2977 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2978 void *peer_ctx, int ecn) 2979{ 2980 const unsigned char *const packet_begin = packet_in_data; 2981 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2982 struct packin_parse_state ppstate; 2983 lsquic_packet_in_t *packet_in; 2984 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2985 int is_server, unsigned cid_len, struct packin_parse_state *); 2986 unsigned n_zeroes; 2987 int s, is_ietf; 2988 lsquic_cid_t cid; 2989 2990 ENGINE_CALLS_INCR(engine); 2991 2992 if (engine->flags & ENG_SERVER) 2993 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2994 else if (engine->flags & ENG_CONNS_BY_ADDR) 2995 { 2996 struct lsquic_hash_elem *el; 2997 const struct lsquic_conn *conn; 2998 el = find_conn_by_addr(engine->conns_hash, sa_local); 2999 if (!el) 3000 return -1; 3001 conn = lsquic_hashelem_getdata(el); 3002 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 3003 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 3004 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 3005 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 3006 else if (conn->cn_version == LSQVER_050) 3007 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 3008 else 3009 { 3010#if LSQUIC_USE_Q098 3011 assert(conn->cn_version == LSQVER_046 || conn->cn_version == LSQVER_098); 3012#else 3013 assert(conn->cn_version == LSQVER_046); 3014#endif 3015 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 3016 } 3017 } 3018 else 3019 parse_packet_in_begin = lsquic_parse_packet_in_begin; 3020 3021 engine->curr_conn = NULL; 3022 n_zeroes = 0; 3023 is_ietf = 0; 3024#ifdef _MSC_VER 3025 s = 0; 3026 cid.len = 0; 3027 cid.idbuf[0] = 0; 3028#endif 3029 do 3030 { 3031 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 3032 if (!packet_in) 3033 return -1; 3034 /* Library does not modify packet_in_data, it is not referenced after 3035 * this function returns and subsequent release of pi_data is guarded 3036 * by PI_OWN_DATA flag. 3037 */ 3038 packet_in->pi_data = (unsigned char *) packet_in_data; 3039 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 3040 engine->flags & ENG_SERVER, 3041 engine->pub.enp_settings.es_scid_len, &ppstate)) 3042 { 3043 LSQ_DEBUG("Cannot parse incoming packet's header"); 3044 maybe_count_garbage(engine, packet_end - packet_in_data); 3045 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 3046 s = 1; 3047 break; 3048 } 3049 3050 /* [draft-ietf-quic-transport-30] Section 12.2: 3051 * " Receivers SHOULD ignore any subsequent packets with a different 3052 * " Destination Connection ID than the first packet in the datagram. 3053 */ 3054 if (is_ietf && packet_in_data > packet_begin) 3055 { 3056 if (!((packet_in->pi_flags & (PI_GQUIC|PI_CONN_ID)) == PI_CONN_ID 3057 && LSQUIC_CIDS_EQ(&packet_in->pi_dcid, &cid))) 3058 { 3059 packet_in_data += packet_in->pi_data_sz; 3060 maybe_count_garbage(engine, packet_in->pi_data_sz); 3061 continue; 3062 } 3063 } 3064 3065 is_ietf = 0 == (packet_in->pi_flags & PI_GQUIC); 3066 packet_in_data += packet_in->pi_data_sz; 3067 if (is_ietf && packet_in_data < packet_end) 3068 { 3069 cid = packet_in->pi_dcid; 3070 if (packet_begin == packet_in->pi_data) /* Only log once: */ 3071 LSQ_DEBUGC("received coalesced datagram of %zd bytes for " 3072 "connection %"CID_FMT, packet_in_size, CID_BITS(&cid)); 3073 } 3074 packet_in->pi_received = lsquic_time_now(); 3075 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 3076 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 3077 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 3078 peer_ctx, packet_in_size); 3079 n_zeroes += s == 0; 3080 } 3081 while (0 == s && packet_in_data < packet_end); 3082 3083 return n_zeroes > 0 ? 0 : s; 3084} 3085 3086 3087#if __GNUC__ && !defined(NDEBUG) 3088__attribute__((weak)) 3089#endif 3090unsigned 3091lsquic_engine_quic_versions (const lsquic_engine_t *engine) 3092{ 3093 return engine->pub.enp_settings.es_versions; 3094} 3095 3096 3097void 3098lsquic_engine_cooldown (lsquic_engine_t *engine) 3099{ 3100 struct lsquic_hash_elem *el; 3101 lsquic_conn_t *conn; 3102 3103 if (engine->flags & ENG_COOLDOWN) 3104 /* AFAICT, there is no harm in calling this function more than once, 3105 * but log it just in case, as it may indicate an error in the caller. 3106 */ 3107 LSQ_INFO("cooldown called again"); 3108 engine->flags |= ENG_COOLDOWN; 3109 LSQ_INFO("entering cooldown mode"); 3110 if (engine->flags & ENG_SERVER) 3111 drop_all_mini_conns(engine); 3112 for (el = lsquic_hash_first(engine->conns_hash); el; 3113 el = lsquic_hash_next(engine->conns_hash)) 3114 { 3115 conn = lsquic_hashelem_getdata(el); 3116 lsquic_conn_going_away(conn); 3117 } 3118} 3119 3120 3121#if LSQUIC_CONN_STATS 3122static void 3123update_busy_detector (struct lsquic_engine *engine, struct lsquic_conn *conn, 3124 int immed) 3125{ 3126 char cidstr[MAX_CID_LEN * 2 + 1]; 3127 3128 if (!(LSQ_LOG_ENABLED_EXT(LSQ_LOG_NOTICE, LSQLM_CONN_STATS) 3129 && conn->cn_if->ci_log_stats)) 3130 return; 3131 3132 if (conn == engine->busy.last_conn) 3133 { 3134 engine->busy.immed_ticks <<= 1u; 3135 engine->busy.immed_ticks |= immed; 3136 if (MAX_IMMED_TICKS == engine->busy.immed_ticks) 3137 { 3138 if (engine->busy.current != conn) 3139 lsquic_logger_log1(LSQ_LOG_NOTICE, LSQLM_CONN_STATS, 3140 "connection %s marked busy: it's had %u immediate ticks " 3141 "in a row", 3142 (lsquic_cid2str(lsquic_conn_log_cid(conn), cidstr), cidstr), 3143 (unsigned) (sizeof(engine->busy.immed_ticks) * 8)); 3144 engine->busy.current = conn; 3145 } 3146 } 3147 else 3148 engine->busy.immed_ticks <<= 1; 3149 3150 engine->busy.last_conn = conn; 3151} 3152 3153 3154#endif 3155 3156 3157int 3158lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 3159{ 3160 const struct attq_elem *next_attq; 3161 lsquic_time_t now, next_time; 3162#if LSQUIC_DEBUG_NEXT_ADV_TICK || LSQUIC_CONN_STATS 3163 struct lsquic_conn *conn; 3164 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 3165#endif 3166 3167 ENGINE_CALLS_INCR(engine); 3168 3169 if ((engine->flags & ENG_PAST_DEADLINE) 3170 && lsquic_mh_count(&engine->conns_out)) 3171 { 3172#if LSQUIC_DEBUG_NEXT_ADV_TICK 3173 conn = lsquic_mh_peek(&engine->conns_out); 3174 engine->last_logged_conn = 0; 3175 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 3176 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 3177 lsquic_mh_count(&engine->conns_out), 3178 lsquic_mh_count(&engine->conns_out) != 1, "s", 3179 CID_BITS(lsquic_conn_log_cid(conn))); 3180#endif 3181#if LSQUIC_CONN_STATS 3182 conn = lsquic_mh_peek(&engine->conns_out); 3183 update_busy_detector(engine, conn, 1); 3184#endif 3185 *diff = 0; 3186 return 1; 3187 } 3188 3189 if (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 3190 { 3191#if LSQUIC_DEBUG_NEXT_ADV_TICK 3192 engine->last_logged_conn = 0; 3193 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 3194#endif 3195 *diff = 0; 3196 return 1; 3197 } 3198 3199 if (lsquic_mh_count(&engine->conns_tickable)) 3200 { 3201#if LSQUIC_DEBUG_NEXT_ADV_TICK 3202 conn = lsquic_mh_peek(&engine->conns_tickable); 3203 engine->last_logged_conn = 0; 3204 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 3205 "connection%.*s (%"CID_FMT" first)", 3206 lsquic_mh_count(&engine->conns_tickable), 3207 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 3208 CID_BITS(lsquic_conn_log_cid(conn))); 3209#endif 3210#if LSQUIC_CONN_STATS 3211 conn = lsquic_mh_peek(&engine->conns_tickable); 3212 update_busy_detector(engine, conn, 1); 3213#endif 3214 *diff = 0; 3215 return 1; 3216 } 3217 3218 next_attq = lsquic_attq_next(engine->attq); 3219 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 3220 { 3221 if (next_attq) 3222 next_time = next_attq->ae_adv_time; 3223 else 3224 return 0; 3225 } 3226 else 3227 { 3228 if (next_attq) 3229 { 3230 next_time = next_attq->ae_adv_time; 3231 if (engine->resume_sending_at < next_time) 3232 { 3233 next_time = engine->resume_sending_at; 3234 next_attq = NULL; 3235 } 3236 } 3237 else 3238 next_time = engine->resume_sending_at; 3239 } 3240 3241 now = lsquic_time_now(); 3242 *diff = (int) ((int64_t) next_time - (int64_t) now); 3243#if LSQUIC_DEBUG_NEXT_ADV_TICK 3244 if (next_attq) 3245 { 3246 /* Deduplicate consecutive log messages about the same reason for the 3247 * same connection. 3248 * If diff is always zero or diff reset to a higher value, event is 3249 * still logged. 3250 */ 3251 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 3252 && (uintptr_t) next_attq->ae_conn 3253 == engine->last_logged_conn 3254 && *diff < engine->last_tick_diff)) 3255 { 3256 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 3257 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 3258 engine->last_tick_diff = *diff; 3259 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 3260 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 3261 lsquic_attq_why2str(next_attq->ae_why)); 3262 } 3263 } 3264 else 3265 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 3266#endif 3267 3268#if LSQUIC_CONN_STATS 3269 if (next_attq) 3270 update_busy_detector(engine, next_attq->ae_conn, 3271 /* Immediate if: a) time is now or in the past */ 3272 *diff <= 0 3273 /* b) next event is pacer, which means that the 3274 * connection wants to send, but is prevented 3275 * by the pacer from doing so. 3276 */ 3277 || next_attq->ae_why == AEW_PACER 3278 /* c) next event is to retransmit data (meaning 3279 * that there is data in flight) and the 3280 * time is small, which implies a small RTT. 3281 */ 3282 || (next_attq->ae_why == N_AEWS + AL_RETX_APP 3283 && *diff < 5000)); 3284#endif 3285 3286 return 1; 3287} 3288 3289 3290unsigned 3291lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 3292{ 3293 lsquic_time_t now; 3294 ENGINE_CALLS_INCR(engine); 3295 now = lsquic_time_now(); 3296 if (from_now < 0) 3297 now -= from_now; 3298 else 3299 now += from_now; 3300 return lsquic_attq_count_before(engine->attq, now); 3301} 3302 3303 3304int 3305lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 3306 struct lsquic_conn *conn, unsigned cce_idx) 3307{ 3308 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 3309 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 3310 void *peer_ctx; 3311 3312 assert(cce_idx < conn->cn_n_cces); 3313 assert(conn->cn_cces_mask & (1 << cce_idx)); 3314 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 3315 3316 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 3317 cce->cce_cid.len, conn, &cce->cce_hash_el)) 3318 { 3319 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 3320 CID_BITS(&cce->cce_cid)); 3321 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 3322 cce->cce_flags |= CCE_REG; 3323 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 3324 return 0; 3325 } 3326 else 3327 { 3328 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 3329 CID_BITS(&cce->cce_cid)); 3330 return -1; 3331 } 3332} 3333 3334 3335void 3336lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 3337 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now, 3338 lsquic_time_t drain_time) 3339{ 3340 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 3341 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 3342 struct purga_el *puel; 3343 void *peer_ctx; 3344 3345 assert(cce_idx < conn->cn_n_cces); 3346 3347 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 3348 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 3349 3350 if (engine->purga) 3351 { 3352 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 3353 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 3354 PUTY_CID_RETIRED, now); 3355 if (puel) 3356 puel->puel_time = now + drain_time; 3357 } 3358 conn->cn_cces_mask &= ~(1u << cce_idx); 3359 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 3360} 3361 3362 3363