lsquic_engine.c revision fb73393f
1/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <limits.h> 10#include <stdint.h> 11#include <stdio.h> 12#include <stdlib.h> 13#include <string.h> 14#include <sys/queue.h> 15#include <time.h> 16#ifndef WIN32 17#include <sys/time.h> 18#include <netinet/in.h> 19#include <sys/types.h> 20#include <sys/stat.h> 21#include <fcntl.h> 22#include <unistd.h> 23#include <netdb.h> 24#endif 25 26#ifndef NDEBUG 27#include <sys/types.h> 28#include <regex.h> /* For code that loses packets */ 29#endif 30 31#if LOG_PACKET_CHECKSUM 32#include <zlib.h> 33#endif 34 35#include <openssl/aead.h> 36 37#include "lsquic.h" 38#include "lsquic_types.h" 39#include "lsquic_int_types.h" 40#include "lsquic_sizes.h" 41#include "lsquic_parse_common.h" 42#include "lsquic_parse.h" 43#include "lsquic_packet_in.h" 44#include "lsquic_packet_out.h" 45#include "lsquic_senhist.h" 46#include "lsquic_rtt.h" 47#include "lsquic_cubic.h" 48#include "lsquic_pacer.h" 49#include "lsquic_bw_sampler.h" 50#include "lsquic_minmax.h" 51#include "lsquic_bbr.h" 52#include "lsquic_send_ctl.h" 53#include "lsquic_set.h" 54#include "lsquic_conn_flow.h" 55#include "lsquic_sfcw.h" 56#include "lsquic_hash.h" 57#include "lsquic_conn.h" 58#include "lsquic_full_conn.h" 59#include "lsquic_util.h" 60#include "lsquic_qtags.h" 61#include "lsquic_enc_sess.h" 62#include "lsquic_mm.h" 63#include "lsquic_engine_public.h" 64#include "lsquic_eng_hist.h" 65#include "lsquic_ev_log.h" 66#include "lsquic_version.h" 67#include "lsquic_pr_queue.h" 68#include "lsquic_mini_conn.h" 69#include "lsquic_mini_conn_ietf.h" 70#include "lsquic_stock_shi.h" 71#include "lsquic_purga.h" 72#include "lsquic_tokgen.h" 73#include "lsquic_attq.h" 74#include "lsquic_min_heap.h" 75#include "lsquic_http1x_if.h" 76#include "lsquic_handshake.h" 77#include "lsquic_crand.h" 78#include "lsquic_ietf.h" 79 80#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 81#include "lsquic_logger.h" 82 83#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK 84#define LSQUIC_DEBUG_NEXT_ADV_TICK 1 85#endif 86 87#if LSQUIC_DEBUG_NEXT_ADV_TICK 88#include "lsquic_alarmset.h" 89#endif 90 91#define MIN(a, b) ((a) < (b) ? (a) : (b)) 92 93/* The batch of outgoing packets grows and shrinks dynamically */ 94#define MAX_OUT_BATCH_SIZE 1024 95#define MIN_OUT_BATCH_SIZE 4 96#define INITIAL_OUT_BATCH_SIZE 32 97 98struct out_batch 99{ 100 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 101 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 102 unsigned pack_off[MAX_OUT_BATCH_SIZE]; 103 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE * 2]; 104 struct iovec iov [MAX_OUT_BATCH_SIZE * 2]; 105}; 106 107typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 108 109static void 110process_connections (struct lsquic_engine *engine, conn_iter_f iter, 111 lsquic_time_t now); 112 113static void 114engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 115 116static lsquic_conn_t * 117engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 118 enum lsquic_conn_flags flag); 119 120static void 121force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 122 123#if LSQUIC_COUNT_ENGINE_CALLS 124#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0) 125#else 126#define ENGINE_CALLS_INCR(e) 127#endif 128 129/* Nested calls to LSQUIC are not supported */ 130#define ENGINE_IN(e) do { \ 131 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 132 (e)->pub.enp_flags |= ENPUB_PROC; \ 133 ENGINE_CALLS_INCR(e); \ 134} while (0) 135 136#define ENGINE_OUT(e) do { \ 137 assert((e)->pub.enp_flags & ENPUB_PROC); \ 138 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 139} while (0) 140 141/* A connection can be referenced from one of six places: 142 * 143 * 1. A hash is used to find connections in order to dispatch an incoming 144 * packet. Connections can be hashed by CIDs or by address. In the 145 * former case, each connection has one or more mappings in the hash 146 * table. IETF QUIC connections have up to eight (in our implementation) 147 * source CIDs and each of those would have a mapping. In client mode, 148 * depending on QUIC versions and options selected, it is may be 149 * necessary to hash connections by address, in which case incoming 150 * packets are delivered to connections based on the address. 151 * 152 * 2. Outgoing queue. 153 * 154 * 3. Tickable queue 155 * 156 * 4. Advisory Tick Time queue. 157 * 158 * 5. Closing connections queue. This is a transient queue -- it only 159 * exists for the duration of process_connections() function call. 160 * 161 * 6. Ticked connections queue. Another transient queue, similar to (5). 162 * 163 * The idea is to destroy the connection when it is no longer referenced. 164 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 165 * that case, the connection is referenced from two places: (2) and (5). 166 * After its packets are sent, it is only referenced in (5), and at the 167 * end of the function call, when it is removed from (5), reference count 168 * goes to zero and the connection is destroyed. If not all packets can 169 * be sent, at the end of the function call, the connection is referenced 170 * by (2) and will only be removed once all outgoing packets have been 171 * sent. 172 */ 173#define CONN_REF_FLAGS (LSCONN_HASHED \ 174 |LSCONN_HAS_OUTGOING \ 175 |LSCONN_TICKABLE \ 176 |LSCONN_TICKED \ 177 |LSCONN_CLOSING \ 178 |LSCONN_ATTQ) 179 180 181 182 183struct cid_update_batch 184{ 185 lsquic_cids_update_f cub_update_cids; 186 void *cub_update_ctx; 187 unsigned cub_count; 188 lsquic_cid_t cub_cids[20]; 189 void *cub_peer_ctxs[20]; 190}; 191 192static void 193cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *); 194 195 196struct lsquic_engine 197{ 198 struct lsquic_engine_public pub; 199 enum { 200 ENG_SERVER = LSENG_SERVER, 201 ENG_HTTP = LSENG_HTTP, 202 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 203 ENG_PAST_DEADLINE 204 = (1 << 8), /* Previous call to a processing 205 * function went past time threshold. 206 */ 207 ENG_CONNS_BY_ADDR 208 = (1 << 9), /* Connections are hashed by address */ 209#ifndef NDEBUG 210 ENG_COALESCE = (1 << 24), /* Packet coalescing is enabled */ 211 ENG_LOSE_PACKETS= (1 << 25), /* Lose *some* outgoing packets */ 212 ENG_DTOR = (1 << 26), /* Engine destructor */ 213#endif 214 } flags; 215 lsquic_packets_out_f packets_out; 216 void *packets_out_ctx; 217 lsquic_cids_update_f report_new_scids; 218 lsquic_cids_update_f report_live_scids; 219 lsquic_cids_update_f report_old_scids; 220 void *scids_ctx; 221 struct lsquic_hash *conns_hash; 222 struct min_heap conns_tickable; 223 struct min_heap conns_out; 224 struct eng_hist history; 225 unsigned batch_size; 226 struct lsquic_conn *curr_conn; 227 struct pr_queue *pr_queue; 228 struct attq *attq; 229 /* Track time last time a packet was sent to give new connections 230 * priority lower than that of existing connections. 231 */ 232 lsquic_time_t last_sent; 233#ifndef NDEBUG 234 regex_t lose_packets_re; 235 const char *lose_packets_str; 236#endif 237 unsigned n_conns; 238 lsquic_time_t deadline; 239 lsquic_time_t resume_sending_at; 240 unsigned mini_conns_count; 241 struct lsquic_purga *purga; 242#if LSQUIC_CONN_STATS 243 struct { 244 unsigned conns; 245 } stats; 246 struct conn_stats conn_stats_sum; 247 FILE *stats_fh; 248#endif 249 struct cid_update_batch new_scids; 250 struct out_batch out_batch; 251#if LSQUIC_COUNT_ENGINE_CALLS 252 unsigned long n_engine_calls; 253#endif 254#if LSQUIC_DEBUG_NEXT_ADV_TICK 255 uintptr_t last_logged_conn; 256 unsigned last_logged_ae_why; 257 int last_tick_diff; 258#endif 259 struct crand crand; 260 EVP_AEAD_CTX retry_aead_ctx; 261}; 262 263 264void 265lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 266 unsigned flags) 267{ 268 memset(settings, 0, sizeof(*settings)); 269 settings->es_versions = LSQUIC_DF_VERSIONS; 270 if (flags & ENG_SERVER) 271 { 272 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 273 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 274 settings->es_init_max_data 275 = LSQUIC_DF_INIT_MAX_DATA_SERVER; 276 settings->es_init_max_stream_data_bidi_remote 277 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER; 278 settings->es_init_max_stream_data_bidi_local 279 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER; 280 settings->es_init_max_stream_data_uni 281 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER; 282 settings->es_init_max_streams_uni 283 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER; 284 settings->es_ping_period = 0; 285 } 286 else 287 { 288 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 289 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 290 settings->es_init_max_data 291 = LSQUIC_DF_INIT_MAX_DATA_CLIENT; 292 settings->es_init_max_stream_data_bidi_remote 293 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT; 294 settings->es_init_max_stream_data_bidi_local 295 = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT; 296 settings->es_init_max_stream_data_uni 297 = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT; 298 settings->es_init_max_streams_uni 299 = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT; 300 settings->es_ping_period = LSQUIC_DF_PING_PERIOD; 301 } 302 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 303 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 304 settings->es_idle_timeout = LSQUIC_DF_IDLE_TIMEOUT; 305 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 306 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 307 settings->es_max_header_list_size 308 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 309 settings->es_ua = LSQUIC_DF_UA; 310 settings->es_ecn = LSQUIC_DF_ECN; 311 312 settings->es_pdmd = QTAG_X509; 313 settings->es_aead = QTAG_AESG; 314 settings->es_kexs = QTAG_C255; 315 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 316 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 317 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 318 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 319 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 320 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 321 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 322 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 323 settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY; 324 settings->es_max_inchoate = LSQUIC_DF_MAX_INCHOATE; 325 settings->es_send_prst = LSQUIC_DF_SEND_PRST; 326 settings->es_sttl = LSQUIC_DF_STTL; 327 settings->es_init_max_streams_bidi 328 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI; 329 settings->es_scid_len = LSQUIC_DF_SCID_LEN; 330 settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE; 331 settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE; 332 settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED; 333 settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE; 334 settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED; 335 settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION; 336 settings->es_ql_bits = LSQUIC_DF_QL_BITS; 337 settings->es_spin = LSQUIC_DF_SPIN; 338 settings->es_delayed_acks = LSQUIC_DF_DELAYED_ACKS; 339 settings->es_timestamps = LSQUIC_DF_TIMESTAMPS; 340} 341 342 343/* Note: if returning an error, err_buf must be valid if non-NULL */ 344int 345lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 346 unsigned flags, 347 char *err_buf, size_t err_buf_sz) 348{ 349 if (settings->es_cfcw < LSQUIC_MIN_FCW || 350 settings->es_sfcw < LSQUIC_MIN_FCW) 351 { 352 if (err_buf) 353 snprintf(err_buf, err_buf_sz, "%s", 354 "flow control window set too low"); 355 return -1; 356 } 357 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 358 { 359 if (err_buf) 360 snprintf(err_buf, err_buf_sz, "%s", 361 "No supported QUIC versions specified"); 362 return -1; 363 } 364 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 365 { 366 if (err_buf) 367 snprintf(err_buf, err_buf_sz, "%s", 368 "one or more unsupported QUIC version is specified"); 369 return -1; 370 } 371 if (flags & ENG_SERVER) 372 { 373 if (settings->es_handshake_to > 374 MAX_MINI_CONN_LIFESPAN_IN_USEC) 375 { 376 if (err_buf) 377 snprintf(err_buf, err_buf_sz, "handshake timeout %lu" 378 " usec is too large. The maximum for server is %u usec", 379 settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC); 380 return -1; 381 } 382 } 383 if (settings->es_idle_timeout > 600) 384 { 385 if (err_buf) 386 snprintf(err_buf, err_buf_sz, "%s", 387 "The maximum value of idle timeout is 600 seconds"); 388 return -1; 389 } 390 if (settings->es_scid_len > MAX_CID_LEN) 391 { 392 if (err_buf) 393 snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u " 394 "bytes long; it must be between 0 and %u.", 395 settings->es_scid_len, MAX_CID_LEN); 396 return -1; 397 } 398 399 if (settings->es_cc_algo > 2) 400 { 401 if (err_buf) 402 snprintf(err_buf, err_buf_sz, "Invalid congestion control " 403 "algorithm value %u", settings->es_cc_algo); 404 return -1; 405 } 406 407 if (!(settings->es_ql_bits >= 0 && settings->es_ql_bits <= 2)) 408 { 409 if (err_buf) 410 snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ", 411 settings->es_ql_bits); 412 return -1; 413 } 414 415 if (!(settings->es_spin == 0 || settings->es_spin == 1)) 416 { 417 if (err_buf) 418 snprintf(err_buf, err_buf_sz, "Invalid spin value %d", 419 settings->es_spin); 420 return -1; 421 } 422 423 return 0; 424} 425 426 427static void 428free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6) 429{ 430 free(packet_data); 431} 432 433 434static void * 435malloc_buf (void *ctx, void *conn_ctx, unsigned short size, char is_ipv6) 436{ 437 return malloc(size); 438} 439 440 441static const struct lsquic_packout_mem_if stock_pmi = 442{ 443 malloc_buf, free_packet, free_packet, 444}; 445 446 447static int 448hash_conns_by_addr (const struct lsquic_engine *engine) 449{ 450 if (engine->flags & ENG_SERVER) 451 return 0; 452 if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS) 453 return 1; 454 if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS) 455 && engine->pub.enp_settings.es_support_tcid0) 456 return 1; 457 if (engine->pub.enp_settings.es_scid_len == 0) 458 return 1; 459 return 0; 460} 461 462 463lsquic_engine_t * 464lsquic_engine_new (unsigned flags, 465 const struct lsquic_engine_api *api) 466{ 467 lsquic_engine_t *engine; 468 size_t alpn_len; 469 char err_buf[100]; 470 471 if (!api->ea_packets_out) 472 { 473 LSQ_ERROR("packets_out callback is not specified"); 474 return NULL; 475 } 476 477 if (!api->ea_stream_if) 478 { 479 LSQ_ERROR("stream interface is not specified"); 480 return NULL; 481 } 482 483 if (!(flags & LSENG_HTTP) && api->ea_alpn) 484 { 485 alpn_len = strlen(api->ea_alpn); 486 if (alpn_len < 1 || alpn_len > 255) 487 { 488 LSQ_ERROR("ALPN string length invalid: %zd bytes", alpn_len); 489 return NULL; 490 } 491 } 492 else 493 alpn_len = 0; 494 495 if (api->ea_settings && 496 0 != lsquic_engine_check_settings(api->ea_settings, flags, 497 err_buf, sizeof(err_buf))) 498 { 499 LSQ_ERROR("cannot create engine: %s", err_buf); 500 return NULL; 501 } 502 503 engine = calloc(1, sizeof(*engine)); 504 if (!engine) 505 return NULL; 506 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 507 { 508 free(engine); 509 return NULL; 510 } 511 if (api->ea_settings) 512 engine->pub.enp_settings = *api->ea_settings; 513 else 514 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 515 int tag_buf_len; 516 tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf, 517 sizeof(engine->pub.enp_ver_tags_buf), 518 engine->pub.enp_settings.es_versions); 519 if (tag_buf_len <= 0) 520 { 521 LSQ_ERROR("cannot generate version tags buffer"); 522 free(engine); 523 return NULL; 524 } 525 engine->pub.enp_ver_tags_len = tag_buf_len; 526 engine->pub.enp_flags = ENPUB_CAN_SEND; 527 engine->pub.enp_stream_if = api->ea_stream_if; 528 engine->pub.enp_stream_if_ctx = api->ea_stream_if_ctx; 529 530 engine->flags = flags; 531#ifndef NDEBUG 532 engine->flags |= ENG_COALESCE; 533#endif 534 engine->packets_out = api->ea_packets_out; 535 engine->packets_out_ctx = api->ea_packets_out_ctx; 536 engine->report_new_scids = api->ea_new_scids; 537 engine->report_live_scids = api->ea_live_scids; 538 engine->report_old_scids = api->ea_old_scids; 539 engine->scids_ctx = api->ea_cids_update_ctx; 540 cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx); 541 engine->pub.enp_lookup_cert = api->ea_lookup_cert; 542 engine->pub.enp_cert_lu_ctx = api->ea_cert_lu_ctx; 543 engine->pub.enp_get_ssl_ctx = api->ea_get_ssl_ctx; 544 if (api->ea_shi) 545 { 546 engine->pub.enp_shi = api->ea_shi; 547 engine->pub.enp_shi_ctx = api->ea_shi_ctx; 548 } 549 else 550 { 551 engine->pub.enp_shi = &stock_shi; 552 engine->pub.enp_shi_ctx = lsquic_stock_shared_hash_new(); 553 if (!engine->pub.enp_shi_ctx) 554 { 555 free(engine); 556 return NULL; 557 } 558 } 559 if (api->ea_hsi_if) 560 { 561 engine->pub.enp_hsi_if = api->ea_hsi_if; 562 engine->pub.enp_hsi_ctx = api->ea_hsi_ctx; 563 } 564 else 565 { 566 engine->pub.enp_hsi_if = lsquic_http1x_if; 567 engine->pub.enp_hsi_ctx = NULL; 568 } 569 if (api->ea_pmi) 570 { 571 engine->pub.enp_pmi = api->ea_pmi; 572 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 573 } 574 else 575 { 576 engine->pub.enp_pmi = &stock_pmi; 577 engine->pub.enp_pmi_ctx = NULL; 578 } 579 engine->pub.enp_verify_cert = api->ea_verify_cert; 580 engine->pub.enp_verify_ctx = api->ea_verify_ctx; 581 engine->pub.enp_kli = api->ea_keylog_if; 582 engine->pub.enp_kli_ctx = api->ea_keylog_ctx; 583 engine->pub.enp_engine = engine; 584 if (hash_conns_by_addr(engine)) 585 engine->flags |= ENG_CONNS_BY_ADDR; 586 engine->conns_hash = lsquic_hash_create(); 587 engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub); 588 if (!engine->pub.enp_tokgen) 589 return NULL; 590 engine->pub.enp_crand = &engine->crand; 591 if (flags & ENG_SERVER) 592 { 593 engine->pr_queue = lsquic_prq_create( 594 10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE, 595 &engine->pub); 596 if (!engine->pr_queue) 597 { 598 lsquic_tg_destroy(engine->pub.enp_tokgen); 599 return NULL; 600 } 601 engine->purga = lsquic_purga_new(30 * 1000 * 1000, 602 engine->report_old_scids, engine->scids_ctx); 603 if (!engine->purga) 604 { 605 lsquic_tg_destroy(engine->pub.enp_tokgen); 606 lsquic_prq_destroy(engine->pr_queue); 607 return NULL; 608 } 609 } 610 engine->attq = lsquic_attq_create(); 611 eng_hist_init(&engine->history); 612 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 613 if (engine->pub.enp_settings.es_honor_prst) 614 { 615 engine->pub.enp_srst_hash = lsquic_hash_create(); 616 if (!engine->pub.enp_srst_hash) 617 { 618 lsquic_engine_destroy(engine); 619 return NULL; 620 } 621 } 622 623 if (alpn_len) 624 { 625 engine->pub.enp_alpn = malloc(alpn_len + 1); 626 if (!engine->pub.enp_alpn) 627 { 628 lsquic_engine_destroy(engine); 629 return NULL; 630 } 631 engine->pub.enp_alpn[0] = alpn_len; 632 memcpy(engine->pub.enp_alpn + 1, api->ea_alpn, alpn_len); 633 } 634 635 if (flags & LSENG_HTTP) 636 engine->pub.enp_flags |= ENPUB_HTTP; 637 638#ifndef NDEBUG 639 { 640 const char *env; 641 env = getenv("LSQUIC_LOSE_PACKETS_RE"); 642 if (env) 643 { 644 if (0 != regcomp(&engine->lose_packets_re, env, 645 REG_EXTENDED|REG_NOSUB)) 646 { 647 LSQ_ERROR("could not compile lost packet regex `%s'", env); 648 return NULL; 649 } 650 engine->flags |= ENG_LOSE_PACKETS; 651 engine->lose_packets_str = env; 652 LSQ_WARN("will lose packets that match the following regex: %s", 653 env); 654 } 655 env = getenv("LSQUIC_COALESCE"); 656 if (env) 657 { 658 engine->flags &= ~ENG_COALESCE; 659 if (atoi(env)) 660 { 661 engine->flags |= ENG_COALESCE; 662 LSQ_NOTICE("will coalesce packets"); 663 } 664 else 665 LSQ_NOTICE("will not coalesce packets"); 666 } 667 } 668#endif 669#if LSQUIC_CONN_STATS 670 engine->stats_fh = api->ea_stats_fh; 671#endif 672 if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx, EVP_aead_aes_128_gcm(), 673 IETF_RETRY_KEY_BUF, IETF_RETRY_KEY_SZ, 16, NULL)) 674 { 675 LSQ_ERROR("could not initialize retry AEAD ctx"); 676 lsquic_engine_destroy(engine); 677 return NULL; 678 } 679 engine->pub.enp_retry_aead_ctx = &engine->retry_aead_ctx; 680 681 LSQ_INFO("instantiated engine"); 682 return engine; 683} 684 685 686#if LOG_PACKET_CHECKSUM 687static void 688log_packet_checksum (const lsquic_cid_t *cid, const char *direction, 689 const unsigned char *buf, size_t bufsz) 690{ 691 EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction, 692 (uint32_t) crc32(0, buf, bufsz)); 693} 694 695 696#endif 697 698 699static void 700grow_batch_size (struct lsquic_engine *engine) 701{ 702 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 703} 704 705 706static void 707shrink_batch_size (struct lsquic_engine *engine) 708{ 709 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 710} 711 712 713struct cce_cid_iter 714{ 715 const struct lsquic_conn *conn; 716 unsigned todo, n; 717}; 718 719 720static struct conn_cid_elem * 721cce_iter_next (struct cce_cid_iter *citer) 722{ 723 struct conn_cid_elem *cce; 724 725 while (citer->todo) 726 if (citer->todo & (1 << citer->n)) 727 { 728 citer->todo &= ~(1 << citer->n); 729 cce = &citer->conn->cn_cces[ citer->n++ ]; 730 if (!(cce->cce_flags & CCE_PORT)) 731 return cce; 732 } 733 else 734 ++citer->n; 735 736 return NULL; 737} 738 739 740static struct conn_cid_elem * 741cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn) 742{ 743 citer->conn = conn; 744 citer->todo = conn->cn_cces_mask; 745 citer->n = 0; 746 return cce_iter_next(citer); 747} 748 749 750#if LSQUIC_CONN_STATS 751void 752update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn) 753{ 754 unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum; 755 const unsigned long *src; 756 const struct conn_stats *stats; 757 unsigned i; 758 759 if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn))) 760 { 761 ++engine->stats.conns; 762 src = (unsigned long *) stats; 763 for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i) 764 dst[i] += src[i]; 765 } 766} 767 768 769#endif 770 771 772/* Wrapper to make sure important things occur before the connection is 773 * really destroyed. 774 */ 775static void 776destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn, 777 lsquic_time_t now) 778{ 779 struct cce_cid_iter citer; 780 const struct conn_cid_elem *cce; 781 lsquic_time_t drain_time; 782 struct purga_el *puel; 783 784 engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI); 785 if (engine->purga 786 /* Blacklist all CIDs except for promoted mini connections */ 787 && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) 788 != (LSCONN_MINI|LSCONN_PROMOTED)) 789 { 790 if (!(conn->cn_flags & LSCONN_IMMED_CLOSE) 791 && conn->cn_if->ci_drain_time && 792 (drain_time = conn->cn_if->ci_drain_time(conn), drain_time)) 793 { 794 for (cce = cce_iter_first(&citer, conn); cce; 795 cce = cce_iter_next(&citer)) 796 { 797 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 798 lsquic_conn_get_peer_ctx(conn, NULL), 799 PUTY_CONN_DRAIN, now); 800 if (puel) 801 puel->puel_time = now + drain_time; 802 } 803 } 804 else 805 { 806 for (cce = cce_iter_first(&citer, conn); cce; 807 cce = cce_iter_next(&citer)) 808 { 809 puel = lsquic_purga_add(engine->purga, &cce->cce_cid, 810 lsquic_conn_get_peer_ctx(conn, NULL), 811 PUTY_CONN_DELETED, now); 812 if (puel) 813 { 814 puel->puel_time = now; 815 puel->puel_count = 0; 816 } 817 } 818 } 819 } 820#if LSQUIC_CONN_STATS 821 update_stats_sum(engine, conn); 822#endif 823 --engine->n_conns; 824 conn->cn_flags |= LSCONN_NEVER_TICKABLE; 825 conn->cn_if->ci_destroy(conn); 826} 827 828 829static int 830maybe_grow_conn_heaps (struct lsquic_engine *engine) 831{ 832 struct min_heap_elem *els; 833 unsigned count; 834 835 if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable)) 836 return 0; /* Nothing to do */ 837 838 if (lsquic_mh_nalloc(&engine->conns_tickable)) 839 count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2; 840 else 841 count = 8; 842 843 els = malloc(sizeof(els[0]) * count); 844 if (!els) 845 { 846 LSQ_ERROR("%s: malloc failed", __func__); 847 return -1; 848 } 849 850 LSQ_DEBUG("grew heaps to %u elements", count / 2); 851 memcpy(&els[0], engine->conns_tickable.mh_elems, 852 sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable)); 853 memcpy(&els[count / 2], engine->conns_out.mh_elems, 854 sizeof(els[0]) * lsquic_mh_count(&engine->conns_out)); 855 free(engine->conns_tickable.mh_elems); 856 engine->conns_tickable.mh_elems = els; 857 engine->conns_out.mh_elems = &els[count / 2]; 858 engine->conns_tickable.mh_nalloc = count / 2; 859 engine->conns_out.mh_nalloc = count / 2; 860 return 0; 861} 862 863 864static void 865remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn, 866 unsigned todo) 867{ 868 unsigned n; 869 870 for (n = 0; todo; todo &= ~(1 << n++)) 871 if ((todo & (1 << n)) && 872 (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED)) 873 lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el); 874} 875 876 877static void 878remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn) 879{ 880 remove_cces_from_hash(hash, conn, conn->cn_cces_mask); 881} 882 883 884static void 885cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx); 886 887 888static int 889insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 890 void *peer_ctx) 891{ 892 struct conn_cid_elem *cce; 893 unsigned todo, done, n; 894 895 for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++)) 896 if (todo & (1 << n)) 897 { 898 cce = &conn->cn_cces[n]; 899 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 900 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 901 cce->cce_cid.len, conn, &cce->cce_hash_el)) 902 done |= 1 << n; 903 else 904 goto err; 905 if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG)) 906 { 907 cce->cce_flags |= CCE_REG; 908 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 909 } 910 } 911 912 return 0; 913 914 err: 915 remove_cces_from_hash(engine->conns_hash, conn, done); 916 return -1; 917} 918 919 920static lsquic_conn_t * 921new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn, 922 lsquic_time_t now) 923{ 924 const lsquic_cid_t *cid; 925 server_conn_ctor_f ctor; 926 lsquic_conn_t *conn; 927 unsigned flags; 928 if (0 != maybe_grow_conn_heaps(engine)) 929 return NULL; 930 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 931 932 if (mini_conn->cn_flags & LSCONN_IETF) 933 ctor = lsquic_ietf_full_conn_server_new; 934 else 935 ctor = lsquic_gquic_full_conn_server_new; 936 937 conn = ctor(&engine->pub, flags, mini_conn); 938 if (!conn) 939 { 940 /* Otherwise, full_conn_server_new prints its own warnings */ 941 if (ENOMEM == errno) 942 { 943 cid = lsquic_conn_log_cid(mini_conn); 944 LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s", 945 CID_BITS(cid), strerror(errno)); 946 } 947 return NULL; 948 } 949 ++engine->n_conns; 950 if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL))) 951 { 952 cid = lsquic_conn_log_cid(conn); 953 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 954 CID_BITS(cid)); 955 destroy_conn(engine, conn, now); 956 return NULL; 957 } 958 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 959 conn->cn_flags |= LSCONN_HASHED; 960 return conn; 961} 962 963 964static enum 965{ 966 VER_NOT_SPECIFIED, 967 VER_SUPPORTED, 968 VER_UNSUPPORTED, 969} 970 971 972version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in, 973 enum lsquic_version *pversion) 974{ 975 lsquic_ver_tag_t ver_tag; 976 enum lsquic_version version; 977 978 if (!packet_in->pi_quic_ver) 979 { 980 LSQ_DEBUG("packet does not specify version"); 981 return VER_NOT_SPECIFIED; 982 } 983 984 memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag)); 985 version = lsquic_tag2ver(ver_tag); 986 if (version < N_LSQVER) 987 { 988 if (engine->pub.enp_settings.es_versions & (1 << version)) 989 { 990 LSQ_DEBUG("client-supplied version %s is supported", 991 lsquic_ver2str[version]); 992 *pversion = version; 993 return VER_SUPPORTED; 994 } 995 else 996 LSQ_DEBUG("client-supplied version %s is not supported", 997 lsquic_ver2str[version]); 998 } 999 else 1000 LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized", 1001 ver_tag); 1002 1003 return VER_UNSUPPORTED; 1004} 1005 1006 1007static void 1008schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type, 1009 const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local, 1010 const struct sockaddr *sa_peer, void *peer_ctx) 1011{ 1012 assert(engine->pr_queue); 1013 if (0 == lsquic_prq_new_req(engine->pr_queue, type, packet_in, peer_ctx, 1014 sa_local, sa_peer)) 1015 LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT, 1016 lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id)); 1017 else 1018 LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]); 1019} 1020 1021 1022static unsigned short 1023sa2port (const struct sockaddr *sa) 1024{ 1025 if (sa->sa_family == AF_INET) 1026 { 1027 struct sockaddr_in *const sa4 = (void *) sa; 1028 return sa4->sin_port; 1029 } 1030 else 1031 { 1032 struct sockaddr_in6 *const sa6 = (void *) sa; 1033 return sa6->sin6_port; 1034 } 1035} 1036 1037 1038static struct lsquic_hash_elem * 1039find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa) 1040{ 1041 unsigned short port; 1042 1043 port = sa2port(sa); 1044 return lsquic_hash_find(hash, &port, sizeof(port)); 1045} 1046 1047 1048static lsquic_conn_t * 1049find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1050 struct packin_parse_state *ppstate, const struct sockaddr *sa_local) 1051{ 1052 struct lsquic_hash_elem *el; 1053 lsquic_conn_t *conn; 1054 1055 if (engine->flags & ENG_CONNS_BY_ADDR) 1056 el = find_conn_by_addr(engine->conns_hash, sa_local); 1057 else if (packet_in->pi_flags & PI_CONN_ID) 1058 el = lsquic_hash_find(engine->conns_hash, 1059 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1060 else 1061 { 1062 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1063 return NULL; 1064 } 1065 1066 if (!el) 1067 return NULL; 1068 1069 conn = lsquic_hashelem_getdata(el); 1070 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1071 if ((engine->flags & ENG_CONNS_BY_ADDR) 1072 && !(conn->cn_flags & LSCONN_IETF) 1073 && (packet_in->pi_flags & PI_CONN_ID) 1074 && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id)) 1075 { 1076 LSQ_DEBUG("connection IDs do not match"); 1077 return NULL; 1078 } 1079 1080 return conn; 1081} 1082 1083 1084static lsquic_conn_t * 1085find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1086 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1087 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1088{ 1089 struct lsquic_hash_elem *el; 1090 struct purga_el *puel; 1091 lsquic_conn_t *conn; 1092 1093 if (!(packet_in->pi_flags & PI_CONN_ID)) 1094 { 1095 LSQ_DEBUG("packet header does not have connection ID: discarding"); 1096 return NULL; 1097 } 1098 el = lsquic_hash_find(engine->conns_hash, 1099 packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len); 1100 1101 if (el) 1102 { 1103 conn = lsquic_hashelem_getdata(el); 1104 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 1105 return conn; 1106 } 1107 1108 if (engine->flags & ENG_COOLDOWN) 1109 { /* Do not create incoming connections during cooldown */ 1110 LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)"); 1111 return NULL; 1112 } 1113 1114 if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate) 1115 { 1116 LSQ_DEBUG("reached limit of %u inchoate connections", 1117 engine->pub.enp_settings.es_max_inchoate); 1118 return NULL; 1119 } 1120 1121 1122 if (engine->purga 1123 && (puel = lsquic_purga_contains(engine->purga, 1124 &packet_in->pi_conn_id), puel)) 1125 { 1126 switch (puel->puel_type) 1127 { 1128 case PUTY_CID_RETIRED: 1129 LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet", 1130 CID_BITS(&packet_in->pi_conn_id)); 1131 return NULL; 1132 case PUTY_CONN_DRAIN: 1133 LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64, 1134 puel->puel_time, packet_in->pi_received); 1135 if (puel->puel_time > packet_in->pi_received) 1136 { 1137 LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet", 1138 CID_BITS(&packet_in->pi_conn_id)); 1139 return NULL; 1140 } 1141 LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted", 1142 CID_BITS(&packet_in->pi_conn_id)); 1143 puel->puel_type = PUTY_CONN_DELETED; 1144 puel->puel_count = 0; 1145 puel->puel_time = 0; 1146 /* fall-through */ 1147 case PUTY_CONN_DELETED: 1148 LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted", 1149 CID_BITS(&packet_in->pi_conn_id)); 1150 if (puel->puel_time < packet_in->pi_received) 1151 { 1152 puel->puel_time = packet_in->pi_received 1153 /* Exponential back-off */ 1154 + 1000000ull * (1 << MIN(puel->puel_count, 4)); 1155 ++puel->puel_count; 1156 goto maybe_send_prst; 1157 } 1158 return NULL; 1159 default: 1160 assert(0); 1161 return NULL; 1162 } 1163 } 1164 1165 if (engine->pub.enp_settings.es_send_prst 1166 && !(packet_in->pi_flags & PI_GQUIC) 1167 && HETY_NOT_SET == packet_in->pi_header_type) 1168 goto maybe_send_prst; 1169 1170 if (0 != maybe_grow_conn_heaps(engine)) 1171 return NULL; 1172 1173 const struct parse_funcs *pf; 1174 enum lsquic_version version; 1175 switch (version_matches(engine, packet_in, &version)) 1176 { 1177 case VER_UNSUPPORTED: 1178 if (engine->flags & ENG_SERVER) 1179 schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in, 1180 sa_local, sa_peer, peer_ctx); 1181 return NULL; 1182 case VER_NOT_SPECIFIED: 1183 maybe_send_prst: 1184 if ((engine->flags & ENG_SERVER) && 1185 engine->pub.enp_settings.es_send_prst) 1186 schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in, 1187 sa_local, sa_peer, peer_ctx); 1188 return NULL; 1189 case VER_SUPPORTED: 1190 pf = select_pf_by_ver(version); 1191 pf->pf_parse_packet_in_finish(packet_in, ppstate); 1192 break; 1193 } 1194 1195 1196 if ((1 << version) & LSQUIC_IETF_VERSIONS) 1197 { 1198 conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version, 1199 sa_peer->sa_family == AF_INET, NULL, packet_in_size); 1200 } 1201 else 1202 { 1203 conn = lsquic_mini_conn_new(&engine->pub, packet_in, version); 1204 } 1205 if (!conn) 1206 return NULL; 1207 ++engine->mini_conns_count; 1208 ++engine->n_conns; 1209 if (0 != insert_conn_into_hash(engine, conn, peer_ctx)) 1210 { 1211 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1212 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1213 CID_BITS(cid)); 1214 destroy_conn(engine, conn, packet_in->pi_received); 1215 return NULL; 1216 } 1217 assert(!(conn->cn_flags & CONN_REF_FLAGS)); 1218 conn->cn_flags |= LSCONN_HASHED; 1219 eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns); 1220 conn->cn_last_sent = engine->last_sent; 1221 return conn; 1222} 1223 1224 1225lsquic_conn_t * 1226lsquic_engine_find_conn (const struct lsquic_engine_public *engine, 1227 const lsquic_cid_t *cid) 1228{ 1229 struct lsquic_hash_elem *el; 1230 lsquic_conn_t *conn = NULL; 1231 el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len); 1232 1233 if (el) 1234 conn = lsquic_hashelem_getdata(el); 1235 return conn; 1236} 1237 1238 1239#if !defined(NDEBUG) && __GNUC__ 1240__attribute__((weak)) 1241#endif 1242void 1243lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub, 1244 lsquic_conn_t *conn) 1245{ 1246 if (0 == (enpub->enp_flags & ENPUB_PROC) && 1247 0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE))) 1248 { 1249 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 1250 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1251 engine_incref_conn(conn, LSCONN_TICKABLE); 1252 } 1253} 1254 1255 1256void 1257lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 1258 lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why) 1259{ 1260 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 1261 if (conn->cn_flags & LSCONN_TICKABLE) 1262 { 1263 /* Optimization: no need to add the connection to the Advisory Tick 1264 * Time Queue: it is about to be ticked, after which it its next tick 1265 * time may be queried again. 1266 */; 1267 } 1268 else if (conn->cn_flags & LSCONN_ATTQ) 1269 { 1270 if (lsquic_conn_adv_time(conn) != tick_time) 1271 { 1272 lsquic_attq_remove(engine->attq, conn); 1273 if (0 != lsquic_attq_add(engine->attq, conn, tick_time, why)) 1274 engine_decref_conn(engine, conn, LSCONN_ATTQ); 1275 } 1276 } 1277 else if (0 == lsquic_attq_add(engine->attq, conn, tick_time, why)) 1278 engine_incref_conn(conn, LSCONN_ATTQ); 1279} 1280 1281 1282static struct lsquic_conn * 1283find_conn_by_srst (struct lsquic_engine *engine, 1284 const struct lsquic_packet_in *packet_in) 1285{ 1286 struct lsquic_hash_elem *el; 1287 struct lsquic_conn *conn; 1288 1289 if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE 1290 || (packet_in->pi_data[0] & 0xC0) != 0x40) 1291 return NULL; 1292 1293 el = lsquic_hash_find(engine->pub.enp_srst_hash, 1294 packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ, 1295 IQUIC_SRESET_TOKEN_SZ); 1296 if (!el) 1297 return NULL; 1298 1299 conn = lsquic_hashelem_getdata(el); 1300 return conn; 1301} 1302 1303 1304/* Return 0 if packet is being processed by a real connection (mini or full), 1305 * otherwise return 1. 1306 */ 1307static int 1308process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 1309 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 1310 const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size) 1311{ 1312 lsquic_conn_t *conn; 1313 const unsigned char *packet_in_data; 1314 1315 if (lsquic_packet_in_is_gquic_prst(packet_in) 1316 && !engine->pub.enp_settings.es_honor_prst) 1317 { 1318 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1319 LSQ_DEBUG("public reset packet: discarding"); 1320 return 1; 1321 } 1322 1323 if (engine->flags & ENG_SERVER) 1324 { 1325 conn = find_or_create_conn(engine, packet_in, ppstate, sa_local, 1326 sa_peer, peer_ctx, packet_in_size); 1327 if (!engine->curr_conn) 1328 engine->curr_conn = conn; 1329 } 1330 else 1331 conn = find_conn(engine, packet_in, ppstate, sa_local); 1332 1333 if (!conn) 1334 { 1335 if (engine->pub.enp_settings.es_honor_prst 1336 && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */ 1337 && !(packet_in->pi_flags & PI_GQUIC) 1338 && engine->pub.enp_srst_hash 1339 && (conn = find_conn_by_srst(engine, packet_in))) 1340 { 1341 LSQ_DEBUGC("got stateless reset for connection %"CID_FMT, 1342 CID_BITS(lsquic_conn_log_cid(conn))); 1343 conn->cn_if->ci_stateless_reset(conn); 1344 if (!(conn->cn_flags & LSCONN_TICKABLE) 1345 && conn->cn_if->ci_is_tickable(conn)) 1346 { 1347 lsquic_mh_insert(&engine->conns_tickable, conn, 1348 conn->cn_last_ticked); 1349 engine_incref_conn(conn, LSCONN_TICKABLE); 1350 } 1351 /* Even though the connection processes this packet, we return 1352 * 1 so that the caller does not add reset packet's random 1353 * bytes to the list of valid CIDs. 1354 */ 1355 } 1356 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1357 return 1; 1358 } 1359 1360 if (0 == (conn->cn_flags & LSCONN_TICKABLE)) 1361 { 1362 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1363 engine_incref_conn(conn, LSCONN_TICKABLE); 1364 } 1365 packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx, 1366 sa_local, sa_peer); 1367 lsquic_packet_in_upref(packet_in); 1368#if LOG_PACKET_CHECKSUM 1369 log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data, 1370 packet_in->pi_data_sz); 1371#endif 1372 /* Note on QLog: 1373 * For the PACKET_RX QLog event, we are interested in logging these things: 1374 * - raw packet (however it comes in, encrypted or not) 1375 * - frames (list of frame names) 1376 * - packet type and number 1377 * - packet rx timestamp 1378 * 1379 * Since only some of these items are available at this code 1380 * juncture, we will wait until after the packet has been 1381 * decrypted (if necessary) and parsed to call the log functions. 1382 * 1383 * Once the PACKET_RX event is finally logged, the timestamp 1384 * will come from packet_in->pi_received. For correct sequential 1385 * ordering of QLog events, be sure to process the QLogs downstream. 1386 * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.) 1387 */ 1388 packet_in_data = packet_in->pi_data; 1389 packet_in_size = packet_in->pi_data_sz; 1390 conn->cn_if->ci_packet_in(conn, packet_in); 1391 QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size); 1392 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 1393 return 0; 1394} 1395 1396 1397void 1398lsquic_engine_destroy (lsquic_engine_t *engine) 1399{ 1400 struct lsquic_hash_elem *el; 1401 lsquic_conn_t *conn; 1402 1403 LSQ_DEBUG("destroying engine"); 1404#ifndef NDEBUG 1405 engine->flags |= ENG_DTOR; 1406#endif 1407 1408 while ((conn = lsquic_mh_pop(&engine->conns_out))) 1409 { 1410 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 1411 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1412 } 1413 1414 while ((conn = lsquic_mh_pop(&engine->conns_tickable))) 1415 { 1416 assert(conn->cn_flags & LSCONN_TICKABLE); 1417 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1418 } 1419 1420 for (el = lsquic_hash_first(engine->conns_hash); el; 1421 el = lsquic_hash_next(engine->conns_hash)) 1422 { 1423 conn = lsquic_hashelem_getdata(el); 1424 force_close_conn(engine, conn); 1425 } 1426 lsquic_hash_destroy(engine->conns_hash); 1427 1428 assert(0 == engine->n_conns); 1429 assert(0 == engine->mini_conns_count); 1430 if (engine->pr_queue) 1431 lsquic_prq_destroy(engine->pr_queue); 1432 if (engine->purga) 1433 lsquic_purga_destroy(engine->purga); 1434 lsquic_attq_destroy(engine->attq); 1435 1436 assert(0 == lsquic_mh_count(&engine->conns_out)); 1437 assert(0 == lsquic_mh_count(&engine->conns_tickable)); 1438 if (engine->pub.enp_shi == &stock_shi) 1439 lsquic_stock_shared_hash_destroy(engine->pub.enp_shi_ctx); 1440 lsquic_mm_cleanup(&engine->pub.enp_mm); 1441 free(engine->conns_tickable.mh_elems); 1442#ifndef NDEBUG 1443 if (engine->flags & ENG_LOSE_PACKETS) 1444 regfree(&engine->lose_packets_re); 1445#endif 1446 if (engine->pub.enp_tokgen) 1447 lsquic_tg_destroy(engine->pub.enp_tokgen); 1448#if LSQUIC_CONN_STATS 1449 if (engine->stats_fh) 1450 { 1451 const struct conn_stats *const stats = &engine->conn_stats_sum; 1452 fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n"); 1453 fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns); 1454 fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks); 1455 fprintf(engine->stats_fh, "In:\n"); 1456 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->in.bytes); 1457 fprintf(engine->stats_fh, " packets: %lu\n", stats->in.packets); 1458 fprintf(engine->stats_fh, " undecryptable packets: %lu\n", stats->in.undec_packets); 1459 fprintf(engine->stats_fh, " duplicate packets: %lu\n", stats->in.dup_packets); 1460 fprintf(engine->stats_fh, " error packets: %lu\n", stats->in.err_packets); 1461 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->in.stream_frames); 1462 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->in.stream_data_sz); 1463 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1464 stats->in.headers_comp, stats->in.headers_uncomp, 1465 stats->in.headers_uncomp ? 1466 (double) stats->in.headers_comp / (double) stats->in.headers_uncomp 1467 : 0); 1468 fprintf(engine->stats_fh, " ACK frames: %lu\n", stats->in.n_acks); 1469 fprintf(engine->stats_fh, " ACK frames processed: %lu\n", stats->in.n_acks_proc); 1470 fprintf(engine->stats_fh, " ACK frames merged to new: %lu\n", stats->in.n_acks_merged[0]); 1471 fprintf(engine->stats_fh, " ACK frames merged to old: %lu\n", stats->in.n_acks_merged[1]); 1472 fprintf(engine->stats_fh, "Out:\n"); 1473 fprintf(engine->stats_fh, " Total bytes: %lu\n", stats->out.bytes); 1474 fprintf(engine->stats_fh, " packets: %lu\n", stats->out.packets); 1475 fprintf(engine->stats_fh, " acked via loss record: %lu\n", stats->out.acked_via_loss); 1476 fprintf(engine->stats_fh, " acks: %lu\n", stats->out.acks); 1477 fprintf(engine->stats_fh, " retx packets: %lu\n", stats->out.retx_packets); 1478 fprintf(engine->stats_fh, " STREAM frame count: %lu\n", stats->out.stream_frames); 1479 fprintf(engine->stats_fh, " STREAM payload size: %lu\n", stats->out.stream_data_sz); 1480 fprintf(engine->stats_fh, " Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n", 1481 stats->out.headers_comp, stats->out.headers_uncomp, 1482 stats->out.headers_uncomp ? 1483 (double) stats->out.headers_comp / (double) stats->out.headers_uncomp 1484 : 0); 1485 fprintf(engine->stats_fh, " ACKs: %lu\n", stats->out.acks); 1486 } 1487#endif 1488 if (engine->pub.enp_srst_hash) 1489 lsquic_hash_destroy(engine->pub.enp_srst_hash); 1490#if LSQUIC_COUNT_ENGINE_CALLS 1491 LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls); 1492#endif 1493 if (engine->pub.enp_retry_aead_ctx) 1494 EVP_AEAD_CTX_cleanup(engine->pub.enp_retry_aead_ctx); 1495 free(engine->pub.enp_alpn); 1496 free(engine); 1497} 1498 1499 1500static struct conn_cid_elem * 1501find_free_cce (struct lsquic_conn *conn) 1502{ 1503 struct conn_cid_elem *cce; 1504 1505 for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce) 1506 if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces)))) 1507 return cce; 1508 1509 return NULL; 1510} 1511 1512 1513static int 1514add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn, 1515 const struct sockaddr *local_sa, void *peer_ctx) 1516{ 1517 struct conn_cid_elem *cce; 1518 1519 if (engine->flags & ENG_CONNS_BY_ADDR) 1520 { 1521 cce = find_free_cce(conn); 1522 if (!cce) 1523 { 1524 LSQ_ERROR("cannot find free CCE"); 1525 return -1; 1526 } 1527 cce->cce_port = sa2port(local_sa); 1528 cce->cce_flags = CCE_PORT; 1529 if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port, 1530 sizeof(cce->cce_port), conn, &cce->cce_hash_el)) 1531 { 1532 conn->cn_cces_mask |= 1 << (cce - conn->cn_cces); 1533 return 0; 1534 } 1535 else 1536 return -1; 1537 1538 } 1539 else 1540 return insert_conn_into_hash(engine, conn, peer_ctx); 1541} 1542 1543 1544lsquic_conn_t * 1545lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version, 1546 const struct sockaddr *local_sa, 1547 const struct sockaddr *peer_sa, 1548 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 1549 const char *hostname, unsigned short max_packet_size, 1550 const unsigned char *zero_rtt, size_t zero_rtt_len, 1551 const unsigned char *token, size_t token_sz) 1552{ 1553 lsquic_conn_t *conn; 1554 unsigned flags, versions; 1555 int is_ipv4; 1556 1557 ENGINE_IN(engine); 1558 1559 if (engine->flags & ENG_SERVER) 1560 { 1561 LSQ_ERROR("`%s' must only be called in client mode", __func__); 1562 goto err; 1563 } 1564 1565 if (engine->flags & ENG_CONNS_BY_ADDR 1566 && find_conn_by_addr(engine->conns_hash, local_sa)) 1567 { 1568 LSQ_ERROR("cannot have more than one connection on the same port"); 1569 goto err; 1570 } 1571 1572 if (0 != maybe_grow_conn_heaps(engine)) 1573 return NULL; 1574 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 1575 is_ipv4 = peer_sa->sa_family == AF_INET; 1576 if (zero_rtt && zero_rtt_len) 1577 { 1578 version = lsquic_zero_rtt_version(zero_rtt, zero_rtt_len); 1579 if (version >= N_LSQVER) 1580 { 1581 LSQ_INFO("zero-rtt version is bad, won't use"); 1582 zero_rtt = NULL; 1583 zero_rtt_len = 0; 1584 } 1585 } 1586 if (version >= N_LSQVER) 1587 { 1588 if (version > N_LSQVER) 1589 LSQ_WARN("invalid version specified, engine will pick"); 1590 versions = engine->pub.enp_settings.es_versions; 1591 } 1592 else 1593 versions = 1u << version; 1594 if (versions & LSQUIC_IETF_VERSIONS) 1595 conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions, 1596 flags, hostname, max_packet_size, 1597 is_ipv4, zero_rtt, zero_rtt_len, token, token_sz); 1598 else 1599 conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions, 1600 flags, hostname, max_packet_size, is_ipv4, 1601 zero_rtt, zero_rtt_len); 1602 if (!conn) 1603 goto err; 1604 EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa); 1605 EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed", 1606 lsquic_ver2str[conn->cn_version]); 1607 ++engine->n_conns; 1608 lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa); 1609 if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx)) 1610 { 1611 const lsquic_cid_t *cid = lsquic_conn_log_cid(conn); 1612 LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy", 1613 CID_BITS(cid)); 1614 destroy_conn(engine, conn, lsquic_time_now()); 1615 goto err; 1616 } 1617 assert(!(conn->cn_flags & 1618 (CONN_REF_FLAGS 1619 & ~LSCONN_TICKABLE /* This flag may be set as effect of user 1620 callbacks */ 1621 ))); 1622 conn->cn_flags |= LSCONN_HASHED; 1623 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1624 engine_incref_conn(conn, LSCONN_TICKABLE); 1625 lsquic_conn_set_ctx(conn, conn_ctx); 1626 conn->cn_if->ci_client_call_on_new(conn); 1627 end: 1628 ENGINE_OUT(engine); 1629 return conn; 1630 err: 1631 conn = NULL; 1632 goto end; 1633} 1634 1635 1636static void 1637remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 1638{ 1639 remove_all_cces_from_hash(engine->conns_hash, conn); 1640 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 1641} 1642 1643 1644static void 1645refflags2str (enum lsquic_conn_flags flags, char s[7]) 1646{ 1647 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 1648 *s = 'H'; s += !!(flags & LSCONN_HASHED); 1649 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 1650 *s = 'T'; s += !!(flags & LSCONN_TICKABLE); 1651 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 1652 *s = 'K'; s += !!(flags & LSCONN_TICKED); 1653 *s = '\0'; 1654} 1655 1656 1657static void 1658engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 1659{ 1660 char str[2][7]; 1661 assert(flag & CONN_REF_FLAGS); 1662 assert(!(conn->cn_flags & flag)); 1663 conn->cn_flags |= flag; 1664 LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'", 1665 CID_BITS(lsquic_conn_log_cid(conn)), 1666 (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]), 1667 (refflags2str(conn->cn_flags, str[1]), str[1])); 1668} 1669 1670 1671static lsquic_conn_t * 1672engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 1673 enum lsquic_conn_flags flags) 1674{ 1675 char str[2][7]; 1676 lsquic_time_t now; 1677 assert(flags & CONN_REF_FLAGS); 1678 assert(conn->cn_flags & flags); 1679#ifndef NDEBUG 1680 if (flags & LSCONN_CLOSING) 1681 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 1682#endif 1683 conn->cn_flags &= ~flags; 1684 LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'", 1685 CID_BITS(lsquic_conn_log_cid(conn)), 1686 (refflags2str(conn->cn_flags | flags, str[0]), str[0]), 1687 (refflags2str(conn->cn_flags, str[1]), str[1])); 1688 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 1689 { 1690 now = lsquic_time_now(); 1691 if (conn->cn_flags & LSCONN_MINI) 1692 eng_hist_inc(&engine->history, now, sl_del_mini_conns); 1693 else 1694 eng_hist_inc(&engine->history, now, sl_del_full_conns); 1695 destroy_conn(engine, conn, now); 1696 return NULL; 1697 } 1698 else 1699 return conn; 1700} 1701 1702 1703/* This is not a general-purpose function. Only call from engine dtor. */ 1704static void 1705force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 1706{ 1707 assert(engine->flags & ENG_DTOR); 1708 const enum lsquic_conn_flags flags = conn->cn_flags; 1709 assert(conn->cn_flags & CONN_REF_FLAGS); 1710 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 1711 assert(!(flags & LSCONN_TICKABLE)); /* Should be removed already */ 1712 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 1713 if (flags & LSCONN_ATTQ) 1714 { 1715 lsquic_attq_remove(engine->attq, conn); 1716 (void) engine_decref_conn(engine, conn, LSCONN_ATTQ); 1717 } 1718 if (flags & LSCONN_HASHED) 1719 remove_conn_from_hash(engine, conn); 1720} 1721 1722 1723/* Iterator for tickable connections (those on the Tickable Queue). Before 1724 * a connection is returned, it is removed from the Advisory Tick Time queue 1725 * if necessary. 1726 */ 1727static lsquic_conn_t * 1728conn_iter_next_tickable (struct lsquic_engine *engine) 1729{ 1730 lsquic_conn_t *conn; 1731 1732 if (engine->flags & ENG_SERVER) 1733 while (1) 1734 { 1735 conn = lsquic_mh_pop(&engine->conns_tickable); 1736 if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC)) 1737 (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1738 else 1739 break; 1740 } 1741 else 1742 conn = lsquic_mh_pop(&engine->conns_tickable); 1743 1744 if (conn) 1745 conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE); 1746 if (conn && (conn->cn_flags & LSCONN_ATTQ)) 1747 { 1748 lsquic_attq_remove(engine->attq, conn); 1749 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1750 } 1751 1752 return conn; 1753} 1754 1755 1756static void 1757cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update, 1758 void *update_ctx) 1759{ 1760 cub->cub_update_cids = update; 1761 cub->cub_update_ctx = update_ctx; 1762 cub->cub_count = 0; 1763} 1764 1765 1766static void 1767cub_flush (struct cid_update_batch *cub) 1768{ 1769 if (cub->cub_count > 0 && cub->cub_update_cids) 1770 cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs, 1771 cub->cub_cids, cub->cub_count); 1772 cub->cub_count = 0; 1773} 1774 1775 1776static void 1777cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx) 1778{ 1779 cub->cub_cids [ cub->cub_count ] = *cid; 1780 cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx; 1781 ++cub->cub_count; 1782 if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0])) 1783 cub_flush(cub); 1784} 1785 1786 1787/* Process registered CIDs */ 1788static void 1789cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn) 1790{ 1791 struct cce_cid_iter citer; 1792 struct conn_cid_elem *cce; 1793 void *peer_ctx; 1794 1795 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 1796 for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer)) 1797 if (cce->cce_flags & CCE_REG) 1798 cub_add(cub, &cce->cce_cid, peer_ctx); 1799} 1800 1801 1802static void 1803drop_all_mini_conns (lsquic_engine_t *engine) 1804{ 1805 struct lsquic_hash_elem *el; 1806 lsquic_conn_t *conn; 1807 struct cid_update_batch cub; 1808 1809 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 1810 1811 for (el = lsquic_hash_first(engine->conns_hash); el; 1812 el = lsquic_hash_next(engine->conns_hash)) 1813 { 1814 conn = lsquic_hashelem_getdata(el); 1815 if (conn->cn_flags & LSCONN_MINI) 1816 { 1817 /* If promoted, why is it still in this hash? */ 1818 assert(!(conn->cn_flags & LSCONN_PROMOTED)); 1819 if (!(conn->cn_flags & LSCONN_PROMOTED)) 1820 cub_add_cids_from_cces(&cub, conn); 1821 remove_conn_from_hash(engine, conn); 1822 } 1823 } 1824 1825 cub_flush(&cub); 1826} 1827 1828 1829void 1830lsquic_engine_process_conns (lsquic_engine_t *engine) 1831{ 1832 lsquic_conn_t *conn; 1833 lsquic_time_t now; 1834 1835 ENGINE_IN(engine); 1836 1837 now = lsquic_time_now(); 1838 while ((conn = lsquic_attq_pop(engine->attq, now))) 1839 { 1840 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1841 if (conn && !(conn->cn_flags & LSCONN_TICKABLE)) 1842 { 1843 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 1844 engine_incref_conn(conn, LSCONN_TICKABLE); 1845 } 1846 } 1847 1848 process_connections(engine, conn_iter_next_tickable, now); 1849 ENGINE_OUT(engine); 1850} 1851 1852 1853static void 1854release_or_return_enc_data (struct lsquic_engine *engine, 1855 void (*pmi_rel_or_ret) (void *, void *, void *, char), 1856 struct lsquic_conn *conn, struct lsquic_packet_out *packet_out) 1857{ 1858 pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1859 packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out)); 1860 packet_out->po_flags &= ~PO_ENCRYPTED; 1861 packet_out->po_enc_data = NULL; 1862} 1863 1864 1865static void 1866release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1867 struct lsquic_packet_out *packet_out) 1868{ 1869 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release, 1870 conn, packet_out); 1871} 1872 1873 1874static void 1875return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn, 1876 struct lsquic_packet_out *packet_out) 1877{ 1878 release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return, 1879 conn, packet_out); 1880} 1881 1882 1883static int 1884copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn, 1885 struct lsquic_packet_out *packet_out) 1886{ 1887 int ipv6; 1888 1889 ipv6 = NP_IS_IPv6(packet_out->po_path); 1890 if (packet_out->po_flags & PO_ENCRYPTED) 1891 { 1892 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1893 && packet_out->po_data_sz == packet_out->po_enc_data_sz 1894 && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data, 1895 packet_out->po_data_sz)) 1896 return 0; 1897 if (ipv6 == lsquic_packet_out_ipv6(packet_out) 1898 && packet_out->po_data_sz <= packet_out->po_enc_data_sz) 1899 goto copy; 1900 return_enc_data(engine, conn, packet_out); 1901 } 1902 1903 packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate( 1904 engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx, 1905 packet_out->po_data_sz, ipv6); 1906 if (!packet_out->po_enc_data) 1907 { 1908 LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet " 1909 "of size %hu", packet_out->po_data_sz); 1910 return -1; 1911 } 1912 1913 copy: 1914 memcpy(packet_out->po_enc_data, packet_out->po_data, 1915 packet_out->po_data_sz); 1916 packet_out->po_enc_data_sz = packet_out->po_data_sz; 1917 packet_out->po_sent_sz = packet_out->po_data_sz; 1918 packet_out->po_flags &= ~PO_IPv6; 1919 packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT); 1920 1921 return 0; 1922} 1923 1924 1925STAILQ_HEAD(conns_stailq, lsquic_conn); 1926TAILQ_HEAD(conns_tailq, lsquic_conn); 1927 1928 1929struct conns_out_iter 1930{ 1931 struct min_heap *coi_heap; 1932 struct pr_queue *coi_prq; 1933 TAILQ_HEAD(, lsquic_conn) coi_active_list, 1934 coi_inactive_list; 1935 lsquic_conn_t *coi_next; 1936#ifndef NDEBUG 1937 lsquic_time_t coi_last_sent; 1938#endif 1939}; 1940 1941 1942static void 1943coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 1944{ 1945 iter->coi_heap = &engine->conns_out; 1946 iter->coi_prq = engine->pr_queue; 1947 iter->coi_next = NULL; 1948 TAILQ_INIT(&iter->coi_active_list); 1949 TAILQ_INIT(&iter->coi_inactive_list); 1950#ifndef NDEBUG 1951 iter->coi_last_sent = 0; 1952#endif 1953} 1954 1955 1956static lsquic_conn_t * 1957coi_next (struct conns_out_iter *iter) 1958{ 1959 lsquic_conn_t *conn; 1960 1961 if (lsquic_mh_count(iter->coi_heap) > 0) 1962 { 1963 conn = lsquic_mh_pop(iter->coi_heap); 1964 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1965 conn->cn_flags |= LSCONN_COI_ACTIVE; 1966#ifndef NDEBUG 1967 if (iter->coi_last_sent) 1968 assert(iter->coi_last_sent <= conn->cn_last_sent); 1969 iter->coi_last_sent = conn->cn_last_sent; 1970#endif 1971 return conn; 1972 } 1973 else if (iter->coi_prq && (conn = lsquic_prq_next_conn(iter->coi_prq))) 1974 { 1975 return conn; 1976 } 1977 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 1978 { 1979 iter->coi_prq = NULL; /* Save function call in previous conditional */ 1980 conn = iter->coi_next; 1981 if (!conn) 1982 conn = TAILQ_FIRST(&iter->coi_active_list); 1983 if (conn) 1984 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 1985 return conn; 1986 } 1987 else 1988 return NULL; 1989} 1990 1991 1992static void 1993coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1994{ 1995 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 1996 { 1997 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 1998 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1999 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2000 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 2001 conn->cn_flags |= LSCONN_COI_INACTIVE; 2002 } 2003} 2004 2005 2006static void 2007coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 2008{ 2009 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 2010 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2011 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2012 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 2013 conn->cn_flags |= LSCONN_COI_ACTIVE; 2014} 2015 2016 2017static void 2018coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 2019{ 2020 lsquic_conn_t *conn; 2021 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 2022 { 2023 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 2024 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 2025 if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING 2026 && !(conn->cn_flags & LSCONN_IMMED_CLOSE)) 2027 lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent); 2028 else /* Closed connection gets one shot at sending packets */ 2029 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2030 } 2031 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 2032 { 2033 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 2034 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 2035 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 2036 } 2037} 2038 2039 2040#ifndef NDEBUG 2041static void 2042lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch, 2043 unsigned n) 2044{ 2045 const lsquic_cid_t *cid; 2046 struct iovec *iov; 2047 unsigned i; 2048 char packno_str[22]; 2049 2050 for (i = 0; i < n; ++i) 2051 { 2052 snprintf(packno_str, sizeof(packno_str), "%"PRIu64, 2053 batch->packets[i]->po_packno); 2054 if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0)) 2055 { 2056 for (iov = batch->outs[i].iov; iov < 2057 batch->outs[i].iov + batch->outs[i].iovlen; ++iov) 2058 batch->outs[i].iov->iov_len -= 1; 2059 cid = lsquic_conn_log_cid(batch->conns[i]); 2060 LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str, 2061 CID_BITS(cid)); 2062 } 2063 } 2064} 2065 2066 2067#endif 2068 2069 2070#ifdef NDEBUG 2071#define CONST_BATCH const 2072#else 2073#define CONST_BATCH 2074#endif 2075 2076 2077struct send_batch_ctx { 2078 struct conns_stailq *closed_conns; 2079 struct conns_tailq *ticked_conns; 2080 struct conns_out_iter *conns_iter; 2081 CONST_BATCH struct out_batch *batch; 2082}; 2083 2084 2085static void 2086close_conn_immediately (struct lsquic_engine *engine, 2087 const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn) 2088{ 2089 conn->cn_flags |= LSCONN_IMMED_CLOSE; 2090 if (!(conn->cn_flags & LSCONN_CLOSING)) 2091 { 2092 STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn); 2093 engine_incref_conn(conn, LSCONN_CLOSING); 2094 if (conn->cn_flags & LSCONN_HASHED) 2095 remove_conn_from_hash(engine, conn); 2096 } 2097 if (conn->cn_flags & LSCONN_TICKED) 2098 { 2099 TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked); 2100 engine_decref_conn(engine, conn, LSCONN_TICKED); 2101 } 2102} 2103 2104 2105static void 2106close_conn_on_send_error (struct lsquic_engine *engine, 2107 const struct send_batch_ctx *sb_ctx, int n, int e_val) 2108{ 2109 const struct out_batch *batch = sb_ctx->batch; 2110 struct lsquic_conn *const conn = batch->conns[n]; 2111 char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")]; 2112 2113 LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; " 2114 "src: %s; dst: %s; errno: %d", 2115 conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" : 2116 conn->cn_flags & LSCONN_MINI ? "mini" : "regular", 2117 CID_BITS(lsquic_conn_log_cid(conn)), 2118 SA2STR(batch->outs[n].local_sa, buf[0]), 2119 SA2STR(batch->outs[n].dest_sa, buf[1]), 2120 e_val); 2121 if (conn->cn_flags & LSCONN_EVANESCENT) 2122 lsquic_prq_drop(conn); 2123 else 2124 close_conn_immediately(engine, sb_ctx, conn); 2125} 2126 2127 2128static unsigned 2129send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx, 2130 unsigned n_to_send) 2131{ 2132 int n_sent, i, e_val; 2133 lsquic_time_t now; 2134 unsigned off; 2135 size_t count; 2136 CONST_BATCH struct out_batch *const batch = sb_ctx->batch; 2137 struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end; 2138 2139#ifndef NDEBUG 2140 if (engine->flags & ENG_LOSE_PACKETS) 2141 lose_matching_packets(engine, batch, n_to_send); 2142#endif 2143 /* Set sent time before the write to avoid underestimating RTT */ 2144 now = lsquic_time_now(); 2145 for (i = 0; i < (int) n_to_send; ++i) 2146 { 2147 off = batch->pack_off[i]; 2148 count = batch->outs[i].iovlen; 2149 assert(count > 0); 2150 packet_out = &batch->packets[off]; 2151 end = packet_out + count; 2152 do 2153 (*packet_out)->po_sent = now; 2154 while (++packet_out < end); 2155 } 2156 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs, 2157 n_to_send); 2158 e_val = errno; 2159 if (n_sent < (int) n_to_send) 2160 { 2161 engine->pub.enp_flags &= ~ENPUB_CAN_SEND; 2162 engine->resume_sending_at = now + 1000000; 2163 LSQ_DEBUG("cannot send packets"); 2164 EV_LOG_GENERIC_EVENT("cannot send packets"); 2165 if (!(EAGAIN == e_val || EWOULDBLOCK == e_val)) 2166 close_conn_on_send_error(engine, sb_ctx, 2167 n_sent < 0 ? 0 : n_sent, e_val); 2168 } 2169 if (n_sent >= 0) 2170 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, n_to_send); 2171 else 2172 { 2173 LSQ_DEBUG("packets out returned an error: %s", strerror(e_val)); 2174 n_sent = 0; 2175 } 2176 if (n_sent > 0) 2177 engine->last_sent = now + n_sent; 2178 for (i = 0; i < n_sent; ++i) 2179 { 2180 eng_hist_inc(&engine->history, now, sl_packets_out); 2181 /* `i' is added to maintain relative order */ 2182 batch->conns[i]->cn_last_sent = now + i; 2183 2184 off = batch->pack_off[i]; 2185 count = batch->outs[i].iovlen; 2186 assert(count > 0); 2187 packet_out = &batch->packets[off]; 2188 end = packet_out + count; 2189 do 2190 { 2191#if LOG_PACKET_CHECKSUM 2192 log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out", 2193 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base, 2194 batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len); 2195#endif 2196 EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]), 2197 *packet_out); 2198 /* Release packet out buffer as soon as the packet is sent 2199 * successfully. If not successfully sent, we hold on to 2200 * this buffer until the packet sending is attempted again 2201 * or until it times out and regenerated. 2202 */ 2203 if ((*packet_out)->po_flags & PO_ENCRYPTED) 2204 release_enc_data(engine, batch->conns[i], *packet_out); 2205 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 2206 *packet_out); 2207 } 2208 while (++packet_out < end); 2209 } 2210 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 2211 for ( ; i < (int) n_to_send; ++i) 2212 { 2213 off = batch->pack_off[i]; 2214 count = batch->outs[i].iovlen; 2215 assert(count > 0); 2216 packet_out = &batch->packets[off]; 2217 end = packet_out + count; 2218 do 2219 EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]), 2220 *packet_out); 2221 while (++packet_out < end); 2222 } 2223 /* Return packets to the connection in reverse order so that the packet 2224 * ordering is maintained. 2225 */ 2226 for (i = (int) n_to_send - 1; i >= n_sent; --i) 2227 { 2228 off = batch->pack_off[i]; 2229 count = batch->outs[i].iovlen; 2230 assert(count > 0); 2231 packet_out = &batch->packets[off + count - 1]; 2232 end = &batch->packets[off - 1]; 2233 do 2234 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 2235 *packet_out); 2236 while (--packet_out > end); 2237 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 2238 coi_reactivate(sb_ctx->conns_iter, batch->conns[i]); 2239 } 2240 return n_sent; 2241} 2242 2243 2244/* Return 1 if went past deadline, 0 otherwise */ 2245static int 2246check_deadline (lsquic_engine_t *engine) 2247{ 2248 if (engine->pub.enp_settings.es_proc_time_thresh && 2249 lsquic_time_now() > engine->deadline) 2250 { 2251 LSQ_INFO("went past threshold of %u usec, stop sending", 2252 engine->pub.enp_settings.es_proc_time_thresh); 2253 engine->flags |= ENG_PAST_DEADLINE; 2254 return 1; 2255 } 2256 else 2257 return 0; 2258} 2259 2260 2261static size_t 2262iov_size (const struct iovec *iov, const struct iovec *const end) 2263{ 2264 size_t size; 2265 2266 assert(iov < end); 2267 2268 size = 0; 2269 do 2270 size += iov->iov_len; 2271 while (++iov < end); 2272 2273 return size; 2274} 2275 2276 2277static void 2278send_packets_out (struct lsquic_engine *engine, 2279 struct conns_tailq *ticked_conns, 2280 struct conns_stailq *closed_conns) 2281{ 2282 unsigned n, w, n_sent, n_batches_sent; 2283 lsquic_packet_out_t *packet_out; 2284 struct lsquic_packet_out **packet; 2285 lsquic_conn_t *conn; 2286 struct out_batch *const batch = &engine->out_batch; 2287 struct iovec *iov, *packet_iov; 2288 struct conns_out_iter conns_iter; 2289 int shrink, deadline_exceeded; 2290 const struct send_batch_ctx sb_ctx = { 2291 closed_conns, 2292 ticked_conns, 2293 &conns_iter, 2294 &engine->out_batch, 2295 }; 2296 2297 coi_init(&conns_iter, engine); 2298 n_batches_sent = 0; 2299 n_sent = 0, n = 0; 2300 shrink = 0; 2301 deadline_exceeded = 0; 2302 iov = batch->iov; 2303 packet = batch->packets; 2304 2305 while ((conn = coi_next(&conns_iter))) 2306 { 2307 packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0); 2308 if (!packet_out) { 2309 /* Evanescent connection always has a packet to send: */ 2310 assert(!(conn->cn_flags & LSCONN_EVANESCENT)); 2311 LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT, 2312 (conn->cn_flags & LSCONN_MINI ? "mini" : "full"), 2313 CID_BITS(lsquic_conn_log_cid(conn))); 2314 coi_deactivate(&conns_iter, conn); 2315 continue; 2316 } 2317 batch->outs[n].iov = packet_iov = iov; 2318 next_coa: 2319 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 2320 { 2321 switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session, 2322 &engine->pub, conn, packet_out)) 2323 { 2324 case ENCPA_NOMEM: 2325 /* Send what we have and wait for a more opportune moment */ 2326 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2327 goto end_for; 2328 case ENCPA_BADCRYPT: 2329 /* This is pretty bad: close connection immediately */ 2330 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2331 LSQ_INFOC("conn %"CID_FMT" has unsendable packets", 2332 CID_BITS(lsquic_conn_log_cid(conn))); 2333 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 2334 { 2335 close_conn_immediately(engine, &sb_ctx, conn); 2336 coi_deactivate(&conns_iter, conn); 2337 } 2338 continue; 2339 case ENCPA_OK: 2340 break; 2341 } 2342 } 2343 else if ((packet_out->po_flags & PO_NOENCRYPT) 2344 && engine->pub.enp_pmi != &stock_pmi) 2345 { 2346 if (0 != copy_packet(engine, conn, packet_out)) 2347 { 2348 /* Copy can only fail if packet could not be allocated */ 2349 conn->cn_if->ci_packet_not_sent(conn, packet_out); 2350 goto end_for; 2351 } 2352 } 2353 LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT, 2354 packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn))); 2355 if (packet_out->po_flags & PO_ENCRYPTED) 2356 { 2357 iov->iov_base = packet_out->po_enc_data; 2358 iov->iov_len = packet_out->po_enc_data_sz; 2359 } 2360 else 2361 { 2362 iov->iov_base = packet_out->po_data; 2363 iov->iov_len = packet_out->po_data_sz; 2364 } 2365 if (packet_iov == iov) 2366 { 2367 batch->pack_off[n] = packet - batch->packets; 2368 batch->outs [n].ecn = lsquic_packet_out_ecn(packet_out); 2369 batch->outs [n].peer_ctx = packet_out->po_path->np_peer_ctx; 2370 batch->outs [n].local_sa = NP_LOCAL_SA(packet_out->po_path); 2371 batch->outs [n].dest_sa = NP_PEER_SA(packet_out->po_path); 2372 batch->conns [n] = conn; 2373 } 2374 *packet = packet_out; 2375 ++packet; 2376 ++iov; 2377 if ((conn->cn_flags & LSCONN_IETF) 2378 && ((1 << packet_out->po_header_type) 2379 & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT))) 2380#ifndef NDEBUG 2381 && (engine->flags & ENG_COALESCE) 2382#endif 2383 && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2384 { 2385 const size_t size = iov_size(packet_iov, iov); 2386 packet_out = conn->cn_if->ci_next_packet_to_send(conn, size); 2387 if (packet_out) 2388 goto next_coa; 2389 } 2390 batch->outs [n].iovlen = iov - packet_iov; 2391 ++n; 2392 if (n == engine->batch_size 2393 || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0])) 2394 { 2395 w = send_batch(engine, &sb_ctx, n); 2396 n = 0; 2397 iov = batch->iov; 2398 packet = batch->packets; 2399 ++n_batches_sent; 2400 n_sent += w; 2401 if (w < engine->batch_size) 2402 { 2403 shrink = 1; 2404 break; 2405 } 2406 deadline_exceeded = check_deadline(engine); 2407 if (deadline_exceeded) 2408 break; 2409 grow_batch_size(engine); 2410 } 2411 } 2412 end_for: 2413 2414 if (n > 0) { 2415 w = send_batch(engine, &sb_ctx, n); 2416 n_sent += w; 2417 shrink = w < n; 2418 ++n_batches_sent; 2419 } 2420 2421 if (shrink) 2422 shrink_batch_size(engine); 2423 else if (n_batches_sent > 1) 2424 { 2425 deadline_exceeded = check_deadline(engine); 2426 if (!deadline_exceeded) 2427 grow_batch_size(engine); 2428 } 2429 2430 coi_reheap(&conns_iter, engine); 2431 2432 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 2433} 2434 2435 2436int 2437lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 2438{ 2439 return lsquic_mh_count(&engine->conns_out) > 0 2440 || (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 2441 ; 2442} 2443 2444 2445static void 2446reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 2447{ 2448 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 2449 engine->flags &= ~ENG_PAST_DEADLINE; 2450} 2451 2452 2453static void 2454check_tickable_conns_again (struct lsquic_engine *engine) 2455{ 2456 struct lsquic_hash_elem *el; 2457 struct lsquic_conn *conn; 2458 unsigned count; 2459 2460 count = 0; 2461 for (el = lsquic_hash_first(engine->conns_hash); el; 2462 el = lsquic_hash_next(engine->conns_hash)) 2463 { 2464 conn = lsquic_hashelem_getdata(el); 2465 if (!(conn->cn_flags & LSCONN_TICKABLE) 2466 && conn->cn_if->ci_is_tickable(conn)) 2467 { 2468 lsquic_mh_insert(&engine->conns_tickable, conn, 2469 conn->cn_last_ticked); 2470 engine_incref_conn(conn, LSCONN_TICKABLE); 2471 ++count; 2472 } 2473 } 2474 LSQ_DEBUG("%u connection%s tickable again after sending has been " 2475 "re-enabled", count, count == 1 ? " is" : "s are"); 2476} 2477 2478 2479void 2480lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 2481{ 2482 lsquic_conn_t *conn; 2483 struct conns_stailq closed_conns; 2484 struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns); 2485 struct cid_update_batch cub; 2486 2487 ENGINE_IN(engine); 2488 cub_init(&cub, engine->report_old_scids, engine->scids_ctx); 2489 STAILQ_INIT(&closed_conns); 2490 reset_deadline(engine, lsquic_time_now()); 2491 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)) 2492 { 2493 LSQ_DEBUG("can send again"); 2494 EV_LOG_GENERIC_EVENT("can send again"); 2495 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2496 check_tickable_conns_again(engine); 2497 } 2498 2499 send_packets_out(engine, &ticked_conns, &closed_conns); 2500 2501 while ((conn = STAILQ_FIRST(&closed_conns))) { 2502 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2503 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2504 cub_add_cids_from_cces(&cub, conn); 2505 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2506 } 2507 2508 cub_flush(&cub); 2509 ENGINE_OUT(engine); 2510} 2511 2512 2513static lsquic_conn_t * 2514next_new_full_conn (struct conns_stailq *new_full_conns) 2515{ 2516 lsquic_conn_t *conn; 2517 2518 conn = STAILQ_FIRST(new_full_conns); 2519 if (conn) 2520 STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full); 2521 return conn; 2522} 2523 2524 2525static void 2526process_connections (lsquic_engine_t *engine, conn_iter_f next_conn, 2527 lsquic_time_t now) 2528{ 2529 lsquic_conn_t *conn; 2530 enum tick_st tick_st; 2531 unsigned i, why; 2532 lsquic_time_t next_tick_time; 2533 struct conns_stailq closed_conns; 2534 struct conns_tailq ticked_conns; 2535 struct conns_stailq new_full_conns; 2536 struct cid_update_batch cub_old, cub_live; 2537 cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx); 2538 cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx); 2539 2540 eng_hist_tick(&engine->history, now); 2541 2542 STAILQ_INIT(&closed_conns); 2543 TAILQ_INIT(&ticked_conns); 2544 reset_deadline(engine, now); 2545 STAILQ_INIT(&new_full_conns); 2546 2547 if (!(engine->pub.enp_flags & ENPUB_CAN_SEND) 2548 && now > engine->resume_sending_at) 2549 { 2550 LSQ_NOTICE("failsafe activated: resume sending packets again after " 2551 "timeout"); 2552 EV_LOG_GENERIC_EVENT("resume sending packets again after timeout"); 2553 engine->pub.enp_flags |= ENPUB_CAN_SEND; 2554 } 2555 2556 i = 0; 2557 while ((conn = next_conn(engine)) 2558 || (conn = next_new_full_conn(&new_full_conns))) 2559 { 2560 tick_st = conn->cn_if->ci_tick(conn, now); 2561 conn->cn_last_ticked = now + i /* Maintain relative order */ ++; 2562 if (tick_st & TICK_PROMOTE) 2563 { 2564 lsquic_conn_t *new_conn; 2565 EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn), 2566 "scheduled for promotion"); 2567 assert(conn->cn_flags & LSCONN_MINI); 2568 new_conn = new_full_conn_server(engine, conn, now); 2569 if (new_conn) 2570 { 2571 STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full); 2572 new_conn->cn_last_sent = engine->last_sent; 2573 eng_hist_inc(&engine->history, now, sl_new_full_conns); 2574 conn->cn_flags |= LSCONN_PROMOTED; 2575 } 2576 tick_st |= TICK_CLOSE; /* Destroy mini connection */ 2577 } 2578 if (tick_st & TICK_SEND) 2579 { 2580 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 2581 { 2582 lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent); 2583 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 2584 } 2585 } 2586 if (tick_st & TICK_CLOSE) 2587 { 2588 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 2589 engine_incref_conn(conn, LSCONN_CLOSING); 2590 if (conn->cn_flags & LSCONN_HASHED) 2591 remove_conn_from_hash(engine, conn); 2592 } 2593 else 2594 { 2595 TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked); 2596 engine_incref_conn(conn, LSCONN_TICKED); 2597 if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live 2598 && conn->cn_if->ci_report_live(conn, now)) 2599 cub_add_cids_from_cces(&cub_live, conn); 2600 } 2601 } 2602 2603 if ((engine->pub.enp_flags & ENPUB_CAN_SEND) 2604 && lsquic_engine_has_unsent_packets(engine)) 2605 send_packets_out(engine, &ticked_conns, &closed_conns); 2606 2607 while ((conn = STAILQ_FIRST(&closed_conns))) { 2608 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 2609 if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI) 2610 cub_add_cids_from_cces(&cub_old, conn); 2611 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 2612 } 2613 2614 while ((conn = TAILQ_FIRST(&ticked_conns))) 2615 { 2616 TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked); 2617 engine_decref_conn(engine, conn, LSCONN_TICKED); 2618 if (!(conn->cn_flags & LSCONN_TICKABLE) 2619 && conn->cn_if->ci_is_tickable(conn)) 2620 { 2621 /* Floyd heapification is not faster, don't bother. */ 2622 lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked); 2623 engine_incref_conn(conn, LSCONN_TICKABLE); 2624 } 2625 else if (!(conn->cn_flags & LSCONN_ATTQ)) 2626 { 2627 next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why); 2628 if (next_tick_time) 2629 { 2630 if (0 == lsquic_attq_add(engine->attq, conn, next_tick_time, 2631 why)) 2632 engine_incref_conn(conn, LSCONN_ATTQ); 2633 } 2634 else 2635 /* In all other cases, the idle timeout would make the next 2636 * tick time non-zero: 2637 */ 2638 assert((conn->cn_flags & LSCONN_IETF) 2639 && engine->pub.enp_settings.es_idle_timeout == 0); 2640 } 2641 } 2642 2643 cub_flush(&engine->new_scids); 2644 cub_flush(&cub_live); 2645 cub_flush(&cub_old); 2646} 2647 2648 2649static void 2650maybe_count_garbage (struct lsquic_engine *engine, size_t garbage_sz) 2651{ 2652 /* This is not very pretty (action at a distance via engine->curr_conn), 2653 * but it's the cheapest I can come up with to handle the "count garbage 2654 * toward amplification limit" requirement in 2655 * [draft-ietf-quic-transport-28] Section 8.1. 2656 */ 2657 if (engine->curr_conn && engine->curr_conn->cn_if->ci_count_garbage) 2658 engine->curr_conn->cn_if->ci_count_garbage(engine->curr_conn, 2659 garbage_sz); 2660} 2661 2662 2663/* Return 0 if packet is being processed by a real connection, 1 if the 2664 * packet was processed, but not by a connection, and -1 on error. 2665 */ 2666int 2667lsquic_engine_packet_in (lsquic_engine_t *engine, 2668 const unsigned char *packet_in_data, size_t packet_in_size, 2669 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 2670 void *peer_ctx, int ecn) 2671{ 2672 const unsigned char *const packet_begin = packet_in_data; 2673 const unsigned char *const packet_end = packet_in_data + packet_in_size; 2674 struct packin_parse_state ppstate; 2675 lsquic_packet_in_t *packet_in; 2676 int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length, 2677 int is_server, unsigned cid_len, struct packin_parse_state *); 2678 unsigned n_zeroes; 2679 int s, is_ietf; 2680 lsquic_cid_t cid; 2681 2682 ENGINE_CALLS_INCR(engine); 2683 2684 if (engine->flags & ENG_SERVER) 2685 parse_packet_in_begin = lsquic_parse_packet_in_server_begin; 2686 else if (engine->flags & ENG_CONNS_BY_ADDR) 2687 { 2688 struct lsquic_hash_elem *el; 2689 const struct lsquic_conn *conn; 2690 el = find_conn_by_addr(engine->conns_hash, sa_local); 2691 if (!el) 2692 return -1; 2693 conn = lsquic_hashelem_getdata(el); 2694 if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS) 2695 parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin; 2696 else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS) 2697 parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin; 2698 else if (conn->cn_version == LSQVER_050) 2699 parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin; 2700 else 2701 { 2702 assert(conn->cn_version == LSQVER_046 2703#if LSQUIC_USE_Q098 2704 || conn->cn_version == LSQVER_098 2705#endif 2706 2707 ); 2708 parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin; 2709 } 2710 } 2711 else 2712 parse_packet_in_begin = lsquic_parse_packet_in_begin; 2713 2714 engine->curr_conn = NULL; 2715 n_zeroes = 0; 2716 is_ietf = 0; 2717 do 2718 { 2719 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 2720 if (!packet_in) 2721 return -1; 2722 /* Library does not modify packet_in_data, it is not referenced after 2723 * this function returns and subsequent release of pi_data is guarded 2724 * by PI_OWN_DATA flag. 2725 */ 2726 packet_in->pi_data = (unsigned char *) packet_in_data; 2727 if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data, 2728 engine->flags & ENG_SERVER, 2729 engine->pub.enp_settings.es_scid_len, &ppstate)) 2730 { 2731 LSQ_DEBUG("Cannot parse incoming packet's header"); 2732 maybe_count_garbage(engine, packet_end - packet_in_data); 2733 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 2734 s = 1; 2735 break; 2736 } 2737 2738 /* [draft-ietf-quic-transport-27] Section 12.2: 2739 * " Receivers SHOULD ignore any subsequent packets with a different 2740 * " Destination Connection ID than the first packet in the datagram. 2741 */ 2742 if (is_ietf && packet_in_data > packet_begin) 2743 { 2744 if (!((packet_in->pi_flags & (PI_GQUIC|PI_CONN_ID)) == PI_CONN_ID 2745 && LSQUIC_CIDS_EQ(&packet_in->pi_dcid, &cid))) 2746 { 2747 packet_in_data += packet_in->pi_data_sz; 2748 maybe_count_garbage(engine, packet_in->pi_data_sz); 2749 continue; 2750 } 2751 } 2752 2753 is_ietf = 0 == (packet_in->pi_flags & PI_GQUIC); 2754 packet_in_data += packet_in->pi_data_sz; 2755 if (is_ietf && packet_in_data < packet_end) 2756 cid = packet_in->pi_dcid; 2757 packet_in->pi_received = lsquic_time_now(); 2758 packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT; 2759 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 2760 s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 2761 peer_ctx, packet_in_size); 2762 n_zeroes += s == 0; 2763 } 2764 while (0 == s && packet_in_data < packet_end); 2765 2766 return n_zeroes > 0 ? 0 : s; 2767} 2768 2769 2770#if __GNUC__ && !defined(NDEBUG) 2771__attribute__((weak)) 2772#endif 2773unsigned 2774lsquic_engine_quic_versions (const lsquic_engine_t *engine) 2775{ 2776 return engine->pub.enp_settings.es_versions; 2777} 2778 2779 2780void 2781lsquic_engine_cooldown (lsquic_engine_t *engine) 2782{ 2783 struct lsquic_hash_elem *el; 2784 lsquic_conn_t *conn; 2785 2786 if (engine->flags & ENG_COOLDOWN) 2787 /* AFAICT, there is no harm in calling this function more than once, 2788 * but log it just in case, as it may indicate an error in the caller. 2789 */ 2790 LSQ_INFO("cooldown called again"); 2791 engine->flags |= ENG_COOLDOWN; 2792 LSQ_INFO("entering cooldown mode"); 2793 if (engine->flags & ENG_SERVER) 2794 drop_all_mini_conns(engine); 2795 for (el = lsquic_hash_first(engine->conns_hash); el; 2796 el = lsquic_hash_next(engine->conns_hash)) 2797 { 2798 conn = lsquic_hashelem_getdata(el); 2799 lsquic_conn_going_away(conn); 2800 } 2801} 2802 2803 2804int 2805lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 2806{ 2807 const struct attq_elem *next_attq; 2808 lsquic_time_t now, next_time; 2809#if LSQUIC_DEBUG_NEXT_ADV_TICK 2810 const struct lsquic_conn *conn; 2811 const enum lsq_log_level L = LSQ_LOG_DEBUG; /* Easy toggle */ 2812#endif 2813 2814 ENGINE_CALLS_INCR(engine); 2815 2816 if ((engine->flags & ENG_PAST_DEADLINE) 2817 && lsquic_mh_count(&engine->conns_out)) 2818 { 2819#if LSQUIC_DEBUG_NEXT_ADV_TICK 2820 conn = lsquic_mh_peek(&engine->conns_out); 2821 engine->last_logged_conn = 0; 2822 LSQ_LOGC(L, "next advisory tick is now: went past deadline last time " 2823 "and have %u outgoing connection%.*s (%"CID_FMT" first)", 2824 lsquic_mh_count(&engine->conns_out), 2825 lsquic_mh_count(&engine->conns_out) != 1, "s", 2826 CID_BITS(lsquic_conn_log_cid(conn))); 2827#endif 2828 *diff = 0; 2829 return 1; 2830 } 2831 2832 if (engine->pr_queue && lsquic_prq_have_pending(engine->pr_queue)) 2833 { 2834#if LSQUIC_DEBUG_NEXT_ADV_TICK 2835 engine->last_logged_conn = 0; 2836 LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements"); 2837#endif 2838 *diff = 0; 2839 return 1; 2840 } 2841 2842 if (lsquic_mh_count(&engine->conns_tickable)) 2843 { 2844#if LSQUIC_DEBUG_NEXT_ADV_TICK 2845 conn = lsquic_mh_peek(&engine->conns_tickable); 2846 engine->last_logged_conn = 0; 2847 LSQ_LOGC(L, "next advisory tick is now: have %u tickable " 2848 "connection%.*s (%"CID_FMT" first)", 2849 lsquic_mh_count(&engine->conns_tickable), 2850 lsquic_mh_count(&engine->conns_tickable) != 1, "s", 2851 CID_BITS(lsquic_conn_log_cid(conn))); 2852#endif 2853 *diff = 0; 2854 return 1; 2855 } 2856 2857 next_attq = lsquic_attq_next(engine->attq); 2858 if (engine->pub.enp_flags & ENPUB_CAN_SEND) 2859 { 2860 if (next_attq) 2861 next_time = next_attq->ae_adv_time; 2862 else 2863 return 0; 2864 } 2865 else 2866 { 2867 if (next_attq) 2868 { 2869 next_time = next_attq->ae_adv_time; 2870 if (engine->resume_sending_at < next_time) 2871 { 2872 next_time = engine->resume_sending_at; 2873 next_attq = NULL; 2874 } 2875 } 2876 else 2877 next_time = engine->resume_sending_at; 2878 } 2879 2880 now = lsquic_time_now(); 2881 *diff = (int) ((int64_t) next_time - (int64_t) now); 2882#if LSQUIC_DEBUG_NEXT_ADV_TICK 2883 if (next_attq) 2884 { 2885 /* Deduplicate consecutive log messages about the same reason for the 2886 * same connection. 2887 * If diff is always zero or diff reset to a higher value, event is 2888 * still logged. 2889 */ 2890 if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why 2891 && (uintptr_t) next_attq->ae_conn 2892 == engine->last_logged_conn 2893 && *diff < engine->last_tick_diff)) 2894 { 2895 engine->last_logged_conn = (uintptr_t) next_attq->ae_conn; 2896 engine->last_logged_ae_why = (unsigned) next_attq->ae_why; 2897 engine->last_tick_diff = *diff; 2898 LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT 2899 ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)), 2900 lsquic_attq_why2str(next_attq->ae_why)); 2901 } 2902 } 2903 else 2904 LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff); 2905#endif 2906 return 1; 2907} 2908 2909 2910unsigned 2911lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 2912{ 2913 lsquic_time_t now; 2914 ENGINE_CALLS_INCR(engine); 2915 now = lsquic_time_now(); 2916 if (from_now < 0) 2917 now -= from_now; 2918 else 2919 now += from_now; 2920 return lsquic_attq_count_before(engine->attq, now); 2921} 2922 2923 2924int 2925lsquic_engine_add_cid (struct lsquic_engine_public *enpub, 2926 struct lsquic_conn *conn, unsigned cce_idx) 2927{ 2928 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2929 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2930 void *peer_ctx; 2931 2932 assert(cce_idx < conn->cn_n_cces); 2933 assert(conn->cn_cces_mask & (1 << cce_idx)); 2934 assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED)); 2935 2936 if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf, 2937 cce->cce_cid.len, conn, &cce->cce_hash_el)) 2938 { 2939 LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs", 2940 CID_BITS(&cce->cce_cid)); 2941 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2942 cce->cce_flags |= CCE_REG; 2943 cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx); 2944 return 0; 2945 } 2946 else 2947 { 2948 LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash", 2949 CID_BITS(&cce->cce_cid)); 2950 return -1; 2951 } 2952} 2953 2954 2955void 2956lsquic_engine_retire_cid (struct lsquic_engine_public *enpub, 2957 struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now) 2958{ 2959 struct lsquic_engine *const engine = (struct lsquic_engine *) enpub; 2960 struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx]; 2961 void *peer_ctx; 2962 2963 assert(cce_idx < conn->cn_n_cces); 2964 2965 if (cce->cce_hash_el.qhe_flags & QHE_HASHED) 2966 lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el); 2967 2968 if (engine->purga) 2969 { 2970 peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL); 2971 lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx, 2972 PUTY_CID_RETIRED, now); 2973 } 2974 conn->cn_cces_mask &= ~(1u << cce_idx); 2975 LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid)); 2976} 2977 2978 2979