lsquic_engine.c revision 461e84d8
1/* Copyright (c) 2017 LiteSpeed Technologies Inc. See LICENSE. */ 2/* 3 * lsquic_engine.c - QUIC engine 4 */ 5 6#include <assert.h> 7#include <errno.h> 8#include <inttypes.h> 9#include <stdint.h> 10#include <stdio.h> 11#include <stdlib.h> 12#include <string.h> 13#include <sys/queue.h> 14#include <time.h> 15#ifndef WIN32 16#include <sys/time.h> 17#include <netinet/in.h> 18#include <sys/types.h> 19#include <sys/stat.h> 20#include <fcntl.h> 21#include <unistd.h> 22#include <netdb.h> 23#endif 24 25 26 27#include "lsquic.h" 28#include "lsquic_types.h" 29#include "lsquic_alarmset.h" 30#include "lsquic_parse.h" 31#include "lsquic_packet_in.h" 32#include "lsquic_packet_out.h" 33#include "lsquic_senhist.h" 34#include "lsquic_rtt.h" 35#include "lsquic_cubic.h" 36#include "lsquic_pacer.h" 37#include "lsquic_send_ctl.h" 38#include "lsquic_set.h" 39#include "lsquic_conn_flow.h" 40#include "lsquic_sfcw.h" 41#include "lsquic_stream.h" 42#include "lsquic_conn.h" 43#include "lsquic_full_conn.h" 44#include "lsquic_util.h" 45#include "lsquic_qtags.h" 46#include "lsquic_str.h" 47#include "lsquic_handshake.h" 48#include "lsquic_mm.h" 49#include "lsquic_conn_hash.h" 50#include "lsquic_engine_public.h" 51#include "lsquic_eng_hist.h" 52#include "lsquic_ev_log.h" 53#include "lsquic_version.h" 54#include "lsquic_hash.h" 55#include "lsquic_attq.h" 56 57#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE 58#include "lsquic_logger.h" 59 60 61/* The batch of outgoing packets grows and shrinks dynamically */ 62#define MAX_OUT_BATCH_SIZE 1024 63#define MIN_OUT_BATCH_SIZE 256 64#define INITIAL_OUT_BATCH_SIZE 512 65 66typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *); 67 68static void 69process_connections (struct lsquic_engine *engine, conn_iter_f iter); 70 71static void 72engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag); 73 74static lsquic_conn_t * 75engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 76 enum lsquic_conn_flags flag); 77 78static void 79force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn); 80 81/* Nested calls to LSQUIC are not supported */ 82#define ENGINE_IN(e) do { \ 83 assert(!((e)->pub.enp_flags & ENPUB_PROC)); \ 84 (e)->pub.enp_flags |= ENPUB_PROC; \ 85} while (0) 86 87#define ENGINE_OUT(e) do { \ 88 assert((e)->pub.enp_flags & ENPUB_PROC); \ 89 (e)->pub.enp_flags &= ~ENPUB_PROC; \ 90} while (0) 91 92/* A connection can be referenced from one of six places: 93 * 94 * 1. Connection hash: a connection starts its life in one of those. 95 * 96 * 2. Outgoing queue. 97 * 98 * 3. Incoming queue. 99 * 100 * 4. Pending RW Events queue. 101 * 102 * 5. Advisory Tick Time queue. 103 * 104 * 6. Closing connections queue. This is a transient queue -- it only 105 * exists for the duration of process_connections() function call. 106 * 107 * The idea is to destroy the connection when it is no longer referenced. 108 * For example, a connection tick may return TICK_SEND|TICK_CLOSE. In 109 * that case, the connection is referenced from two places: (2) and (6). 110 * After its packets are sent, it is only referenced in (6), and at the 111 * end of the function call, when it is removed from (6), reference count 112 * goes to zero and the connection is destroyed. If not all packets can 113 * be sent, at the end of the function call, the connection is referenced 114 * by (2) and will only be removed once all outgoing packets have been 115 * sent. 116 */ 117#define CONN_REF_FLAGS (LSCONN_HASHED \ 118 |LSCONN_HAS_OUTGOING \ 119 |LSCONN_HAS_INCOMING \ 120 |LSCONN_RW_PENDING \ 121 |LSCONN_CLOSING \ 122 |LSCONN_ATTQ) 123 124 125struct out_heap_elem 126{ 127 struct lsquic_conn *ohe_conn; 128 lsquic_time_t ohe_last_sent; 129}; 130 131 132struct out_heap 133{ 134 struct out_heap_elem *oh_elems; 135 unsigned oh_nalloc, 136 oh_nelem; 137}; 138 139 140 141 142struct lsquic_engine 143{ 144 struct lsquic_engine_public pub; 145 enum { 146 ENG_SERVER = LSENG_SERVER, 147 ENG_HTTP = LSENG_HTTP, 148 ENG_COOLDOWN = (1 << 7), /* Cooldown: no new connections */ 149 ENG_PAST_DEADLINE 150 = (1 << 8), /* Previous call to a processing 151 * function went past time threshold. 152 */ 153#ifndef NDEBUG 154 ENG_DTOR = (1 << 26), /* Engine destructor */ 155#endif 156 } flags; 157 const struct lsquic_stream_if *stream_if; 158 void *stream_if_ctx; 159 lsquic_packets_out_f packets_out; 160 void *packets_out_ctx; 161 void *bad_handshake_ctx; 162 struct conn_hash full_conns; 163 TAILQ_HEAD(, lsquic_conn) conns_in, conns_pend_rw; 164 struct out_heap conns_out; 165 /* Use a union because only one iterator is being used at any one time */ 166 union { 167 struct { 168 /* This iterator does not have any state: it uses `conns_in' */ 169 int ignore; 170 } conn_in; 171 struct { 172 /* This iterator does not have any state: it uses `conns_pend_rw' */ 173 int ignore; 174 } rw_pend; 175 struct { 176 /* Iterator state to process connections in Advisory Tick Time 177 * queue. 178 */ 179 lsquic_time_t cutoff; 180 } attq; 181 struct { 182 /* Iterator state to process all connections */ 183 int ignore; 184 } all; 185 struct { 186 lsquic_conn_t *conn; 187 } one; 188 } iter_state; 189 struct eng_hist history; 190 unsigned batch_size; 191 unsigned time_until_desired_tick; 192 struct attq *attq; 193 lsquic_time_t proc_time; 194 /* Track time last time a packet was sent to give new connections 195 * priority lower than that of existing connections. 196 */ 197 lsquic_time_t last_sent; 198 lsquic_time_t deadline; 199}; 200 201 202#define OHE_PARENT(i) ((i - 1) / 2) 203#define OHE_LCHILD(i) (2 * i + 1) 204#define OHE_RCHILD(i) (2 * i + 2) 205 206 207static void 208heapify_out_heap (struct out_heap *heap, unsigned i) 209{ 210 struct out_heap_elem el; 211 unsigned smallest; 212 213 assert(i < heap->oh_nelem); 214 215 if (OHE_LCHILD(i) < heap->oh_nelem) 216 { 217 if (heap->oh_elems[ OHE_LCHILD(i) ].ohe_last_sent < 218 heap->oh_elems[ i ].ohe_last_sent) 219 smallest = OHE_LCHILD(i); 220 else 221 smallest = i; 222 if (OHE_RCHILD(i) < heap->oh_nelem && 223 heap->oh_elems[ OHE_RCHILD(i) ].ohe_last_sent < 224 heap->oh_elems[ smallest ].ohe_last_sent) 225 smallest = OHE_RCHILD(i); 226 } 227 else 228 smallest = i; 229 230 if (smallest != i) 231 { 232 el = heap->oh_elems[ smallest ]; 233 heap->oh_elems[ smallest ] = heap->oh_elems[ i ]; 234 heap->oh_elems[ i ] = el; 235 heapify_out_heap(heap, smallest); 236 } 237} 238 239 240static void 241oh_insert (struct out_heap *heap, lsquic_conn_t *conn) 242{ 243 struct out_heap_elem el; 244 unsigned nalloc, i; 245 246 if (heap->oh_nelem == heap->oh_nalloc) 247 { 248 if (0 == heap->oh_nalloc) 249 nalloc = 4; 250 else 251 nalloc = heap->oh_nalloc * 2; 252 heap->oh_elems = realloc(heap->oh_elems, 253 nalloc * sizeof(heap->oh_elems[0])); 254 if (!heap->oh_elems) 255 { /* Not much we can do here */ 256 LSQ_ERROR("realloc failed"); 257 return; 258 } 259 heap->oh_nalloc = nalloc; 260 } 261 262 heap->oh_elems[ heap->oh_nelem ].ohe_conn = conn; 263 heap->oh_elems[ heap->oh_nelem ].ohe_last_sent = conn->cn_last_sent; 264 ++heap->oh_nelem; 265 266 i = heap->oh_nelem - 1; 267 while (i > 0 && heap->oh_elems[ OHE_PARENT(i) ].ohe_last_sent > 268 heap->oh_elems[ i ].ohe_last_sent) 269 { 270 el = heap->oh_elems[ OHE_PARENT(i) ]; 271 heap->oh_elems[ OHE_PARENT(i) ] = heap->oh_elems[ i ]; 272 heap->oh_elems[ i ] = el; 273 i = OHE_PARENT(i); 274 } 275} 276 277 278static struct lsquic_conn * 279oh_pop (struct out_heap *heap) 280{ 281 struct lsquic_conn *conn; 282 283 assert(heap->oh_nelem); 284 285 conn = heap->oh_elems[0].ohe_conn; 286 --heap->oh_nelem; 287 if (heap->oh_nelem > 0) 288 { 289 heap->oh_elems[0] = heap->oh_elems[ heap->oh_nelem ]; 290 heapify_out_heap(heap, 0); 291 } 292 293 return conn; 294} 295 296 297void 298lsquic_engine_init_settings (struct lsquic_engine_settings *settings, 299 unsigned flags) 300{ 301 memset(settings, 0, sizeof(*settings)); 302 settings->es_versions = LSQUIC_DF_VERSIONS; 303 if (flags & ENG_SERVER) 304 { 305 settings->es_cfcw = LSQUIC_DF_CFCW_SERVER; 306 settings->es_sfcw = LSQUIC_DF_SFCW_SERVER; 307 settings->es_support_srej= LSQUIC_DF_SUPPORT_SREJ_SERVER; 308 } 309 else 310 { 311 settings->es_cfcw = LSQUIC_DF_CFCW_CLIENT; 312 settings->es_sfcw = LSQUIC_DF_SFCW_CLIENT; 313 settings->es_support_srej= LSQUIC_DF_SUPPORT_SREJ_CLIENT; 314 } 315 settings->es_max_streams_in = LSQUIC_DF_MAX_STREAMS_IN; 316 settings->es_idle_conn_to = LSQUIC_DF_IDLE_CONN_TO; 317 settings->es_handshake_to = LSQUIC_DF_HANDSHAKE_TO; 318 settings->es_silent_close = LSQUIC_DF_SILENT_CLOSE; 319 settings->es_max_header_list_size 320 = LSQUIC_DF_MAX_HEADER_LIST_SIZE; 321 settings->es_ua = LSQUIC_DF_UA; 322 323 settings->es_pdmd = QTAG_X509; 324 settings->es_aead = QTAG_AESG; 325 settings->es_kexs = QTAG_C255; 326 settings->es_support_push = LSQUIC_DF_SUPPORT_PUSH; 327 settings->es_support_tcid0 = LSQUIC_DF_SUPPORT_TCID0; 328 settings->es_support_nstp = LSQUIC_DF_SUPPORT_NSTP; 329 settings->es_honor_prst = LSQUIC_DF_HONOR_PRST; 330 settings->es_progress_check = LSQUIC_DF_PROGRESS_CHECK; 331 settings->es_pendrw_check = LSQUIC_DF_PENDRW_CHECK; 332 settings->es_rw_once = LSQUIC_DF_RW_ONCE; 333 settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH; 334 settings->es_pace_packets = LSQUIC_DF_PACE_PACKETS; 335} 336 337 338/* Note: if returning an error, err_buf must be valid if non-NULL */ 339int 340lsquic_engine_check_settings (const struct lsquic_engine_settings *settings, 341 unsigned flags, 342 char *err_buf, size_t err_buf_sz) 343{ 344 if (settings->es_cfcw < LSQUIC_MIN_FCW || 345 settings->es_sfcw < LSQUIC_MIN_FCW) 346 { 347 if (err_buf) 348 snprintf(err_buf, err_buf_sz, "%s", 349 "flow control window set too low"); 350 return -1; 351 } 352 if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS)) 353 { 354 if (err_buf) 355 snprintf(err_buf, err_buf_sz, "%s", 356 "No supported QUIC versions specified"); 357 return -1; 358 } 359 if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS) 360 { 361 if (err_buf) 362 snprintf(err_buf, err_buf_sz, "%s", 363 "one or more unsupported QUIC version is specified"); 364 return -1; 365 } 366 return 0; 367} 368 369 370static void 371free_packet (void *ctx, unsigned char *packet_data) 372{ 373 free(packet_data); 374} 375 376 377static void * 378malloc_buf (void *ctx, size_t size) 379{ 380 return malloc(size); 381} 382 383 384static const struct lsquic_packout_mem_if stock_pmi = 385{ 386 malloc_buf, (void(*)(void *, void *)) free_packet, 387}; 388 389 390lsquic_engine_t * 391lsquic_engine_new (unsigned flags, 392 const struct lsquic_engine_api *api) 393{ 394 lsquic_engine_t *engine; 395 int tag_buf_len; 396 char err_buf[100]; 397 398 if (!api->ea_packets_out) 399 { 400 LSQ_ERROR("packets_out callback is not specified"); 401 return NULL; 402 } 403 404 if (api->ea_settings && 405 0 != lsquic_engine_check_settings(api->ea_settings, flags, 406 err_buf, sizeof(err_buf))) 407 { 408 LSQ_ERROR("cannot create engine: %s", err_buf); 409 return NULL; 410 } 411 412 engine = calloc(1, sizeof(*engine)); 413 if (!engine) 414 return NULL; 415 if (0 != lsquic_mm_init(&engine->pub.enp_mm)) 416 { 417 free(engine); 418 return NULL; 419 } 420 if (api->ea_settings) 421 engine->pub.enp_settings = *api->ea_settings; 422 else 423 lsquic_engine_init_settings(&engine->pub.enp_settings, flags); 424 tag_buf_len = gen_ver_tags(engine->pub.enp_ver_tags_buf, 425 sizeof(engine->pub.enp_ver_tags_buf), 426 engine->pub.enp_settings.es_versions); 427 if (tag_buf_len <= 0) 428 { 429 LSQ_ERROR("cannot generate version tags buffer"); 430 free(engine); 431 return NULL; 432 } 433 engine->pub.enp_ver_tags_len = tag_buf_len; 434 435 engine->flags = flags; 436 engine->stream_if = api->ea_stream_if; 437 engine->stream_if_ctx = api->ea_stream_if_ctx; 438 engine->packets_out = api->ea_packets_out; 439 engine->packets_out_ctx = api->ea_packets_out_ctx; 440 if (api->ea_pmi) 441 { 442 engine->pub.enp_pmi = api->ea_pmi; 443 engine->pub.enp_pmi_ctx = api->ea_pmi_ctx; 444 } 445 else 446 { 447 engine->pub.enp_pmi = &stock_pmi; 448 engine->pub.enp_pmi_ctx = NULL; 449 } 450 engine->pub.enp_engine = engine; 451 TAILQ_INIT(&engine->conns_in); 452 TAILQ_INIT(&engine->conns_pend_rw); 453 conn_hash_init(&engine->full_conns, ~0); 454 engine->attq = attq_create(); 455 eng_hist_init(&engine->history); 456 engine->batch_size = INITIAL_OUT_BATCH_SIZE; 457 458 459 LSQ_INFO("instantiated engine"); 460 return engine; 461} 462 463 464static void 465grow_batch_size (struct lsquic_engine *engine) 466{ 467 engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE; 468} 469 470 471static void 472shrink_batch_size (struct lsquic_engine *engine) 473{ 474 engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE; 475} 476 477 478/* Wrapper to make sure important things occur before the connection is 479 * really destroyed. 480 */ 481static void 482destroy_conn (struct lsquic_engine *engine, lsquic_conn_t *conn) 483{ 484 conn->cn_flags |= LSCONN_NEVER_PEND_RW; 485 conn->cn_if->ci_destroy(conn); 486} 487 488 489static lsquic_conn_t * 490new_full_conn_client (lsquic_engine_t *engine, const char *hostname, 491 unsigned short max_packet_size) 492{ 493 lsquic_conn_t *conn; 494 unsigned flags; 495 flags = engine->flags & (ENG_SERVER|ENG_HTTP); 496 conn = full_conn_client_new(&engine->pub, engine->stream_if, 497 engine->stream_if_ctx, flags, hostname, max_packet_size); 498 if (!conn) 499 return NULL; 500 if (0 != conn_hash_add(&engine->full_conns, conn)) 501 { 502 LSQ_WARN("cannot add connection %"PRIu64" to hash - destroy", 503 conn->cn_cid); 504 destroy_conn(engine, conn); 505 return NULL; 506 } 507 assert(!(conn->cn_flags & 508 (CONN_REF_FLAGS 509 & ~LSCONN_RW_PENDING /* This flag may be set as effect of user 510 callbacks */ 511 ))); 512 conn->cn_flags |= LSCONN_HASHED; 513 return conn; 514} 515 516 517static lsquic_conn_t * 518find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 519 struct packin_parse_state *ppstate, const struct sockaddr *sa_peer, 520 void *peer_ctx) 521{ 522 lsquic_conn_t *conn; 523 524 if (lsquic_packet_in_is_prst(packet_in) 525 && !engine->pub.enp_settings.es_honor_prst) 526 { 527 LSQ_DEBUG("public reset packet: discarding"); 528 return NULL; 529 } 530 531 if (!(packet_in->pi_flags & PI_CONN_ID)) 532 { 533 LSQ_DEBUG("packet header does not have connection ID: discarding"); 534 return NULL; 535 } 536 537 conn = conn_hash_find(&engine->full_conns, packet_in->pi_conn_id); 538 if (conn) 539 { 540 conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate); 541 return conn; 542 } 543 544 return conn; 545} 546 547 548static void 549add_conn_to_pend_rw (lsquic_engine_t *engine, lsquic_conn_t *conn, 550 enum rw_reason reason) 551{ 552 int hist_idx; 553 554 TAILQ_INSERT_TAIL(&engine->conns_pend_rw, conn, cn_next_pend_rw); 555 engine_incref_conn(conn, LSCONN_RW_PENDING); 556 557 hist_idx = conn->cn_rw_hist_idx & ((1 << RW_HIST_BITS) - 1); 558 conn->cn_rw_hist_buf[ hist_idx ] = reason; 559 ++conn->cn_rw_hist_idx; 560 561 if ((int) sizeof(conn->cn_rw_hist_buf) - 1 == hist_idx) 562 EV_LOG_CONN_EVENT(conn->cn_cid, "added to pending RW queue ('%c'), " 563 "rw_hist: %.*s", (char) reason, 564 (int) sizeof(conn->cn_rw_hist_buf), conn->cn_rw_hist_buf); 565 else 566 EV_LOG_CONN_EVENT(conn->cn_cid, "added to pending RW queue ('%c')", 567 (char) reason); 568} 569 570 571#if !defined(NDEBUG) && __GNUC__ 572__attribute__((weak)) 573#endif 574void 575lsquic_engine_add_conn_to_pend_rw (struct lsquic_engine_public *enpub, 576 lsquic_conn_t *conn, enum rw_reason reason) 577{ 578 if (0 == (enpub->enp_flags & ENPUB_PROC) && 579 0 == (conn->cn_flags & (LSCONN_RW_PENDING|LSCONN_NEVER_PEND_RW))) 580 { 581 lsquic_engine_t *engine = (lsquic_engine_t *) enpub; 582 add_conn_to_pend_rw(engine, conn, reason); 583 } 584} 585 586 587void 588lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub, 589 lsquic_conn_t *conn, lsquic_time_t tick_time) 590{ 591 lsquic_engine_t *const engine = (lsquic_engine_t *) enpub; 592 /* Instead of performing an update, we simply remove the connection from 593 * the queue and add it back. This should not happen in at the time of 594 * this writing. 595 */ 596 if (conn->cn_flags & LSCONN_ATTQ) 597 { 598 attq_remove(engine->attq, conn); 599 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 600 } 601 if (conn && !(conn->cn_flags & LSCONN_ATTQ) && 602 0 == attq_maybe_add(engine->attq, conn, tick_time)) 603 engine_incref_conn(conn, LSCONN_ATTQ); 604} 605 606 607static void 608update_pend_rw_progress (lsquic_engine_t *engine, lsquic_conn_t *conn, 609 int progress_made) 610{ 611 rw_hist_idx_t hist_idx; 612 const unsigned char *empty; 613 const unsigned pendrw_check = engine->pub.enp_settings.es_pendrw_check; 614 615 if (!pendrw_check) 616 return; 617 618 /* Convert previous entry to uppercase: */ 619 hist_idx = (conn->cn_rw_hist_idx - 1) & ((1 << RW_HIST_BITS) - 1); 620 conn->cn_rw_hist_buf[ hist_idx ] -= 0x20; 621 622 LSQ_DEBUG("conn %"PRIu64": progress: %d", conn->cn_cid, !!progress_made); 623 if (progress_made) 624 { 625 conn->cn_noprogress_count = 0; 626 return; 627 } 628 629 EV_LOG_CONN_EVENT(conn->cn_cid, "Pending RW Queue processing made " 630 "no progress"); 631 ++conn->cn_noprogress_count; 632 if (conn->cn_noprogress_count <= pendrw_check) 633 return; 634 635 conn->cn_flags |= LSCONN_NEVER_PEND_RW; 636 empty = memchr(conn->cn_rw_hist_buf, RW_REASON_EMPTY, 637 sizeof(conn->cn_rw_hist_buf)); 638 if (empty) 639 LSQ_WARN("conn %"PRIu64" noprogress count reached %u " 640 "(rw_hist: %.*s): will not put it onto Pend RW queue again", 641 conn->cn_cid, conn->cn_noprogress_count, 642 (int) (empty - conn->cn_rw_hist_buf), conn->cn_rw_hist_buf); 643 else 644 { 645 hist_idx = conn->cn_rw_hist_idx & ((1 << RW_HIST_BITS) - 1); 646 LSQ_WARN("conn %"PRIu64" noprogress count reached %u " 647 "(rw_hist: %.*s%.*s): will not put it onto Pend RW queue again", 648 conn->cn_cid, conn->cn_noprogress_count, 649 /* First part of history: */ 650 (int) (sizeof(conn->cn_rw_hist_buf) - hist_idx), 651 conn->cn_rw_hist_buf + hist_idx, 652 /* Second part of history: */ 653 hist_idx, conn->cn_rw_hist_buf); 654 } 655} 656 657 658/* Return 0 if packet is being processed by a connections, otherwise return 1 */ 659static int 660process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in, 661 struct packin_parse_state *ppstate, const struct sockaddr *sa_local, 662 const struct sockaddr *sa_peer, void *peer_ctx) 663{ 664 lsquic_conn_t *conn; 665 666 conn = find_or_create_conn(engine, packet_in, ppstate, sa_peer, peer_ctx); 667 if (!conn) 668 { 669 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 670 return 1; 671 } 672 673 if (0 == (conn->cn_flags & LSCONN_HAS_INCOMING)) { 674 TAILQ_INSERT_TAIL(&engine->conns_in, conn, cn_next_in); 675 engine_incref_conn(conn, LSCONN_HAS_INCOMING); 676 } 677 lsquic_conn_record_sockaddr(conn, sa_local, sa_peer); 678 lsquic_packet_in_upref(packet_in); 679 conn->cn_peer_ctx = peer_ctx; 680 conn->cn_if->ci_packet_in(conn, packet_in); 681 lsquic_packet_in_put(&engine->pub.enp_mm, packet_in); 682 return 0; 683} 684 685 686static int 687conn_attq_expired (const struct lsquic_engine *engine, 688 const lsquic_conn_t *conn) 689{ 690 assert(conn->cn_attq_elem); 691 return lsquic_conn_adv_time(conn) < engine->proc_time; 692} 693 694 695/* Iterator for connections with incoming packets */ 696static lsquic_conn_t * 697conn_iter_next_incoming (struct lsquic_engine *engine) 698{ 699 enum lsquic_conn_flags addl_flags; 700 lsquic_conn_t *conn; 701 while ((conn = TAILQ_FIRST(&engine->conns_in))) 702 { 703 TAILQ_REMOVE(&engine->conns_in, conn, cn_next_in); 704 if (conn->cn_flags & LSCONN_RW_PENDING) 705 { 706 TAILQ_REMOVE(&engine->conns_pend_rw, conn, cn_next_pend_rw); 707 EV_LOG_CONN_EVENT(conn->cn_cid, 708 "removed from pending RW queue (processing incoming)"); 709 } 710 if ((conn->cn_flags & LSCONN_ATTQ) && conn_attq_expired(engine, conn)) 711 { 712 addl_flags = LSCONN_ATTQ; 713 attq_remove(engine->attq, conn); 714 } 715 else 716 addl_flags = 0; 717 conn = engine_decref_conn(engine, conn, 718 LSCONN_RW_PENDING|LSCONN_HAS_INCOMING|addl_flags); 719 if (conn) 720 break; 721 } 722 return conn; 723} 724 725 726/* Iterator for connections with that have pending read/write events */ 727static lsquic_conn_t * 728conn_iter_next_rw_pend (struct lsquic_engine *engine) 729{ 730 enum lsquic_conn_flags addl_flags; 731 lsquic_conn_t *conn; 732 while ((conn = TAILQ_FIRST(&engine->conns_pend_rw))) 733 { 734 TAILQ_REMOVE(&engine->conns_pend_rw, conn, cn_next_pend_rw); 735 EV_LOG_CONN_EVENT(conn->cn_cid, 736 "removed from pending RW queue (processing pending RW conns)"); 737 if (conn->cn_flags & LSCONN_HAS_INCOMING) 738 TAILQ_REMOVE(&engine->conns_in, conn, cn_next_in); 739 if ((conn->cn_flags & LSCONN_ATTQ) && conn_attq_expired(engine, conn)) 740 { 741 addl_flags = LSCONN_ATTQ; 742 attq_remove(engine->attq, conn); 743 } 744 else 745 addl_flags = 0; 746 conn = engine_decref_conn(engine, conn, 747 LSCONN_RW_PENDING|LSCONN_HAS_INCOMING|addl_flags); 748 if (conn) 749 break; 750 } 751 return conn; 752} 753 754 755void 756lsquic_engine_process_conns_with_incoming (lsquic_engine_t *engine) 757{ 758 LSQ_DEBUG("process connections with incoming packets"); 759 ENGINE_IN(engine); 760 process_connections(engine, conn_iter_next_incoming); 761 assert(TAILQ_EMPTY(&engine->conns_in)); 762 ENGINE_OUT(engine); 763} 764 765 766int 767lsquic_engine_has_pend_rw (lsquic_engine_t *engine) 768{ 769 return !(engine->flags & ENG_PAST_DEADLINE) 770 && !TAILQ_EMPTY(&engine->conns_pend_rw); 771} 772 773 774void 775lsquic_engine_process_conns_with_pend_rw (lsquic_engine_t *engine) 776{ 777 LSQ_DEBUG("process connections with pending RW events"); 778 ENGINE_IN(engine); 779 process_connections(engine, conn_iter_next_rw_pend); 780 ENGINE_OUT(engine); 781} 782 783 784void 785lsquic_engine_destroy (lsquic_engine_t *engine) 786{ 787 lsquic_conn_t *conn; 788 789 LSQ_DEBUG("destroying engine"); 790#ifndef NDEBUG 791 engine->flags |= ENG_DTOR; 792#endif 793 794 while (engine->conns_out.oh_nelem > 0) 795 { 796 --engine->conns_out.oh_nelem; 797 conn = engine->conns_out.oh_elems[ 798 engine->conns_out.oh_nelem ].ohe_conn; 799 assert(conn->cn_flags & LSCONN_HAS_OUTGOING); 800 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 801 } 802 803 for (conn = conn_hash_first(&engine->full_conns); conn; 804 conn = conn_hash_next(&engine->full_conns)) 805 force_close_conn(engine, conn); 806 conn_hash_cleanup(&engine->full_conns); 807 808 809 attq_destroy(engine->attq); 810 811 assert(0 == engine->conns_out.oh_nelem); 812 assert(TAILQ_EMPTY(&engine->conns_pend_rw)); 813 lsquic_mm_cleanup(&engine->pub.enp_mm); 814 free(engine->conns_out.oh_elems); 815 free(engine); 816} 817 818 819#if __GNUC__ 820__attribute__((nonnull(3))) 821#endif 822static lsquic_conn_t * 823remove_from_inc_andor_pend_rw (lsquic_engine_t *engine, 824 lsquic_conn_t *conn, const char *reason) 825{ 826 assert(conn->cn_flags & (LSCONN_HAS_INCOMING|LSCONN_RW_PENDING)); 827 if (conn->cn_flags & LSCONN_HAS_INCOMING) 828 TAILQ_REMOVE(&engine->conns_in, conn, cn_next_in); 829 if (conn->cn_flags & LSCONN_RW_PENDING) 830 { 831 TAILQ_REMOVE(&engine->conns_pend_rw, conn, cn_next_pend_rw); 832 EV_LOG_CONN_EVENT(conn->cn_cid, 833 "removed from pending RW queue (%s)", reason); 834 } 835 conn = engine_decref_conn(engine, conn, 836 LSCONN_HAS_INCOMING|LSCONN_RW_PENDING); 837 assert(conn); 838 return conn; 839} 840 841 842static lsquic_conn_t * 843conn_iter_next_one (lsquic_engine_t *engine) 844{ 845 lsquic_conn_t *conn = engine->iter_state.one.conn; 846 if (conn) 847 { 848 if (conn->cn_flags & (LSCONN_HAS_INCOMING|LSCONN_RW_PENDING)) 849 conn = remove_from_inc_andor_pend_rw(engine, conn, "connect"); 850 if (conn && (conn->cn_flags & LSCONN_ATTQ) && 851 conn_attq_expired(engine, conn)) 852 { 853 attq_remove(engine->attq, conn); 854 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 855 } 856 engine->iter_state.one.conn = NULL; 857 } 858 return conn; 859} 860 861 862lsquic_conn_t * 863lsquic_engine_connect (lsquic_engine_t *engine, const struct sockaddr *peer_sa, 864 void *peer_ctx, lsquic_conn_ctx_t *conn_ctx, 865 const char *hostname, unsigned short max_packet_size) 866{ 867 lsquic_conn_t *conn; 868 869 if (engine->flags & ENG_SERVER) 870 { 871 LSQ_ERROR("`%s' must only be called in client mode", __func__); 872 return NULL; 873 } 874 875 if (0 == max_packet_size) 876 { 877 switch (peer_sa->sa_family) 878 { 879 case AF_INET: 880 max_packet_size = QUIC_MAX_IPv4_PACKET_SZ; 881 break; 882 default: 883 max_packet_size = QUIC_MAX_IPv6_PACKET_SZ; 884 break; 885 } 886 } 887 888 conn = new_full_conn_client(engine, hostname, max_packet_size); 889 if (!conn) 890 return NULL; 891 ENGINE_IN(engine); 892 lsquic_conn_record_peer_sa(conn, peer_sa); 893 conn->cn_peer_ctx = peer_ctx; 894 lsquic_conn_set_ctx(conn, conn_ctx); 895 engine->iter_state.one.conn = conn; 896 full_conn_client_call_on_new(conn); 897 process_connections(engine, conn_iter_next_one); 898 ENGINE_OUT(engine); 899 return conn; 900} 901 902 903static void 904remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn) 905{ 906 conn_hash_remove(&engine->full_conns, conn); 907 (void) engine_decref_conn(engine, conn, LSCONN_HASHED); 908} 909 910 911static void 912refflags2str (enum lsquic_conn_flags flags, char s[7]) 913{ 914 *s = 'C'; s += !!(flags & LSCONN_CLOSING); 915 *s = 'H'; s += !!(flags & LSCONN_HASHED); 916 *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING); 917 *s = 'I'; s += !!(flags & LSCONN_HAS_INCOMING); 918 *s = 'R'; s += !!(flags & LSCONN_RW_PENDING); 919 *s = 'A'; s += !!(flags & LSCONN_ATTQ); 920 *s = '\0'; 921} 922 923 924static void 925engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag) 926{ 927 char str[7]; 928 assert(flag & CONN_REF_FLAGS); 929 assert(!(conn->cn_flags & flag)); 930 conn->cn_flags |= flag; 931 LSQ_DEBUG("incref conn %"PRIu64", now '%s'", conn->cn_cid, 932 (refflags2str(conn->cn_flags, str), str)); 933} 934 935 936static lsquic_conn_t * 937engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn, 938 enum lsquic_conn_flags flags) 939{ 940 char str[7]; 941 assert(flags & CONN_REF_FLAGS); 942 assert(conn->cn_flags & flags); 943#ifndef NDEBUG 944 if (flags & LSCONN_CLOSING) 945 assert(0 == (conn->cn_flags & LSCONN_HASHED)); 946#endif 947 conn->cn_flags &= ~flags; 948 LSQ_DEBUG("decref conn %"PRIu64", now '%s'", conn->cn_cid, 949 (refflags2str(conn->cn_flags, str), str)); 950 if (0 == (conn->cn_flags & CONN_REF_FLAGS)) 951 { 952 eng_hist_inc(&engine->history, 0, sl_del_full_conns); 953 destroy_conn(engine, conn); 954 return NULL; 955 } 956 else 957 return conn; 958} 959 960 961/* This is not a general-purpose function. Only call from engine dtor. */ 962static void 963force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn) 964{ 965 assert(engine->flags & ENG_DTOR); 966 const enum lsquic_conn_flags flags = conn->cn_flags; 967 assert(conn->cn_flags & CONN_REF_FLAGS); 968 assert(!(flags & LSCONN_HAS_OUTGOING)); /* Should be removed already */ 969 assert(!(flags & LSCONN_CLOSING)); /* It is in transient queue? */ 970 if (flags & LSCONN_HAS_INCOMING) 971 { 972 TAILQ_REMOVE(&engine->conns_in, conn, cn_next_in); 973 (void) engine_decref_conn(engine, conn, LSCONN_HAS_INCOMING); 974 } 975 if (flags & LSCONN_RW_PENDING) 976 { 977 TAILQ_REMOVE(&engine->conns_pend_rw, conn, cn_next_pend_rw); 978 EV_LOG_CONN_EVENT(conn->cn_cid, 979 "removed from pending RW queue (engine destruction)"); 980 (void) engine_decref_conn(engine, conn, LSCONN_RW_PENDING); 981 } 982 if (flags & LSCONN_ATTQ) 983 attq_remove(engine->attq, conn); 984 if (flags & LSCONN_HASHED) 985 remove_conn_from_hash(engine, conn); 986} 987 988 989/* Iterator for all connections. 990 * Returned connections are removed from the Incoming, Pending RW Event, 991 * and Advisory Tick Time queues if necessary. 992 */ 993static lsquic_conn_t * 994conn_iter_next_all (struct lsquic_engine *engine) 995{ 996 lsquic_conn_t *conn; 997 998 conn = conn_hash_next(&engine->full_conns); 999 1000 if (conn && (conn->cn_flags & (LSCONN_HAS_INCOMING|LSCONN_RW_PENDING))) 1001 conn = remove_from_inc_andor_pend_rw(engine, conn, "process all"); 1002 if (conn && (conn->cn_flags & LSCONN_ATTQ) 1003 && conn_attq_expired(engine, conn)) 1004 { 1005 attq_remove(engine->attq, conn); 1006 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1007 } 1008 1009 return conn; 1010} 1011 1012 1013static lsquic_conn_t * 1014conn_iter_next_attq (struct lsquic_engine *engine) 1015{ 1016 lsquic_conn_t *conn; 1017 1018 conn = attq_pop(engine->attq, engine->iter_state.attq.cutoff); 1019 if (conn) 1020 { 1021 assert(conn->cn_flags & LSCONN_ATTQ); 1022 if (conn->cn_flags & (LSCONN_HAS_INCOMING|LSCONN_RW_PENDING)) 1023 conn = remove_from_inc_andor_pend_rw(engine, conn, "process attq"); 1024 conn = engine_decref_conn(engine, conn, LSCONN_ATTQ); 1025 } 1026 1027 return conn; 1028} 1029 1030 1031void 1032lsquic_engine_proc_all (lsquic_engine_t *engine) 1033{ 1034 ENGINE_IN(engine); 1035 /* We poke each connection every time as initial implementation. If it 1036 * proves to be too inefficient, we will need to figure out 1037 * a) when to stop processing; and 1038 * b) how to remember state between calls. 1039 */ 1040 conn_hash_reset_iter(&engine->full_conns); 1041 process_connections(engine, conn_iter_next_all); 1042 ENGINE_OUT(engine); 1043} 1044 1045 1046void 1047lsquic_engine_process_conns_to_tick (lsquic_engine_t *engine) 1048{ 1049 lsquic_time_t prev_min, now; 1050 1051 now = lsquic_time_now(); 1052 if (LSQ_LOG_ENABLED(LSQ_LOG_DEBUG)) 1053 { 1054 const lsquic_time_t *expected_time; 1055 int64_t diff; 1056 expected_time = attq_next_time(engine->attq); 1057 if (expected_time) 1058 diff = *expected_time - now; 1059 else 1060 diff = -1; 1061 LSQ_DEBUG("process connections in attq; time diff: %"PRIi64, diff); 1062 } 1063 1064 ENGINE_IN(engine); 1065 prev_min = attq_set_min(engine->attq, now); /* Prevent infinite loop */ 1066 engine->iter_state.attq.cutoff = now; 1067 process_connections(engine, conn_iter_next_attq); 1068 attq_set_min(engine->attq, prev_min); /* Restore previos value */ 1069 ENGINE_OUT(engine); 1070} 1071 1072 1073static int 1074generate_header (const lsquic_packet_out_t *packet_out, 1075 const struct parse_funcs *pf, lsquic_cid_t cid, 1076 unsigned char *buf, size_t bufsz) 1077{ 1078 return pf->pf_gen_reg_pkt_header(buf, bufsz, 1079 packet_out->po_flags & PO_CONN_ID ? &cid : NULL, 1080 packet_out->po_flags & PO_VERSION ? &packet_out->po_ver_tag : NULL, 1081 packet_out->po_flags & PO_NONCE ? packet_out->po_nonce : NULL, 1082 packet_out->po_packno, lsquic_packet_out_packno_bits(packet_out)); 1083} 1084 1085 1086static ssize_t 1087really_encrypt_packet (const lsquic_conn_t *conn, 1088 const lsquic_packet_out_t *packet_out, 1089 unsigned char *buf, size_t bufsz) 1090{ 1091 int enc, header_sz, is_hello_packet; 1092 size_t packet_sz; 1093 unsigned char header_buf[QUIC_MAX_PUBHDR_SZ]; 1094 1095 header_sz = generate_header(packet_out, conn->cn_pf, conn->cn_cid, 1096 header_buf, sizeof(header_buf)); 1097 if (header_sz < 0) 1098 return -1; 1099 1100 is_hello_packet = !!(packet_out->po_flags & PO_HELLO); 1101 enc = conn->cn_esf->esf_encrypt(conn->cn_enc_session, conn->cn_version, 0, 1102 packet_out->po_packno, header_buf, header_sz, 1103 packet_out->po_data, packet_out->po_data_sz, 1104 buf, bufsz, &packet_sz, is_hello_packet); 1105 if (0 == enc) 1106 { 1107 LSQ_DEBUG("encrypted packet %"PRIu64"; plaintext is %u bytes, " 1108 "ciphertext is %zd bytes", 1109 packet_out->po_packno, 1110 lsquic_po_header_length(packet_out->po_flags) + 1111 packet_out->po_data_sz, 1112 packet_sz); 1113 return packet_sz; 1114 } 1115 else 1116 return -1; 1117} 1118 1119 1120static enum { ENCPA_OK, ENCPA_NOMEM, ENCPA_BADCRYPT, } 1121encrypt_packet (lsquic_engine_t *engine, const lsquic_conn_t *conn, 1122 lsquic_packet_out_t *packet_out) 1123{ 1124 ssize_t enc_sz; 1125 size_t bufsz; 1126 unsigned char *buf; 1127 1128 bufsz = lsquic_po_header_length(packet_out->po_flags) + 1129 packet_out->po_data_sz + QUIC_PACKET_HASH_SZ; 1130 buf = engine->pub.enp_pmi->pmi_allocate(engine->pub.enp_pmi_ctx, bufsz); 1131 if (!buf) 1132 { 1133 LSQ_DEBUG("could not allocate memory for outgoing packet of size %zd", 1134 bufsz); 1135 return ENCPA_NOMEM; 1136 } 1137 1138 enc_sz = really_encrypt_packet(conn, packet_out, buf, bufsz); 1139 1140 if (enc_sz < 0) 1141 { 1142 engine->pub.enp_pmi->pmi_release(engine->pub.enp_pmi_ctx, buf); 1143 return ENCPA_BADCRYPT; 1144 } 1145 1146 packet_out->po_enc_data = buf; 1147 packet_out->po_enc_data_sz = enc_sz; 1148 packet_out->po_flags |= PO_ENCRYPTED; 1149 1150 return ENCPA_OK; 1151} 1152 1153 1154struct out_batch 1155{ 1156 lsquic_conn_t *conns [MAX_OUT_BATCH_SIZE]; 1157 lsquic_packet_out_t *packets[MAX_OUT_BATCH_SIZE]; 1158 struct lsquic_out_spec outs [MAX_OUT_BATCH_SIZE]; 1159}; 1160 1161 1162STAILQ_HEAD(closed_conns, lsquic_conn); 1163 1164 1165struct conns_out_iter 1166{ 1167 struct out_heap *coi_heap; 1168 TAILQ_HEAD(, lsquic_conn) coi_active_list, 1169 coi_inactive_list; 1170 lsquic_conn_t *coi_next; 1171#ifndef NDEBUG 1172 lsquic_time_t coi_last_sent; 1173#endif 1174}; 1175 1176 1177static void 1178coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine) 1179{ 1180 iter->coi_heap = &engine->conns_out; 1181 iter->coi_next = NULL; 1182 TAILQ_INIT(&iter->coi_active_list); 1183 TAILQ_INIT(&iter->coi_inactive_list); 1184#ifndef NDEBUG 1185 iter->coi_last_sent = 0; 1186#endif 1187} 1188 1189 1190static lsquic_conn_t * 1191coi_next (struct conns_out_iter *iter) 1192{ 1193 lsquic_conn_t *conn; 1194 1195 if (iter->coi_heap->oh_nelem > 0) 1196 { 1197 conn = oh_pop(iter->coi_heap); 1198 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1199 conn->cn_flags |= LSCONN_COI_ACTIVE; 1200#ifndef NDEBUG 1201 if (iter->coi_last_sent) 1202 assert(iter->coi_last_sent <= conn->cn_last_sent); 1203 iter->coi_last_sent = conn->cn_last_sent; 1204#endif 1205 return conn; 1206 } 1207 else if (!TAILQ_EMPTY(&iter->coi_active_list)) 1208 { 1209 conn = iter->coi_next; 1210 if (!conn) 1211 conn = TAILQ_FIRST(&iter->coi_active_list); 1212 if (conn) 1213 iter->coi_next = TAILQ_NEXT(conn, cn_next_out); 1214 return conn; 1215 } 1216 else 1217 return NULL; 1218} 1219 1220 1221static void 1222coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1223{ 1224 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 1225 { 1226 assert(!TAILQ_EMPTY(&iter->coi_active_list)); 1227 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1228 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1229 TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out); 1230 conn->cn_flags |= LSCONN_COI_INACTIVE; 1231 } 1232} 1233 1234 1235static void 1236coi_remove (struct conns_out_iter *iter, lsquic_conn_t *conn) 1237{ 1238 assert(conn->cn_flags & LSCONN_COI_ACTIVE); 1239 if (conn->cn_flags & LSCONN_COI_ACTIVE) 1240 { 1241 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1242 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1243 } 1244} 1245 1246 1247static void 1248coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn) 1249{ 1250 assert(conn->cn_flags & LSCONN_COI_INACTIVE); 1251 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 1252 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 1253 TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out); 1254 conn->cn_flags |= LSCONN_COI_ACTIVE; 1255} 1256 1257 1258static void 1259coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine) 1260{ 1261 lsquic_conn_t *conn; 1262 while ((conn = TAILQ_FIRST(&iter->coi_active_list))) 1263 { 1264 TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out); 1265 conn->cn_flags &= ~LSCONN_COI_ACTIVE; 1266 oh_insert(iter->coi_heap, conn); 1267 } 1268 while ((conn = TAILQ_FIRST(&iter->coi_inactive_list))) 1269 { 1270 TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out); 1271 conn->cn_flags &= ~LSCONN_COI_INACTIVE; 1272 (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING); 1273 } 1274} 1275 1276 1277static unsigned 1278send_batch (lsquic_engine_t *engine, struct conns_out_iter *conns_iter, 1279 struct out_batch *batch, unsigned n_to_send) 1280{ 1281 int n_sent, i; 1282 lsquic_time_t now; 1283 1284 /* Set sent time before the write to avoid underestimating RTT */ 1285 now = lsquic_time_now(); 1286 for (i = 0; i < (int) n_to_send; ++i) 1287 batch->packets[i]->po_sent = now; 1288 n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs, 1289 n_to_send); 1290 if (n_sent >= 0) 1291 LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, n_to_send); 1292 else 1293 { 1294 LSQ_DEBUG("packets out returned an error: %s", strerror(errno)); 1295 n_sent = 0; 1296 } 1297 if (n_sent > 0) 1298 engine->last_sent = now + n_sent; 1299 for (i = 0; i < n_sent; ++i) 1300 { 1301 eng_hist_inc(&engine->history, now, sl_packets_out); 1302 EV_LOG_PACKET_SENT(batch->conns[i]->cn_cid, batch->packets[i]); 1303 batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i], 1304 batch->packets[i]); 1305 /* `i' is added to maintain relative order */ 1306 batch->conns[i]->cn_last_sent = now + i; 1307 /* Release packet out buffer as soon as the packet is sent 1308 * successfully. If not successfully sent, we hold on to 1309 * this buffer until the packet sending is attempted again 1310 * or until it times out and regenerated. 1311 */ 1312 if (batch->packets[i]->po_flags & PO_ENCRYPTED) 1313 { 1314 batch->packets[i]->po_flags &= ~PO_ENCRYPTED; 1315 engine->pub.enp_pmi->pmi_release(engine->pub.enp_pmi_ctx, 1316 batch->packets[i]->po_enc_data); 1317 batch->packets[i]->po_enc_data = NULL; /* JIC */ 1318 } 1319 } 1320 if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT)) 1321 for ( ; i < (int) n_to_send; ++i) 1322 EV_LOG_PACKET_NOT_SENT(batch->conns[i]->cn_cid, batch->packets[i]); 1323 /* Return packets to the connection in reverse order so that the packet 1324 * ordering is maintained. 1325 */ 1326 for (i = (int) n_to_send - 1; i >= n_sent; --i) 1327 { 1328 batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i], 1329 batch->packets[i]); 1330 if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT))) 1331 coi_reactivate(conns_iter, batch->conns[i]); 1332 } 1333 return n_sent; 1334} 1335 1336 1337/* Return 1 if went past deadline, 0 otherwise */ 1338static int 1339check_deadline (lsquic_engine_t *engine) 1340{ 1341 if (engine->pub.enp_settings.es_proc_time_thresh && 1342 lsquic_time_now() > engine->deadline) 1343 { 1344 LSQ_INFO("went past threshold of %u usec, stop sending", 1345 engine->pub.enp_settings.es_proc_time_thresh); 1346 engine->flags |= ENG_PAST_DEADLINE; 1347 return 1; 1348 } 1349 else 1350 return 0; 1351} 1352 1353 1354static void 1355send_packets_out (struct lsquic_engine *engine, 1356 struct closed_conns *closed_conns) 1357{ 1358 unsigned n, w, n_sent, n_batches_sent; 1359 lsquic_packet_out_t *packet_out; 1360 lsquic_conn_t *conn; 1361 struct out_batch batch; 1362 struct conns_out_iter conns_iter; 1363 int shrink, deadline_exceeded; 1364 1365 coi_init(&conns_iter, engine); 1366 n_batches_sent = 0; 1367 n_sent = 0, n = 0; 1368 shrink = 0; 1369 deadline_exceeded = 0; 1370 1371 while ((conn = coi_next(&conns_iter))) 1372 { 1373 packet_out = conn->cn_if->ci_next_packet_to_send(conn); 1374 if (!packet_out) { 1375 LSQ_DEBUG("batched all outgoing packets for conn %"PRIu64, 1376 conn->cn_cid); 1377 coi_deactivate(&conns_iter, conn); 1378 continue; 1379 } 1380 if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT))) 1381 { 1382 switch (encrypt_packet(engine, conn, packet_out)) 1383 { 1384 case ENCPA_NOMEM: 1385 /* Send what we have and wait for a more opportune moment */ 1386 conn->cn_if->ci_packet_not_sent(conn, packet_out); 1387 goto end_for; 1388 case ENCPA_BADCRYPT: 1389 /* This is pretty bad: close connection immediately */ 1390 conn->cn_if->ci_packet_not_sent(conn, packet_out); 1391 LSQ_INFO("conn %"PRIu64" has unsendable packets", conn->cn_cid); 1392 if (!(conn->cn_flags & LSCONN_EVANESCENT)) 1393 { 1394 if (!(conn->cn_flags & LSCONN_CLOSING)) 1395 { 1396 STAILQ_INSERT_TAIL(closed_conns, conn, cn_next_closed_conn); 1397 engine_incref_conn(conn, LSCONN_CLOSING); 1398 if (conn->cn_flags & LSCONN_HASHED) 1399 remove_conn_from_hash(engine, conn); 1400 } 1401 coi_remove(&conns_iter, conn); 1402 } 1403 continue; 1404 case ENCPA_OK: 1405 break; 1406 } 1407 } 1408 LSQ_DEBUG("batched packet %"PRIu64" for connection %"PRIu64, 1409 packet_out->po_packno, conn->cn_cid); 1410 assert(conn->cn_flags & LSCONN_HAS_PEER_SA); 1411 if (packet_out->po_flags & PO_ENCRYPTED) 1412 { 1413 batch.outs[n].buf = packet_out->po_enc_data; 1414 batch.outs[n].sz = packet_out->po_enc_data_sz; 1415 } 1416 else 1417 { 1418 batch.outs[n].buf = packet_out->po_data; 1419 batch.outs[n].sz = packet_out->po_data_sz; 1420 } 1421 batch.outs [n].peer_ctx = conn->cn_peer_ctx; 1422 batch.outs [n].local_sa = (struct sockaddr *) conn->cn_local_addr; 1423 batch.outs [n].dest_sa = (struct sockaddr *) conn->cn_peer_addr; 1424 batch.conns [n] = conn; 1425 batch.packets[n] = packet_out; 1426 ++n; 1427 if (n == engine->batch_size) 1428 { 1429 n = 0; 1430 w = send_batch(engine, &conns_iter, &batch, engine->batch_size); 1431 ++n_batches_sent; 1432 n_sent += w; 1433 if (w < engine->batch_size) 1434 { 1435 shrink = 1; 1436 break; 1437 } 1438 deadline_exceeded = check_deadline(engine); 1439 if (deadline_exceeded) 1440 break; 1441 grow_batch_size(engine); 1442 } 1443 } 1444 end_for: 1445 1446 if (n > 0) { 1447 w = send_batch(engine, &conns_iter, &batch, n); 1448 n_sent += w; 1449 shrink = w < n; 1450 ++n_batches_sent; 1451 deadline_exceeded = check_deadline(engine); 1452 } 1453 1454 if (shrink) 1455 shrink_batch_size(engine); 1456 else if (n_batches_sent > 1 && !deadline_exceeded) 1457 grow_batch_size(engine); 1458 1459 coi_reheap(&conns_iter, engine); 1460 1461 LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s"); 1462} 1463 1464 1465int 1466lsquic_engine_has_unsent_packets (lsquic_engine_t *engine) 1467{ 1468 return !(engine->flags & ENG_PAST_DEADLINE) 1469 && ( engine->conns_out.oh_nelem > 0 1470 ) 1471 ; 1472} 1473 1474 1475static void 1476reset_deadline (lsquic_engine_t *engine, lsquic_time_t now) 1477{ 1478 engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh; 1479 engine->flags &= ~ENG_PAST_DEADLINE; 1480} 1481 1482 1483/* TODO: this is a user-facing function, account for load */ 1484void 1485lsquic_engine_send_unsent_packets (lsquic_engine_t *engine) 1486{ 1487 lsquic_conn_t *conn; 1488 struct closed_conns closed_conns; 1489 1490 STAILQ_INIT(&closed_conns); 1491 reset_deadline(engine, lsquic_time_now()); 1492 1493 send_packets_out(engine, &closed_conns); 1494 1495 while ((conn = STAILQ_FIRST(&closed_conns))) { 1496 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 1497 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 1498 } 1499 1500} 1501 1502 1503static void 1504process_connections (lsquic_engine_t *engine, conn_iter_f next_conn) 1505{ 1506 lsquic_conn_t *conn; 1507 enum tick_st tick_st; 1508 lsquic_time_t now = lsquic_time_now(); 1509 struct closed_conns closed_conns; 1510 1511 engine->proc_time = now; 1512 eng_hist_tick(&engine->history, now); 1513 1514 STAILQ_INIT(&closed_conns); 1515 reset_deadline(engine, now); 1516 1517 while ((conn = next_conn(engine))) 1518 { 1519 tick_st = conn->cn_if->ci_tick(conn, now); 1520 if (conn_iter_next_rw_pend == next_conn) 1521 update_pend_rw_progress(engine, conn, tick_st & TICK_PROGRESS); 1522 if (tick_st & TICK_SEND) 1523 { 1524 if (!(conn->cn_flags & LSCONN_HAS_OUTGOING)) 1525 { 1526 oh_insert(&engine->conns_out, conn); 1527 engine_incref_conn(conn, LSCONN_HAS_OUTGOING); 1528 } 1529 } 1530 if (tick_st & TICK_CLOSE) 1531 { 1532 STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn); 1533 engine_incref_conn(conn, LSCONN_CLOSING); 1534 if (conn->cn_flags & LSCONN_HASHED) 1535 remove_conn_from_hash(engine, conn); 1536 } 1537 } 1538 1539 if (lsquic_engine_has_unsent_packets(engine)) 1540 send_packets_out(engine, &closed_conns); 1541 1542 while ((conn = STAILQ_FIRST(&closed_conns))) { 1543 STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn); 1544 (void) engine_decref_conn(engine, conn, LSCONN_CLOSING); 1545 } 1546 1547} 1548 1549 1550/* Return 0 if packet is being processed by a real connection, 1 if the 1551 * packet was processed, but not by a connection, and -1 on error. 1552 */ 1553int 1554lsquic_engine_packet_in (lsquic_engine_t *engine, 1555 const unsigned char *packet_in_data, size_t packet_in_size, 1556 const struct sockaddr *sa_local, const struct sockaddr *sa_peer, 1557 void *peer_ctx) 1558{ 1559 struct packin_parse_state ppstate; 1560 lsquic_packet_in_t *packet_in; 1561 1562 if (packet_in_size > QUIC_MAX_PACKET_SZ) 1563 { 1564 LSQ_DEBUG("Cannot handle packet_in_size(%zd) > %d packet incoming " 1565 "packet's header", packet_in_size, QUIC_MAX_PACKET_SZ); 1566 errno = E2BIG; 1567 return -1; 1568 } 1569 1570 packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm); 1571 if (!packet_in) 1572 return -1; 1573 1574 /* Library does not modify packet_in_data, it is not referenced after 1575 * this function returns and subsequent release of pi_data is guarded 1576 * by PI_OWN_DATA flag. 1577 */ 1578 packet_in->pi_data = (unsigned char *) packet_in_data; 1579 if (0 != parse_packet_in_begin(packet_in, packet_in_size, 1580 engine->flags & ENG_SERVER, &ppstate)) 1581 { 1582 LSQ_DEBUG("Cannot parse incoming packet's header"); 1583 lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in); 1584 errno = EINVAL; 1585 return -1; 1586 } 1587 1588 packet_in->pi_received = lsquic_time_now(); 1589 eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in); 1590 return process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer, 1591 peer_ctx); 1592} 1593 1594 1595#if __GNUC__ && !defined(NDEBUG) 1596__attribute__((weak)) 1597#endif 1598unsigned 1599lsquic_engine_quic_versions (const lsquic_engine_t *engine) 1600{ 1601 return engine->pub.enp_settings.es_versions; 1602} 1603 1604 1605int 1606lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff) 1607{ 1608 const lsquic_time_t *next_time; 1609 lsquic_time_t now; 1610 1611 next_time = attq_next_time(engine->attq); 1612 if (!next_time) 1613 return 0; 1614 1615 now = lsquic_time_now(); 1616 *diff = (int) ((int64_t) *next_time - (int64_t) now); 1617 return 1; 1618} 1619 1620 1621unsigned 1622lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now) 1623{ 1624 lsquic_time_t now; 1625 now = lsquic_time_now(); 1626 if (from_now < 0) 1627 now -= from_now; 1628 else 1629 now += from_now; 1630 return attq_count_before(engine->attq, now); 1631} 1632 1633 1634