lsquic_engine.c revision feca77f5
1/* Copyright (c) 2017 - 2020 LiteSpeed Technologies Inc.  See LICENSE. */
2/*
3 * lsquic_engine.c - QUIC engine
4 */
5
6#include <assert.h>
7#include <errno.h>
8#include <inttypes.h>
9#include <limits.h>
10#include <stdint.h>
11#include <stdio.h>
12#include <stdlib.h>
13#include <string.h>
14#include <sys/queue.h>
15#include <time.h>
16#ifndef WIN32
17#include <sys/time.h>
18#include <netinet/in.h>
19#include <sys/types.h>
20#include <sys/stat.h>
21#include <fcntl.h>
22#include <unistd.h>
23#include <netdb.h>
24#endif
25
26#ifndef NDEBUG
27#include <sys/types.h>
28#include <regex.h>      /* For code that loses packets */
29#endif
30
31#if LOG_PACKET_CHECKSUM
32#include <zlib.h>
33#endif
34
35#include <openssl/aead.h>
36
37#include "lsquic.h"
38#include "lsquic_types.h"
39#include "lsquic_int_types.h"
40#include "lsquic_sizes.h"
41#include "lsquic_parse_common.h"
42#include "lsquic_parse.h"
43#include "lsquic_packet_in.h"
44#include "lsquic_packet_out.h"
45#include "lsquic_senhist.h"
46#include "lsquic_rtt.h"
47#include "lsquic_cubic.h"
48#include "lsquic_pacer.h"
49#include "lsquic_bw_sampler.h"
50#include "lsquic_minmax.h"
51#include "lsquic_bbr.h"
52#include "lsquic_send_ctl.h"
53#include "lsquic_set.h"
54#include "lsquic_conn_flow.h"
55#include "lsquic_sfcw.h"
56#include "lsquic_hash.h"
57#include "lsquic_conn.h"
58#include "lsquic_full_conn.h"
59#include "lsquic_util.h"
60#include "lsquic_qtags.h"
61#include "lsquic_enc_sess.h"
62#include "lsquic_mm.h"
63#include "lsquic_engine_public.h"
64#include "lsquic_eng_hist.h"
65#include "lsquic_ev_log.h"
66#include "lsquic_version.h"
67#include "lsquic_pr_queue.h"
68#include "lsquic_mini_conn.h"
69#include "lsquic_mini_conn_ietf.h"
70#include "lsquic_stock_shi.h"
71#include "lsquic_purga.h"
72#include "lsquic_tokgen.h"
73#include "lsquic_attq.h"
74#include "lsquic_min_heap.h"
75#include "lsquic_http1x_if.h"
76#include "lsquic_handshake.h"
77#include "lsquic_crand.h"
78#include "lsquic_ietf.h"
79
80#define LSQUIC_LOGGER_MODULE LSQLM_ENGINE
81#include "lsquic_logger.h"
82
83#ifndef LSQUIC_DEBUG_NEXT_ADV_TICK
84#define LSQUIC_DEBUG_NEXT_ADV_TICK 1
85#endif
86
87#if LSQUIC_DEBUG_NEXT_ADV_TICK
88#include "lsquic_alarmset.h"
89#endif
90
91#define MIN(a, b) ((a) < (b) ? (a) : (b))
92
93/* The batch of outgoing packets grows and shrinks dynamically */
94#define MAX_OUT_BATCH_SIZE 1024
95#define MIN_OUT_BATCH_SIZE 4
96#define INITIAL_OUT_BATCH_SIZE 32
97
98struct out_batch
99{
100    lsquic_conn_t           *conns  [MAX_OUT_BATCH_SIZE];
101    struct lsquic_out_spec   outs   [MAX_OUT_BATCH_SIZE];
102    unsigned                 pack_off[MAX_OUT_BATCH_SIZE];
103    lsquic_packet_out_t     *packets[MAX_OUT_BATCH_SIZE * 2];
104    struct iovec             iov    [MAX_OUT_BATCH_SIZE * 2];
105};
106
107typedef struct lsquic_conn * (*conn_iter_f)(struct lsquic_engine *);
108
109static void
110process_connections (struct lsquic_engine *engine, conn_iter_f iter,
111                     lsquic_time_t now);
112
113static void
114engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag);
115
116static lsquic_conn_t *
117engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn,
118                                        enum lsquic_conn_flags flag);
119
120static void
121force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn);
122
123#if LSQUIC_COUNT_ENGINE_CALLS
124#define ENGINE_CALLS_INCR(e) do { ++(e)->n_engine_calls; } while (0)
125#else
126#define ENGINE_CALLS_INCR(e)
127#endif
128
129/* Nested calls to LSQUIC are not supported */
130#define ENGINE_IN(e) do {                               \
131    assert(!((e)->pub.enp_flags & ENPUB_PROC));         \
132    (e)->pub.enp_flags |= ENPUB_PROC;                   \
133    ENGINE_CALLS_INCR(e);                               \
134} while (0)
135
136#define ENGINE_OUT(e) do {                              \
137    assert((e)->pub.enp_flags & ENPUB_PROC);            \
138    (e)->pub.enp_flags &= ~ENPUB_PROC;                  \
139} while (0)
140
141/* A connection can be referenced from one of six places:
142 *
143 *   1. A hash is used to find connections in order to dispatch an incoming
144 *      packet.  Connections can be hashed by CIDs or by address.  In the
145 *      former case, each connection has one or more mappings in the hash
146 *      table.  IETF QUIC connections have up to eight (in our implementation)
147 *      source CIDs and each of those would have a mapping.  In client mode,
148 *      depending on QUIC versions and options selected, it is may be
149 *      necessary to hash connections by address, in which case incoming
150 *      packets are delivered to connections based on the address.
151 *
152 *   2. Outgoing queue.
153 *
154 *   3. Tickable queue
155 *
156 *   4. Advisory Tick Time queue.
157 *
158 *   5. Closing connections queue.  This is a transient queue -- it only
159 *      exists for the duration of process_connections() function call.
160 *
161 *   6. Ticked connections queue.  Another transient queue, similar to (5).
162 *
163 * The idea is to destroy the connection when it is no longer referenced.
164 * For example, a connection tick may return TICK_SEND|TICK_CLOSE.  In
165 * that case, the connection is referenced from two places: (2) and (5).
166 * After its packets are sent, it is only referenced in (5), and at the
167 * end of the function call, when it is removed from (5), reference count
168 * goes to zero and the connection is destroyed.  If not all packets can
169 * be sent, at the end of the function call, the connection is referenced
170 * by (2) and will only be removed once all outgoing packets have been
171 * sent.
172 */
173#define CONN_REF_FLAGS  (LSCONN_HASHED          \
174                        |LSCONN_HAS_OUTGOING    \
175                        |LSCONN_TICKABLE        \
176                        |LSCONN_TICKED          \
177                        |LSCONN_CLOSING         \
178                        |LSCONN_ATTQ)
179
180
181
182
183struct cid_update_batch
184{
185    lsquic_cids_update_f    cub_update_cids;
186    void                   *cub_update_ctx;
187    unsigned                cub_count;
188    lsquic_cid_t            cub_cids[20];
189    void                   *cub_peer_ctxs[20];
190};
191
192static void
193cub_init (struct cid_update_batch *, lsquic_cids_update_f, void *);
194
195
196struct lsquic_engine
197{
198    struct lsquic_engine_public        pub;
199    enum {
200        ENG_SERVER      = LSENG_SERVER,
201        ENG_HTTP        = LSENG_HTTP,
202        ENG_COOLDOWN    = (1 <<  7),    /* Cooldown: no new connections */
203        ENG_PAST_DEADLINE
204                        = (1 <<  8),    /* Previous call to a processing
205                                         * function went past time threshold.
206                                         */
207        ENG_CONNS_BY_ADDR
208                        = (1 <<  9),    /* Connections are hashed by address */
209#ifndef NDEBUG
210        ENG_COALESCE    = (1 << 24),    /* Packet coalescing is enabled */
211        ENG_LOSE_PACKETS= (1 << 25),    /* Lose *some* outgoing packets */
212        ENG_DTOR        = (1 << 26),    /* Engine destructor */
213#endif
214    }                                  flags;
215    lsquic_packets_out_f               packets_out;
216    void                              *packets_out_ctx;
217    lsquic_cids_update_f               report_new_scids;
218    lsquic_cids_update_f               report_live_scids;
219    lsquic_cids_update_f               report_old_scids;
220    void                              *scids_ctx;
221    struct lsquic_hash                *conns_hash;
222    struct min_heap                    conns_tickable;
223    struct min_heap                    conns_out;
224    struct eng_hist                    history;
225    unsigned                           batch_size;
226    struct pr_queue                   *pr_queue;
227    struct attq                       *attq;
228    /* Track time last time a packet was sent to give new connections
229     * priority lower than that of existing connections.
230     */
231    lsquic_time_t                      last_sent;
232#ifndef NDEBUG
233    regex_t                            lose_packets_re;
234    const char                        *lose_packets_str;
235#endif
236    unsigned                           n_conns;
237    lsquic_time_t                      deadline;
238    lsquic_time_t                      resume_sending_at;
239    unsigned                           mini_conns_count;
240    struct lsquic_purga               *purga;
241#if LSQUIC_CONN_STATS
242    struct {
243        unsigned                conns;
244    }                                  stats;
245    struct conn_stats                  conn_stats_sum;
246    FILE                              *stats_fh;
247#endif
248    struct cid_update_batch            new_scids;
249    struct out_batch                   out_batch;
250#if LSQUIC_COUNT_ENGINE_CALLS
251    unsigned long                      n_engine_calls;
252#endif
253#if LSQUIC_DEBUG_NEXT_ADV_TICK
254    uintptr_t                          last_logged_conn;
255    unsigned                           last_logged_ae_why;
256    int                                last_tick_diff;
257#endif
258    struct crand                       crand;
259    EVP_AEAD_CTX                       retry_aead_ctx;
260};
261
262
263void
264lsquic_engine_init_settings (struct lsquic_engine_settings *settings,
265                             unsigned flags)
266{
267    memset(settings, 0, sizeof(*settings));
268    settings->es_versions        = LSQUIC_DF_VERSIONS;
269    if (flags & ENG_SERVER)
270    {
271        settings->es_cfcw        = LSQUIC_DF_CFCW_SERVER;
272        settings->es_sfcw        = LSQUIC_DF_SFCW_SERVER;
273        settings->es_init_max_data
274                                 = LSQUIC_DF_INIT_MAX_DATA_SERVER;
275        settings->es_init_max_stream_data_bidi_remote
276                         = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_SERVER;
277        settings->es_init_max_stream_data_bidi_local
278                         = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_SERVER;
279        settings->es_init_max_stream_data_uni
280                         = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_SERVER;
281        settings->es_init_max_streams_uni
282                         = LSQUIC_DF_INIT_MAX_STREAMS_UNI_SERVER;
283        settings->es_ping_period = 0;
284    }
285    else
286    {
287        settings->es_cfcw        = LSQUIC_DF_CFCW_CLIENT;
288        settings->es_sfcw        = LSQUIC_DF_SFCW_CLIENT;
289        settings->es_init_max_data
290                                 = LSQUIC_DF_INIT_MAX_DATA_CLIENT;
291        settings->es_init_max_stream_data_bidi_remote
292                         = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_REMOTE_CLIENT;
293        settings->es_init_max_stream_data_bidi_local
294                         = LSQUIC_DF_INIT_MAX_STREAM_DATA_BIDI_LOCAL_CLIENT;
295        settings->es_init_max_stream_data_uni
296                         = LSQUIC_DF_INIT_MAX_STREAM_DATA_UNI_CLIENT;
297        settings->es_init_max_streams_uni
298                         = LSQUIC_DF_INIT_MAX_STREAMS_UNI_CLIENT;
299        settings->es_ping_period = LSQUIC_DF_PING_PERIOD;
300    }
301    settings->es_max_streams_in  = LSQUIC_DF_MAX_STREAMS_IN;
302    settings->es_idle_conn_to    = LSQUIC_DF_IDLE_CONN_TO;
303    settings->es_idle_timeout    = LSQUIC_DF_IDLE_TIMEOUT;
304    settings->es_handshake_to    = LSQUIC_DF_HANDSHAKE_TO;
305    settings->es_silent_close    = LSQUIC_DF_SILENT_CLOSE;
306    settings->es_max_header_list_size
307                                 = LSQUIC_DF_MAX_HEADER_LIST_SIZE;
308    settings->es_ua              = LSQUIC_DF_UA;
309    settings->es_ecn             = LSQUIC_DF_ECN;
310
311    settings->es_pdmd            = QTAG_X509;
312    settings->es_aead            = QTAG_AESG;
313    settings->es_kexs            = QTAG_C255;
314    settings->es_support_push    = LSQUIC_DF_SUPPORT_PUSH;
315    settings->es_support_tcid0   = LSQUIC_DF_SUPPORT_TCID0;
316    settings->es_support_nstp    = LSQUIC_DF_SUPPORT_NSTP;
317    settings->es_honor_prst      = LSQUIC_DF_HONOR_PRST;
318    settings->es_progress_check  = LSQUIC_DF_PROGRESS_CHECK;
319    settings->es_rw_once         = LSQUIC_DF_RW_ONCE;
320    settings->es_proc_time_thresh= LSQUIC_DF_PROC_TIME_THRESH;
321    settings->es_pace_packets    = LSQUIC_DF_PACE_PACKETS;
322    settings->es_clock_granularity = LSQUIC_DF_CLOCK_GRANULARITY;
323    settings->es_max_inchoate    = LSQUIC_DF_MAX_INCHOATE;
324    settings->es_send_prst       = LSQUIC_DF_SEND_PRST;
325    settings->es_sttl            = LSQUIC_DF_STTL;
326    settings->es_init_max_streams_bidi
327                                 = LSQUIC_DF_INIT_MAX_STREAMS_BIDI;
328    settings->es_scid_len        = LSQUIC_DF_SCID_LEN;
329    settings->es_scid_iss_rate = LSQUIC_DF_SCID_ISS_RATE;
330    settings->es_qpack_dec_max_size = LSQUIC_DF_QPACK_DEC_MAX_SIZE;
331    settings->es_qpack_dec_max_blocked = LSQUIC_DF_QPACK_DEC_MAX_BLOCKED;
332    settings->es_qpack_enc_max_size = LSQUIC_DF_QPACK_ENC_MAX_SIZE;
333    settings->es_qpack_enc_max_blocked = LSQUIC_DF_QPACK_ENC_MAX_BLOCKED;
334    settings->es_allow_migration = LSQUIC_DF_ALLOW_MIGRATION;
335    settings->es_ql_bits         = LSQUIC_DF_QL_BITS;
336    settings->es_spin            = LSQUIC_DF_SPIN;
337    settings->es_delayed_acks    = LSQUIC_DF_DELAYED_ACKS;
338}
339
340
341/* Note: if returning an error, err_buf must be valid if non-NULL */
342int
343lsquic_engine_check_settings (const struct lsquic_engine_settings *settings,
344                              unsigned flags,
345                              char *err_buf, size_t err_buf_sz)
346{
347    if (settings->es_cfcw < LSQUIC_MIN_FCW ||
348        settings->es_sfcw < LSQUIC_MIN_FCW)
349    {
350        if (err_buf)
351            snprintf(err_buf, err_buf_sz, "%s",
352                                            "flow control window set too low");
353        return -1;
354    }
355    if (0 == (settings->es_versions & LSQUIC_SUPPORTED_VERSIONS))
356    {
357        if (err_buf)
358            snprintf(err_buf, err_buf_sz, "%s",
359                        "No supported QUIC versions specified");
360        return -1;
361    }
362    if (settings->es_versions & ~LSQUIC_SUPPORTED_VERSIONS)
363    {
364        if (err_buf)
365            snprintf(err_buf, err_buf_sz, "%s",
366                        "one or more unsupported QUIC version is specified");
367        return -1;
368    }
369    if (flags & ENG_SERVER)
370    {
371        if (settings->es_handshake_to >
372                                    MAX_MINI_CONN_LIFESPAN_IN_USEC)
373        {
374            if (err_buf)
375                snprintf(err_buf, err_buf_sz, "handshake timeout %lu"
376                    " usec is too large.  The maximum for server is %u usec",
377                    settings->es_handshake_to, MAX_MINI_CONN_LIFESPAN_IN_USEC);
378            return -1;
379        }
380    }
381    if (settings->es_idle_timeout > 600)
382    {
383        if (err_buf)
384            snprintf(err_buf, err_buf_sz, "%s",
385                        "The maximum value of idle timeout is 600 seconds");
386        return -1;
387    }
388    if (settings->es_scid_len > MAX_CID_LEN)
389    {
390        if (err_buf)
391            snprintf(err_buf, err_buf_sz, "Source connection ID cannot be %u "
392                        "bytes long; it must be between 0 and %u.",
393                        settings->es_scid_len, MAX_CID_LEN);
394        return -1;
395    }
396
397    if (settings->es_cc_algo > 2)
398    {
399        if (err_buf)
400            snprintf(err_buf, err_buf_sz, "Invalid congestion control "
401                "algorithm value %u", settings->es_cc_algo);
402        return -1;
403    }
404
405    if (!(settings->es_ql_bits >= 0 && settings->es_ql_bits <= 2))
406    {
407        if (err_buf)
408            snprintf(err_buf, err_buf_sz, "Invalid QL bits value %d ",
409                settings->es_ql_bits);
410        return -1;
411    }
412
413    if (!(settings->es_spin == 0 || settings->es_spin == 1))
414    {
415        if (err_buf)
416            snprintf(err_buf, err_buf_sz, "Invalid spin value %d",
417                settings->es_spin);
418        return -1;
419    }
420
421    return 0;
422}
423
424
425static void
426free_packet (void *ctx, void *conn_ctx, void *packet_data, char is_ipv6)
427{
428    free(packet_data);
429}
430
431
432static void *
433malloc_buf (void *ctx, void *conn_ctx, unsigned short size, char is_ipv6)
434{
435    return malloc(size);
436}
437
438
439static const struct lsquic_packout_mem_if stock_pmi =
440{
441    malloc_buf, free_packet, free_packet,
442};
443
444
445static int
446hash_conns_by_addr (const struct lsquic_engine *engine)
447{
448    if (engine->flags & ENG_SERVER)
449        return 0;
450    if (engine->pub.enp_settings.es_versions & LSQUIC_FORCED_TCID0_VERSIONS)
451        return 1;
452    if ((engine->pub.enp_settings.es_versions & LSQUIC_GQUIC_HEADER_VERSIONS)
453                                && engine->pub.enp_settings.es_support_tcid0)
454        return 1;
455    if (engine->pub.enp_settings.es_scid_len == 0)
456        return 1;
457    return 0;
458}
459
460
461lsquic_engine_t *
462lsquic_engine_new (unsigned flags,
463                   const struct lsquic_engine_api *api)
464{
465    lsquic_engine_t *engine;
466    char err_buf[100];
467
468    if (!api->ea_packets_out)
469    {
470        LSQ_ERROR("packets_out callback is not specified");
471        return NULL;
472    }
473
474    if (!api->ea_stream_if)
475    {
476        LSQ_ERROR("stream interface is not specified");
477        return NULL;
478    }
479
480    if (api->ea_settings &&
481                0 != lsquic_engine_check_settings(api->ea_settings, flags,
482                                                    err_buf, sizeof(err_buf)))
483    {
484        LSQ_ERROR("cannot create engine: %s", err_buf);
485        return NULL;
486    }
487
488    engine = calloc(1, sizeof(*engine));
489    if (!engine)
490        return NULL;
491    if (0 != lsquic_mm_init(&engine->pub.enp_mm))
492    {
493        free(engine);
494        return NULL;
495    }
496    if (api->ea_settings)
497        engine->pub.enp_settings        = *api->ea_settings;
498    else
499        lsquic_engine_init_settings(&engine->pub.enp_settings, flags);
500    int tag_buf_len;
501    tag_buf_len = lsquic_gen_ver_tags(engine->pub.enp_ver_tags_buf,
502                                    sizeof(engine->pub.enp_ver_tags_buf),
503                                    engine->pub.enp_settings.es_versions);
504    if (tag_buf_len <= 0)
505    {
506        LSQ_ERROR("cannot generate version tags buffer");
507        free(engine);
508        return NULL;
509    }
510    engine->pub.enp_ver_tags_len = tag_buf_len;
511    engine->pub.enp_flags = ENPUB_CAN_SEND;
512    engine->pub.enp_stream_if       = api->ea_stream_if;
513    engine->pub.enp_stream_if_ctx   = api->ea_stream_if_ctx;
514
515    engine->flags           = flags;
516#ifndef NDEBUG
517    engine->flags          |= ENG_COALESCE;
518#endif
519    engine->packets_out     = api->ea_packets_out;
520    engine->packets_out_ctx = api->ea_packets_out_ctx;
521    engine->report_new_scids  = api->ea_new_scids;
522    engine->report_live_scids = api->ea_live_scids;
523    engine->report_old_scids  = api->ea_old_scids;
524    engine->scids_ctx         = api->ea_cids_update_ctx;
525    cub_init(&engine->new_scids, engine->report_new_scids, engine->scids_ctx);
526    engine->pub.enp_lookup_cert  = api->ea_lookup_cert;
527    engine->pub.enp_cert_lu_ctx  = api->ea_cert_lu_ctx;
528    engine->pub.enp_get_ssl_ctx  = api->ea_get_ssl_ctx;
529    if (api->ea_shi)
530    {
531        engine->pub.enp_shi      = api->ea_shi;
532        engine->pub.enp_shi_ctx  = api->ea_shi_ctx;
533    }
534    else
535    {
536        engine->pub.enp_shi      = &stock_shi;
537        engine->pub.enp_shi_ctx  = stock_shared_hash_new();
538        if (!engine->pub.enp_shi_ctx)
539        {
540            free(engine);
541            return NULL;
542        }
543    }
544    if (api->ea_hsi_if)
545    {
546        engine->pub.enp_hsi_if  = api->ea_hsi_if;
547        engine->pub.enp_hsi_ctx = api->ea_hsi_ctx;
548    }
549    else
550    {
551        engine->pub.enp_hsi_if  = lsquic_http1x_if;
552        engine->pub.enp_hsi_ctx = NULL;
553    }
554    if (api->ea_pmi)
555    {
556        engine->pub.enp_pmi      = api->ea_pmi;
557        engine->pub.enp_pmi_ctx  = api->ea_pmi_ctx;
558    }
559    else
560    {
561        engine->pub.enp_pmi      = &stock_pmi;
562        engine->pub.enp_pmi_ctx  = NULL;
563    }
564    engine->pub.enp_verify_cert  = api->ea_verify_cert;
565    engine->pub.enp_verify_ctx   = api->ea_verify_ctx;
566    engine->pub.enp_kli          = api->ea_keylog_if;
567    engine->pub.enp_kli_ctx      = api->ea_keylog_ctx;
568    engine->pub.enp_engine = engine;
569    if (hash_conns_by_addr(engine))
570        engine->flags |= ENG_CONNS_BY_ADDR;
571    engine->conns_hash = lsquic_hash_create();
572    engine->pub.enp_tokgen = lsquic_tg_new(&engine->pub);
573    if (!engine->pub.enp_tokgen)
574        return NULL;
575    engine->pub.enp_crand = &engine->crand;
576    if (flags & ENG_SERVER)
577    {
578        engine->pr_queue = prq_create(
579            10000 /* TODO: make configurable */, MAX_OUT_BATCH_SIZE,
580            &engine->pub);
581        if (!engine->pr_queue)
582        {
583            lsquic_tg_destroy(engine->pub.enp_tokgen);
584            return NULL;
585        }
586        engine->purga = lsquic_purga_new(30 * 1000 * 1000,
587                            engine->report_old_scids, engine->scids_ctx);
588        if (!engine->purga)
589        {
590            lsquic_tg_destroy(engine->pub.enp_tokgen);
591            prq_destroy(engine->pr_queue);
592            return NULL;
593        }
594    }
595    engine->attq = attq_create();
596    eng_hist_init(&engine->history);
597    engine->batch_size = INITIAL_OUT_BATCH_SIZE;
598    if (engine->pub.enp_settings.es_honor_prst)
599    {
600        engine->pub.enp_srst_hash = lsquic_hash_create();
601        if (!engine->pub.enp_srst_hash)
602        {
603            lsquic_engine_destroy(engine);
604            return NULL;
605        }
606    }
607
608#ifndef NDEBUG
609    {
610        const char *env;
611        env = getenv("LSQUIC_LOSE_PACKETS_RE");
612        if (env)
613        {
614            if (0 != regcomp(&engine->lose_packets_re, env,
615                                                    REG_EXTENDED|REG_NOSUB))
616            {
617                LSQ_ERROR("could not compile lost packet regex `%s'", env);
618                return NULL;
619            }
620            engine->flags |= ENG_LOSE_PACKETS;
621            engine->lose_packets_str = env;
622            LSQ_WARN("will lose packets that match the following regex: %s",
623                                                                        env);
624        }
625        env = getenv("LSQUIC_COALESCE");
626        if (env)
627        {
628            engine->flags &= ~ENG_COALESCE;
629            if (atoi(env))
630            {
631                engine->flags |= ENG_COALESCE;
632                LSQ_NOTICE("will coalesce packets");
633            }
634            else
635                LSQ_NOTICE("will not coalesce packets");
636        }
637    }
638#endif
639#if LSQUIC_CONN_STATS
640    engine->stats_fh = api->ea_stats_fh;
641#endif
642    if (1 != EVP_AEAD_CTX_init(&engine->retry_aead_ctx, EVP_aead_aes_128_gcm(),
643                            IETF_RETRY_KEY_BUF, IETF_RETRY_KEY_SZ, 16, NULL))
644    {
645        LSQ_ERROR("could not initialize retry AEAD ctx");
646        lsquic_engine_destroy(engine);
647        return NULL;
648    }
649    engine->pub.enp_retry_aead_ctx = &engine->retry_aead_ctx;
650
651    LSQ_INFO("instantiated engine");
652    return engine;
653}
654
655
656#if LOG_PACKET_CHECKSUM
657static void
658log_packet_checksum (const lsquic_cid_t *cid, const char *direction,
659                     const unsigned char *buf, size_t bufsz)
660{
661    EV_LOG_CONN_EVENT(cid, "packet %s checksum: %08X", direction,
662                                        (uint32_t) crc32(0, buf, bufsz));
663}
664
665
666#endif
667
668
669static void
670grow_batch_size (struct lsquic_engine *engine)
671{
672    engine->batch_size <<= engine->batch_size < MAX_OUT_BATCH_SIZE;
673}
674
675
676static void
677shrink_batch_size (struct lsquic_engine *engine)
678{
679    engine->batch_size >>= engine->batch_size > MIN_OUT_BATCH_SIZE;
680}
681
682
683struct cce_cid_iter
684{
685    const struct lsquic_conn   *conn;
686    unsigned                    todo, n;
687};
688
689
690static struct conn_cid_elem *
691cce_iter_next (struct cce_cid_iter *citer)
692{
693    struct conn_cid_elem *cce;
694
695    while (citer->todo)
696        if (citer->todo & (1 << citer->n))
697        {
698            citer->todo &= ~(1 << citer->n);
699            cce = &citer->conn->cn_cces[ citer->n++ ];
700            if (!(cce->cce_flags & CCE_PORT))
701                return cce;
702        }
703        else
704            ++citer->n;
705
706    return NULL;
707}
708
709
710static struct conn_cid_elem *
711cce_iter_first (struct cce_cid_iter *citer, const struct lsquic_conn *conn)
712{
713    citer->conn = conn;
714    citer->todo = conn->cn_cces_mask;
715    citer->n    = 0;
716    return cce_iter_next(citer);
717}
718
719
720#if LSQUIC_CONN_STATS
721void
722update_stats_sum (struct lsquic_engine *engine, struct lsquic_conn *conn)
723{
724    unsigned long *const dst = (unsigned long *) &engine->conn_stats_sum;
725    const unsigned long *src;
726    const struct conn_stats *stats;
727    unsigned i;
728
729    if (conn->cn_if->ci_get_stats && (stats = conn->cn_if->ci_get_stats(conn)))
730    {
731        ++engine->stats.conns;
732        src = (unsigned long *) stats;
733        for (i = 0; i < sizeof(*stats) / sizeof(unsigned long); ++i)
734            dst[i] += src[i];
735    }
736}
737
738
739#endif
740
741
742/* Wrapper to make sure important things occur before the connection is
743 * really destroyed.
744 */
745static void
746destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn,
747                                                            lsquic_time_t now)
748{
749    struct cce_cid_iter citer;
750    const struct conn_cid_elem *cce;
751    lsquic_time_t drain_time;
752    struct purga_el *puel;
753
754    engine->mini_conns_count -= !!(conn->cn_flags & LSCONN_MINI);
755    if (engine->purga
756        /* Blacklist all CIDs except for promoted mini connections */
757            && (conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED))
758                                        != (LSCONN_MINI|LSCONN_PROMOTED))
759    {
760        if (!(conn->cn_flags & LSCONN_IMMED_CLOSE)
761            && conn->cn_if->ci_drain_time &&
762            (drain_time = conn->cn_if->ci_drain_time(conn), drain_time))
763        {
764            for (cce = cce_iter_first(&citer, conn); cce;
765                                                cce = cce_iter_next(&citer))
766            {
767                puel = lsquic_purga_add(engine->purga, &cce->cce_cid,
768                                    lsquic_conn_get_peer_ctx(conn, NULL),
769                                    PUTY_CONN_DRAIN, now);
770                if (puel)
771                    puel->puel_time = now + drain_time;
772            }
773        }
774        else
775        {
776            for (cce = cce_iter_first(&citer, conn); cce;
777                                                cce = cce_iter_next(&citer))
778            {
779                puel = lsquic_purga_add(engine->purga, &cce->cce_cid,
780                                    lsquic_conn_get_peer_ctx(conn, NULL),
781                                    PUTY_CONN_DELETED, now);
782                if (puel)
783                {
784                    puel->puel_time = now;
785                    puel->puel_count = 0;
786                }
787            }
788        }
789    }
790#if LSQUIC_CONN_STATS
791    update_stats_sum(engine, conn);
792#endif
793    --engine->n_conns;
794    conn->cn_flags |= LSCONN_NEVER_TICKABLE;
795    conn->cn_if->ci_destroy(conn);
796}
797
798
799static int
800maybe_grow_conn_heaps (struct lsquic_engine *engine)
801{
802    struct min_heap_elem *els;
803    unsigned count;
804
805    if (engine->n_conns < lsquic_mh_nalloc(&engine->conns_tickable))
806        return 0;   /* Nothing to do */
807
808    if (lsquic_mh_nalloc(&engine->conns_tickable))
809        count = lsquic_mh_nalloc(&engine->conns_tickable) * 2 * 2;
810    else
811        count = 8;
812
813    els = malloc(sizeof(els[0]) * count);
814    if (!els)
815    {
816        LSQ_ERROR("%s: malloc failed", __func__);
817        return -1;
818    }
819
820    LSQ_DEBUG("grew heaps to %u elements", count / 2);
821    memcpy(&els[0], engine->conns_tickable.mh_elems,
822                sizeof(els[0]) * lsquic_mh_count(&engine->conns_tickable));
823    memcpy(&els[count / 2], engine->conns_out.mh_elems,
824                sizeof(els[0]) * lsquic_mh_count(&engine->conns_out));
825    free(engine->conns_tickable.mh_elems);
826    engine->conns_tickable.mh_elems = els;
827    engine->conns_out.mh_elems = &els[count / 2];
828    engine->conns_tickable.mh_nalloc = count / 2;
829    engine->conns_out.mh_nalloc = count / 2;
830    return 0;
831}
832
833
834static void
835remove_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn,
836                                                                unsigned todo)
837{
838    unsigned n;
839
840    for (n = 0; todo; todo &= ~(1 << n++))
841        if ((todo & (1 << n)) &&
842                        (conn->cn_cces[n].cce_hash_el.qhe_flags & QHE_HASHED))
843            lsquic_hash_erase(hash, &conn->cn_cces[n].cce_hash_el);
844}
845
846
847static void
848remove_all_cces_from_hash (struct lsquic_hash *hash, struct lsquic_conn *conn)
849{
850    remove_cces_from_hash(hash, conn, conn->cn_cces_mask);
851}
852
853
854static void
855cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx);
856
857
858static int
859insert_conn_into_hash (struct lsquic_engine *engine, struct lsquic_conn *conn,
860                                                                void *peer_ctx)
861{
862    struct conn_cid_elem *cce;
863    unsigned todo, done, n;
864
865    for (todo = conn->cn_cces_mask, done = 0, n = 0; todo; todo &= ~(1 << n++))
866        if (todo & (1 << n))
867        {
868            cce = &conn->cn_cces[n];
869            assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED));
870            if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf,
871                                    cce->cce_cid.len, conn, &cce->cce_hash_el))
872                done |= 1 << n;
873            else
874                goto err;
875            if ((engine->flags & ENG_SERVER) && 0 == (cce->cce_flags & CCE_REG))
876            {
877                cce->cce_flags |= CCE_REG;
878                cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx);
879            }
880        }
881
882    return 0;
883
884  err:
885    remove_cces_from_hash(engine->conns_hash, conn, done);
886    return -1;
887}
888
889
890static lsquic_conn_t *
891new_full_conn_server (lsquic_engine_t *engine, lsquic_conn_t *mini_conn,
892                                                        lsquic_time_t now)
893{
894    const lsquic_cid_t *cid;
895    server_conn_ctor_f ctor;
896    lsquic_conn_t *conn;
897    unsigned flags;
898    if (0 != maybe_grow_conn_heaps(engine))
899        return NULL;
900    flags = engine->flags & (ENG_SERVER|ENG_HTTP);
901
902    if (mini_conn->cn_flags & LSCONN_IETF)
903    {
904        if (mini_conn->cn_version == LSQVER_ID24)
905            ctor = lsquic_id24_full_conn_server_new;
906        else
907            ctor = lsquic_ietf_full_conn_server_new;
908    }
909    else
910        ctor = lsquic_gquic_full_conn_server_new;
911
912    conn = ctor(&engine->pub, flags, mini_conn);
913    if (!conn)
914    {
915        /* Otherwise, full_conn_server_new prints its own warnings */
916        if (ENOMEM == errno)
917        {
918            cid = lsquic_conn_log_cid(mini_conn);
919            LSQ_WARNC("could not allocate full connection for %"CID_FMT": %s",
920                                               CID_BITS(cid), strerror(errno));
921        }
922        return NULL;
923    }
924    ++engine->n_conns;
925    if (0 != insert_conn_into_hash(engine, conn, lsquic_conn_get_peer_ctx(conn, NULL)))
926    {
927        cid = lsquic_conn_log_cid(conn);
928        LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy",
929            CID_BITS(cid));
930        destroy_conn(engine, conn, now);
931        return NULL;
932    }
933    assert(!(conn->cn_flags & CONN_REF_FLAGS));
934    conn->cn_flags |= LSCONN_HASHED;
935    return conn;
936}
937
938
939static enum
940{
941    VER_NOT_SPECIFIED,
942    VER_SUPPORTED,
943    VER_UNSUPPORTED,
944}
945
946
947version_matches (lsquic_engine_t *engine, const lsquic_packet_in_t *packet_in,
948                 enum lsquic_version *pversion)
949{
950    lsquic_ver_tag_t ver_tag;
951    enum lsquic_version version;
952
953    if (!packet_in->pi_quic_ver)
954    {
955        LSQ_DEBUG("packet does not specify version");
956        return VER_NOT_SPECIFIED;
957    }
958
959    memcpy(&ver_tag, packet_in->pi_data + packet_in->pi_quic_ver, sizeof(ver_tag));
960    version = lsquic_tag2ver(ver_tag);
961    if (version < N_LSQVER)
962    {
963        if (engine->pub.enp_settings.es_versions & (1 << version))
964        {
965            LSQ_DEBUG("client-supplied version %s is supported",
966                                                lsquic_ver2str[version]);
967            *pversion = version;
968            return VER_SUPPORTED;
969        }
970        else
971            LSQ_DEBUG("client-supplied version %s is not supported",
972                                                lsquic_ver2str[version]);
973    }
974    else
975        LSQ_DEBUG("client-supplied version tag 0x%08X is not recognized",
976                                                ver_tag);
977
978    return VER_UNSUPPORTED;
979}
980
981
982static void
983schedule_req_packet (struct lsquic_engine *engine, enum packet_req_type type,
984    const struct lsquic_packet_in *packet_in, const struct sockaddr *sa_local,
985    const struct sockaddr *sa_peer, void *peer_ctx)
986{
987    assert(engine->pr_queue);
988    if (0 == prq_new_req(engine->pr_queue, type, packet_in, peer_ctx,
989                                                            sa_local, sa_peer))
990        LSQ_DEBUGC("scheduled %s packet for cid %"CID_FMT,
991                    lsquic_preqt2str[type], CID_BITS(&packet_in->pi_conn_id));
992    else
993        LSQ_DEBUG("cannot schedule %s packet", lsquic_preqt2str[type]);
994}
995
996
997static unsigned short
998sa2port (const struct sockaddr *sa)
999{
1000    if (sa->sa_family == AF_INET)
1001    {
1002        struct sockaddr_in *const sa4 = (void *) sa;
1003        return sa4->sin_port;
1004    }
1005    else
1006    {
1007        struct sockaddr_in6 *const sa6 = (void *) sa;
1008        return sa6->sin6_port;
1009    }
1010}
1011
1012
1013static struct lsquic_hash_elem *
1014find_conn_by_addr (struct lsquic_hash *hash, const struct sockaddr *sa)
1015{
1016    unsigned short port;
1017
1018    port = sa2port(sa);
1019    return lsquic_hash_find(hash, &port, sizeof(port));
1020}
1021
1022
1023static lsquic_conn_t *
1024find_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in,
1025         struct packin_parse_state *ppstate, const struct sockaddr *sa_local)
1026{
1027    struct lsquic_hash_elem *el;
1028    lsquic_conn_t *conn;
1029
1030    if (engine->flags & ENG_CONNS_BY_ADDR)
1031        el = find_conn_by_addr(engine->conns_hash, sa_local);
1032    else if (packet_in->pi_flags & PI_CONN_ID)
1033        el = lsquic_hash_find(engine->conns_hash,
1034                    packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len);
1035    else
1036    {
1037        LSQ_DEBUG("packet header does not have connection ID: discarding");
1038        return NULL;
1039    }
1040
1041    if (!el)
1042        return NULL;
1043
1044    conn = lsquic_hashelem_getdata(el);
1045    conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate);
1046    if ((engine->flags & ENG_CONNS_BY_ADDR)
1047        && !(conn->cn_flags & LSCONN_IETF)
1048        && (packet_in->pi_flags & PI_CONN_ID)
1049        && !LSQUIC_CIDS_EQ(CN_SCID(conn), &packet_in->pi_conn_id))
1050    {
1051        LSQ_DEBUG("connection IDs do not match");
1052        return NULL;
1053    }
1054
1055    return conn;
1056}
1057
1058
1059static lsquic_conn_t *
1060find_or_create_conn (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in,
1061         struct packin_parse_state *ppstate, const struct sockaddr *sa_local,
1062         const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size)
1063{
1064    struct lsquic_hash_elem *el;
1065    struct purga_el *puel;
1066    lsquic_conn_t *conn;
1067
1068    if (!(packet_in->pi_flags & PI_CONN_ID))
1069    {
1070        LSQ_DEBUG("packet header does not have connection ID: discarding");
1071        return NULL;
1072    }
1073    el = lsquic_hash_find(engine->conns_hash,
1074                    packet_in->pi_conn_id.idbuf, packet_in->pi_conn_id.len);
1075
1076    if (el)
1077    {
1078        conn = lsquic_hashelem_getdata(el);
1079        conn->cn_pf->pf_parse_packet_in_finish(packet_in, ppstate);
1080        return conn;
1081    }
1082
1083    if (engine->flags & ENG_COOLDOWN)
1084    {   /* Do not create incoming connections during cooldown */
1085        LSQ_DEBUG("dropping inbound packet for unknown connection (cooldown)");
1086        return NULL;
1087    }
1088
1089    if (engine->mini_conns_count >= engine->pub.enp_settings.es_max_inchoate)
1090    {
1091        LSQ_DEBUG("reached limit of %u inchoate connections",
1092                                    engine->pub.enp_settings.es_max_inchoate);
1093        return NULL;
1094    }
1095
1096
1097    if (engine->purga
1098        && (puel = lsquic_purga_contains(engine->purga,
1099                                        &packet_in->pi_conn_id), puel))
1100    {
1101        switch (puel->puel_type)
1102        {
1103        case PUTY_CID_RETIRED:
1104            LSQ_DEBUGC("CID %"CID_FMT" was retired, ignore packet",
1105                                            CID_BITS(&packet_in->pi_conn_id));
1106            return NULL;
1107        case PUTY_CONN_DRAIN:
1108            LSQ_DEBUG("drain till: %"PRIu64"; now: %"PRIu64,
1109                puel->puel_time, packet_in->pi_received);
1110            if (puel->puel_time > packet_in->pi_received)
1111            {
1112                LSQ_DEBUGC("CID %"CID_FMT" is in drain state, ignore packet",
1113                                            CID_BITS(&packet_in->pi_conn_id));
1114                return NULL;
1115            }
1116            LSQ_DEBUGC("CID %"CID_FMT" goes from drain state to deleted",
1117                                            CID_BITS(&packet_in->pi_conn_id));
1118            puel->puel_type = PUTY_CONN_DELETED;
1119            puel->puel_count = 0;
1120            puel->puel_time = 0;
1121            /* fall-through */
1122        case PUTY_CONN_DELETED:
1123            LSQ_DEBUGC("Connection with CID %"CID_FMT" was deleted",
1124                                            CID_BITS(&packet_in->pi_conn_id));
1125            if (puel->puel_time < packet_in->pi_received)
1126            {
1127                puel->puel_time = packet_in->pi_received
1128                            /* Exponential back-off */
1129                            + 1000000ull * (1 << MIN(puel->puel_count, 4));
1130                ++puel->puel_count;
1131                goto maybe_send_prst;
1132            }
1133            return NULL;
1134        default:
1135            assert(0);
1136            return NULL;
1137        }
1138    }
1139
1140    if (engine->pub.enp_settings.es_send_prst
1141            && !(packet_in->pi_flags & PI_GQUIC)
1142            && HETY_NOT_SET == packet_in->pi_header_type)
1143        goto maybe_send_prst;
1144
1145    if (0 != maybe_grow_conn_heaps(engine))
1146        return NULL;
1147
1148    const struct parse_funcs *pf;
1149    enum lsquic_version version;
1150    switch (version_matches(engine, packet_in, &version))
1151    {
1152    case VER_UNSUPPORTED:
1153        if (engine->flags & ENG_SERVER)
1154            schedule_req_packet(engine, PACKET_REQ_VERNEG, packet_in,
1155                                                sa_local, sa_peer, peer_ctx);
1156        return NULL;
1157    case VER_NOT_SPECIFIED:
1158  maybe_send_prst:
1159        if ((engine->flags & ENG_SERVER) &&
1160                                        engine->pub.enp_settings.es_send_prst)
1161            schedule_req_packet(engine, PACKET_REQ_PUBRES, packet_in,
1162                                                sa_local, sa_peer, peer_ctx);
1163        return NULL;
1164    case VER_SUPPORTED:
1165        pf = select_pf_by_ver(version);
1166        pf->pf_parse_packet_in_finish(packet_in, ppstate);
1167        break;
1168    }
1169
1170
1171    if ((1 << version) & LSQUIC_IETF_VERSIONS)
1172    {
1173        conn = lsquic_mini_conn_ietf_new(&engine->pub, packet_in, version,
1174                    sa_peer->sa_family == AF_INET, NULL, packet_in_size);
1175    }
1176    else
1177    {
1178        conn = mini_conn_new(&engine->pub, packet_in, version);
1179    }
1180    if (!conn)
1181        return NULL;
1182    ++engine->mini_conns_count;
1183    ++engine->n_conns;
1184    if (0 != insert_conn_into_hash(engine, conn, peer_ctx))
1185    {
1186        const lsquic_cid_t *cid = lsquic_conn_log_cid(conn);
1187        LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy",
1188            CID_BITS(cid));
1189        destroy_conn(engine, conn, packet_in->pi_received);
1190        return NULL;
1191    }
1192    assert(!(conn->cn_flags & CONN_REF_FLAGS));
1193    conn->cn_flags |= LSCONN_HASHED;
1194    eng_hist_inc(&engine->history, packet_in->pi_received, sl_new_mini_conns);
1195    conn->cn_last_sent = engine->last_sent;
1196    return conn;
1197}
1198
1199
1200lsquic_conn_t *
1201lsquic_engine_find_conn (const struct lsquic_engine_public *engine,
1202                         const lsquic_cid_t *cid)
1203{
1204    struct lsquic_hash_elem *el;
1205    lsquic_conn_t *conn = NULL;
1206    el = lsquic_hash_find(engine->enp_engine->conns_hash, cid->idbuf, cid->len);
1207
1208    if (el)
1209        conn = lsquic_hashelem_getdata(el);
1210    return conn;
1211}
1212
1213
1214#if !defined(NDEBUG) && __GNUC__
1215__attribute__((weak))
1216#endif
1217void
1218lsquic_engine_add_conn_to_tickable (struct lsquic_engine_public *enpub,
1219                                    lsquic_conn_t *conn)
1220{
1221    if (0 == (enpub->enp_flags & ENPUB_PROC) &&
1222        0 == (conn->cn_flags & (LSCONN_TICKABLE|LSCONN_NEVER_TICKABLE)))
1223    {
1224        lsquic_engine_t *engine = (lsquic_engine_t *) enpub;
1225        lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked);
1226        engine_incref_conn(conn, LSCONN_TICKABLE);
1227    }
1228}
1229
1230
1231void
1232lsquic_engine_add_conn_to_attq (struct lsquic_engine_public *enpub,
1233                    lsquic_conn_t *conn, lsquic_time_t tick_time, unsigned why)
1234{
1235    lsquic_engine_t *const engine = (lsquic_engine_t *) enpub;
1236    if (conn->cn_flags & LSCONN_TICKABLE)
1237    {
1238        /* Optimization: no need to add the connection to the Advisory Tick
1239         * Time Queue: it is about to be ticked, after which it its next tick
1240         * time may be queried again.
1241         */;
1242    }
1243    else if (conn->cn_flags & LSCONN_ATTQ)
1244    {
1245        if (lsquic_conn_adv_time(conn) != tick_time)
1246        {
1247            attq_remove(engine->attq, conn);
1248            if (0 != attq_add(engine->attq, conn, tick_time, why))
1249                engine_decref_conn(engine, conn, LSCONN_ATTQ);
1250        }
1251    }
1252    else if (0 == attq_add(engine->attq, conn, tick_time, why))
1253        engine_incref_conn(conn, LSCONN_ATTQ);
1254}
1255
1256
1257static struct lsquic_conn *
1258find_conn_by_srst (struct lsquic_engine *engine,
1259                                    const struct lsquic_packet_in *packet_in)
1260{
1261    struct lsquic_hash_elem *el;
1262    struct lsquic_conn *conn;
1263
1264    if (packet_in->pi_data_sz < IQUIC_MIN_SRST_SIZE
1265                            || (packet_in->pi_data[0] & 0xC0) != 0x40)
1266        return NULL;
1267
1268    el = lsquic_hash_find(engine->pub.enp_srst_hash,
1269            packet_in->pi_data + packet_in->pi_data_sz - IQUIC_SRESET_TOKEN_SZ,
1270            IQUIC_SRESET_TOKEN_SZ);
1271    if (!el)
1272        return NULL;
1273
1274    conn = lsquic_hashelem_getdata(el);
1275    return conn;
1276}
1277
1278
1279/* Return 0 if packet is being processed by a real connection (mini or full),
1280 * otherwise return 1.
1281 */
1282static int
1283process_packet_in (lsquic_engine_t *engine, lsquic_packet_in_t *packet_in,
1284       struct packin_parse_state *ppstate, const struct sockaddr *sa_local,
1285       const struct sockaddr *sa_peer, void *peer_ctx, size_t packet_in_size)
1286{
1287    lsquic_conn_t *conn;
1288    const unsigned char *packet_in_data;
1289
1290    if (lsquic_packet_in_is_gquic_prst(packet_in)
1291                                && !engine->pub.enp_settings.es_honor_prst)
1292    {
1293        lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in);
1294        LSQ_DEBUG("public reset packet: discarding");
1295        return 1;
1296    }
1297
1298    if (engine->flags & ENG_SERVER)
1299        conn = find_or_create_conn(engine, packet_in, ppstate, sa_local,
1300                                            sa_peer, peer_ctx, packet_in_size);
1301    else
1302        conn = find_conn(engine, packet_in, ppstate, sa_local);
1303
1304    if (!conn)
1305    {
1306        if (engine->pub.enp_settings.es_honor_prst
1307                && packet_in_size == packet_in->pi_data_sz /* Full UDP packet */
1308                && !(packet_in->pi_flags & PI_GQUIC)
1309                && engine->pub.enp_srst_hash
1310                && (conn = find_conn_by_srst(engine, packet_in)))
1311        {
1312            LSQ_DEBUGC("got stateless reset for connection %"CID_FMT,
1313                CID_BITS(lsquic_conn_log_cid(conn)));
1314            conn->cn_if->ci_stateless_reset(conn);
1315            if (!(conn->cn_flags & LSCONN_TICKABLE)
1316                && conn->cn_if->ci_is_tickable(conn))
1317            {
1318                lsquic_mh_insert(&engine->conns_tickable, conn,
1319                                                        conn->cn_last_ticked);
1320                engine_incref_conn(conn, LSCONN_TICKABLE);
1321            }
1322            /* Even though the connection processes this packet, we return
1323             * 1 so that the caller does not add reset packet's random
1324             * bytes to the list of valid CIDs.
1325             */
1326        }
1327        lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in);
1328        return 1;
1329    }
1330
1331    if (0 == (conn->cn_flags & LSCONN_TICKABLE))
1332    {
1333        lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked);
1334        engine_incref_conn(conn, LSCONN_TICKABLE);
1335    }
1336    packet_in->pi_path_id = lsquic_conn_record_sockaddr(conn, peer_ctx,
1337                                                        sa_local, sa_peer);
1338    lsquic_packet_in_upref(packet_in);
1339#if LOG_PACKET_CHECKSUM
1340    log_packet_checksum(lsquic_conn_log_cid(conn), "in", packet_in->pi_data,
1341                                                    packet_in->pi_data_sz);
1342#endif
1343    /* Note on QLog:
1344     * For the PACKET_RX QLog event, we are interested in logging these things:
1345     *  - raw packet (however it comes in, encrypted or not)
1346     *  - frames (list of frame names)
1347     *  - packet type and number
1348     *  - packet rx timestamp
1349     *
1350     * Since only some of these items are available at this code
1351     * juncture, we will wait until after the packet has been
1352     * decrypted (if necessary) and parsed to call the log functions.
1353     *
1354     * Once the PACKET_RX event is finally logged, the timestamp
1355     * will come from packet_in->pi_received. For correct sequential
1356     * ordering of QLog events, be sure to process the QLogs downstream.
1357     * (Hint: Use the qlog_parser.py tool in tools/ for full QLog processing.)
1358     */
1359    packet_in_data = packet_in->pi_data;
1360    packet_in_size = packet_in->pi_data_sz;
1361    conn->cn_if->ci_packet_in(conn, packet_in);
1362    QLOG_PACKET_RX(lsquic_conn_log_cid(conn), packet_in, packet_in_data, packet_in_size);
1363    lsquic_packet_in_put(&engine->pub.enp_mm, packet_in);
1364    return 0;
1365}
1366
1367
1368void
1369lsquic_engine_destroy (lsquic_engine_t *engine)
1370{
1371    struct lsquic_hash_elem *el;
1372    lsquic_conn_t *conn;
1373
1374    LSQ_DEBUG("destroying engine");
1375#ifndef NDEBUG
1376    engine->flags |= ENG_DTOR;
1377#endif
1378
1379    while ((conn = lsquic_mh_pop(&engine->conns_out)))
1380    {
1381        assert(conn->cn_flags & LSCONN_HAS_OUTGOING);
1382        (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING);
1383    }
1384
1385    while ((conn = lsquic_mh_pop(&engine->conns_tickable)))
1386    {
1387        assert(conn->cn_flags & LSCONN_TICKABLE);
1388        (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE);
1389    }
1390
1391    for (el = lsquic_hash_first(engine->conns_hash); el;
1392                                el = lsquic_hash_next(engine->conns_hash))
1393    {
1394        conn = lsquic_hashelem_getdata(el);
1395        force_close_conn(engine, conn);
1396    }
1397    lsquic_hash_destroy(engine->conns_hash);
1398
1399    assert(0 == engine->n_conns);
1400    assert(0 == engine->mini_conns_count);
1401    if (engine->pr_queue)
1402        prq_destroy(engine->pr_queue);
1403    if (engine->purga)
1404        lsquic_purga_destroy(engine->purga);
1405    attq_destroy(engine->attq);
1406
1407    assert(0 == lsquic_mh_count(&engine->conns_out));
1408    assert(0 == lsquic_mh_count(&engine->conns_tickable));
1409    if (engine->pub.enp_shi == &stock_shi)
1410        stock_shared_hash_destroy(engine->pub.enp_shi_ctx);
1411    lsquic_mm_cleanup(&engine->pub.enp_mm);
1412    free(engine->conns_tickable.mh_elems);
1413#ifndef NDEBUG
1414    if (engine->flags & ENG_LOSE_PACKETS)
1415        regfree(&engine->lose_packets_re);
1416#endif
1417    if (engine->pub.enp_tokgen)
1418        lsquic_tg_destroy(engine->pub.enp_tokgen);
1419#if LSQUIC_CONN_STATS
1420    if (engine->stats_fh)
1421    {
1422        const struct conn_stats *const stats = &engine->conn_stats_sum;
1423        fprintf(engine->stats_fh, "Aggregate connection stats collected by engine:\n");
1424        fprintf(engine->stats_fh, "Connections: %u\n", engine->stats.conns);
1425        fprintf(engine->stats_fh, "Ticks: %lu\n", stats->n_ticks);
1426        fprintf(engine->stats_fh, "In:\n");
1427        fprintf(engine->stats_fh, "    Total bytes: %lu\n", stats->in.bytes);
1428        fprintf(engine->stats_fh, "    packets: %lu\n", stats->in.packets);
1429        fprintf(engine->stats_fh, "    undecryptable packets: %lu\n", stats->in.undec_packets);
1430        fprintf(engine->stats_fh, "    duplicate packets: %lu\n", stats->in.dup_packets);
1431        fprintf(engine->stats_fh, "    error packets: %lu\n", stats->in.err_packets);
1432        fprintf(engine->stats_fh, "    STREAM frame count: %lu\n", stats->in.stream_frames);
1433        fprintf(engine->stats_fh, "    STREAM payload size: %lu\n", stats->in.stream_data_sz);
1434        fprintf(engine->stats_fh, "    Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n",
1435            stats->in.headers_comp, stats->in.headers_uncomp,
1436            stats->in.headers_uncomp ?
1437            (double) stats->in.headers_comp / (double) stats->in.headers_uncomp
1438            : 0);
1439        fprintf(engine->stats_fh, "    ACK frames: %lu\n", stats->in.n_acks);
1440        fprintf(engine->stats_fh, "    ACK frames processed: %lu\n", stats->in.n_acks_proc);
1441        fprintf(engine->stats_fh, "    ACK frames merged to new: %lu\n", stats->in.n_acks_merged[0]);
1442        fprintf(engine->stats_fh, "    ACK frames merged to old: %lu\n", stats->in.n_acks_merged[1]);
1443        fprintf(engine->stats_fh, "Out:\n");
1444        fprintf(engine->stats_fh, "    Total bytes: %lu\n", stats->out.bytes);
1445        fprintf(engine->stats_fh, "    packets: %lu\n", stats->out.packets);
1446        fprintf(engine->stats_fh, "    acked via loss record: %lu\n", stats->out.acked_via_loss);
1447        fprintf(engine->stats_fh, "    acks: %lu\n", stats->out.acks);
1448        fprintf(engine->stats_fh, "    retx packets: %lu\n", stats->out.retx_packets);
1449        fprintf(engine->stats_fh, "    STREAM frame count: %lu\n", stats->out.stream_frames);
1450        fprintf(engine->stats_fh, "    STREAM payload size: %lu\n", stats->out.stream_data_sz);
1451        fprintf(engine->stats_fh, "    Header bytes: %lu; uncompressed: %lu; ratio %.3lf\n",
1452            stats->out.headers_comp, stats->out.headers_uncomp,
1453            stats->out.headers_uncomp ?
1454            (double) stats->out.headers_comp / (double) stats->out.headers_uncomp
1455            : 0);
1456        fprintf(engine->stats_fh, "    ACKs: %lu\n", stats->out.acks);
1457    }
1458#endif
1459    if (engine->pub.enp_srst_hash)
1460        lsquic_hash_destroy(engine->pub.enp_srst_hash);
1461#if LSQUIC_COUNT_ENGINE_CALLS
1462    LSQ_NOTICE("number of calls into the engine: %lu", engine->n_engine_calls);
1463#endif
1464    if (engine->pub.enp_retry_aead_ctx)
1465        EVP_AEAD_CTX_cleanup(engine->pub.enp_retry_aead_ctx);
1466    free(engine);
1467}
1468
1469
1470static struct conn_cid_elem *
1471find_free_cce (struct lsquic_conn *conn)
1472{
1473    struct conn_cid_elem *cce;
1474
1475    for (cce = conn->cn_cces; cce < END_OF_CCES(conn); ++cce)
1476        if (!(conn->cn_cces_mask & (1 << (cce - conn->cn_cces))))
1477            return cce;
1478
1479    return NULL;
1480}
1481
1482
1483static int
1484add_conn_to_hash (struct lsquic_engine *engine, struct lsquic_conn *conn,
1485                                const struct sockaddr *local_sa, void *peer_ctx)
1486{
1487    struct conn_cid_elem *cce;
1488
1489    if (engine->flags & ENG_CONNS_BY_ADDR)
1490    {
1491        cce = find_free_cce(conn);
1492        if (!cce)
1493        {
1494            LSQ_ERROR("cannot find free CCE");
1495            return -1;
1496        }
1497        cce->cce_port = sa2port(local_sa);
1498        cce->cce_flags = CCE_PORT;
1499        if (lsquic_hash_insert(engine->conns_hash, &cce->cce_port,
1500                                sizeof(cce->cce_port), conn, &cce->cce_hash_el))
1501        {
1502            conn->cn_cces_mask |= 1 << (cce - conn->cn_cces);
1503            return 0;
1504        }
1505        else
1506            return -1;
1507
1508    }
1509    else
1510        return insert_conn_into_hash(engine, conn, peer_ctx);
1511}
1512
1513
1514lsquic_conn_t *
1515lsquic_engine_connect (lsquic_engine_t *engine, enum lsquic_version version,
1516                       const struct sockaddr *local_sa,
1517                       const struct sockaddr *peer_sa,
1518                       void *peer_ctx, lsquic_conn_ctx_t *conn_ctx,
1519                       const char *hostname, unsigned short max_packet_size,
1520                       const unsigned char *zero_rtt, size_t zero_rtt_len,
1521                       const unsigned char *token, size_t token_sz)
1522{
1523    lsquic_conn_t *conn;
1524    unsigned flags, versions;
1525    int is_ipv4;
1526
1527    ENGINE_IN(engine);
1528
1529    if (engine->flags & ENG_SERVER)
1530    {
1531        LSQ_ERROR("`%s' must only be called in client mode", __func__);
1532        goto err;
1533    }
1534
1535    if (engine->flags & ENG_CONNS_BY_ADDR
1536                        && find_conn_by_addr(engine->conns_hash, local_sa))
1537    {
1538        LSQ_ERROR("cannot have more than one connection on the same port");
1539        goto err;
1540    }
1541
1542    if (0 != maybe_grow_conn_heaps(engine))
1543        return NULL;
1544    flags = engine->flags & (ENG_SERVER|ENG_HTTP);
1545    is_ipv4 = peer_sa->sa_family == AF_INET;
1546    if (zero_rtt && zero_rtt_len)
1547    {
1548        version = lsquic_zero_rtt_version(zero_rtt, zero_rtt_len);
1549        if (version >= N_LSQVER)
1550        {
1551            LSQ_INFO("zero-rtt version is bad, won't use");
1552            zero_rtt = NULL;
1553            zero_rtt_len = 0;
1554        }
1555    }
1556    if (version >= N_LSQVER)
1557    {
1558        if (version > N_LSQVER)
1559            LSQ_WARN("invalid version specified, engine will pick");
1560        versions = engine->pub.enp_settings.es_versions;
1561    }
1562    else
1563        versions = 1u << version;
1564    if (versions & LSQUIC_IETF_VERSIONS)
1565    {
1566        if (version == LSQVER_ID24)
1567            conn = lsquic_id24_full_conn_client_new(&engine->pub, versions,
1568                        flags, hostname, max_packet_size,
1569                        is_ipv4, zero_rtt, zero_rtt_len, token, token_sz);
1570        else
1571            conn = lsquic_ietf_full_conn_client_new(&engine->pub, versions,
1572                        flags, hostname, max_packet_size,
1573                        is_ipv4, zero_rtt, zero_rtt_len, token, token_sz);
1574    }
1575    else
1576        conn = lsquic_gquic_full_conn_client_new(&engine->pub, versions,
1577                            flags, hostname, max_packet_size, is_ipv4,
1578                            zero_rtt, zero_rtt_len);
1579    if (!conn)
1580        goto err;
1581    EV_LOG_CREATE_CONN(lsquic_conn_log_cid(conn), local_sa, peer_sa);
1582    EV_LOG_VER_NEG(lsquic_conn_log_cid(conn), "proposed",
1583                                            lsquic_ver2str[conn->cn_version]);
1584    ++engine->n_conns;
1585    lsquic_conn_record_sockaddr(conn, peer_ctx, local_sa, peer_sa);
1586    if (0 != add_conn_to_hash(engine, conn, local_sa, peer_ctx))
1587    {
1588        const lsquic_cid_t *cid = lsquic_conn_log_cid(conn);
1589        LSQ_WARNC("cannot add connection %"CID_FMT" to hash - destroy",
1590            CID_BITS(cid));
1591        destroy_conn(engine, conn, lsquic_time_now());
1592        goto err;
1593    }
1594    assert(!(conn->cn_flags &
1595        (CONN_REF_FLAGS
1596         & ~LSCONN_TICKABLE /* This flag may be set as effect of user
1597                                 callbacks */
1598                             )));
1599    conn->cn_flags |= LSCONN_HASHED;
1600    lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked);
1601    engine_incref_conn(conn, LSCONN_TICKABLE);
1602    lsquic_conn_set_ctx(conn, conn_ctx);
1603    conn->cn_if->ci_client_call_on_new(conn);
1604  end:
1605    ENGINE_OUT(engine);
1606    return conn;
1607  err:
1608    conn = NULL;
1609    goto end;
1610}
1611
1612
1613static void
1614remove_conn_from_hash (lsquic_engine_t *engine, lsquic_conn_t *conn)
1615{
1616    remove_all_cces_from_hash(engine->conns_hash, conn);
1617    (void) engine_decref_conn(engine, conn, LSCONN_HASHED);
1618}
1619
1620
1621static void
1622refflags2str (enum lsquic_conn_flags flags, char s[6])
1623{
1624    *s = 'C'; s += !!(flags & LSCONN_CLOSING);
1625    *s = 'H'; s += !!(flags & LSCONN_HASHED);
1626    *s = 'O'; s += !!(flags & LSCONN_HAS_OUTGOING);
1627    *s = 'T'; s += !!(flags & LSCONN_TICKABLE);
1628    *s = 'A'; s += !!(flags & LSCONN_ATTQ);
1629    *s = 'K'; s += !!(flags & LSCONN_TICKED);
1630    *s = '\0';
1631}
1632
1633
1634static void
1635engine_incref_conn (lsquic_conn_t *conn, enum lsquic_conn_flags flag)
1636{
1637    char str[2][7];
1638    assert(flag & CONN_REF_FLAGS);
1639    assert(!(conn->cn_flags & flag));
1640    conn->cn_flags |= flag;
1641    LSQ_DEBUGC("incref conn %"CID_FMT", '%s' -> '%s'",
1642                    CID_BITS(lsquic_conn_log_cid(conn)),
1643                    (refflags2str(conn->cn_flags & ~flag, str[0]), str[0]),
1644                    (refflags2str(conn->cn_flags, str[1]), str[1]));
1645}
1646
1647
1648static lsquic_conn_t *
1649engine_decref_conn (lsquic_engine_t *engine, lsquic_conn_t *conn,
1650                                        enum lsquic_conn_flags flags)
1651{
1652    char str[2][7];
1653    lsquic_time_t now;
1654    assert(flags & CONN_REF_FLAGS);
1655    assert(conn->cn_flags & flags);
1656#ifndef NDEBUG
1657    if (flags & LSCONN_CLOSING)
1658        assert(0 == (conn->cn_flags & LSCONN_HASHED));
1659#endif
1660    conn->cn_flags &= ~flags;
1661    LSQ_DEBUGC("decref conn %"CID_FMT", '%s' -> '%s'",
1662                    CID_BITS(lsquic_conn_log_cid(conn)),
1663                    (refflags2str(conn->cn_flags | flags, str[0]), str[0]),
1664                    (refflags2str(conn->cn_flags, str[1]), str[1]));
1665    if (0 == (conn->cn_flags & CONN_REF_FLAGS))
1666    {
1667        now = lsquic_time_now();
1668        if (conn->cn_flags & LSCONN_MINI)
1669            eng_hist_inc(&engine->history, now, sl_del_mini_conns);
1670        else
1671            eng_hist_inc(&engine->history, now, sl_del_full_conns);
1672        destroy_conn(engine, conn, now);
1673        return NULL;
1674    }
1675    else
1676        return conn;
1677}
1678
1679
1680/* This is not a general-purpose function.  Only call from engine dtor. */
1681static void
1682force_close_conn (lsquic_engine_t *engine, lsquic_conn_t *conn)
1683{
1684    assert(engine->flags & ENG_DTOR);
1685    const enum lsquic_conn_flags flags = conn->cn_flags;
1686    assert(conn->cn_flags & CONN_REF_FLAGS);
1687    assert(!(flags & LSCONN_HAS_OUTGOING));  /* Should be removed already */
1688    assert(!(flags & LSCONN_TICKABLE));    /* Should be removed already */
1689    assert(!(flags & LSCONN_CLOSING));  /* It is in transient queue? */
1690    if (flags & LSCONN_ATTQ)
1691    {
1692        attq_remove(engine->attq, conn);
1693        (void) engine_decref_conn(engine, conn, LSCONN_ATTQ);
1694    }
1695    if (flags & LSCONN_HASHED)
1696        remove_conn_from_hash(engine, conn);
1697}
1698
1699
1700/* Iterator for tickable connections (those on the Tickable Queue).  Before
1701 * a connection is returned, it is removed from the Advisory Tick Time queue
1702 * if necessary.
1703 */
1704static lsquic_conn_t *
1705conn_iter_next_tickable (struct lsquic_engine *engine)
1706{
1707    lsquic_conn_t *conn;
1708
1709    if (engine->flags & ENG_SERVER)
1710        while (1)
1711        {
1712            conn = lsquic_mh_pop(&engine->conns_tickable);
1713            if (conn && (conn->cn_flags & LSCONN_SKIP_ON_PROC))
1714                (void) engine_decref_conn(engine, conn, LSCONN_TICKABLE);
1715            else
1716                break;
1717        }
1718    else
1719        conn = lsquic_mh_pop(&engine->conns_tickable);
1720
1721    if (conn)
1722        conn = engine_decref_conn(engine, conn, LSCONN_TICKABLE);
1723    if (conn && (conn->cn_flags & LSCONN_ATTQ))
1724    {
1725        attq_remove(engine->attq, conn);
1726        conn = engine_decref_conn(engine, conn, LSCONN_ATTQ);
1727    }
1728
1729    return conn;
1730}
1731
1732
1733static void
1734cub_init (struct cid_update_batch *cub, lsquic_cids_update_f update,
1735                                                        void *update_ctx)
1736{
1737    cub->cub_update_cids = update;
1738    cub->cub_update_ctx  = update_ctx;
1739    cub->cub_count       = 0;
1740}
1741
1742
1743static void
1744cub_flush (struct cid_update_batch *cub)
1745{
1746    if (cub->cub_count > 0 && cub->cub_update_cids)
1747        cub->cub_update_cids(cub->cub_update_ctx, cub->cub_peer_ctxs,
1748                                                cub->cub_cids, cub->cub_count);
1749    cub->cub_count = 0;
1750}
1751
1752
1753static void
1754cub_add (struct cid_update_batch *cub, const lsquic_cid_t *cid, void *peer_ctx)
1755{
1756    cub->cub_cids     [ cub->cub_count ] = *cid;
1757    cub->cub_peer_ctxs[ cub->cub_count ] = peer_ctx;
1758    ++cub->cub_count;
1759    if (cub->cub_count == sizeof(cub->cub_cids) / sizeof(cub->cub_cids[0]))
1760        cub_flush(cub);
1761}
1762
1763
1764/* Process registered CIDs */
1765static void
1766cub_add_cids_from_cces (struct cid_update_batch *cub, struct lsquic_conn *conn)
1767{
1768    struct cce_cid_iter citer;
1769    struct conn_cid_elem *cce;
1770    void *peer_ctx;
1771
1772    peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL);
1773    for (cce = cce_iter_first(&citer, conn); cce; cce = cce_iter_next(&citer))
1774        if (cce->cce_flags & CCE_REG)
1775            cub_add(cub, &cce->cce_cid, peer_ctx);
1776}
1777
1778
1779static void
1780drop_all_mini_conns (lsquic_engine_t *engine)
1781{
1782    struct lsquic_hash_elem *el;
1783    lsquic_conn_t *conn;
1784    struct cid_update_batch cub;
1785
1786    cub_init(&cub, engine->report_old_scids, engine->scids_ctx);
1787
1788    for (el = lsquic_hash_first(engine->conns_hash); el;
1789                                el = lsquic_hash_next(engine->conns_hash))
1790    {
1791        conn = lsquic_hashelem_getdata(el);
1792        if (conn->cn_flags & LSCONN_MINI)
1793        {
1794            /* If promoted, why is it still in this hash? */
1795            assert(!(conn->cn_flags & LSCONN_PROMOTED));
1796            if (!(conn->cn_flags & LSCONN_PROMOTED))
1797                cub_add_cids_from_cces(&cub, conn);
1798            remove_conn_from_hash(engine, conn);
1799        }
1800    }
1801
1802    cub_flush(&cub);
1803}
1804
1805
1806void
1807lsquic_engine_process_conns (lsquic_engine_t *engine)
1808{
1809    lsquic_conn_t *conn;
1810    lsquic_time_t now;
1811
1812    ENGINE_IN(engine);
1813
1814    now = lsquic_time_now();
1815    while ((conn = attq_pop(engine->attq, now)))
1816    {
1817        conn = engine_decref_conn(engine, conn, LSCONN_ATTQ);
1818        if (conn && !(conn->cn_flags & LSCONN_TICKABLE))
1819        {
1820            lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked);
1821            engine_incref_conn(conn, LSCONN_TICKABLE);
1822        }
1823    }
1824
1825    process_connections(engine, conn_iter_next_tickable, now);
1826    ENGINE_OUT(engine);
1827}
1828
1829
1830static void
1831release_or_return_enc_data (struct lsquic_engine *engine,
1832                void (*pmi_rel_or_ret) (void *, void *, void *, char),
1833                struct lsquic_conn *conn, struct lsquic_packet_out *packet_out)
1834{
1835    pmi_rel_or_ret(engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx,
1836                packet_out->po_enc_data, lsquic_packet_out_ipv6(packet_out));
1837    packet_out->po_flags &= ~PO_ENCRYPTED;
1838    packet_out->po_enc_data = NULL;
1839}
1840
1841
1842static void
1843release_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn,
1844                                        struct lsquic_packet_out *packet_out)
1845{
1846    release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_release,
1847                                conn, packet_out);
1848}
1849
1850
1851static void
1852return_enc_data (struct lsquic_engine *engine, struct lsquic_conn *conn,
1853                                        struct lsquic_packet_out *packet_out)
1854{
1855    release_or_return_enc_data(engine, engine->pub.enp_pmi->pmi_return,
1856                                conn, packet_out);
1857}
1858
1859
1860static int
1861copy_packet (struct lsquic_engine *engine, struct lsquic_conn *conn,
1862                                        struct lsquic_packet_out *packet_out)
1863{
1864    int ipv6;
1865
1866    ipv6 = NP_IS_IPv6(packet_out->po_path);
1867    if (packet_out->po_flags & PO_ENCRYPTED)
1868    {
1869        if (ipv6 == lsquic_packet_out_ipv6(packet_out)
1870            && packet_out->po_data_sz == packet_out->po_enc_data_sz
1871            && 0 == memcmp(packet_out->po_data, packet_out->po_enc_data,
1872                                                        packet_out->po_data_sz))
1873            return 0;
1874        if (ipv6 == lsquic_packet_out_ipv6(packet_out)
1875            && packet_out->po_data_sz <= packet_out->po_enc_data_sz)
1876            goto copy;
1877        return_enc_data(engine, conn, packet_out);
1878    }
1879
1880    packet_out->po_enc_data = engine->pub.enp_pmi->pmi_allocate(
1881                    engine->pub.enp_pmi_ctx, packet_out->po_path->np_peer_ctx,
1882                    packet_out->po_data_sz, ipv6);
1883    if (!packet_out->po_enc_data)
1884    {
1885        LSQ_DEBUG("could not allocate memory for outgoing unencrypted packet "
1886                                        "of size %hu", packet_out->po_data_sz);
1887        return -1;
1888    }
1889
1890  copy:
1891    memcpy(packet_out->po_enc_data, packet_out->po_data,
1892                                                    packet_out->po_data_sz);
1893    packet_out->po_enc_data_sz = packet_out->po_data_sz;
1894    packet_out->po_sent_sz     = packet_out->po_data_sz;
1895    packet_out->po_flags &= ~PO_IPv6;
1896    packet_out->po_flags |= PO_ENCRYPTED|PO_SENT_SZ|(ipv6 << POIPv6_SHIFT);
1897
1898    return 0;
1899}
1900
1901
1902STAILQ_HEAD(conns_stailq, lsquic_conn);
1903TAILQ_HEAD(conns_tailq, lsquic_conn);
1904
1905
1906struct conns_out_iter
1907{
1908    struct min_heap            *coi_heap;
1909    struct pr_queue            *coi_prq;
1910    TAILQ_HEAD(, lsquic_conn)   coi_active_list,
1911                                coi_inactive_list;
1912    lsquic_conn_t              *coi_next;
1913#ifndef NDEBUG
1914    lsquic_time_t               coi_last_sent;
1915#endif
1916};
1917
1918
1919static void
1920coi_init (struct conns_out_iter *iter, struct lsquic_engine *engine)
1921{
1922    iter->coi_heap = &engine->conns_out;
1923    iter->coi_prq = engine->pr_queue;
1924    iter->coi_next = NULL;
1925    TAILQ_INIT(&iter->coi_active_list);
1926    TAILQ_INIT(&iter->coi_inactive_list);
1927#ifndef NDEBUG
1928    iter->coi_last_sent = 0;
1929#endif
1930}
1931
1932
1933static lsquic_conn_t *
1934coi_next (struct conns_out_iter *iter)
1935{
1936    lsquic_conn_t *conn;
1937
1938    if (lsquic_mh_count(iter->coi_heap) > 0)
1939    {
1940        conn = lsquic_mh_pop(iter->coi_heap);
1941        TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out);
1942        conn->cn_flags |= LSCONN_COI_ACTIVE;
1943#ifndef NDEBUG
1944        if (iter->coi_last_sent)
1945            assert(iter->coi_last_sent <= conn->cn_last_sent);
1946        iter->coi_last_sent = conn->cn_last_sent;
1947#endif
1948        return conn;
1949    }
1950    else if (iter->coi_prq && (conn = prq_next_conn(iter->coi_prq)))
1951    {
1952        return conn;
1953    }
1954    else if (!TAILQ_EMPTY(&iter->coi_active_list))
1955    {
1956        iter->coi_prq = NULL; /* Save function call in previous conditional */
1957        conn = iter->coi_next;
1958        if (!conn)
1959            conn = TAILQ_FIRST(&iter->coi_active_list);
1960        if (conn)
1961            iter->coi_next = TAILQ_NEXT(conn, cn_next_out);
1962        return conn;
1963    }
1964    else
1965        return NULL;
1966}
1967
1968
1969static void
1970coi_deactivate (struct conns_out_iter *iter, lsquic_conn_t *conn)
1971{
1972    if (!(conn->cn_flags & LSCONN_EVANESCENT))
1973    {
1974        assert(!TAILQ_EMPTY(&iter->coi_active_list));
1975        TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out);
1976        conn->cn_flags &= ~LSCONN_COI_ACTIVE;
1977        TAILQ_INSERT_TAIL(&iter->coi_inactive_list, conn, cn_next_out);
1978        conn->cn_flags |= LSCONN_COI_INACTIVE;
1979    }
1980}
1981
1982
1983static void
1984coi_reactivate (struct conns_out_iter *iter, lsquic_conn_t *conn)
1985{
1986    assert(conn->cn_flags & LSCONN_COI_INACTIVE);
1987    TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out);
1988    conn->cn_flags &= ~LSCONN_COI_INACTIVE;
1989    TAILQ_INSERT_TAIL(&iter->coi_active_list, conn, cn_next_out);
1990    conn->cn_flags |= LSCONN_COI_ACTIVE;
1991}
1992
1993
1994static void
1995coi_reheap (struct conns_out_iter *iter, lsquic_engine_t *engine)
1996{
1997    lsquic_conn_t *conn;
1998    while ((conn = TAILQ_FIRST(&iter->coi_active_list)))
1999    {
2000        TAILQ_REMOVE(&iter->coi_active_list, conn, cn_next_out);
2001        conn->cn_flags &= ~LSCONN_COI_ACTIVE;
2002        if ((conn->cn_flags & CONN_REF_FLAGS) != LSCONN_HAS_OUTGOING
2003                                && !(conn->cn_flags & LSCONN_IMMED_CLOSE))
2004            lsquic_mh_insert(iter->coi_heap, conn, conn->cn_last_sent);
2005        else    /* Closed connection gets one shot at sending packets */
2006            (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING);
2007    }
2008    while ((conn = TAILQ_FIRST(&iter->coi_inactive_list)))
2009    {
2010        TAILQ_REMOVE(&iter->coi_inactive_list, conn, cn_next_out);
2011        conn->cn_flags &= ~LSCONN_COI_INACTIVE;
2012        (void) engine_decref_conn(engine, conn, LSCONN_HAS_OUTGOING);
2013    }
2014}
2015
2016
2017#ifndef NDEBUG
2018static void
2019lose_matching_packets (const lsquic_engine_t *engine, struct out_batch *batch,
2020                                                                    unsigned n)
2021{
2022    const lsquic_cid_t *cid;
2023    struct iovec *iov;
2024    unsigned i;
2025    char packno_str[22];
2026
2027    for (i = 0; i < n; ++i)
2028    {
2029        snprintf(packno_str, sizeof(packno_str), "%"PRIu64,
2030                                                batch->packets[i]->po_packno);
2031        if (0 == regexec(&engine->lose_packets_re, packno_str, 0, NULL, 0))
2032        {
2033            for (iov = batch->outs[i].iov; iov <
2034                            batch->outs[i].iov + batch->outs[i].iovlen; ++iov)
2035                batch->outs[i].iov->iov_len -= 1;
2036            cid = lsquic_conn_log_cid(batch->conns[i]);
2037            LSQ_WARNC("losing packet %s for connection %"CID_FMT, packno_str,
2038                CID_BITS(cid));
2039        }
2040    }
2041}
2042
2043
2044#endif
2045
2046
2047#ifdef NDEBUG
2048#define CONST_BATCH const
2049#else
2050#define CONST_BATCH
2051#endif
2052
2053
2054struct send_batch_ctx {
2055    struct conns_stailq                 *closed_conns;
2056    struct conns_tailq                  *ticked_conns;
2057    struct conns_out_iter               *conns_iter;
2058    CONST_BATCH struct out_batch        *batch;
2059};
2060
2061
2062static void
2063close_conn_immediately (struct lsquic_engine *engine,
2064                const struct send_batch_ctx *sb_ctx, struct lsquic_conn *conn)
2065{
2066    conn->cn_flags |= LSCONN_IMMED_CLOSE;
2067    if (!(conn->cn_flags & LSCONN_CLOSING))
2068    {
2069        STAILQ_INSERT_TAIL(sb_ctx->closed_conns, conn, cn_next_closed_conn);
2070        engine_incref_conn(conn, LSCONN_CLOSING);
2071        if (conn->cn_flags & LSCONN_HASHED)
2072            remove_conn_from_hash(engine, conn);
2073    }
2074    if (conn->cn_flags & LSCONN_TICKED)
2075    {
2076        TAILQ_REMOVE(sb_ctx->ticked_conns, conn, cn_next_ticked);
2077        engine_decref_conn(engine, conn, LSCONN_TICKED);
2078    }
2079}
2080
2081
2082static void
2083close_conn_on_send_error (struct lsquic_engine *engine,
2084                          const struct send_batch_ctx *sb_ctx, int n, int e_val)
2085{
2086    const struct out_batch *batch = sb_ctx->batch;
2087    struct lsquic_conn *const conn = batch->conns[n];
2088    char buf[2][INET6_ADDRSTRLEN + sizeof(":65535")];
2089
2090    LSQ_WARNC("error sending packet for %s connection %"CID_FMT" - close it; "
2091        "src: %s; dst: %s; errno: %d",
2092        conn->cn_flags & LSCONN_EVANESCENT ? "evanecsent" :
2093        conn->cn_flags & LSCONN_MINI ? "mini" : "regular",
2094        CID_BITS(lsquic_conn_log_cid(conn)),
2095        SA2STR(batch->outs[n].local_sa, buf[0]),
2096        SA2STR(batch->outs[n].dest_sa, buf[1]),
2097        e_val);
2098    if (conn->cn_flags & LSCONN_EVANESCENT)
2099        lsquic_prq_drop(conn);
2100    else
2101        close_conn_immediately(engine, sb_ctx, conn);
2102}
2103
2104
2105static unsigned
2106send_batch (lsquic_engine_t *engine, const struct send_batch_ctx *sb_ctx,
2107            unsigned n_to_send)
2108{
2109    int n_sent, i, e_val;
2110    lsquic_time_t now;
2111    unsigned off;
2112    size_t count;
2113    CONST_BATCH struct out_batch *const batch = sb_ctx->batch;
2114    struct lsquic_packet_out *CONST_BATCH *packet_out, *CONST_BATCH *end;
2115
2116#ifndef NDEBUG
2117    if (engine->flags & ENG_LOSE_PACKETS)
2118        lose_matching_packets(engine, batch, n_to_send);
2119#endif
2120    /* Set sent time before the write to avoid underestimating RTT */
2121    now = lsquic_time_now();
2122    for (i = 0; i < (int) n_to_send; ++i)
2123    {
2124        off = batch->pack_off[i];
2125        count = batch->outs[i].iovlen;
2126        assert(count > 0);
2127        packet_out = &batch->packets[off];
2128        end = packet_out + count;
2129        do
2130            (*packet_out)->po_sent = now;
2131        while (++packet_out < end);
2132    }
2133    n_sent = engine->packets_out(engine->packets_out_ctx, batch->outs,
2134                                                                n_to_send);
2135    e_val = errno;
2136    if (n_sent < (int) n_to_send)
2137    {
2138        engine->pub.enp_flags &= ~ENPUB_CAN_SEND;
2139        engine->resume_sending_at = now + 1000000;
2140        LSQ_DEBUG("cannot send packets");
2141        EV_LOG_GENERIC_EVENT("cannot send packets");
2142        if (!(EAGAIN == e_val || EWOULDBLOCK == e_val))
2143            close_conn_on_send_error(engine, sb_ctx,
2144                                        n_sent < 0 ? 0 : n_sent, e_val);
2145    }
2146    if (n_sent >= 0)
2147        LSQ_DEBUG("packets out returned %d (out of %u)", n_sent, n_to_send);
2148    else
2149    {
2150        LSQ_DEBUG("packets out returned an error: %s", strerror(e_val));
2151        n_sent = 0;
2152    }
2153    if (n_sent > 0)
2154        engine->last_sent = now + n_sent;
2155    for (i = 0; i < n_sent; ++i)
2156    {
2157        eng_hist_inc(&engine->history, now, sl_packets_out);
2158        /* `i' is added to maintain relative order */
2159        batch->conns[i]->cn_last_sent = now + i;
2160
2161        off = batch->pack_off[i];
2162        count = batch->outs[i].iovlen;
2163        assert(count > 0);
2164        packet_out = &batch->packets[off];
2165        end = packet_out + count;
2166        do
2167        {
2168#if LOG_PACKET_CHECKSUM
2169            log_packet_checksum(lsquic_conn_log_cid(batch->conns[i]), "out",
2170                batch->outs[i].iov[packet_out - &batch->packets[off]].iov_base,
2171                batch->outs[i].iov[packet_out - &batch->packets[off]].iov_len);
2172#endif
2173            EV_LOG_PACKET_SENT(lsquic_conn_log_cid(batch->conns[i]),
2174                                                        *packet_out);
2175            /* Release packet out buffer as soon as the packet is sent
2176             * successfully.  If not successfully sent, we hold on to
2177             * this buffer until the packet sending is attempted again
2178             * or until it times out and regenerated.
2179             */
2180            if ((*packet_out)->po_flags & PO_ENCRYPTED)
2181                release_enc_data(engine, batch->conns[i], *packet_out);
2182            batch->conns[i]->cn_if->ci_packet_sent(batch->conns[i],
2183                                                        *packet_out);
2184        }
2185        while (++packet_out < end);
2186    }
2187    if (LSQ_LOG_ENABLED_EXT(LSQ_LOG_DEBUG, LSQLM_EVENT))
2188        for ( ; i < (int) n_to_send; ++i)
2189        {
2190            off = batch->pack_off[i];
2191            count = batch->outs[i].iovlen;
2192            assert(count > 0);
2193            packet_out = &batch->packets[off];
2194            end = packet_out + count;
2195            do
2196                EV_LOG_PACKET_NOT_SENT(lsquic_conn_log_cid(batch->conns[i]),
2197                                                                *packet_out);
2198            while (++packet_out < end);
2199        }
2200    /* Return packets to the connection in reverse order so that the packet
2201     * ordering is maintained.
2202     */
2203    for (i = (int) n_to_send - 1; i >= n_sent; --i)
2204    {
2205        off = batch->pack_off[i];
2206        count = batch->outs[i].iovlen;
2207        assert(count > 0);
2208        packet_out = &batch->packets[off + count - 1];
2209        end = &batch->packets[off - 1];
2210        do
2211            batch->conns[i]->cn_if->ci_packet_not_sent(batch->conns[i],
2212                                                                *packet_out);
2213        while (--packet_out > end);
2214        if (!(batch->conns[i]->cn_flags & (LSCONN_COI_ACTIVE|LSCONN_EVANESCENT)))
2215            coi_reactivate(sb_ctx->conns_iter, batch->conns[i]);
2216    }
2217    return n_sent;
2218}
2219
2220
2221/* Return 1 if went past deadline, 0 otherwise */
2222static int
2223check_deadline (lsquic_engine_t *engine)
2224{
2225    if (engine->pub.enp_settings.es_proc_time_thresh &&
2226                                lsquic_time_now() > engine->deadline)
2227    {
2228        LSQ_INFO("went past threshold of %u usec, stop sending",
2229                            engine->pub.enp_settings.es_proc_time_thresh);
2230        engine->flags |= ENG_PAST_DEADLINE;
2231        return 1;
2232    }
2233    else
2234        return 0;
2235}
2236
2237
2238static size_t
2239iov_size (const struct iovec *iov, const struct iovec *const end)
2240{
2241    size_t size;
2242
2243    assert(iov < end);
2244
2245    size = 0;
2246    do
2247        size += iov->iov_len;
2248    while (++iov < end);
2249
2250    return size;
2251}
2252
2253
2254static void
2255send_packets_out (struct lsquic_engine *engine,
2256                  struct conns_tailq *ticked_conns,
2257                  struct conns_stailq *closed_conns)
2258{
2259    unsigned n, w, n_sent, n_batches_sent;
2260    lsquic_packet_out_t *packet_out;
2261    struct lsquic_packet_out **packet;
2262    lsquic_conn_t *conn;
2263    struct out_batch *const batch = &engine->out_batch;
2264    struct iovec *iov, *packet_iov;
2265    struct conns_out_iter conns_iter;
2266    int shrink, deadline_exceeded;
2267    const struct send_batch_ctx sb_ctx = {
2268        closed_conns,
2269        ticked_conns,
2270        &conns_iter,
2271        &engine->out_batch,
2272    };
2273
2274    coi_init(&conns_iter, engine);
2275    n_batches_sent = 0;
2276    n_sent = 0, n = 0;
2277    shrink = 0;
2278    deadline_exceeded = 0;
2279    iov = batch->iov;
2280    packet = batch->packets;
2281
2282    while ((conn = coi_next(&conns_iter)))
2283    {
2284        packet_out = conn->cn_if->ci_next_packet_to_send(conn, 0);
2285        if (!packet_out) {
2286            /* Evanescent connection always has a packet to send: */
2287            assert(!(conn->cn_flags & LSCONN_EVANESCENT));
2288            LSQ_DEBUGC("batched all outgoing packets for %s conn %"CID_FMT,
2289                (conn->cn_flags & LSCONN_MINI   ? "mini" : "full"),
2290                CID_BITS(lsquic_conn_log_cid(conn)));
2291            coi_deactivate(&conns_iter, conn);
2292            continue;
2293        }
2294        batch->outs[n].iov = packet_iov = iov;
2295  next_coa:
2296        if (!(packet_out->po_flags & (PO_ENCRYPTED|PO_NOENCRYPT)))
2297        {
2298            switch (conn->cn_esf_c->esf_encrypt_packet(conn->cn_enc_session,
2299                                            &engine->pub, conn, packet_out))
2300            {
2301            case ENCPA_NOMEM:
2302                /* Send what we have and wait for a more opportune moment */
2303                conn->cn_if->ci_packet_not_sent(conn, packet_out);
2304                goto end_for;
2305            case ENCPA_BADCRYPT:
2306                /* This is pretty bad: close connection immediately */
2307                conn->cn_if->ci_packet_not_sent(conn, packet_out);
2308                LSQ_INFOC("conn %"CID_FMT" has unsendable packets",
2309                                        CID_BITS(lsquic_conn_log_cid(conn)));
2310                if (!(conn->cn_flags & LSCONN_EVANESCENT))
2311                {
2312                    close_conn_immediately(engine, &sb_ctx, conn);
2313                    coi_deactivate(&conns_iter, conn);
2314                }
2315                continue;
2316            case ENCPA_OK:
2317                break;
2318            }
2319        }
2320        else if ((packet_out->po_flags & PO_NOENCRYPT)
2321                                         && engine->pub.enp_pmi != &stock_pmi)
2322        {
2323            if (0 != copy_packet(engine, conn, packet_out))
2324            {
2325                /* Copy can only fail if packet could not be allocated */
2326                conn->cn_if->ci_packet_not_sent(conn, packet_out);
2327                goto end_for;
2328            }
2329        }
2330        LSQ_DEBUGC("batched packet %"PRIu64" for connection %"CID_FMT,
2331                    packet_out->po_packno, CID_BITS(lsquic_conn_log_cid(conn)));
2332        if (packet_out->po_flags & PO_ENCRYPTED)
2333        {
2334            iov->iov_base          = packet_out->po_enc_data;
2335            iov->iov_len           = packet_out->po_enc_data_sz;
2336        }
2337        else
2338        {
2339            iov->iov_base          = packet_out->po_data;
2340            iov->iov_len           = packet_out->po_data_sz;
2341        }
2342        if (packet_iov == iov)
2343        {
2344            batch->pack_off[n]         = packet - batch->packets;
2345            batch->outs   [n].ecn      = lsquic_packet_out_ecn(packet_out);
2346            batch->outs   [n].peer_ctx = packet_out->po_path->np_peer_ctx;
2347            batch->outs   [n].local_sa = NP_LOCAL_SA(packet_out->po_path);
2348            batch->outs   [n].dest_sa  = NP_PEER_SA(packet_out->po_path);
2349            batch->conns  [n]          = conn;
2350        }
2351        *packet = packet_out;
2352        ++packet;
2353        ++iov;
2354        if ((conn->cn_flags & LSCONN_IETF)
2355            && ((1 << packet_out->po_header_type)
2356              & ((1 << HETY_INITIAL)|(1 << HETY_HANDSHAKE)|(1 << HETY_0RTT)))
2357#ifndef NDEBUG
2358            && (engine->flags & ENG_COALESCE)
2359#endif
2360            && iov < batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0]))
2361        {
2362            const size_t size = iov_size(packet_iov, iov);
2363            packet_out = conn->cn_if->ci_next_packet_to_send(conn, size);
2364            if (packet_out)
2365                goto next_coa;
2366        }
2367        batch->outs   [n].iovlen = iov - packet_iov;
2368        ++n;
2369        if (n == engine->batch_size
2370            || iov >= batch->iov + sizeof(batch->iov) / sizeof(batch->iov[0]))
2371        {
2372            w = send_batch(engine, &sb_ctx, n);
2373            n = 0;
2374            iov = batch->iov;
2375            packet = batch->packets;
2376            ++n_batches_sent;
2377            n_sent += w;
2378            if (w < engine->batch_size)
2379            {
2380                shrink = 1;
2381                break;
2382            }
2383            deadline_exceeded = check_deadline(engine);
2384            if (deadline_exceeded)
2385                break;
2386            grow_batch_size(engine);
2387        }
2388    }
2389  end_for:
2390
2391    if (n > 0) {
2392        w = send_batch(engine, &sb_ctx, n);
2393        n_sent += w;
2394        shrink = w < n;
2395        ++n_batches_sent;
2396    }
2397
2398    if (shrink)
2399        shrink_batch_size(engine);
2400    else if (n_batches_sent > 1)
2401    {
2402        deadline_exceeded = check_deadline(engine);
2403        if (!deadline_exceeded)
2404            grow_batch_size(engine);
2405    }
2406
2407    coi_reheap(&conns_iter, engine);
2408
2409    LSQ_DEBUG("%s: sent %u packet%.*s", __func__, n_sent, n_sent != 1, "s");
2410}
2411
2412
2413int
2414lsquic_engine_has_unsent_packets (lsquic_engine_t *engine)
2415{
2416    return lsquic_mh_count(&engine->conns_out) > 0
2417             || (engine->pr_queue && prq_have_pending(engine->pr_queue))
2418    ;
2419}
2420
2421
2422static void
2423reset_deadline (lsquic_engine_t *engine, lsquic_time_t now)
2424{
2425    engine->deadline = now + engine->pub.enp_settings.es_proc_time_thresh;
2426    engine->flags &= ~ENG_PAST_DEADLINE;
2427}
2428
2429
2430void
2431lsquic_engine_send_unsent_packets (lsquic_engine_t *engine)
2432{
2433    lsquic_conn_t *conn;
2434    struct conns_stailq closed_conns;
2435    struct conns_tailq ticked_conns = TAILQ_HEAD_INITIALIZER(ticked_conns);
2436    struct cid_update_batch cub;
2437
2438    ENGINE_IN(engine);
2439    cub_init(&cub, engine->report_old_scids, engine->scids_ctx);
2440    STAILQ_INIT(&closed_conns);
2441    reset_deadline(engine, lsquic_time_now());
2442    if (!(engine->pub.enp_flags & ENPUB_CAN_SEND))
2443    {
2444        LSQ_DEBUG("can send again");
2445        EV_LOG_GENERIC_EVENT("can send again");
2446        engine->pub.enp_flags |= ENPUB_CAN_SEND;
2447    }
2448
2449    send_packets_out(engine, &ticked_conns, &closed_conns);
2450
2451    while ((conn = STAILQ_FIRST(&closed_conns))) {
2452        STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn);
2453        if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI)
2454            cub_add_cids_from_cces(&cub, conn);
2455        (void) engine_decref_conn(engine, conn, LSCONN_CLOSING);
2456    }
2457
2458    cub_flush(&cub);
2459    ENGINE_OUT(engine);
2460}
2461
2462
2463static lsquic_conn_t *
2464next_new_full_conn (struct conns_stailq *new_full_conns)
2465{
2466    lsquic_conn_t *conn;
2467
2468    conn = STAILQ_FIRST(new_full_conns);
2469    if (conn)
2470        STAILQ_REMOVE_HEAD(new_full_conns, cn_next_new_full);
2471    return conn;
2472}
2473
2474
2475static void
2476process_connections (lsquic_engine_t *engine, conn_iter_f next_conn,
2477                     lsquic_time_t now)
2478{
2479    lsquic_conn_t *conn;
2480    enum tick_st tick_st;
2481    unsigned i, why;
2482    lsquic_time_t next_tick_time;
2483    struct conns_stailq closed_conns;
2484    struct conns_tailq ticked_conns;
2485    struct conns_stailq new_full_conns;
2486    struct cid_update_batch cub_old, cub_live;
2487    cub_init(&cub_old, engine->report_old_scids, engine->scids_ctx);
2488    cub_init(&cub_live, engine->report_live_scids, engine->scids_ctx);
2489
2490    eng_hist_tick(&engine->history, now);
2491
2492    STAILQ_INIT(&closed_conns);
2493    TAILQ_INIT(&ticked_conns);
2494    reset_deadline(engine, now);
2495    STAILQ_INIT(&new_full_conns);
2496
2497    if (!(engine->pub.enp_flags & ENPUB_CAN_SEND)
2498                                        && now > engine->resume_sending_at)
2499    {
2500        LSQ_NOTICE("failsafe activated: resume sending packets again after "
2501                    "timeout");
2502        EV_LOG_GENERIC_EVENT("resume sending packets again after timeout");
2503        engine->pub.enp_flags |= ENPUB_CAN_SEND;
2504    }
2505
2506    i = 0;
2507    while ((conn = next_conn(engine))
2508                            || (conn = next_new_full_conn(&new_full_conns)))
2509    {
2510        tick_st = conn->cn_if->ci_tick(conn, now);
2511        conn->cn_last_ticked = now + i /* Maintain relative order */ ++;
2512        if (tick_st & TICK_PROMOTE)
2513        {
2514            lsquic_conn_t *new_conn;
2515            EV_LOG_CONN_EVENT(lsquic_conn_log_cid(conn),
2516                                                "scheduled for promotion");
2517            assert(conn->cn_flags & LSCONN_MINI);
2518            new_conn = new_full_conn_server(engine, conn, now);
2519            if (new_conn)
2520            {
2521                STAILQ_INSERT_TAIL(&new_full_conns, new_conn, cn_next_new_full);
2522                new_conn->cn_last_sent = engine->last_sent;
2523                eng_hist_inc(&engine->history, now, sl_new_full_conns);
2524                conn->cn_flags |= LSCONN_PROMOTED;
2525            }
2526            tick_st |= TICK_CLOSE;  /* Destroy mini connection */
2527        }
2528        if (tick_st & TICK_SEND)
2529        {
2530            if (!(conn->cn_flags & LSCONN_HAS_OUTGOING))
2531            {
2532                lsquic_mh_insert(&engine->conns_out, conn, conn->cn_last_sent);
2533                engine_incref_conn(conn, LSCONN_HAS_OUTGOING);
2534            }
2535        }
2536        if (tick_st & TICK_CLOSE)
2537        {
2538            STAILQ_INSERT_TAIL(&closed_conns, conn, cn_next_closed_conn);
2539            engine_incref_conn(conn, LSCONN_CLOSING);
2540            if (conn->cn_flags & LSCONN_HASHED)
2541                remove_conn_from_hash(engine, conn);
2542        }
2543        else
2544        {
2545            TAILQ_INSERT_TAIL(&ticked_conns, conn, cn_next_ticked);
2546            engine_incref_conn(conn, LSCONN_TICKED);
2547            if ((engine->flags & ENG_SERVER) && conn->cn_if->ci_report_live
2548                                    && conn->cn_if->ci_report_live(conn, now))
2549                cub_add_cids_from_cces(&cub_live, conn);
2550        }
2551    }
2552
2553    if ((engine->pub.enp_flags & ENPUB_CAN_SEND)
2554                        && lsquic_engine_has_unsent_packets(engine))
2555        send_packets_out(engine, &ticked_conns, &closed_conns);
2556
2557    while ((conn = STAILQ_FIRST(&closed_conns))) {
2558        STAILQ_REMOVE_HEAD(&closed_conns, cn_next_closed_conn);
2559        if ((conn->cn_flags & (LSCONN_MINI|LSCONN_PROMOTED)) == LSCONN_MINI)
2560            cub_add_cids_from_cces(&cub_old, conn);
2561        (void) engine_decref_conn(engine, conn, LSCONN_CLOSING);
2562    }
2563
2564    while ((conn = TAILQ_FIRST(&ticked_conns)))
2565    {
2566        TAILQ_REMOVE(&ticked_conns, conn, cn_next_ticked);
2567        engine_decref_conn(engine, conn, LSCONN_TICKED);
2568        if (!(conn->cn_flags & LSCONN_TICKABLE)
2569            && conn->cn_if->ci_is_tickable(conn))
2570        {
2571            /* Floyd heapification is not faster, don't bother. */
2572            lsquic_mh_insert(&engine->conns_tickable, conn, conn->cn_last_ticked);
2573            engine_incref_conn(conn, LSCONN_TICKABLE);
2574        }
2575        else if (!(conn->cn_flags & LSCONN_ATTQ))
2576        {
2577            next_tick_time = conn->cn_if->ci_next_tick_time(conn, &why);
2578            if (next_tick_time)
2579            {
2580                if (0 == attq_add(engine->attq, conn, next_tick_time, why))
2581                    engine_incref_conn(conn, LSCONN_ATTQ);
2582            }
2583            else
2584                /* In all other cases, the idle timeout would make the next
2585                 * tick time non-zero:
2586                 */
2587                assert((conn->cn_flags & LSCONN_IETF)
2588                    && engine->pub.enp_settings.es_idle_timeout == 0);
2589        }
2590    }
2591
2592    cub_flush(&engine->new_scids);
2593    cub_flush(&cub_live);
2594    cub_flush(&cub_old);
2595}
2596
2597
2598/* Return 0 if packet is being processed by a real connection, 1 if the
2599 * packet was processed, but not by a connection, and -1 on error.
2600 */
2601int
2602lsquic_engine_packet_in (lsquic_engine_t *engine,
2603    const unsigned char *packet_in_data, size_t packet_in_size,
2604    const struct sockaddr *sa_local, const struct sockaddr *sa_peer,
2605    void *peer_ctx, int ecn)
2606{
2607    const unsigned char *const packet_end = packet_in_data + packet_in_size;
2608    struct packin_parse_state ppstate;
2609    lsquic_packet_in_t *packet_in;
2610    int (*parse_packet_in_begin) (struct lsquic_packet_in *, size_t length,
2611                int is_server, unsigned cid_len, struct packin_parse_state *);
2612    unsigned n_zeroes;
2613    int s;
2614
2615    ENGINE_CALLS_INCR(engine);
2616
2617    if (engine->flags & ENG_SERVER)
2618        parse_packet_in_begin = lsquic_parse_packet_in_server_begin;
2619    else if (engine->flags & ENG_CONNS_BY_ADDR)
2620    {
2621        struct lsquic_hash_elem *el;
2622        const struct lsquic_conn *conn;
2623        el = find_conn_by_addr(engine->conns_hash, sa_local);
2624        if (!el)
2625            return -1;
2626        conn = lsquic_hashelem_getdata(el);
2627        if ((1 << conn->cn_version) & LSQUIC_GQUIC_HEADER_VERSIONS)
2628            parse_packet_in_begin = lsquic_gquic_parse_packet_in_begin;
2629        else if ((1 << conn->cn_version) & LSQUIC_IETF_VERSIONS)
2630            parse_packet_in_begin = lsquic_ietf_v1_parse_packet_in_begin;
2631        else if (conn->cn_version == LSQVER_050)
2632            parse_packet_in_begin = lsquic_Q050_parse_packet_in_begin;
2633        else
2634        {
2635            assert(conn->cn_version == LSQVER_046
2636#if LSQUIC_USE_Q098
2637                   || conn->cn_version == LSQVER_098
2638#endif
2639
2640                                                    );
2641            parse_packet_in_begin = lsquic_Q046_parse_packet_in_begin;
2642        }
2643    }
2644    else
2645        parse_packet_in_begin = lsquic_parse_packet_in_begin;
2646
2647    n_zeroes = 0;
2648    do
2649    {
2650        packet_in = lsquic_mm_get_packet_in(&engine->pub.enp_mm);
2651        if (!packet_in)
2652            return -1;
2653        /* Library does not modify packet_in_data, it is not referenced after
2654         * this function returns and subsequent release of pi_data is guarded
2655         * by PI_OWN_DATA flag.
2656         */
2657        packet_in->pi_data = (unsigned char *) packet_in_data;
2658        if (0 != parse_packet_in_begin(packet_in, packet_end - packet_in_data,
2659                                engine->flags & ENG_SERVER,
2660                                engine->pub.enp_settings.es_scid_len, &ppstate))
2661        {
2662            LSQ_DEBUG("Cannot parse incoming packet's header");
2663            lsquic_mm_put_packet_in(&engine->pub.enp_mm, packet_in);
2664            s = 1;
2665            break;
2666        }
2667
2668        packet_in_data += packet_in->pi_data_sz;
2669        packet_in->pi_received = lsquic_time_now();
2670        packet_in->pi_flags |= (3 & ecn) << PIBIT_ECN_SHIFT;
2671        eng_hist_inc(&engine->history, packet_in->pi_received, sl_packets_in);
2672        s = process_packet_in(engine, packet_in, &ppstate, sa_local, sa_peer,
2673                            peer_ctx, packet_in_size);
2674        n_zeroes += s == 0;
2675    }
2676    while (0 == s && packet_in_data < packet_end);
2677
2678    return n_zeroes > 0 ? 0 : s;
2679}
2680
2681
2682#if __GNUC__ && !defined(NDEBUG)
2683__attribute__((weak))
2684#endif
2685unsigned
2686lsquic_engine_quic_versions (const lsquic_engine_t *engine)
2687{
2688    return engine->pub.enp_settings.es_versions;
2689}
2690
2691
2692void
2693lsquic_engine_cooldown (lsquic_engine_t *engine)
2694{
2695    struct lsquic_hash_elem *el;
2696    lsquic_conn_t *conn;
2697
2698    if (engine->flags & ENG_COOLDOWN)
2699        /* AFAICT, there is no harm in calling this function more than once,
2700         * but log it just in case, as it may indicate an error in the caller.
2701         */
2702        LSQ_INFO("cooldown called again");
2703    engine->flags |= ENG_COOLDOWN;
2704    LSQ_INFO("entering cooldown mode");
2705    if (engine->flags & ENG_SERVER)
2706        drop_all_mini_conns(engine);
2707    for (el = lsquic_hash_first(engine->conns_hash); el;
2708                                el = lsquic_hash_next(engine->conns_hash))
2709    {
2710        conn = lsquic_hashelem_getdata(el);
2711        lsquic_conn_going_away(conn);
2712    }
2713}
2714
2715
2716int
2717lsquic_engine_earliest_adv_tick (lsquic_engine_t *engine, int *diff)
2718{
2719    const struct attq_elem *next_attq;
2720    lsquic_time_t now, next_time;
2721#if LSQUIC_DEBUG_NEXT_ADV_TICK
2722    const struct lsquic_conn *conn;
2723    const enum lsq_log_level L = LSQ_LOG_DEBUG;  /* Easy toggle */
2724#endif
2725
2726    ENGINE_CALLS_INCR(engine);
2727
2728    if ((engine->flags & ENG_PAST_DEADLINE)
2729                                    && lsquic_mh_count(&engine->conns_out))
2730    {
2731#if LSQUIC_DEBUG_NEXT_ADV_TICK
2732        conn = lsquic_mh_peek(&engine->conns_out);
2733        engine->last_logged_conn = 0;
2734        LSQ_LOGC(L, "next advisory tick is now: went past deadline last time "
2735            "and have %u outgoing connection%.*s (%"CID_FMT" first)",
2736            lsquic_mh_count(&engine->conns_out),
2737            lsquic_mh_count(&engine->conns_out) != 1, "s",
2738            CID_BITS(lsquic_conn_log_cid(conn)));
2739#endif
2740        *diff = 0;
2741        return 1;
2742    }
2743
2744    if (engine->pr_queue && prq_have_pending(engine->pr_queue))
2745    {
2746#if LSQUIC_DEBUG_NEXT_ADV_TICK
2747        engine->last_logged_conn = 0;
2748        LSQ_LOG(L, "next advisory tick is now: have pending PRQ elements");
2749#endif
2750        *diff = 0;
2751        return 1;
2752    }
2753
2754    if (lsquic_mh_count(&engine->conns_tickable))
2755    {
2756#if LSQUIC_DEBUG_NEXT_ADV_TICK
2757        conn = lsquic_mh_peek(&engine->conns_tickable);
2758        engine->last_logged_conn = 0;
2759        LSQ_LOGC(L, "next advisory tick is now: have %u tickable "
2760            "connection%.*s (%"CID_FMT" first)",
2761            lsquic_mh_count(&engine->conns_tickable),
2762            lsquic_mh_count(&engine->conns_tickable) != 1, "s",
2763            CID_BITS(lsquic_conn_log_cid(conn)));
2764#endif
2765        *diff = 0;
2766        return 1;
2767    }
2768
2769    next_attq = attq_next(engine->attq);
2770    if (engine->pub.enp_flags & ENPUB_CAN_SEND)
2771    {
2772        if (next_attq)
2773            next_time = next_attq->ae_adv_time;
2774        else
2775            return 0;
2776    }
2777    else
2778    {
2779        if (next_attq)
2780        {
2781            next_time = next_attq->ae_adv_time;
2782            if (engine->resume_sending_at < next_time)
2783            {
2784                next_time = engine->resume_sending_at;
2785                next_attq = NULL;
2786            }
2787        }
2788        else
2789            next_time = engine->resume_sending_at;
2790    }
2791
2792    now = lsquic_time_now();
2793    *diff = (int) ((int64_t) next_time - (int64_t) now);
2794#if LSQUIC_DEBUG_NEXT_ADV_TICK
2795    if (next_attq)
2796    {
2797        /* Deduplicate consecutive log messages about the same reason for the
2798         * same connection.
2799         * If diff is always zero or diff reset to a higher value, event is
2800         * still logged.
2801         */
2802        if (!((unsigned) next_attq->ae_why == engine->last_logged_ae_why
2803                    && (uintptr_t) next_attq->ae_conn
2804                                            == engine->last_logged_conn
2805                    && *diff < engine->last_tick_diff))
2806        {
2807            engine->last_logged_conn = (uintptr_t) next_attq->ae_conn;
2808            engine->last_logged_ae_why = (unsigned) next_attq->ae_why;
2809            engine->last_tick_diff = *diff;
2810            LSQ_LOGC(L, "next advisory tick is %d usec away: conn %"CID_FMT
2811                ": %s", *diff, CID_BITS(lsquic_conn_log_cid(next_attq->ae_conn)),
2812                lsquic_attq_why2str(next_attq->ae_why));
2813        }
2814    }
2815    else
2816        LSQ_LOG(L, "next advisory tick is %d usec away: resume sending", *diff);
2817#endif
2818    return 1;
2819}
2820
2821
2822unsigned
2823lsquic_engine_count_attq (lsquic_engine_t *engine, int from_now)
2824{
2825    lsquic_time_t now;
2826    ENGINE_CALLS_INCR(engine);
2827    now = lsquic_time_now();
2828    if (from_now < 0)
2829        now -= from_now;
2830    else
2831        now += from_now;
2832    return attq_count_before(engine->attq, now);
2833}
2834
2835
2836int
2837lsquic_engine_add_cid (struct lsquic_engine_public *enpub,
2838                              struct lsquic_conn *conn, unsigned cce_idx)
2839{
2840    struct lsquic_engine *const engine = (struct lsquic_engine *) enpub;
2841    struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx];
2842    void *peer_ctx;
2843
2844    assert(cce_idx < conn->cn_n_cces);
2845    assert(conn->cn_cces_mask & (1 << cce_idx));
2846    assert(!(cce->cce_hash_el.qhe_flags & QHE_HASHED));
2847
2848    if (lsquic_hash_insert(engine->conns_hash, cce->cce_cid.idbuf,
2849                                    cce->cce_cid.len, conn, &cce->cce_hash_el))
2850    {
2851        LSQ_DEBUGC("add %"CID_FMT" to the list of SCIDs",
2852                                                    CID_BITS(&cce->cce_cid));
2853        peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL);
2854        cce->cce_flags |= CCE_REG;
2855        cub_add(&engine->new_scids, &cce->cce_cid, peer_ctx);
2856        return 0;
2857    }
2858    else
2859    {
2860        LSQ_WARNC("could not add new cid %"CID_FMT" to the SCID hash",
2861                                                    CID_BITS(&cce->cce_cid));
2862        return -1;
2863    }
2864}
2865
2866
2867void
2868lsquic_engine_retire_cid (struct lsquic_engine_public *enpub,
2869              struct lsquic_conn *conn, unsigned cce_idx, lsquic_time_t now)
2870{
2871    struct lsquic_engine *const engine = (struct lsquic_engine *) enpub;
2872    struct conn_cid_elem *const cce = &conn->cn_cces[cce_idx];
2873    void *peer_ctx;
2874
2875    assert(cce_idx < conn->cn_n_cces);
2876
2877    if (cce->cce_hash_el.qhe_flags & QHE_HASHED)
2878        lsquic_hash_erase(engine->conns_hash, &cce->cce_hash_el);
2879
2880    if (engine->purga)
2881    {
2882        peer_ctx = lsquic_conn_get_peer_ctx(conn, NULL);
2883        lsquic_purga_add(engine->purga, &cce->cce_cid, peer_ctx,
2884                                                    PUTY_CID_RETIRED, now);
2885    }
2886    conn->cn_cces_mask &= ~(1u << cce_idx);
2887    LSQ_DEBUGC("retire CID %"CID_FMT, CID_BITS(&cce->cce_cid));
2888}
2889
2890
2891