lsquic_mm.c revision de46bf2f
1229fce07SDmitri Tikhonov/* Copyright (c) 2017 - 2019 LiteSpeed Technologies Inc.  See LICENSE. */
250aadb33SDmitri Tikhonov/*
350aadb33SDmitri Tikhonov * lsquic_mm.c -- Memory manager.
450aadb33SDmitri Tikhonov */
550aadb33SDmitri Tikhonov
650aadb33SDmitri Tikhonov#include <assert.h>
750aadb33SDmitri Tikhonov#include <errno.h>
850aadb33SDmitri Tikhonov#include <stddef.h>
950aadb33SDmitri Tikhonov#include <stdlib.h>
1050aadb33SDmitri Tikhonov#include <string.h>
1150aadb33SDmitri Tikhonov#include <sys/queue.h>
1250aadb33SDmitri Tikhonov
1350aadb33SDmitri Tikhonov#include "fiu-local.h"
1450aadb33SDmitri Tikhonov
1550aadb33SDmitri Tikhonov#include "lsquic.h"
1650aadb33SDmitri Tikhonov#include "lsquic_int_types.h"
175392f7a3SLiteSpeed Tech#include "lsquic_sizes.h"
1850aadb33SDmitri Tikhonov#include "lsquic_malo.h"
195392f7a3SLiteSpeed Tech#include "lsquic_hash.h"
2050aadb33SDmitri Tikhonov#include "lsquic_conn.h"
2150aadb33SDmitri Tikhonov#include "lsquic_rtt.h"
2250aadb33SDmitri Tikhonov#include "lsquic_packet_common.h"
235392f7a3SLiteSpeed Tech#include "lsquic_mini_conn.h"
245392f7a3SLiteSpeed Tech#include "lsquic_enc_sess.h"
255392f7a3SLiteSpeed Tech#include "lsquic_mini_conn_ietf.h"
265392f7a3SLiteSpeed Tech#include "lsquic_packet_gquic.h"
2750aadb33SDmitri Tikhonov#include "lsquic_packet_in.h"
2850aadb33SDmitri Tikhonov#include "lsquic_packet_out.h"
2950aadb33SDmitri Tikhonov#include "lsquic_parse.h"
3050aadb33SDmitri Tikhonov#include "lsquic_mm.h"
3150aadb33SDmitri Tikhonov#include "lsquic_engine_public.h"
325392f7a3SLiteSpeed Tech#include "lsquic_full_conn.h"
335392f7a3SLiteSpeed Tech#include "lsquic_varint.h"
345392f7a3SLiteSpeed Tech#include "lsquic_hq.h"
355392f7a3SLiteSpeed Tech#include "lsquic_sfcw.h"
365392f7a3SLiteSpeed Tech#include "lsquic_stream.h"
375392f7a3SLiteSpeed Tech
385392f7a3SLiteSpeed Tech#ifndef LSQUIC_LOG_POOL_STATS
395392f7a3SLiteSpeed Tech#define LSQUIC_LOG_POOL_STATS 0
405392f7a3SLiteSpeed Tech#endif
415392f7a3SLiteSpeed Tech
425392f7a3SLiteSpeed Tech#if LSQUIC_LOG_POOL_STATS
435392f7a3SLiteSpeed Tech#include "lsquic_logger.h"
445392f7a3SLiteSpeed Tech#endif
455392f7a3SLiteSpeed Tech
465392f7a3SLiteSpeed Tech#ifndef LSQUIC_USE_POOLS
475392f7a3SLiteSpeed Tech#define LSQUIC_USE_POOLS 1
485392f7a3SLiteSpeed Tech#endif
4950aadb33SDmitri Tikhonov
5050aadb33SDmitri Tikhonov#define FAIL_NOMEM do { errno = ENOMEM; return NULL; } while (0)
5150aadb33SDmitri Tikhonov
5250aadb33SDmitri Tikhonov
535392f7a3SLiteSpeed Techstruct packet_in_buf
5450aadb33SDmitri Tikhonov{
555392f7a3SLiteSpeed Tech    SLIST_ENTRY(packet_in_buf)  next_pib;
5650aadb33SDmitri Tikhonov};
5750aadb33SDmitri Tikhonov
5850aadb33SDmitri Tikhonovstruct packet_out_buf
5950aadb33SDmitri Tikhonov{
6050aadb33SDmitri Tikhonov    SLIST_ENTRY(packet_out_buf) next_pob;
6150aadb33SDmitri Tikhonov};
6250aadb33SDmitri Tikhonov
6350aadb33SDmitri Tikhonovstruct four_k_page
6450aadb33SDmitri Tikhonov{
6550aadb33SDmitri Tikhonov    SLIST_ENTRY(four_k_page)  next_fkp;
6650aadb33SDmitri Tikhonov};
6750aadb33SDmitri Tikhonov
6850aadb33SDmitri Tikhonovstruct sixteen_k_page
6950aadb33SDmitri Tikhonov{
7050aadb33SDmitri Tikhonov    SLIST_ENTRY(sixteen_k_page)  next_skp;
7150aadb33SDmitri Tikhonov};
7250aadb33SDmitri Tikhonov
7350aadb33SDmitri Tikhonov
7450aadb33SDmitri Tikhonovint
7550aadb33SDmitri Tikhonovlsquic_mm_init (struct lsquic_mm *mm)
7650aadb33SDmitri Tikhonov{
775392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
7850aadb33SDmitri Tikhonov    int i;
795392f7a3SLiteSpeed Tech#endif
8050aadb33SDmitri Tikhonov
8150aadb33SDmitri Tikhonov    mm->acki = malloc(sizeof(*mm->acki));
8250aadb33SDmitri Tikhonov    mm->malo.stream_frame = lsquic_malo_create(sizeof(struct stream_frame));
8350aadb33SDmitri Tikhonov    mm->malo.stream_rec_arr = lsquic_malo_create(sizeof(struct stream_rec_arr));
845392f7a3SLiteSpeed Tech    mm->malo.mini_conn = lsquic_malo_create(sizeof(struct mini_conn));
855392f7a3SLiteSpeed Tech    mm->malo.mini_conn_ietf = lsquic_malo_create(sizeof(struct ietf_mini_conn));
8650aadb33SDmitri Tikhonov    mm->malo.packet_in = lsquic_malo_create(sizeof(struct lsquic_packet_in));
8750aadb33SDmitri Tikhonov    mm->malo.packet_out = lsquic_malo_create(sizeof(struct lsquic_packet_out));
885392f7a3SLiteSpeed Tech    mm->malo.dcid_elem = lsquic_malo_create(sizeof(struct dcid_elem));
895392f7a3SLiteSpeed Tech    mm->malo.stream_hq_frame
905392f7a3SLiteSpeed Tech                        = lsquic_malo_create(sizeof(struct stream_hq_frame));
91de46bf2fSDmitri Tikhonov    mm->ack_str = malloc(MAX_ACKI_STR_SZ);
925392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
9350aadb33SDmitri Tikhonov    TAILQ_INIT(&mm->free_packets_in);
9450aadb33SDmitri Tikhonov    for (i = 0; i < MM_N_OUT_BUCKETS; ++i)
9550aadb33SDmitri Tikhonov        SLIST_INIT(&mm->packet_out_bufs[i]);
965392f7a3SLiteSpeed Tech    for (i = 0; i < MM_N_IN_BUCKETS; ++i)
975392f7a3SLiteSpeed Tech        SLIST_INIT(&mm->packet_in_bufs[i]);
9850aadb33SDmitri Tikhonov    SLIST_INIT(&mm->four_k_pages);
9950aadb33SDmitri Tikhonov    SLIST_INIT(&mm->sixteen_k_pages);
1005392f7a3SLiteSpeed Tech#endif
1015392f7a3SLiteSpeed Tech    if (mm->acki && mm->malo.stream_frame && mm->malo.stream_rec_arr
1025392f7a3SLiteSpeed Tech        && mm->malo.mini_conn && mm->malo.mini_conn_ietf && mm->malo.packet_in
103de46bf2fSDmitri Tikhonov        && mm->malo.packet_out && mm->malo.dcid_elem
104de46bf2fSDmitri Tikhonov        && mm->malo.stream_hq_frame && mm->ack_str)
10550aadb33SDmitri Tikhonov    {
10650aadb33SDmitri Tikhonov        return 0;
10750aadb33SDmitri Tikhonov    }
10850aadb33SDmitri Tikhonov    else
10950aadb33SDmitri Tikhonov        return -1;
11050aadb33SDmitri Tikhonov}
11150aadb33SDmitri Tikhonov
11250aadb33SDmitri Tikhonov
11350aadb33SDmitri Tikhonovvoid
11450aadb33SDmitri Tikhonovlsquic_mm_cleanup (struct lsquic_mm *mm)
11550aadb33SDmitri Tikhonov{
1165392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
11750aadb33SDmitri Tikhonov    int i;
11850aadb33SDmitri Tikhonov    struct packet_out_buf *pob;
1195392f7a3SLiteSpeed Tech    struct packet_in_buf *pib;
12050aadb33SDmitri Tikhonov    struct four_k_page *fkp;
12150aadb33SDmitri Tikhonov    struct sixteen_k_page *skp;
1225392f7a3SLiteSpeed Tech#endif
12350aadb33SDmitri Tikhonov
12450aadb33SDmitri Tikhonov    free(mm->acki);
1255392f7a3SLiteSpeed Tech    lsquic_malo_destroy(mm->malo.stream_hq_frame);
1265392f7a3SLiteSpeed Tech    lsquic_malo_destroy(mm->malo.dcid_elem);
12750aadb33SDmitri Tikhonov    lsquic_malo_destroy(mm->malo.packet_in);
12850aadb33SDmitri Tikhonov    lsquic_malo_destroy(mm->malo.packet_out);
12950aadb33SDmitri Tikhonov    lsquic_malo_destroy(mm->malo.stream_frame);
13050aadb33SDmitri Tikhonov    lsquic_malo_destroy(mm->malo.stream_rec_arr);
1315392f7a3SLiteSpeed Tech    lsquic_malo_destroy(mm->malo.mini_conn);
1325392f7a3SLiteSpeed Tech    lsquic_malo_destroy(mm->malo.mini_conn_ietf);
133de46bf2fSDmitri Tikhonov    free(mm->ack_str);
13450aadb33SDmitri Tikhonov
1355392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
13650aadb33SDmitri Tikhonov    for (i = 0; i < MM_N_OUT_BUCKETS; ++i)
13750aadb33SDmitri Tikhonov        while ((pob = SLIST_FIRST(&mm->packet_out_bufs[i])))
13850aadb33SDmitri Tikhonov        {
13950aadb33SDmitri Tikhonov            SLIST_REMOVE_HEAD(&mm->packet_out_bufs[i], next_pob);
14050aadb33SDmitri Tikhonov            free(pob);
14150aadb33SDmitri Tikhonov        }
14250aadb33SDmitri Tikhonov
1435392f7a3SLiteSpeed Tech    for (i = 0; i < MM_N_IN_BUCKETS; ++i)
1445392f7a3SLiteSpeed Tech        while ((pib = SLIST_FIRST(&mm->packet_in_bufs[i])))
1455392f7a3SLiteSpeed Tech        {
1465392f7a3SLiteSpeed Tech            SLIST_REMOVE_HEAD(&mm->packet_in_bufs[i], next_pib);
1475392f7a3SLiteSpeed Tech            free(pib);
1485392f7a3SLiteSpeed Tech        }
14950aadb33SDmitri Tikhonov
15050aadb33SDmitri Tikhonov    while ((fkp = SLIST_FIRST(&mm->four_k_pages)))
15150aadb33SDmitri Tikhonov    {
15250aadb33SDmitri Tikhonov        SLIST_REMOVE_HEAD(&mm->four_k_pages, next_fkp);
15350aadb33SDmitri Tikhonov        free(fkp);
15450aadb33SDmitri Tikhonov    }
15550aadb33SDmitri Tikhonov
15650aadb33SDmitri Tikhonov    while ((skp = SLIST_FIRST(&mm->sixteen_k_pages)))
15750aadb33SDmitri Tikhonov    {
15850aadb33SDmitri Tikhonov        SLIST_REMOVE_HEAD(&mm->sixteen_k_pages, next_skp);
15950aadb33SDmitri Tikhonov        free(skp);
16050aadb33SDmitri Tikhonov    }
1615392f7a3SLiteSpeed Tech#endif
1625392f7a3SLiteSpeed Tech}
1635392f7a3SLiteSpeed Tech
1645392f7a3SLiteSpeed Tech
1655392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
1665392f7a3SLiteSpeed Techenum {
1675392f7a3SLiteSpeed Tech    PACKET_IN_PAYLOAD_0 = 1370,     /* common QUIC payload size upperbound */
1685392f7a3SLiteSpeed Tech    PACKET_IN_PAYLOAD_1 = 4096,     /* payload size middleground guess */
1695392f7a3SLiteSpeed Tech    PACKET_IN_PAYLOAD_2 = 0xffff,   /* UDP payload size upperbound */
1705392f7a3SLiteSpeed Tech};
1715392f7a3SLiteSpeed Tech
1725392f7a3SLiteSpeed Tech
1735392f7a3SLiteSpeed Techstatic const unsigned packet_in_sizes[] = {
1745392f7a3SLiteSpeed Tech    PACKET_IN_PAYLOAD_0,
1755392f7a3SLiteSpeed Tech    PACKET_IN_PAYLOAD_1,
1765392f7a3SLiteSpeed Tech    PACKET_IN_PAYLOAD_2,
1775392f7a3SLiteSpeed Tech};
1785392f7a3SLiteSpeed Tech
1795392f7a3SLiteSpeed Tech
1805392f7a3SLiteSpeed Techstatic unsigned
1815392f7a3SLiteSpeed Techpacket_in_index (unsigned size)
1825392f7a3SLiteSpeed Tech{
1835392f7a3SLiteSpeed Tech    unsigned idx = (size > PACKET_IN_PAYLOAD_0)
1845392f7a3SLiteSpeed Tech                 + (size > PACKET_IN_PAYLOAD_1);
1855392f7a3SLiteSpeed Tech    return idx;
1865392f7a3SLiteSpeed Tech}
1875392f7a3SLiteSpeed Tech#endif
1885392f7a3SLiteSpeed Tech
1895392f7a3SLiteSpeed Tech
1905392f7a3SLiteSpeed Techvoid
1915392f7a3SLiteSpeed Techlsquic_mm_put_packet_in (struct lsquic_mm *mm,
1925392f7a3SLiteSpeed Tech                                        struct lsquic_packet_in *packet_in)
1935392f7a3SLiteSpeed Tech{
1945392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
1955392f7a3SLiteSpeed Tech    unsigned idx;
1965392f7a3SLiteSpeed Tech    struct packet_in_buf *pib;
1975392f7a3SLiteSpeed Tech
1985392f7a3SLiteSpeed Tech    assert(0 == packet_in->pi_refcnt);
1995392f7a3SLiteSpeed Tech    if (packet_in->pi_flags & PI_OWN_DATA)
2005392f7a3SLiteSpeed Tech    {
2015392f7a3SLiteSpeed Tech        pib = (struct packet_in_buf *) packet_in->pi_data;
2025392f7a3SLiteSpeed Tech        idx = packet_in_index(packet_in->pi_data_sz);
2035392f7a3SLiteSpeed Tech        SLIST_INSERT_HEAD(&mm->packet_in_bufs[idx], pib, next_pib);
2045392f7a3SLiteSpeed Tech    }
2055392f7a3SLiteSpeed Tech    TAILQ_INSERT_HEAD(&mm->free_packets_in, packet_in, pi_next);
2065392f7a3SLiteSpeed Tech#else
2075392f7a3SLiteSpeed Tech    if (packet_in->pi_flags & PI_OWN_DATA)
2085392f7a3SLiteSpeed Tech        free(packet_in->pi_data);
2095392f7a3SLiteSpeed Tech    lsquic_malo_put(packet_in);
2105392f7a3SLiteSpeed Tech#endif
21150aadb33SDmitri Tikhonov}
21250aadb33SDmitri Tikhonov
21350aadb33SDmitri Tikhonov
21450aadb33SDmitri Tikhonovstruct lsquic_packet_in *
21550aadb33SDmitri Tikhonovlsquic_mm_get_packet_in (struct lsquic_mm *mm)
21650aadb33SDmitri Tikhonov{
21750aadb33SDmitri Tikhonov    struct lsquic_packet_in *packet_in;
21850aadb33SDmitri Tikhonov
21950aadb33SDmitri Tikhonov    fiu_do_on("mm/packet_in", FAIL_NOMEM);
22050aadb33SDmitri Tikhonov
2215392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
22250aadb33SDmitri Tikhonov    packet_in = TAILQ_FIRST(&mm->free_packets_in);
22350aadb33SDmitri Tikhonov    if (packet_in)
22450aadb33SDmitri Tikhonov    {
22550aadb33SDmitri Tikhonov        assert(0 == packet_in->pi_refcnt);
22650aadb33SDmitri Tikhonov        TAILQ_REMOVE(&mm->free_packets_in, packet_in, pi_next);
22750aadb33SDmitri Tikhonov    }
22850aadb33SDmitri Tikhonov    else
2295392f7a3SLiteSpeed Tech#endif
23050aadb33SDmitri Tikhonov        packet_in = lsquic_malo_get(mm->malo.packet_in);
23150aadb33SDmitri Tikhonov
23250aadb33SDmitri Tikhonov    if (packet_in)
23350aadb33SDmitri Tikhonov        memset(packet_in, 0, sizeof(*packet_in));
23450aadb33SDmitri Tikhonov
23550aadb33SDmitri Tikhonov    return packet_in;
23650aadb33SDmitri Tikhonov}
23750aadb33SDmitri Tikhonov
23850aadb33SDmitri Tikhonov
2395392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
24050aadb33SDmitri Tikhonov/* Based on commonly used MTUs, ordered from small to large: */
24150aadb33SDmitri Tikhonovenum {
2425392f7a3SLiteSpeed Tech    PACKET_OUT_PAYLOAD_0 = 1280                    - GQUIC_MIN_PACKET_OVERHEAD,
2435392f7a3SLiteSpeed Tech    PACKET_OUT_PAYLOAD_1 = GQUIC_MAX_IPv6_PACKET_SZ - GQUIC_MIN_PACKET_OVERHEAD,
2445392f7a3SLiteSpeed Tech    PACKET_OUT_PAYLOAD_2 = GQUIC_MAX_IPv4_PACKET_SZ - GQUIC_MIN_PACKET_OVERHEAD,
2455392f7a3SLiteSpeed Tech    PACKET_OUT_PAYLOAD_3 = 4096,
2465392f7a3SLiteSpeed Tech    PACKET_OUT_PAYLOAD_4 = 0xffff,
24750aadb33SDmitri Tikhonov};
24850aadb33SDmitri Tikhonov
24950aadb33SDmitri Tikhonov
25050aadb33SDmitri Tikhonovstatic const unsigned packet_out_sizes[] = {
25150aadb33SDmitri Tikhonov    PACKET_OUT_PAYLOAD_0,
25250aadb33SDmitri Tikhonov    PACKET_OUT_PAYLOAD_1,
25350aadb33SDmitri Tikhonov    PACKET_OUT_PAYLOAD_2,
2545392f7a3SLiteSpeed Tech    PACKET_OUT_PAYLOAD_3,
2555392f7a3SLiteSpeed Tech    PACKET_OUT_PAYLOAD_4,
25650aadb33SDmitri Tikhonov};
25750aadb33SDmitri Tikhonov
25850aadb33SDmitri Tikhonov
25950aadb33SDmitri Tikhonovstatic unsigned
26050aadb33SDmitri Tikhonovpacket_out_index (unsigned size)
26150aadb33SDmitri Tikhonov{
26250aadb33SDmitri Tikhonov    unsigned idx = (size > PACKET_OUT_PAYLOAD_0)
2635392f7a3SLiteSpeed Tech                 + (size > PACKET_OUT_PAYLOAD_1)
2645392f7a3SLiteSpeed Tech                 + (size > PACKET_OUT_PAYLOAD_2)
2655392f7a3SLiteSpeed Tech                 + (size > PACKET_OUT_PAYLOAD_3);
26650aadb33SDmitri Tikhonov    return idx;
26750aadb33SDmitri Tikhonov}
2685392f7a3SLiteSpeed Tech#endif
2695392f7a3SLiteSpeed Tech
2705392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
2715392f7a3SLiteSpeed Tech#define POOL_SAMPLE_PERIOD 1024
2725392f7a3SLiteSpeed Tech
2735392f7a3SLiteSpeed Techstatic void
2745392f7a3SLiteSpeed Techpoolst_sample_max (struct pool_stats *poolst)
2755392f7a3SLiteSpeed Tech{
2765392f7a3SLiteSpeed Tech#define ALPHA_SHIFT 3
2775392f7a3SLiteSpeed Tech#define BETA_SHIFT  2
2785392f7a3SLiteSpeed Tech    unsigned diff;
2795392f7a3SLiteSpeed Tech
2805392f7a3SLiteSpeed Tech    if (poolst->ps_max_avg)
2815392f7a3SLiteSpeed Tech    {
2825392f7a3SLiteSpeed Tech        poolst->ps_max_var -= poolst->ps_max_var >> BETA_SHIFT;
2835392f7a3SLiteSpeed Tech        if (poolst->ps_max_avg > poolst->ps_max)
2845392f7a3SLiteSpeed Tech            diff = poolst->ps_max_avg - poolst->ps_max;
2855392f7a3SLiteSpeed Tech        else
2865392f7a3SLiteSpeed Tech            diff = poolst->ps_max - poolst->ps_max_avg;
2875392f7a3SLiteSpeed Tech        poolst->ps_max_var += diff >> BETA_SHIFT;
2885392f7a3SLiteSpeed Tech        poolst->ps_max_avg -= poolst->ps_max_avg >> ALPHA_SHIFT;
2895392f7a3SLiteSpeed Tech        poolst->ps_max_avg += poolst->ps_max >> ALPHA_SHIFT;
2905392f7a3SLiteSpeed Tech    }
2915392f7a3SLiteSpeed Tech    else
2925392f7a3SLiteSpeed Tech    {
2935392f7a3SLiteSpeed Tech        /* First measurement */
2945392f7a3SLiteSpeed Tech        poolst->ps_max_avg  = poolst->ps_max;
2955392f7a3SLiteSpeed Tech        poolst->ps_max_var  = poolst->ps_max / 2;
2965392f7a3SLiteSpeed Tech    }
2975392f7a3SLiteSpeed Tech
2985392f7a3SLiteSpeed Tech    poolst->ps_calls = 0;
2995392f7a3SLiteSpeed Tech    poolst->ps_max = poolst->ps_objs_out;
3005392f7a3SLiteSpeed Tech#if LSQUIC_LOG_POOL_STATS
3015392f7a3SLiteSpeed Tech    LSQ_DEBUG("new sample: max avg: %u; var: %u", poolst->ps_max_avg,
3025392f7a3SLiteSpeed Tech                                                        poolst->ps_max_var);
3035392f7a3SLiteSpeed Tech#endif
3045392f7a3SLiteSpeed Tech}
3055392f7a3SLiteSpeed Tech
3065392f7a3SLiteSpeed Tech
3075392f7a3SLiteSpeed Techstatic void
3085392f7a3SLiteSpeed Techpoolst_allocated (struct pool_stats *poolst, unsigned new)
3095392f7a3SLiteSpeed Tech{
3105392f7a3SLiteSpeed Tech    poolst->ps_objs_out += 1;
3115392f7a3SLiteSpeed Tech    poolst->ps_objs_all += new;
3125392f7a3SLiteSpeed Tech    if (poolst->ps_objs_out > poolst->ps_max)
3135392f7a3SLiteSpeed Tech        poolst->ps_max = poolst->ps_objs_out;
3145392f7a3SLiteSpeed Tech    ++poolst->ps_calls;
3155392f7a3SLiteSpeed Tech    if (0 == poolst->ps_calls % POOL_SAMPLE_PERIOD)
3165392f7a3SLiteSpeed Tech        poolst_sample_max(poolst);
3175392f7a3SLiteSpeed Tech}
3185392f7a3SLiteSpeed Tech
3195392f7a3SLiteSpeed Tech
3205392f7a3SLiteSpeed Techstatic void
3215392f7a3SLiteSpeed Techpoolst_freed (struct pool_stats *poolst)
3225392f7a3SLiteSpeed Tech{
3235392f7a3SLiteSpeed Tech    --poolst->ps_objs_out;
3245392f7a3SLiteSpeed Tech    ++poolst->ps_calls;
3255392f7a3SLiteSpeed Tech    if (0 == poolst->ps_calls % POOL_SAMPLE_PERIOD)
3265392f7a3SLiteSpeed Tech        poolst_sample_max(poolst);
3275392f7a3SLiteSpeed Tech}
3285392f7a3SLiteSpeed Tech
3295392f7a3SLiteSpeed Tech
3305392f7a3SLiteSpeed Techstatic int
3315392f7a3SLiteSpeed Techpoolst_has_new_sample (const struct pool_stats *poolst)
3325392f7a3SLiteSpeed Tech{
3335392f7a3SLiteSpeed Tech    return poolst->ps_calls == 0;
3345392f7a3SLiteSpeed Tech}
3355392f7a3SLiteSpeed Tech
3365392f7a3SLiteSpeed Tech
3375392f7a3SLiteSpeed Tech/* If average maximum falls under 1/4 of all objects allocated, release
3385392f7a3SLiteSpeed Tech * half of the objects allocated.
3395392f7a3SLiteSpeed Tech */
3405392f7a3SLiteSpeed Techstatic void
3415392f7a3SLiteSpeed Techmaybe_shrink_packet_out_bufs (struct lsquic_mm *mm, unsigned idx)
3425392f7a3SLiteSpeed Tech{
3435392f7a3SLiteSpeed Tech    struct pool_stats *poolst;
3445392f7a3SLiteSpeed Tech    struct packet_out_buf *pob;
3455392f7a3SLiteSpeed Tech    unsigned n_to_leave;
3465392f7a3SLiteSpeed Tech
3475392f7a3SLiteSpeed Tech    poolst = &mm->packet_out_bstats[idx];
3485392f7a3SLiteSpeed Tech    if (poolst->ps_max_avg * 4 < poolst->ps_objs_all)
3495392f7a3SLiteSpeed Tech    {
3505392f7a3SLiteSpeed Tech        n_to_leave = poolst->ps_objs_all / 2;
3515392f7a3SLiteSpeed Tech        while (poolst->ps_objs_all > n_to_leave
3525392f7a3SLiteSpeed Tech                        && (pob = SLIST_FIRST(&mm->packet_out_bufs[idx])))
3535392f7a3SLiteSpeed Tech        {
3545392f7a3SLiteSpeed Tech            SLIST_REMOVE_HEAD(&mm->packet_out_bufs[idx], next_pob);
3555392f7a3SLiteSpeed Tech            free(pob);
3565392f7a3SLiteSpeed Tech            --poolst->ps_objs_all;
3575392f7a3SLiteSpeed Tech        }
3585392f7a3SLiteSpeed Tech#if LSQUIC_LOG_POOL_STATS
3595392f7a3SLiteSpeed Tech        LSQ_DEBUG("pool #%u; max avg %u; shrank from %u to %u objs",
3605392f7a3SLiteSpeed Tech                idx, poolst->ps_max_avg, n_to_leave * 2, poolst->ps_objs_all);
3615392f7a3SLiteSpeed Tech#endif
3625392f7a3SLiteSpeed Tech    }
3635392f7a3SLiteSpeed Tech#if LSQUIC_LOG_POOL_STATS
3645392f7a3SLiteSpeed Tech    else
3655392f7a3SLiteSpeed Tech        LSQ_DEBUG("pool #%u; max avg %u; objs: %u; won't shrink",
3665392f7a3SLiteSpeed Tech                                idx, poolst->ps_max_avg, poolst->ps_objs_all);
3675392f7a3SLiteSpeed Tech#endif
3685392f7a3SLiteSpeed Tech}
3695392f7a3SLiteSpeed Tech#endif
37050aadb33SDmitri Tikhonov
37150aadb33SDmitri Tikhonov
37250aadb33SDmitri Tikhonovvoid
37350aadb33SDmitri Tikhonovlsquic_mm_put_packet_out (struct lsquic_mm *mm,
37450aadb33SDmitri Tikhonov                          struct lsquic_packet_out *packet_out)
37550aadb33SDmitri Tikhonov{
3765392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
37750aadb33SDmitri Tikhonov    struct packet_out_buf *pob;
37850aadb33SDmitri Tikhonov    unsigned idx;
37950aadb33SDmitri Tikhonov
38050aadb33SDmitri Tikhonov    assert(packet_out->po_data);
38150aadb33SDmitri Tikhonov    pob = (struct packet_out_buf *) packet_out->po_data;
38250aadb33SDmitri Tikhonov    idx = packet_out_index(packet_out->po_n_alloc);
38350aadb33SDmitri Tikhonov    SLIST_INSERT_HEAD(&mm->packet_out_bufs[idx], pob, next_pob);
3845392f7a3SLiteSpeed Tech    poolst_freed(&mm->packet_out_bstats[idx]);
3855392f7a3SLiteSpeed Tech    if (poolst_has_new_sample(&mm->packet_out_bstats[idx]))
3865392f7a3SLiteSpeed Tech        maybe_shrink_packet_out_bufs(mm, idx);
3875392f7a3SLiteSpeed Tech    if (packet_out->po_bwp_state)
3885392f7a3SLiteSpeed Tech        lsquic_malo_put(packet_out->po_bwp_state);
3895392f7a3SLiteSpeed Tech#else
3905392f7a3SLiteSpeed Tech    free(packet_out->po_data);
3915392f7a3SLiteSpeed Tech#endif
39250aadb33SDmitri Tikhonov    lsquic_malo_put(packet_out);
39350aadb33SDmitri Tikhonov}
39450aadb33SDmitri Tikhonov
39550aadb33SDmitri Tikhonov
39650aadb33SDmitri Tikhonovstruct lsquic_packet_out *
39750aadb33SDmitri Tikhonovlsquic_mm_get_packet_out (struct lsquic_mm *mm, struct malo *malo,
39850aadb33SDmitri Tikhonov                          unsigned short size)
39950aadb33SDmitri Tikhonov{
40050aadb33SDmitri Tikhonov    struct lsquic_packet_out *packet_out;
40150aadb33SDmitri Tikhonov    struct packet_out_buf *pob;
4025392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
40350aadb33SDmitri Tikhonov    unsigned idx;
4045392f7a3SLiteSpeed Tech#endif
40550aadb33SDmitri Tikhonov
40650aadb33SDmitri Tikhonov    fiu_do_on("mm/packet_out", FAIL_NOMEM);
40750aadb33SDmitri Tikhonov
40850aadb33SDmitri Tikhonov    packet_out = lsquic_malo_get(malo ? malo : mm->malo.packet_out);
40950aadb33SDmitri Tikhonov    if (!packet_out)
41050aadb33SDmitri Tikhonov        return NULL;
41150aadb33SDmitri Tikhonov
4125392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
41350aadb33SDmitri Tikhonov    idx = packet_out_index(size);
41450aadb33SDmitri Tikhonov    pob = SLIST_FIRST(&mm->packet_out_bufs[idx]);
41550aadb33SDmitri Tikhonov    if (pob)
4165392f7a3SLiteSpeed Tech    {
41750aadb33SDmitri Tikhonov        SLIST_REMOVE_HEAD(&mm->packet_out_bufs[idx], next_pob);
4185392f7a3SLiteSpeed Tech        poolst_allocated(&mm->packet_out_bstats[idx], 0);
4195392f7a3SLiteSpeed Tech    }
42050aadb33SDmitri Tikhonov    else
42150aadb33SDmitri Tikhonov    {
42250aadb33SDmitri Tikhonov        pob = malloc(packet_out_sizes[idx]);
42350aadb33SDmitri Tikhonov        if (!pob)
42450aadb33SDmitri Tikhonov        {
42550aadb33SDmitri Tikhonov            lsquic_malo_put(packet_out);
42650aadb33SDmitri Tikhonov            return NULL;
42750aadb33SDmitri Tikhonov        }
4285392f7a3SLiteSpeed Tech        poolst_allocated(&mm->packet_out_bstats[idx], 1);
42950aadb33SDmitri Tikhonov    }
4305392f7a3SLiteSpeed Tech    if (poolst_has_new_sample(&mm->packet_out_bstats[idx]))
4315392f7a3SLiteSpeed Tech        maybe_shrink_packet_out_bufs(mm, idx);
4325392f7a3SLiteSpeed Tech#else
4335392f7a3SLiteSpeed Tech    pob = malloc(size);
4345392f7a3SLiteSpeed Tech    if (!pob)
4355392f7a3SLiteSpeed Tech    {
4365392f7a3SLiteSpeed Tech        lsquic_malo_put(packet_out);
4375392f7a3SLiteSpeed Tech        return NULL;
4385392f7a3SLiteSpeed Tech    }
4395392f7a3SLiteSpeed Tech#endif
44050aadb33SDmitri Tikhonov
44150aadb33SDmitri Tikhonov    memset(packet_out, 0, sizeof(*packet_out));
44250aadb33SDmitri Tikhonov    packet_out->po_n_alloc = size;
44350aadb33SDmitri Tikhonov    packet_out->po_data = (unsigned char *) pob;
44450aadb33SDmitri Tikhonov
44550aadb33SDmitri Tikhonov    return packet_out;
44650aadb33SDmitri Tikhonov}
44750aadb33SDmitri Tikhonov
44850aadb33SDmitri Tikhonov
44950aadb33SDmitri Tikhonovvoid *
4505392f7a3SLiteSpeed Techlsquic_mm_get_packet_in_buf (struct lsquic_mm *mm, size_t size)
45150aadb33SDmitri Tikhonov{
4525392f7a3SLiteSpeed Tech    struct packet_in_buf *pib;
4535392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
4545392f7a3SLiteSpeed Tech    unsigned idx;
4555392f7a3SLiteSpeed Tech
4565392f7a3SLiteSpeed Tech    idx = packet_in_index(size);
4575392f7a3SLiteSpeed Tech    pib = SLIST_FIRST(&mm->packet_in_bufs[idx]);
4585392f7a3SLiteSpeed Tech    fiu_do_on("mm/packet_in_buf", FAIL_NOMEM);
4595392f7a3SLiteSpeed Tech    if (pib)
4605392f7a3SLiteSpeed Tech        SLIST_REMOVE_HEAD(&mm->packet_in_bufs[idx], next_pib);
46150aadb33SDmitri Tikhonov    else
4625392f7a3SLiteSpeed Tech        pib = malloc(packet_in_sizes[idx]);
4635392f7a3SLiteSpeed Tech#else
4645392f7a3SLiteSpeed Tech    pib = malloc(size);
4655392f7a3SLiteSpeed Tech#endif
4665392f7a3SLiteSpeed Tech    return pib;
46750aadb33SDmitri Tikhonov}
46850aadb33SDmitri Tikhonov
46950aadb33SDmitri Tikhonov
47050aadb33SDmitri Tikhonovvoid
4715392f7a3SLiteSpeed Techlsquic_mm_put_packet_in_buf (struct lsquic_mm *mm, void *mem, size_t size)
47250aadb33SDmitri Tikhonov{
4735392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
4745392f7a3SLiteSpeed Tech    unsigned idx;
4755392f7a3SLiteSpeed Tech    struct packet_in_buf *pib;
4765392f7a3SLiteSpeed Tech
4775392f7a3SLiteSpeed Tech    pib = (struct packet_in_buf *) mem;
4785392f7a3SLiteSpeed Tech    idx = packet_in_index(size);
4795392f7a3SLiteSpeed Tech    SLIST_INSERT_HEAD(&mm->packet_in_bufs[idx], pib, next_pib);
4805392f7a3SLiteSpeed Tech#else
4815392f7a3SLiteSpeed Tech    free(mem);
4825392f7a3SLiteSpeed Tech#endif
48350aadb33SDmitri Tikhonov}
48450aadb33SDmitri Tikhonov
48550aadb33SDmitri Tikhonov
48650aadb33SDmitri Tikhonovvoid *
48750aadb33SDmitri Tikhonovlsquic_mm_get_4k (struct lsquic_mm *mm)
48850aadb33SDmitri Tikhonov{
4895392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
49050aadb33SDmitri Tikhonov    struct four_k_page *fkp = SLIST_FIRST(&mm->four_k_pages);
49150aadb33SDmitri Tikhonov    fiu_do_on("mm/4k", FAIL_NOMEM);
49250aadb33SDmitri Tikhonov    if (fkp)
49350aadb33SDmitri Tikhonov        SLIST_REMOVE_HEAD(&mm->four_k_pages, next_fkp);
49450aadb33SDmitri Tikhonov    else
49550aadb33SDmitri Tikhonov        fkp = malloc(0x1000);
49650aadb33SDmitri Tikhonov    return fkp;
4975392f7a3SLiteSpeed Tech#else
4985392f7a3SLiteSpeed Tech    return malloc(0x1000);
4995392f7a3SLiteSpeed Tech#endif
50050aadb33SDmitri Tikhonov}
50150aadb33SDmitri Tikhonov
50250aadb33SDmitri Tikhonov
50350aadb33SDmitri Tikhonovvoid
50450aadb33SDmitri Tikhonovlsquic_mm_put_4k (struct lsquic_mm *mm, void *mem)
50550aadb33SDmitri Tikhonov{
5065392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
50750aadb33SDmitri Tikhonov    struct four_k_page *fkp = mem;
50850aadb33SDmitri Tikhonov    SLIST_INSERT_HEAD(&mm->four_k_pages, fkp, next_fkp);
5095392f7a3SLiteSpeed Tech#else
5105392f7a3SLiteSpeed Tech    free(mem);
5115392f7a3SLiteSpeed Tech#endif
51250aadb33SDmitri Tikhonov}
51350aadb33SDmitri Tikhonov
51450aadb33SDmitri Tikhonov
51550aadb33SDmitri Tikhonovvoid *
51650aadb33SDmitri Tikhonovlsquic_mm_get_16k (struct lsquic_mm *mm)
51750aadb33SDmitri Tikhonov{
5185392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
51950aadb33SDmitri Tikhonov    struct sixteen_k_page *skp = SLIST_FIRST(&mm->sixteen_k_pages);
52050aadb33SDmitri Tikhonov    fiu_do_on("mm/16k", FAIL_NOMEM);
52150aadb33SDmitri Tikhonov    if (skp)
52250aadb33SDmitri Tikhonov        SLIST_REMOVE_HEAD(&mm->sixteen_k_pages, next_skp);
52350aadb33SDmitri Tikhonov    else
52450aadb33SDmitri Tikhonov        skp = malloc(16 * 1024);
52550aadb33SDmitri Tikhonov    return skp;
5265392f7a3SLiteSpeed Tech#else
5275392f7a3SLiteSpeed Tech    return malloc(16 * 1024);
5285392f7a3SLiteSpeed Tech#endif
52950aadb33SDmitri Tikhonov}
53050aadb33SDmitri Tikhonov
53150aadb33SDmitri Tikhonov
53250aadb33SDmitri Tikhonovvoid
53350aadb33SDmitri Tikhonovlsquic_mm_put_16k (struct lsquic_mm *mm, void *mem)
53450aadb33SDmitri Tikhonov{
5355392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
53650aadb33SDmitri Tikhonov    struct sixteen_k_page *skp = mem;
53750aadb33SDmitri Tikhonov    SLIST_INSERT_HEAD(&mm->sixteen_k_pages, skp, next_skp);
5385392f7a3SLiteSpeed Tech#else
5395392f7a3SLiteSpeed Tech    free(mem);
5405392f7a3SLiteSpeed Tech#endif
54150aadb33SDmitri Tikhonov}
54250aadb33SDmitri Tikhonov
54350aadb33SDmitri Tikhonov
544c51ce338SDmitri Tikhonovsize_t
545c51ce338SDmitri Tikhonovlsquic_mm_mem_used (const struct lsquic_mm *mm)
546c51ce338SDmitri Tikhonov{
5475392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS
548c51ce338SDmitri Tikhonov    const struct packet_out_buf *pob;
5495392f7a3SLiteSpeed Tech    const struct packet_in_buf *pib;
550c51ce338SDmitri Tikhonov    const struct four_k_page *fkp;
551c51ce338SDmitri Tikhonov    const struct sixteen_k_page *skp;
552c51ce338SDmitri Tikhonov    unsigned i;
553c51ce338SDmitri Tikhonov    size_t size;
554c51ce338SDmitri Tikhonov
555c51ce338SDmitri Tikhonov    size = sizeof(*mm);
556c51ce338SDmitri Tikhonov    size += sizeof(*mm->acki);
557c51ce338SDmitri Tikhonov    size += lsquic_malo_mem_used(mm->malo.stream_frame);
558c51ce338SDmitri Tikhonov    size += lsquic_malo_mem_used(mm->malo.stream_rec_arr);
5595392f7a3SLiteSpeed Tech    size += lsquic_malo_mem_used(mm->malo.mini_conn);
5605392f7a3SLiteSpeed Tech    size += lsquic_malo_mem_used(mm->malo.mini_conn_ietf);
561c51ce338SDmitri Tikhonov    size += lsquic_malo_mem_used(mm->malo.packet_in);
562c51ce338SDmitri Tikhonov    size += lsquic_malo_mem_used(mm->malo.packet_out);
563c51ce338SDmitri Tikhonov
564c51ce338SDmitri Tikhonov    for (i = 0; i < MM_N_OUT_BUCKETS; ++i)
565c51ce338SDmitri Tikhonov        SLIST_FOREACH(pob, &mm->packet_out_bufs[i], next_pob)
566c51ce338SDmitri Tikhonov            size += packet_out_sizes[i];
567c51ce338SDmitri Tikhonov
5685392f7a3SLiteSpeed Tech    for (i = 0; i < MM_N_IN_BUCKETS; ++i)
5695392f7a3SLiteSpeed Tech        SLIST_FOREACH(pib, &mm->packet_in_bufs[i], next_pib)
5705392f7a3SLiteSpeed Tech            size += packet_in_sizes[i];
571c51ce338SDmitri Tikhonov
572c51ce338SDmitri Tikhonov    SLIST_FOREACH(fkp, &mm->four_k_pages, next_fkp)
573c51ce338SDmitri Tikhonov        size += 0x1000;
574c51ce338SDmitri Tikhonov
575c51ce338SDmitri Tikhonov    SLIST_FOREACH(skp, &mm->sixteen_k_pages, next_skp)
576c51ce338SDmitri Tikhonov        size += 0x4000;
577c51ce338SDmitri Tikhonov
578c51ce338SDmitri Tikhonov    return size;
5795392f7a3SLiteSpeed Tech#else
5805392f7a3SLiteSpeed Tech    return sizeof(*mm);
5815392f7a3SLiteSpeed Tech#endif
582c51ce338SDmitri Tikhonov}
583