lsquic_mm.c revision 5392f7a3
1229fce07SDmitri Tikhonov/* Copyright (c) 2017 - 2019 LiteSpeed Technologies Inc. See LICENSE. */ 250aadb33SDmitri Tikhonov/* 350aadb33SDmitri Tikhonov * lsquic_mm.c -- Memory manager. 450aadb33SDmitri Tikhonov */ 550aadb33SDmitri Tikhonov 650aadb33SDmitri Tikhonov#include <assert.h> 750aadb33SDmitri Tikhonov#include <errno.h> 850aadb33SDmitri Tikhonov#include <stddef.h> 950aadb33SDmitri Tikhonov#include <stdlib.h> 1050aadb33SDmitri Tikhonov#include <string.h> 1150aadb33SDmitri Tikhonov#include <sys/queue.h> 1250aadb33SDmitri Tikhonov 1350aadb33SDmitri Tikhonov#include "fiu-local.h" 1450aadb33SDmitri Tikhonov 1550aadb33SDmitri Tikhonov#include "lsquic.h" 1650aadb33SDmitri Tikhonov#include "lsquic_int_types.h" 175392f7a3SLiteSpeed Tech#include "lsquic_sizes.h" 1850aadb33SDmitri Tikhonov#include "lsquic_malo.h" 195392f7a3SLiteSpeed Tech#include "lsquic_hash.h" 2050aadb33SDmitri Tikhonov#include "lsquic_conn.h" 2150aadb33SDmitri Tikhonov#include "lsquic_rtt.h" 2250aadb33SDmitri Tikhonov#include "lsquic_packet_common.h" 235392f7a3SLiteSpeed Tech#include "lsquic_mini_conn.h" 245392f7a3SLiteSpeed Tech#include "lsquic_enc_sess.h" 255392f7a3SLiteSpeed Tech#include "lsquic_mini_conn_ietf.h" 265392f7a3SLiteSpeed Tech#include "lsquic_packet_gquic.h" 2750aadb33SDmitri Tikhonov#include "lsquic_packet_in.h" 2850aadb33SDmitri Tikhonov#include "lsquic_packet_out.h" 2950aadb33SDmitri Tikhonov#include "lsquic_parse.h" 3050aadb33SDmitri Tikhonov#include "lsquic_mm.h" 3150aadb33SDmitri Tikhonov#include "lsquic_engine_public.h" 325392f7a3SLiteSpeed Tech#include "lsquic_full_conn.h" 335392f7a3SLiteSpeed Tech#include "lsquic_varint.h" 345392f7a3SLiteSpeed Tech#include "lsquic_hq.h" 355392f7a3SLiteSpeed Tech#include "lsquic_sfcw.h" 365392f7a3SLiteSpeed Tech#include "lsquic_stream.h" 375392f7a3SLiteSpeed Tech 385392f7a3SLiteSpeed Tech#ifndef LSQUIC_LOG_POOL_STATS 395392f7a3SLiteSpeed Tech#define LSQUIC_LOG_POOL_STATS 0 405392f7a3SLiteSpeed Tech#endif 415392f7a3SLiteSpeed Tech 425392f7a3SLiteSpeed Tech#if LSQUIC_LOG_POOL_STATS 435392f7a3SLiteSpeed Tech#include "lsquic_logger.h" 445392f7a3SLiteSpeed Tech#endif 455392f7a3SLiteSpeed Tech 465392f7a3SLiteSpeed Tech#ifndef LSQUIC_USE_POOLS 475392f7a3SLiteSpeed Tech#define LSQUIC_USE_POOLS 1 485392f7a3SLiteSpeed Tech#endif 4950aadb33SDmitri Tikhonov 5050aadb33SDmitri Tikhonov#define FAIL_NOMEM do { errno = ENOMEM; return NULL; } while (0) 5150aadb33SDmitri Tikhonov 5250aadb33SDmitri Tikhonov 535392f7a3SLiteSpeed Techstruct packet_in_buf 5450aadb33SDmitri Tikhonov{ 555392f7a3SLiteSpeed Tech SLIST_ENTRY(packet_in_buf) next_pib; 5650aadb33SDmitri Tikhonov}; 5750aadb33SDmitri Tikhonov 5850aadb33SDmitri Tikhonovstruct packet_out_buf 5950aadb33SDmitri Tikhonov{ 6050aadb33SDmitri Tikhonov SLIST_ENTRY(packet_out_buf) next_pob; 6150aadb33SDmitri Tikhonov}; 6250aadb33SDmitri Tikhonov 6350aadb33SDmitri Tikhonovstruct four_k_page 6450aadb33SDmitri Tikhonov{ 6550aadb33SDmitri Tikhonov SLIST_ENTRY(four_k_page) next_fkp; 6650aadb33SDmitri Tikhonov}; 6750aadb33SDmitri Tikhonov 6850aadb33SDmitri Tikhonovstruct sixteen_k_page 6950aadb33SDmitri Tikhonov{ 7050aadb33SDmitri Tikhonov SLIST_ENTRY(sixteen_k_page) next_skp; 7150aadb33SDmitri Tikhonov}; 7250aadb33SDmitri Tikhonov 7350aadb33SDmitri Tikhonov 7450aadb33SDmitri Tikhonovint 7550aadb33SDmitri Tikhonovlsquic_mm_init (struct lsquic_mm *mm) 7650aadb33SDmitri Tikhonov{ 775392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 7850aadb33SDmitri Tikhonov int i; 795392f7a3SLiteSpeed Tech#endif 8050aadb33SDmitri Tikhonov 8150aadb33SDmitri Tikhonov mm->acki = malloc(sizeof(*mm->acki)); 8250aadb33SDmitri Tikhonov mm->malo.stream_frame = lsquic_malo_create(sizeof(struct stream_frame)); 8350aadb33SDmitri Tikhonov mm->malo.stream_rec_arr = lsquic_malo_create(sizeof(struct stream_rec_arr)); 845392f7a3SLiteSpeed Tech mm->malo.mini_conn = lsquic_malo_create(sizeof(struct mini_conn)); 855392f7a3SLiteSpeed Tech mm->malo.mini_conn_ietf = lsquic_malo_create(sizeof(struct ietf_mini_conn)); 8650aadb33SDmitri Tikhonov mm->malo.packet_in = lsquic_malo_create(sizeof(struct lsquic_packet_in)); 8750aadb33SDmitri Tikhonov mm->malo.packet_out = lsquic_malo_create(sizeof(struct lsquic_packet_out)); 885392f7a3SLiteSpeed Tech mm->malo.dcid_elem = lsquic_malo_create(sizeof(struct dcid_elem)); 895392f7a3SLiteSpeed Tech mm->malo.stream_hq_frame 905392f7a3SLiteSpeed Tech = lsquic_malo_create(sizeof(struct stream_hq_frame)); 915392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 9250aadb33SDmitri Tikhonov TAILQ_INIT(&mm->free_packets_in); 9350aadb33SDmitri Tikhonov for (i = 0; i < MM_N_OUT_BUCKETS; ++i) 9450aadb33SDmitri Tikhonov SLIST_INIT(&mm->packet_out_bufs[i]); 955392f7a3SLiteSpeed Tech for (i = 0; i < MM_N_IN_BUCKETS; ++i) 965392f7a3SLiteSpeed Tech SLIST_INIT(&mm->packet_in_bufs[i]); 9750aadb33SDmitri Tikhonov SLIST_INIT(&mm->four_k_pages); 9850aadb33SDmitri Tikhonov SLIST_INIT(&mm->sixteen_k_pages); 995392f7a3SLiteSpeed Tech#endif 1005392f7a3SLiteSpeed Tech if (mm->acki && mm->malo.stream_frame && mm->malo.stream_rec_arr 1015392f7a3SLiteSpeed Tech && mm->malo.mini_conn && mm->malo.mini_conn_ietf && mm->malo.packet_in 1025392f7a3SLiteSpeed Tech && mm->malo.stream_hq_frame) 10350aadb33SDmitri Tikhonov { 10450aadb33SDmitri Tikhonov return 0; 10550aadb33SDmitri Tikhonov } 10650aadb33SDmitri Tikhonov else 10750aadb33SDmitri Tikhonov return -1; 10850aadb33SDmitri Tikhonov} 10950aadb33SDmitri Tikhonov 11050aadb33SDmitri Tikhonov 11150aadb33SDmitri Tikhonovvoid 11250aadb33SDmitri Tikhonovlsquic_mm_cleanup (struct lsquic_mm *mm) 11350aadb33SDmitri Tikhonov{ 1145392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 11550aadb33SDmitri Tikhonov int i; 11650aadb33SDmitri Tikhonov struct packet_out_buf *pob; 1175392f7a3SLiteSpeed Tech struct packet_in_buf *pib; 11850aadb33SDmitri Tikhonov struct four_k_page *fkp; 11950aadb33SDmitri Tikhonov struct sixteen_k_page *skp; 1205392f7a3SLiteSpeed Tech#endif 12150aadb33SDmitri Tikhonov 12250aadb33SDmitri Tikhonov free(mm->acki); 1235392f7a3SLiteSpeed Tech lsquic_malo_destroy(mm->malo.stream_hq_frame); 1245392f7a3SLiteSpeed Tech lsquic_malo_destroy(mm->malo.dcid_elem); 12550aadb33SDmitri Tikhonov lsquic_malo_destroy(mm->malo.packet_in); 12650aadb33SDmitri Tikhonov lsquic_malo_destroy(mm->malo.packet_out); 12750aadb33SDmitri Tikhonov lsquic_malo_destroy(mm->malo.stream_frame); 12850aadb33SDmitri Tikhonov lsquic_malo_destroy(mm->malo.stream_rec_arr); 1295392f7a3SLiteSpeed Tech lsquic_malo_destroy(mm->malo.mini_conn); 1305392f7a3SLiteSpeed Tech lsquic_malo_destroy(mm->malo.mini_conn_ietf); 13150aadb33SDmitri Tikhonov 1325392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 13350aadb33SDmitri Tikhonov for (i = 0; i < MM_N_OUT_BUCKETS; ++i) 13450aadb33SDmitri Tikhonov while ((pob = SLIST_FIRST(&mm->packet_out_bufs[i]))) 13550aadb33SDmitri Tikhonov { 13650aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->packet_out_bufs[i], next_pob); 13750aadb33SDmitri Tikhonov free(pob); 13850aadb33SDmitri Tikhonov } 13950aadb33SDmitri Tikhonov 1405392f7a3SLiteSpeed Tech for (i = 0; i < MM_N_IN_BUCKETS; ++i) 1415392f7a3SLiteSpeed Tech while ((pib = SLIST_FIRST(&mm->packet_in_bufs[i]))) 1425392f7a3SLiteSpeed Tech { 1435392f7a3SLiteSpeed Tech SLIST_REMOVE_HEAD(&mm->packet_in_bufs[i], next_pib); 1445392f7a3SLiteSpeed Tech free(pib); 1455392f7a3SLiteSpeed Tech } 14650aadb33SDmitri Tikhonov 14750aadb33SDmitri Tikhonov while ((fkp = SLIST_FIRST(&mm->four_k_pages))) 14850aadb33SDmitri Tikhonov { 14950aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->four_k_pages, next_fkp); 15050aadb33SDmitri Tikhonov free(fkp); 15150aadb33SDmitri Tikhonov } 15250aadb33SDmitri Tikhonov 15350aadb33SDmitri Tikhonov while ((skp = SLIST_FIRST(&mm->sixteen_k_pages))) 15450aadb33SDmitri Tikhonov { 15550aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->sixteen_k_pages, next_skp); 15650aadb33SDmitri Tikhonov free(skp); 15750aadb33SDmitri Tikhonov } 1585392f7a3SLiteSpeed Tech#endif 1595392f7a3SLiteSpeed Tech} 1605392f7a3SLiteSpeed Tech 1615392f7a3SLiteSpeed Tech 1625392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 1635392f7a3SLiteSpeed Techenum { 1645392f7a3SLiteSpeed Tech PACKET_IN_PAYLOAD_0 = 1370, /* common QUIC payload size upperbound */ 1655392f7a3SLiteSpeed Tech PACKET_IN_PAYLOAD_1 = 4096, /* payload size middleground guess */ 1665392f7a3SLiteSpeed Tech PACKET_IN_PAYLOAD_2 = 0xffff, /* UDP payload size upperbound */ 1675392f7a3SLiteSpeed Tech}; 1685392f7a3SLiteSpeed Tech 1695392f7a3SLiteSpeed Tech 1705392f7a3SLiteSpeed Techstatic const unsigned packet_in_sizes[] = { 1715392f7a3SLiteSpeed Tech PACKET_IN_PAYLOAD_0, 1725392f7a3SLiteSpeed Tech PACKET_IN_PAYLOAD_1, 1735392f7a3SLiteSpeed Tech PACKET_IN_PAYLOAD_2, 1745392f7a3SLiteSpeed Tech}; 1755392f7a3SLiteSpeed Tech 1765392f7a3SLiteSpeed Tech 1775392f7a3SLiteSpeed Techstatic unsigned 1785392f7a3SLiteSpeed Techpacket_in_index (unsigned size) 1795392f7a3SLiteSpeed Tech{ 1805392f7a3SLiteSpeed Tech unsigned idx = (size > PACKET_IN_PAYLOAD_0) 1815392f7a3SLiteSpeed Tech + (size > PACKET_IN_PAYLOAD_1); 1825392f7a3SLiteSpeed Tech return idx; 1835392f7a3SLiteSpeed Tech} 1845392f7a3SLiteSpeed Tech#endif 1855392f7a3SLiteSpeed Tech 1865392f7a3SLiteSpeed Tech 1875392f7a3SLiteSpeed Techvoid 1885392f7a3SLiteSpeed Techlsquic_mm_put_packet_in (struct lsquic_mm *mm, 1895392f7a3SLiteSpeed Tech struct lsquic_packet_in *packet_in) 1905392f7a3SLiteSpeed Tech{ 1915392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 1925392f7a3SLiteSpeed Tech unsigned idx; 1935392f7a3SLiteSpeed Tech struct packet_in_buf *pib; 1945392f7a3SLiteSpeed Tech 1955392f7a3SLiteSpeed Tech assert(0 == packet_in->pi_refcnt); 1965392f7a3SLiteSpeed Tech if (packet_in->pi_flags & PI_OWN_DATA) 1975392f7a3SLiteSpeed Tech { 1985392f7a3SLiteSpeed Tech pib = (struct packet_in_buf *) packet_in->pi_data; 1995392f7a3SLiteSpeed Tech idx = packet_in_index(packet_in->pi_data_sz); 2005392f7a3SLiteSpeed Tech SLIST_INSERT_HEAD(&mm->packet_in_bufs[idx], pib, next_pib); 2015392f7a3SLiteSpeed Tech } 2025392f7a3SLiteSpeed Tech TAILQ_INSERT_HEAD(&mm->free_packets_in, packet_in, pi_next); 2035392f7a3SLiteSpeed Tech#else 2045392f7a3SLiteSpeed Tech if (packet_in->pi_flags & PI_OWN_DATA) 2055392f7a3SLiteSpeed Tech free(packet_in->pi_data); 2065392f7a3SLiteSpeed Tech lsquic_malo_put(packet_in); 2075392f7a3SLiteSpeed Tech#endif 20850aadb33SDmitri Tikhonov} 20950aadb33SDmitri Tikhonov 21050aadb33SDmitri Tikhonov 21150aadb33SDmitri Tikhonovstruct lsquic_packet_in * 21250aadb33SDmitri Tikhonovlsquic_mm_get_packet_in (struct lsquic_mm *mm) 21350aadb33SDmitri Tikhonov{ 21450aadb33SDmitri Tikhonov struct lsquic_packet_in *packet_in; 21550aadb33SDmitri Tikhonov 21650aadb33SDmitri Tikhonov fiu_do_on("mm/packet_in", FAIL_NOMEM); 21750aadb33SDmitri Tikhonov 2185392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 21950aadb33SDmitri Tikhonov packet_in = TAILQ_FIRST(&mm->free_packets_in); 22050aadb33SDmitri Tikhonov if (packet_in) 22150aadb33SDmitri Tikhonov { 22250aadb33SDmitri Tikhonov assert(0 == packet_in->pi_refcnt); 22350aadb33SDmitri Tikhonov TAILQ_REMOVE(&mm->free_packets_in, packet_in, pi_next); 22450aadb33SDmitri Tikhonov } 22550aadb33SDmitri Tikhonov else 2265392f7a3SLiteSpeed Tech#endif 22750aadb33SDmitri Tikhonov packet_in = lsquic_malo_get(mm->malo.packet_in); 22850aadb33SDmitri Tikhonov 22950aadb33SDmitri Tikhonov if (packet_in) 23050aadb33SDmitri Tikhonov memset(packet_in, 0, sizeof(*packet_in)); 23150aadb33SDmitri Tikhonov 23250aadb33SDmitri Tikhonov return packet_in; 23350aadb33SDmitri Tikhonov} 23450aadb33SDmitri Tikhonov 23550aadb33SDmitri Tikhonov 2365392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 23750aadb33SDmitri Tikhonov/* Based on commonly used MTUs, ordered from small to large: */ 23850aadb33SDmitri Tikhonovenum { 2395392f7a3SLiteSpeed Tech PACKET_OUT_PAYLOAD_0 = 1280 - GQUIC_MIN_PACKET_OVERHEAD, 2405392f7a3SLiteSpeed Tech PACKET_OUT_PAYLOAD_1 = GQUIC_MAX_IPv6_PACKET_SZ - GQUIC_MIN_PACKET_OVERHEAD, 2415392f7a3SLiteSpeed Tech PACKET_OUT_PAYLOAD_2 = GQUIC_MAX_IPv4_PACKET_SZ - GQUIC_MIN_PACKET_OVERHEAD, 2425392f7a3SLiteSpeed Tech PACKET_OUT_PAYLOAD_3 = 4096, 2435392f7a3SLiteSpeed Tech PACKET_OUT_PAYLOAD_4 = 0xffff, 24450aadb33SDmitri Tikhonov}; 24550aadb33SDmitri Tikhonov 24650aadb33SDmitri Tikhonov 24750aadb33SDmitri Tikhonovstatic const unsigned packet_out_sizes[] = { 24850aadb33SDmitri Tikhonov PACKET_OUT_PAYLOAD_0, 24950aadb33SDmitri Tikhonov PACKET_OUT_PAYLOAD_1, 25050aadb33SDmitri Tikhonov PACKET_OUT_PAYLOAD_2, 2515392f7a3SLiteSpeed Tech PACKET_OUT_PAYLOAD_3, 2525392f7a3SLiteSpeed Tech PACKET_OUT_PAYLOAD_4, 25350aadb33SDmitri Tikhonov}; 25450aadb33SDmitri Tikhonov 25550aadb33SDmitri Tikhonov 25650aadb33SDmitri Tikhonovstatic unsigned 25750aadb33SDmitri Tikhonovpacket_out_index (unsigned size) 25850aadb33SDmitri Tikhonov{ 25950aadb33SDmitri Tikhonov unsigned idx = (size > PACKET_OUT_PAYLOAD_0) 2605392f7a3SLiteSpeed Tech + (size > PACKET_OUT_PAYLOAD_1) 2615392f7a3SLiteSpeed Tech + (size > PACKET_OUT_PAYLOAD_2) 2625392f7a3SLiteSpeed Tech + (size > PACKET_OUT_PAYLOAD_3); 26350aadb33SDmitri Tikhonov return idx; 26450aadb33SDmitri Tikhonov} 2655392f7a3SLiteSpeed Tech#endif 2665392f7a3SLiteSpeed Tech 2675392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 2685392f7a3SLiteSpeed Tech#define POOL_SAMPLE_PERIOD 1024 2695392f7a3SLiteSpeed Tech 2705392f7a3SLiteSpeed Techstatic void 2715392f7a3SLiteSpeed Techpoolst_sample_max (struct pool_stats *poolst) 2725392f7a3SLiteSpeed Tech{ 2735392f7a3SLiteSpeed Tech#define ALPHA_SHIFT 3 2745392f7a3SLiteSpeed Tech#define BETA_SHIFT 2 2755392f7a3SLiteSpeed Tech unsigned diff; 2765392f7a3SLiteSpeed Tech 2775392f7a3SLiteSpeed Tech if (poolst->ps_max_avg) 2785392f7a3SLiteSpeed Tech { 2795392f7a3SLiteSpeed Tech poolst->ps_max_var -= poolst->ps_max_var >> BETA_SHIFT; 2805392f7a3SLiteSpeed Tech if (poolst->ps_max_avg > poolst->ps_max) 2815392f7a3SLiteSpeed Tech diff = poolst->ps_max_avg - poolst->ps_max; 2825392f7a3SLiteSpeed Tech else 2835392f7a3SLiteSpeed Tech diff = poolst->ps_max - poolst->ps_max_avg; 2845392f7a3SLiteSpeed Tech poolst->ps_max_var += diff >> BETA_SHIFT; 2855392f7a3SLiteSpeed Tech poolst->ps_max_avg -= poolst->ps_max_avg >> ALPHA_SHIFT; 2865392f7a3SLiteSpeed Tech poolst->ps_max_avg += poolst->ps_max >> ALPHA_SHIFT; 2875392f7a3SLiteSpeed Tech } 2885392f7a3SLiteSpeed Tech else 2895392f7a3SLiteSpeed Tech { 2905392f7a3SLiteSpeed Tech /* First measurement */ 2915392f7a3SLiteSpeed Tech poolst->ps_max_avg = poolst->ps_max; 2925392f7a3SLiteSpeed Tech poolst->ps_max_var = poolst->ps_max / 2; 2935392f7a3SLiteSpeed Tech } 2945392f7a3SLiteSpeed Tech 2955392f7a3SLiteSpeed Tech poolst->ps_calls = 0; 2965392f7a3SLiteSpeed Tech poolst->ps_max = poolst->ps_objs_out; 2975392f7a3SLiteSpeed Tech#if LSQUIC_LOG_POOL_STATS 2985392f7a3SLiteSpeed Tech LSQ_DEBUG("new sample: max avg: %u; var: %u", poolst->ps_max_avg, 2995392f7a3SLiteSpeed Tech poolst->ps_max_var); 3005392f7a3SLiteSpeed Tech#endif 3015392f7a3SLiteSpeed Tech} 3025392f7a3SLiteSpeed Tech 3035392f7a3SLiteSpeed Tech 3045392f7a3SLiteSpeed Techstatic void 3055392f7a3SLiteSpeed Techpoolst_allocated (struct pool_stats *poolst, unsigned new) 3065392f7a3SLiteSpeed Tech{ 3075392f7a3SLiteSpeed Tech poolst->ps_objs_out += 1; 3085392f7a3SLiteSpeed Tech poolst->ps_objs_all += new; 3095392f7a3SLiteSpeed Tech if (poolst->ps_objs_out > poolst->ps_max) 3105392f7a3SLiteSpeed Tech poolst->ps_max = poolst->ps_objs_out; 3115392f7a3SLiteSpeed Tech ++poolst->ps_calls; 3125392f7a3SLiteSpeed Tech if (0 == poolst->ps_calls % POOL_SAMPLE_PERIOD) 3135392f7a3SLiteSpeed Tech poolst_sample_max(poolst); 3145392f7a3SLiteSpeed Tech} 3155392f7a3SLiteSpeed Tech 3165392f7a3SLiteSpeed Tech 3175392f7a3SLiteSpeed Techstatic void 3185392f7a3SLiteSpeed Techpoolst_freed (struct pool_stats *poolst) 3195392f7a3SLiteSpeed Tech{ 3205392f7a3SLiteSpeed Tech --poolst->ps_objs_out; 3215392f7a3SLiteSpeed Tech ++poolst->ps_calls; 3225392f7a3SLiteSpeed Tech if (0 == poolst->ps_calls % POOL_SAMPLE_PERIOD) 3235392f7a3SLiteSpeed Tech poolst_sample_max(poolst); 3245392f7a3SLiteSpeed Tech} 3255392f7a3SLiteSpeed Tech 3265392f7a3SLiteSpeed Tech 3275392f7a3SLiteSpeed Techstatic int 3285392f7a3SLiteSpeed Techpoolst_has_new_sample (const struct pool_stats *poolst) 3295392f7a3SLiteSpeed Tech{ 3305392f7a3SLiteSpeed Tech return poolst->ps_calls == 0; 3315392f7a3SLiteSpeed Tech} 3325392f7a3SLiteSpeed Tech 3335392f7a3SLiteSpeed Tech 3345392f7a3SLiteSpeed Tech/* If average maximum falls under 1/4 of all objects allocated, release 3355392f7a3SLiteSpeed Tech * half of the objects allocated. 3365392f7a3SLiteSpeed Tech */ 3375392f7a3SLiteSpeed Techstatic void 3385392f7a3SLiteSpeed Techmaybe_shrink_packet_out_bufs (struct lsquic_mm *mm, unsigned idx) 3395392f7a3SLiteSpeed Tech{ 3405392f7a3SLiteSpeed Tech struct pool_stats *poolst; 3415392f7a3SLiteSpeed Tech struct packet_out_buf *pob; 3425392f7a3SLiteSpeed Tech unsigned n_to_leave; 3435392f7a3SLiteSpeed Tech 3445392f7a3SLiteSpeed Tech poolst = &mm->packet_out_bstats[idx]; 3455392f7a3SLiteSpeed Tech if (poolst->ps_max_avg * 4 < poolst->ps_objs_all) 3465392f7a3SLiteSpeed Tech { 3475392f7a3SLiteSpeed Tech n_to_leave = poolst->ps_objs_all / 2; 3485392f7a3SLiteSpeed Tech while (poolst->ps_objs_all > n_to_leave 3495392f7a3SLiteSpeed Tech && (pob = SLIST_FIRST(&mm->packet_out_bufs[idx]))) 3505392f7a3SLiteSpeed Tech { 3515392f7a3SLiteSpeed Tech SLIST_REMOVE_HEAD(&mm->packet_out_bufs[idx], next_pob); 3525392f7a3SLiteSpeed Tech free(pob); 3535392f7a3SLiteSpeed Tech --poolst->ps_objs_all; 3545392f7a3SLiteSpeed Tech } 3555392f7a3SLiteSpeed Tech#if LSQUIC_LOG_POOL_STATS 3565392f7a3SLiteSpeed Tech LSQ_DEBUG("pool #%u; max avg %u; shrank from %u to %u objs", 3575392f7a3SLiteSpeed Tech idx, poolst->ps_max_avg, n_to_leave * 2, poolst->ps_objs_all); 3585392f7a3SLiteSpeed Tech#endif 3595392f7a3SLiteSpeed Tech } 3605392f7a3SLiteSpeed Tech#if LSQUIC_LOG_POOL_STATS 3615392f7a3SLiteSpeed Tech else 3625392f7a3SLiteSpeed Tech LSQ_DEBUG("pool #%u; max avg %u; objs: %u; won't shrink", 3635392f7a3SLiteSpeed Tech idx, poolst->ps_max_avg, poolst->ps_objs_all); 3645392f7a3SLiteSpeed Tech#endif 3655392f7a3SLiteSpeed Tech} 3665392f7a3SLiteSpeed Tech#endif 36750aadb33SDmitri Tikhonov 36850aadb33SDmitri Tikhonov 36950aadb33SDmitri Tikhonovvoid 37050aadb33SDmitri Tikhonovlsquic_mm_put_packet_out (struct lsquic_mm *mm, 37150aadb33SDmitri Tikhonov struct lsquic_packet_out *packet_out) 37250aadb33SDmitri Tikhonov{ 3735392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 37450aadb33SDmitri Tikhonov struct packet_out_buf *pob; 37550aadb33SDmitri Tikhonov unsigned idx; 37650aadb33SDmitri Tikhonov 37750aadb33SDmitri Tikhonov assert(packet_out->po_data); 37850aadb33SDmitri Tikhonov pob = (struct packet_out_buf *) packet_out->po_data; 37950aadb33SDmitri Tikhonov idx = packet_out_index(packet_out->po_n_alloc); 38050aadb33SDmitri Tikhonov SLIST_INSERT_HEAD(&mm->packet_out_bufs[idx], pob, next_pob); 3815392f7a3SLiteSpeed Tech poolst_freed(&mm->packet_out_bstats[idx]); 3825392f7a3SLiteSpeed Tech if (poolst_has_new_sample(&mm->packet_out_bstats[idx])) 3835392f7a3SLiteSpeed Tech maybe_shrink_packet_out_bufs(mm, idx); 3845392f7a3SLiteSpeed Tech if (packet_out->po_bwp_state) 3855392f7a3SLiteSpeed Tech lsquic_malo_put(packet_out->po_bwp_state); 3865392f7a3SLiteSpeed Tech#else 3875392f7a3SLiteSpeed Tech free(packet_out->po_data); 3885392f7a3SLiteSpeed Tech#endif 38950aadb33SDmitri Tikhonov lsquic_malo_put(packet_out); 39050aadb33SDmitri Tikhonov} 39150aadb33SDmitri Tikhonov 39250aadb33SDmitri Tikhonov 39350aadb33SDmitri Tikhonovstruct lsquic_packet_out * 39450aadb33SDmitri Tikhonovlsquic_mm_get_packet_out (struct lsquic_mm *mm, struct malo *malo, 39550aadb33SDmitri Tikhonov unsigned short size) 39650aadb33SDmitri Tikhonov{ 39750aadb33SDmitri Tikhonov struct lsquic_packet_out *packet_out; 39850aadb33SDmitri Tikhonov struct packet_out_buf *pob; 3995392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 40050aadb33SDmitri Tikhonov unsigned idx; 4015392f7a3SLiteSpeed Tech#endif 40250aadb33SDmitri Tikhonov 40350aadb33SDmitri Tikhonov fiu_do_on("mm/packet_out", FAIL_NOMEM); 40450aadb33SDmitri Tikhonov 40550aadb33SDmitri Tikhonov packet_out = lsquic_malo_get(malo ? malo : mm->malo.packet_out); 40650aadb33SDmitri Tikhonov if (!packet_out) 40750aadb33SDmitri Tikhonov return NULL; 40850aadb33SDmitri Tikhonov 4095392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 41050aadb33SDmitri Tikhonov idx = packet_out_index(size); 41150aadb33SDmitri Tikhonov pob = SLIST_FIRST(&mm->packet_out_bufs[idx]); 41250aadb33SDmitri Tikhonov if (pob) 4135392f7a3SLiteSpeed Tech { 41450aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->packet_out_bufs[idx], next_pob); 4155392f7a3SLiteSpeed Tech poolst_allocated(&mm->packet_out_bstats[idx], 0); 4165392f7a3SLiteSpeed Tech } 41750aadb33SDmitri Tikhonov else 41850aadb33SDmitri Tikhonov { 41950aadb33SDmitri Tikhonov pob = malloc(packet_out_sizes[idx]); 42050aadb33SDmitri Tikhonov if (!pob) 42150aadb33SDmitri Tikhonov { 42250aadb33SDmitri Tikhonov lsquic_malo_put(packet_out); 42350aadb33SDmitri Tikhonov return NULL; 42450aadb33SDmitri Tikhonov } 4255392f7a3SLiteSpeed Tech poolst_allocated(&mm->packet_out_bstats[idx], 1); 42650aadb33SDmitri Tikhonov } 4275392f7a3SLiteSpeed Tech if (poolst_has_new_sample(&mm->packet_out_bstats[idx])) 4285392f7a3SLiteSpeed Tech maybe_shrink_packet_out_bufs(mm, idx); 4295392f7a3SLiteSpeed Tech#else 4305392f7a3SLiteSpeed Tech pob = malloc(size); 4315392f7a3SLiteSpeed Tech if (!pob) 4325392f7a3SLiteSpeed Tech { 4335392f7a3SLiteSpeed Tech lsquic_malo_put(packet_out); 4345392f7a3SLiteSpeed Tech return NULL; 4355392f7a3SLiteSpeed Tech } 4365392f7a3SLiteSpeed Tech#endif 43750aadb33SDmitri Tikhonov 43850aadb33SDmitri Tikhonov memset(packet_out, 0, sizeof(*packet_out)); 43950aadb33SDmitri Tikhonov packet_out->po_n_alloc = size; 44050aadb33SDmitri Tikhonov packet_out->po_data = (unsigned char *) pob; 44150aadb33SDmitri Tikhonov 44250aadb33SDmitri Tikhonov return packet_out; 44350aadb33SDmitri Tikhonov} 44450aadb33SDmitri Tikhonov 44550aadb33SDmitri Tikhonov 44650aadb33SDmitri Tikhonovvoid * 4475392f7a3SLiteSpeed Techlsquic_mm_get_packet_in_buf (struct lsquic_mm *mm, size_t size) 44850aadb33SDmitri Tikhonov{ 4495392f7a3SLiteSpeed Tech struct packet_in_buf *pib; 4505392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 4515392f7a3SLiteSpeed Tech unsigned idx; 4525392f7a3SLiteSpeed Tech 4535392f7a3SLiteSpeed Tech idx = packet_in_index(size); 4545392f7a3SLiteSpeed Tech pib = SLIST_FIRST(&mm->packet_in_bufs[idx]); 4555392f7a3SLiteSpeed Tech fiu_do_on("mm/packet_in_buf", FAIL_NOMEM); 4565392f7a3SLiteSpeed Tech if (pib) 4575392f7a3SLiteSpeed Tech SLIST_REMOVE_HEAD(&mm->packet_in_bufs[idx], next_pib); 45850aadb33SDmitri Tikhonov else 4595392f7a3SLiteSpeed Tech pib = malloc(packet_in_sizes[idx]); 4605392f7a3SLiteSpeed Tech#else 4615392f7a3SLiteSpeed Tech pib = malloc(size); 4625392f7a3SLiteSpeed Tech#endif 4635392f7a3SLiteSpeed Tech return pib; 46450aadb33SDmitri Tikhonov} 46550aadb33SDmitri Tikhonov 46650aadb33SDmitri Tikhonov 46750aadb33SDmitri Tikhonovvoid 4685392f7a3SLiteSpeed Techlsquic_mm_put_packet_in_buf (struct lsquic_mm *mm, void *mem, size_t size) 46950aadb33SDmitri Tikhonov{ 4705392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 4715392f7a3SLiteSpeed Tech unsigned idx; 4725392f7a3SLiteSpeed Tech struct packet_in_buf *pib; 4735392f7a3SLiteSpeed Tech 4745392f7a3SLiteSpeed Tech pib = (struct packet_in_buf *) mem; 4755392f7a3SLiteSpeed Tech idx = packet_in_index(size); 4765392f7a3SLiteSpeed Tech SLIST_INSERT_HEAD(&mm->packet_in_bufs[idx], pib, next_pib); 4775392f7a3SLiteSpeed Tech#else 4785392f7a3SLiteSpeed Tech free(mem); 4795392f7a3SLiteSpeed Tech#endif 48050aadb33SDmitri Tikhonov} 48150aadb33SDmitri Tikhonov 48250aadb33SDmitri Tikhonov 48350aadb33SDmitri Tikhonovvoid * 48450aadb33SDmitri Tikhonovlsquic_mm_get_4k (struct lsquic_mm *mm) 48550aadb33SDmitri Tikhonov{ 4865392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 48750aadb33SDmitri Tikhonov struct four_k_page *fkp = SLIST_FIRST(&mm->four_k_pages); 48850aadb33SDmitri Tikhonov fiu_do_on("mm/4k", FAIL_NOMEM); 48950aadb33SDmitri Tikhonov if (fkp) 49050aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->four_k_pages, next_fkp); 49150aadb33SDmitri Tikhonov else 49250aadb33SDmitri Tikhonov fkp = malloc(0x1000); 49350aadb33SDmitri Tikhonov return fkp; 4945392f7a3SLiteSpeed Tech#else 4955392f7a3SLiteSpeed Tech return malloc(0x1000); 4965392f7a3SLiteSpeed Tech#endif 49750aadb33SDmitri Tikhonov} 49850aadb33SDmitri Tikhonov 49950aadb33SDmitri Tikhonov 50050aadb33SDmitri Tikhonovvoid 50150aadb33SDmitri Tikhonovlsquic_mm_put_4k (struct lsquic_mm *mm, void *mem) 50250aadb33SDmitri Tikhonov{ 5035392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 50450aadb33SDmitri Tikhonov struct four_k_page *fkp = mem; 50550aadb33SDmitri Tikhonov SLIST_INSERT_HEAD(&mm->four_k_pages, fkp, next_fkp); 5065392f7a3SLiteSpeed Tech#else 5075392f7a3SLiteSpeed Tech free(mem); 5085392f7a3SLiteSpeed Tech#endif 50950aadb33SDmitri Tikhonov} 51050aadb33SDmitri Tikhonov 51150aadb33SDmitri Tikhonov 51250aadb33SDmitri Tikhonovvoid * 51350aadb33SDmitri Tikhonovlsquic_mm_get_16k (struct lsquic_mm *mm) 51450aadb33SDmitri Tikhonov{ 5155392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 51650aadb33SDmitri Tikhonov struct sixteen_k_page *skp = SLIST_FIRST(&mm->sixteen_k_pages); 51750aadb33SDmitri Tikhonov fiu_do_on("mm/16k", FAIL_NOMEM); 51850aadb33SDmitri Tikhonov if (skp) 51950aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->sixteen_k_pages, next_skp); 52050aadb33SDmitri Tikhonov else 52150aadb33SDmitri Tikhonov skp = malloc(16 * 1024); 52250aadb33SDmitri Tikhonov return skp; 5235392f7a3SLiteSpeed Tech#else 5245392f7a3SLiteSpeed Tech return malloc(16 * 1024); 5255392f7a3SLiteSpeed Tech#endif 52650aadb33SDmitri Tikhonov} 52750aadb33SDmitri Tikhonov 52850aadb33SDmitri Tikhonov 52950aadb33SDmitri Tikhonovvoid 53050aadb33SDmitri Tikhonovlsquic_mm_put_16k (struct lsquic_mm *mm, void *mem) 53150aadb33SDmitri Tikhonov{ 5325392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 53350aadb33SDmitri Tikhonov struct sixteen_k_page *skp = mem; 53450aadb33SDmitri Tikhonov SLIST_INSERT_HEAD(&mm->sixteen_k_pages, skp, next_skp); 5355392f7a3SLiteSpeed Tech#else 5365392f7a3SLiteSpeed Tech free(mem); 5375392f7a3SLiteSpeed Tech#endif 53850aadb33SDmitri Tikhonov} 53950aadb33SDmitri Tikhonov 54050aadb33SDmitri Tikhonov 541c51ce338SDmitri Tikhonovsize_t 542c51ce338SDmitri Tikhonovlsquic_mm_mem_used (const struct lsquic_mm *mm) 543c51ce338SDmitri Tikhonov{ 5445392f7a3SLiteSpeed Tech#if LSQUIC_USE_POOLS 545c51ce338SDmitri Tikhonov const struct packet_out_buf *pob; 5465392f7a3SLiteSpeed Tech const struct packet_in_buf *pib; 547c51ce338SDmitri Tikhonov const struct four_k_page *fkp; 548c51ce338SDmitri Tikhonov const struct sixteen_k_page *skp; 549c51ce338SDmitri Tikhonov unsigned i; 550c51ce338SDmitri Tikhonov size_t size; 551c51ce338SDmitri Tikhonov 552c51ce338SDmitri Tikhonov size = sizeof(*mm); 553c51ce338SDmitri Tikhonov size += sizeof(*mm->acki); 554c51ce338SDmitri Tikhonov size += lsquic_malo_mem_used(mm->malo.stream_frame); 555c51ce338SDmitri Tikhonov size += lsquic_malo_mem_used(mm->malo.stream_rec_arr); 5565392f7a3SLiteSpeed Tech size += lsquic_malo_mem_used(mm->malo.mini_conn); 5575392f7a3SLiteSpeed Tech size += lsquic_malo_mem_used(mm->malo.mini_conn_ietf); 558c51ce338SDmitri Tikhonov size += lsquic_malo_mem_used(mm->malo.packet_in); 559c51ce338SDmitri Tikhonov size += lsquic_malo_mem_used(mm->malo.packet_out); 560c51ce338SDmitri Tikhonov 561c51ce338SDmitri Tikhonov for (i = 0; i < MM_N_OUT_BUCKETS; ++i) 562c51ce338SDmitri Tikhonov SLIST_FOREACH(pob, &mm->packet_out_bufs[i], next_pob) 563c51ce338SDmitri Tikhonov size += packet_out_sizes[i]; 564c51ce338SDmitri Tikhonov 5655392f7a3SLiteSpeed Tech for (i = 0; i < MM_N_IN_BUCKETS; ++i) 5665392f7a3SLiteSpeed Tech SLIST_FOREACH(pib, &mm->packet_in_bufs[i], next_pib) 5675392f7a3SLiteSpeed Tech size += packet_in_sizes[i]; 568c51ce338SDmitri Tikhonov 569c51ce338SDmitri Tikhonov SLIST_FOREACH(fkp, &mm->four_k_pages, next_fkp) 570c51ce338SDmitri Tikhonov size += 0x1000; 571c51ce338SDmitri Tikhonov 572c51ce338SDmitri Tikhonov SLIST_FOREACH(skp, &mm->sixteen_k_pages, next_skp) 573c51ce338SDmitri Tikhonov size += 0x4000; 574c51ce338SDmitri Tikhonov 575c51ce338SDmitri Tikhonov return size; 5765392f7a3SLiteSpeed Tech#else 5775392f7a3SLiteSpeed Tech return sizeof(*mm); 5785392f7a3SLiteSpeed Tech#endif 579c51ce338SDmitri Tikhonov} 580