lsquic_mm.c revision 50aadb33
150aadb33SDmitri Tikhonov/* Copyright (c) 2017 LiteSpeed Technologies Inc. See LICENSE. */ 250aadb33SDmitri Tikhonov/* 350aadb33SDmitri Tikhonov * lsquic_mm.c -- Memory manager. 450aadb33SDmitri Tikhonov */ 550aadb33SDmitri Tikhonov 650aadb33SDmitri Tikhonov#include <assert.h> 750aadb33SDmitri Tikhonov#include <errno.h> 850aadb33SDmitri Tikhonov#include <stddef.h> 950aadb33SDmitri Tikhonov#include <stdlib.h> 1050aadb33SDmitri Tikhonov#include <string.h> 1150aadb33SDmitri Tikhonov#include <sys/queue.h> 1250aadb33SDmitri Tikhonov 1350aadb33SDmitri Tikhonov#include "fiu-local.h" 1450aadb33SDmitri Tikhonov 1550aadb33SDmitri Tikhonov#include "lsquic.h" 1650aadb33SDmitri Tikhonov#include "lsquic_int_types.h" 1750aadb33SDmitri Tikhonov#include "lsquic_malo.h" 1850aadb33SDmitri Tikhonov#include "lsquic_conn.h" 1950aadb33SDmitri Tikhonov#include "lsquic_rtt.h" 2050aadb33SDmitri Tikhonov#include "lsquic_packet_common.h" 2150aadb33SDmitri Tikhonov#include "lsquic_packet_in.h" 2250aadb33SDmitri Tikhonov#include "lsquic_packet_out.h" 2350aadb33SDmitri Tikhonov#include "lsquic_parse.h" 2450aadb33SDmitri Tikhonov#include "lsquic_mm.h" 2550aadb33SDmitri Tikhonov#include "lsquic_engine_public.h" 2650aadb33SDmitri Tikhonov 2750aadb33SDmitri Tikhonov#define FAIL_NOMEM do { errno = ENOMEM; return NULL; } while (0) 2850aadb33SDmitri Tikhonov 2950aadb33SDmitri Tikhonov 3050aadb33SDmitri Tikhonovstruct payload_buf 3150aadb33SDmitri Tikhonov{ 3250aadb33SDmitri Tikhonov SLIST_ENTRY(payload_buf) next_pb; 3350aadb33SDmitri Tikhonov}; 3450aadb33SDmitri Tikhonov 3550aadb33SDmitri Tikhonovstruct packet_out_buf 3650aadb33SDmitri Tikhonov{ 3750aadb33SDmitri Tikhonov SLIST_ENTRY(packet_out_buf) next_pob; 3850aadb33SDmitri Tikhonov}; 3950aadb33SDmitri Tikhonov 4050aadb33SDmitri Tikhonovstruct four_k_page 4150aadb33SDmitri Tikhonov{ 4250aadb33SDmitri Tikhonov SLIST_ENTRY(four_k_page) next_fkp; 4350aadb33SDmitri Tikhonov}; 4450aadb33SDmitri Tikhonov 4550aadb33SDmitri Tikhonovstruct sixteen_k_page 4650aadb33SDmitri Tikhonov{ 4750aadb33SDmitri Tikhonov SLIST_ENTRY(sixteen_k_page) next_skp; 4850aadb33SDmitri Tikhonov}; 4950aadb33SDmitri Tikhonov 5050aadb33SDmitri Tikhonov 5150aadb33SDmitri Tikhonovint 5250aadb33SDmitri Tikhonovlsquic_mm_init (struct lsquic_mm *mm) 5350aadb33SDmitri Tikhonov{ 5450aadb33SDmitri Tikhonov int i; 5550aadb33SDmitri Tikhonov 5650aadb33SDmitri Tikhonov mm->acki = malloc(sizeof(*mm->acki)); 5750aadb33SDmitri Tikhonov mm->malo.stream_frame = lsquic_malo_create(sizeof(struct stream_frame)); 5850aadb33SDmitri Tikhonov mm->malo.stream_rec_arr = lsquic_malo_create(sizeof(struct stream_rec_arr)); 5950aadb33SDmitri Tikhonov mm->malo.packet_in = lsquic_malo_create(sizeof(struct lsquic_packet_in)); 6050aadb33SDmitri Tikhonov mm->malo.packet_out = lsquic_malo_create(sizeof(struct lsquic_packet_out)); 6150aadb33SDmitri Tikhonov TAILQ_INIT(&mm->free_packets_in); 6250aadb33SDmitri Tikhonov for (i = 0; i < MM_N_OUT_BUCKETS; ++i) 6350aadb33SDmitri Tikhonov SLIST_INIT(&mm->packet_out_bufs[i]); 6450aadb33SDmitri Tikhonov SLIST_INIT(&mm->payload_bufs); 6550aadb33SDmitri Tikhonov SLIST_INIT(&mm->four_k_pages); 6650aadb33SDmitri Tikhonov SLIST_INIT(&mm->sixteen_k_pages); 6750aadb33SDmitri Tikhonov if (mm->acki && mm->malo.stream_frame && mm->malo.stream_rec_arr && 6850aadb33SDmitri Tikhonov mm->malo.packet_in) 6950aadb33SDmitri Tikhonov { 7050aadb33SDmitri Tikhonov return 0; 7150aadb33SDmitri Tikhonov } 7250aadb33SDmitri Tikhonov else 7350aadb33SDmitri Tikhonov return -1; 7450aadb33SDmitri Tikhonov} 7550aadb33SDmitri Tikhonov 7650aadb33SDmitri Tikhonov 7750aadb33SDmitri Tikhonovvoid 7850aadb33SDmitri Tikhonovlsquic_mm_cleanup (struct lsquic_mm *mm) 7950aadb33SDmitri Tikhonov{ 8050aadb33SDmitri Tikhonov int i; 8150aadb33SDmitri Tikhonov struct packet_out_buf *pob; 8250aadb33SDmitri Tikhonov struct payload_buf *pb; 8350aadb33SDmitri Tikhonov struct four_k_page *fkp; 8450aadb33SDmitri Tikhonov struct sixteen_k_page *skp; 8550aadb33SDmitri Tikhonov 8650aadb33SDmitri Tikhonov free(mm->acki); 8750aadb33SDmitri Tikhonov lsquic_malo_destroy(mm->malo.packet_in); 8850aadb33SDmitri Tikhonov lsquic_malo_destroy(mm->malo.packet_out); 8950aadb33SDmitri Tikhonov lsquic_malo_destroy(mm->malo.stream_frame); 9050aadb33SDmitri Tikhonov lsquic_malo_destroy(mm->malo.stream_rec_arr); 9150aadb33SDmitri Tikhonov 9250aadb33SDmitri Tikhonov for (i = 0; i < MM_N_OUT_BUCKETS; ++i) 9350aadb33SDmitri Tikhonov while ((pob = SLIST_FIRST(&mm->packet_out_bufs[i]))) 9450aadb33SDmitri Tikhonov { 9550aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->packet_out_bufs[i], next_pob); 9650aadb33SDmitri Tikhonov free(pob); 9750aadb33SDmitri Tikhonov } 9850aadb33SDmitri Tikhonov 9950aadb33SDmitri Tikhonov while ((pb = SLIST_FIRST(&mm->payload_bufs))) 10050aadb33SDmitri Tikhonov { 10150aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->payload_bufs, next_pb); 10250aadb33SDmitri Tikhonov free(pb); 10350aadb33SDmitri Tikhonov } 10450aadb33SDmitri Tikhonov 10550aadb33SDmitri Tikhonov while ((fkp = SLIST_FIRST(&mm->four_k_pages))) 10650aadb33SDmitri Tikhonov { 10750aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->four_k_pages, next_fkp); 10850aadb33SDmitri Tikhonov free(fkp); 10950aadb33SDmitri Tikhonov } 11050aadb33SDmitri Tikhonov 11150aadb33SDmitri Tikhonov while ((skp = SLIST_FIRST(&mm->sixteen_k_pages))) 11250aadb33SDmitri Tikhonov { 11350aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->sixteen_k_pages, next_skp); 11450aadb33SDmitri Tikhonov free(skp); 11550aadb33SDmitri Tikhonov } 11650aadb33SDmitri Tikhonov} 11750aadb33SDmitri Tikhonov 11850aadb33SDmitri Tikhonov 11950aadb33SDmitri Tikhonovstruct lsquic_packet_in * 12050aadb33SDmitri Tikhonovlsquic_mm_get_packet_in (struct lsquic_mm *mm) 12150aadb33SDmitri Tikhonov{ 12250aadb33SDmitri Tikhonov struct lsquic_packet_in *packet_in; 12350aadb33SDmitri Tikhonov 12450aadb33SDmitri Tikhonov fiu_do_on("mm/packet_in", FAIL_NOMEM); 12550aadb33SDmitri Tikhonov 12650aadb33SDmitri Tikhonov packet_in = TAILQ_FIRST(&mm->free_packets_in); 12750aadb33SDmitri Tikhonov if (packet_in) 12850aadb33SDmitri Tikhonov { 12950aadb33SDmitri Tikhonov assert(0 == packet_in->pi_refcnt); 13050aadb33SDmitri Tikhonov TAILQ_REMOVE(&mm->free_packets_in, packet_in, pi_next); 13150aadb33SDmitri Tikhonov } 13250aadb33SDmitri Tikhonov else 13350aadb33SDmitri Tikhonov packet_in = lsquic_malo_get(mm->malo.packet_in); 13450aadb33SDmitri Tikhonov 13550aadb33SDmitri Tikhonov if (packet_in) 13650aadb33SDmitri Tikhonov memset(packet_in, 0, sizeof(*packet_in)); 13750aadb33SDmitri Tikhonov 13850aadb33SDmitri Tikhonov return packet_in; 13950aadb33SDmitri Tikhonov} 14050aadb33SDmitri Tikhonov 14150aadb33SDmitri Tikhonov 14250aadb33SDmitri Tikhonov/* Based on commonly used MTUs, ordered from small to large: */ 14350aadb33SDmitri Tikhonovenum { 14450aadb33SDmitri Tikhonov PACKET_OUT_PAYLOAD_0 = 1280 - QUIC_MIN_PACKET_OVERHEAD, 14550aadb33SDmitri Tikhonov PACKET_OUT_PAYLOAD_1 = QUIC_MAX_IPv6_PACKET_SZ - QUIC_MIN_PACKET_OVERHEAD, 14650aadb33SDmitri Tikhonov PACKET_OUT_PAYLOAD_2 = QUIC_MAX_IPv4_PACKET_SZ - QUIC_MIN_PACKET_OVERHEAD, 14750aadb33SDmitri Tikhonov}; 14850aadb33SDmitri Tikhonov 14950aadb33SDmitri Tikhonov 15050aadb33SDmitri Tikhonovstatic const unsigned packet_out_sizes[] = { 15150aadb33SDmitri Tikhonov PACKET_OUT_PAYLOAD_0, 15250aadb33SDmitri Tikhonov PACKET_OUT_PAYLOAD_1, 15350aadb33SDmitri Tikhonov PACKET_OUT_PAYLOAD_2, 15450aadb33SDmitri Tikhonov}; 15550aadb33SDmitri Tikhonov 15650aadb33SDmitri Tikhonov 15750aadb33SDmitri Tikhonovstatic unsigned 15850aadb33SDmitri Tikhonovpacket_out_index (unsigned size) 15950aadb33SDmitri Tikhonov{ 16050aadb33SDmitri Tikhonov unsigned idx = (size > PACKET_OUT_PAYLOAD_0) 16150aadb33SDmitri Tikhonov + (size > PACKET_OUT_PAYLOAD_1); 16250aadb33SDmitri Tikhonov return idx; 16350aadb33SDmitri Tikhonov} 16450aadb33SDmitri Tikhonov 16550aadb33SDmitri Tikhonov 16650aadb33SDmitri Tikhonovvoid 16750aadb33SDmitri Tikhonovlsquic_mm_put_packet_out (struct lsquic_mm *mm, 16850aadb33SDmitri Tikhonov struct lsquic_packet_out *packet_out) 16950aadb33SDmitri Tikhonov{ 17050aadb33SDmitri Tikhonov struct packet_out_buf *pob; 17150aadb33SDmitri Tikhonov unsigned idx; 17250aadb33SDmitri Tikhonov 17350aadb33SDmitri Tikhonov assert(packet_out->po_data); 17450aadb33SDmitri Tikhonov pob = (struct packet_out_buf *) packet_out->po_data; 17550aadb33SDmitri Tikhonov idx = packet_out_index(packet_out->po_n_alloc); 17650aadb33SDmitri Tikhonov SLIST_INSERT_HEAD(&mm->packet_out_bufs[idx], pob, next_pob); 17750aadb33SDmitri Tikhonov lsquic_malo_put(packet_out); 17850aadb33SDmitri Tikhonov} 17950aadb33SDmitri Tikhonov 18050aadb33SDmitri Tikhonov 18150aadb33SDmitri Tikhonovstruct lsquic_packet_out * 18250aadb33SDmitri Tikhonovlsquic_mm_get_packet_out (struct lsquic_mm *mm, struct malo *malo, 18350aadb33SDmitri Tikhonov unsigned short size) 18450aadb33SDmitri Tikhonov{ 18550aadb33SDmitri Tikhonov struct lsquic_packet_out *packet_out; 18650aadb33SDmitri Tikhonov struct packet_out_buf *pob; 18750aadb33SDmitri Tikhonov unsigned idx; 18850aadb33SDmitri Tikhonov 18950aadb33SDmitri Tikhonov assert(size <= QUIC_MAX_PAYLOAD_SZ); 19050aadb33SDmitri Tikhonov 19150aadb33SDmitri Tikhonov fiu_do_on("mm/packet_out", FAIL_NOMEM); 19250aadb33SDmitri Tikhonov 19350aadb33SDmitri Tikhonov packet_out = lsquic_malo_get(malo ? malo : mm->malo.packet_out); 19450aadb33SDmitri Tikhonov if (!packet_out) 19550aadb33SDmitri Tikhonov return NULL; 19650aadb33SDmitri Tikhonov 19750aadb33SDmitri Tikhonov idx = packet_out_index(size); 19850aadb33SDmitri Tikhonov pob = SLIST_FIRST(&mm->packet_out_bufs[idx]); 19950aadb33SDmitri Tikhonov if (pob) 20050aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->packet_out_bufs[idx], next_pob); 20150aadb33SDmitri Tikhonov else 20250aadb33SDmitri Tikhonov { 20350aadb33SDmitri Tikhonov pob = malloc(packet_out_sizes[idx]); 20450aadb33SDmitri Tikhonov if (!pob) 20550aadb33SDmitri Tikhonov { 20650aadb33SDmitri Tikhonov lsquic_malo_put(packet_out); 20750aadb33SDmitri Tikhonov return NULL; 20850aadb33SDmitri Tikhonov } 20950aadb33SDmitri Tikhonov } 21050aadb33SDmitri Tikhonov 21150aadb33SDmitri Tikhonov memset(packet_out, 0, sizeof(*packet_out)); 21250aadb33SDmitri Tikhonov STAILQ_INIT(&packet_out->po_srec_arrs); 21350aadb33SDmitri Tikhonov packet_out->po_n_alloc = size; 21450aadb33SDmitri Tikhonov packet_out->po_data = (unsigned char *) pob; 21550aadb33SDmitri Tikhonov 21650aadb33SDmitri Tikhonov return packet_out; 21750aadb33SDmitri Tikhonov} 21850aadb33SDmitri Tikhonov 21950aadb33SDmitri Tikhonov 22050aadb33SDmitri Tikhonovvoid * 22150aadb33SDmitri Tikhonovlsquic_mm_get_1370 (struct lsquic_mm *mm) 22250aadb33SDmitri Tikhonov{ 22350aadb33SDmitri Tikhonov struct payload_buf *pb = SLIST_FIRST(&mm->payload_bufs); 22450aadb33SDmitri Tikhonov fiu_do_on("mm/1370", FAIL_NOMEM); 22550aadb33SDmitri Tikhonov if (pb) 22650aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->payload_bufs, next_pb); 22750aadb33SDmitri Tikhonov else 22850aadb33SDmitri Tikhonov pb = malloc(1370); 22950aadb33SDmitri Tikhonov return pb; 23050aadb33SDmitri Tikhonov} 23150aadb33SDmitri Tikhonov 23250aadb33SDmitri Tikhonov 23350aadb33SDmitri Tikhonovvoid 23450aadb33SDmitri Tikhonovlsquic_mm_put_1370 (struct lsquic_mm *mm, void *mem) 23550aadb33SDmitri Tikhonov{ 23650aadb33SDmitri Tikhonov struct payload_buf *pb = mem; 23750aadb33SDmitri Tikhonov SLIST_INSERT_HEAD(&mm->payload_bufs, pb, next_pb); 23850aadb33SDmitri Tikhonov} 23950aadb33SDmitri Tikhonov 24050aadb33SDmitri Tikhonov 24150aadb33SDmitri Tikhonovvoid * 24250aadb33SDmitri Tikhonovlsquic_mm_get_4k (struct lsquic_mm *mm) 24350aadb33SDmitri Tikhonov{ 24450aadb33SDmitri Tikhonov struct four_k_page *fkp = SLIST_FIRST(&mm->four_k_pages); 24550aadb33SDmitri Tikhonov fiu_do_on("mm/4k", FAIL_NOMEM); 24650aadb33SDmitri Tikhonov if (fkp) 24750aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->four_k_pages, next_fkp); 24850aadb33SDmitri Tikhonov else 24950aadb33SDmitri Tikhonov fkp = malloc(0x1000); 25050aadb33SDmitri Tikhonov return fkp; 25150aadb33SDmitri Tikhonov} 25250aadb33SDmitri Tikhonov 25350aadb33SDmitri Tikhonov 25450aadb33SDmitri Tikhonovvoid 25550aadb33SDmitri Tikhonovlsquic_mm_put_4k (struct lsquic_mm *mm, void *mem) 25650aadb33SDmitri Tikhonov{ 25750aadb33SDmitri Tikhonov struct four_k_page *fkp = mem; 25850aadb33SDmitri Tikhonov SLIST_INSERT_HEAD(&mm->four_k_pages, fkp, next_fkp); 25950aadb33SDmitri Tikhonov} 26050aadb33SDmitri Tikhonov 26150aadb33SDmitri Tikhonov 26250aadb33SDmitri Tikhonovvoid * 26350aadb33SDmitri Tikhonovlsquic_mm_get_16k (struct lsquic_mm *mm) 26450aadb33SDmitri Tikhonov{ 26550aadb33SDmitri Tikhonov struct sixteen_k_page *skp = SLIST_FIRST(&mm->sixteen_k_pages); 26650aadb33SDmitri Tikhonov fiu_do_on("mm/16k", FAIL_NOMEM); 26750aadb33SDmitri Tikhonov if (skp) 26850aadb33SDmitri Tikhonov SLIST_REMOVE_HEAD(&mm->sixteen_k_pages, next_skp); 26950aadb33SDmitri Tikhonov else 27050aadb33SDmitri Tikhonov skp = malloc(16 * 1024); 27150aadb33SDmitri Tikhonov return skp; 27250aadb33SDmitri Tikhonov} 27350aadb33SDmitri Tikhonov 27450aadb33SDmitri Tikhonov 27550aadb33SDmitri Tikhonovvoid 27650aadb33SDmitri Tikhonovlsquic_mm_put_16k (struct lsquic_mm *mm, void *mem) 27750aadb33SDmitri Tikhonov{ 27850aadb33SDmitri Tikhonov struct sixteen_k_page *skp = mem; 27950aadb33SDmitri Tikhonov SLIST_INSERT_HEAD(&mm->sixteen_k_pages, skp, next_skp); 28050aadb33SDmitri Tikhonov} 28150aadb33SDmitri Tikhonov 28250aadb33SDmitri Tikhonov 28350aadb33SDmitri Tikhonovvoid 28450aadb33SDmitri Tikhonovlsquic_mm_put_packet_in (struct lsquic_mm *mm, 28550aadb33SDmitri Tikhonov struct lsquic_packet_in *packet_in) 28650aadb33SDmitri Tikhonov{ 28750aadb33SDmitri Tikhonov assert(0 == packet_in->pi_refcnt); 28850aadb33SDmitri Tikhonov if (packet_in->pi_flags & PI_OWN_DATA) 28950aadb33SDmitri Tikhonov lsquic_mm_put_1370(mm, packet_in->pi_data); 29050aadb33SDmitri Tikhonov TAILQ_INSERT_HEAD(&mm->free_packets_in, packet_in, pi_next); 29150aadb33SDmitri Tikhonov} 29250aadb33SDmitri Tikhonov 29350aadb33SDmitri Tikhonov 294