1/*
2 * QPACK Interop -- encode to intermediate format
3 *
4 * https://github.com/quicwg/base-drafts/wiki/QPACK-Offline-Interop
5 */
6
7#include <assert.h>
8
9#if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__NetBSD__)
10#include <sys/endian.h>
11#define bswap_16 bswap16
12#define bswap_32 bswap32
13#define bswap_64 bswap64
14#elif defined(__APPLE__)
15#include <libkern/OSByteOrder.h>
16#define bswap_16 OSSwapInt16
17#define bswap_32 OSSwapInt32
18#define bswap_64 OSSwapInt64
19#elif defined(WIN32)
20#define bswap_16 _byteswap_ushort
21#define bswap_32 _byteswap_ulong
22#define bswap_64 _byteswap_uint64
23#else
24#include <byteswap.h>
25#endif
26
27#include <errno.h>
28#include <stdio.h>
29#include <stdlib.h>
30#include <string.h>
31#ifdef WIN32
32#include <getopt.h>
33#else
34#include <unistd.h>
35#endif
36#include <sys/types.h>
37#include <sys/stat.h>
38#include <fcntl.h>
39#include <inttypes.h>
40
41#include "lsqpack.h"
42#include "lsxpack_header.h"
43
44static int s_verbose;
45
46unsigned char *
47lsqpack_enc_int (unsigned char *dst, unsigned char *const end, uint64_t value,
48                                                        unsigned prefix_bits);
49
50static void
51usage (const char *name)
52{
53    fprintf(stderr,
54"Usage: %s [options] [-i input] [-o output]\n"
55"\n"
56"Options:\n"
57"   -i FILE     Input file.  If not specified or set to `-', the input is\n"
58"                 read from stdin.\n"
59"   -o FILE     Output file.  If not spepcified or set to `-', the output\n"
60"                 is written to stdout.\n"
61"   -s NUMBER   Maximum number of risked streams.  Defaults to %u.\n"
62"   -t NUMBER   Dynamic table size.  Defaults to %u.\n"
63"   -a MODE     Header acknowledgement mode.  0 means headers are never\n"
64"                 acknowledged, non-zero means header blocks are acknowledged\n"
65"                 immediately.  Default value is 0.\n"
66"   -n          Process annotations.\n"
67"   -S          Server mode.\n"
68"   -D          Do not emit \"Duplicate\" instructions.\n"
69"   -A          Aggressive indexing.\n"
70"   -M          Turn off memory guard.\n"
71"   -f          Fast: use maximum output buffers.\n"
72"   -v          Verbose: print various messages to stderr.\n"
73"\n"
74"   -h          Print this help screen and exit\n"
75    , name, LSQPACK_DEF_MAX_RISKED_STREAMS, LSQPACK_DEF_DYN_TABLE_SIZE);
76}
77
78
79static void
80write_enc_stream (FILE *out, const unsigned char *enc_buf, size_t enc_sz)
81{
82    uint64_t stream_id_enc;
83    uint32_t length_enc;
84    size_t written;
85
86    if (enc_sz <= 0)
87        return;
88    stream_id_enc = 0;
89    length_enc = enc_sz;
90#if __BYTE_ORDER == __LITTLE_ENDIAN
91    stream_id_enc = bswap_64(stream_id_enc);
92    length_enc = bswap_32(length_enc);
93#endif
94    written = fwrite(&stream_id_enc, 1, sizeof(stream_id_enc), out);
95    if (written != sizeof(stream_id_enc))
96    {
97        perror("fwrite");
98        exit(EXIT_FAILURE);
99    }
100    written = fwrite(&length_enc, 1, sizeof(length_enc), out);
101    if (written != sizeof(length_enc))
102    {
103        perror("fwrite");
104        exit(EXIT_FAILURE);
105    }
106    written = fwrite(enc_buf, 1, enc_sz, out);
107    if (written != enc_sz)
108    {
109        perror("fwrite");
110        exit(EXIT_FAILURE);
111    }
112}
113
114
115static void
116write_enc_and_header_streams (FILE *out, unsigned stream_id,
117                              const unsigned char *enc_buf, size_t enc_sz,
118                              const unsigned char *pref_buf, size_t pref_sz,
119                              const unsigned char *hea_buf, size_t hea_sz)
120{
121    uint64_t stream_id_enc;
122    uint32_t length_enc;
123    size_t written;
124
125    if (s_verbose)
126        fprintf(stderr, "%s: stream %"PRIu32"\n", __func__, stream_id);
127
128    if (enc_sz)
129    {
130        stream_id_enc = 0;
131        length_enc = enc_sz;
132#if __BYTE_ORDER == __LITTLE_ENDIAN
133        stream_id_enc = bswap_64(stream_id_enc);
134        length_enc = bswap_32(length_enc);
135#endif
136        written = fwrite(&stream_id_enc, 1, sizeof(stream_id_enc), out);
137        if (written != sizeof(stream_id_enc))
138            goto write_err;
139        written = fwrite(&length_enc, 1, sizeof(length_enc), out);
140        if (written != sizeof(length_enc))
141            goto write_err;
142        written = fwrite(enc_buf, 1, enc_sz, out);
143        if (written != enc_sz)
144            goto write_err;
145    }
146
147    stream_id_enc = stream_id;
148    length_enc = pref_sz + hea_sz;
149#if __BYTE_ORDER == __LITTLE_ENDIAN
150    stream_id_enc = bswap_64(stream_id_enc);
151    length_enc = bswap_32(length_enc);
152#endif
153    written = fwrite(&stream_id_enc, 1, sizeof(stream_id_enc), out);
154    if (written != sizeof(stream_id_enc))
155        goto write_err;
156    written = fwrite(&length_enc, 1, sizeof(length_enc), out);
157    if (written != sizeof(length_enc))
158        goto write_err;
159    written = fwrite(pref_buf, 1, pref_sz, out);
160    if (written != pref_sz)
161        goto write_err;
162    written = fwrite(hea_buf, 1, hea_sz, out);
163    if (written != hea_sz)
164        goto write_err;
165    return;
166
167  write_err:
168    perror("fwrite");
169    exit(EXIT_FAILURE);
170}
171
172
173static unsigned s_saved_ins_count;
174static int
175ack_last_entry_id (struct lsqpack_enc *encoder)
176{
177    unsigned char *end_cmd;
178    unsigned char cmd[80];
179    unsigned val;
180
181    if (s_verbose)
182        fprintf(stderr, "ACK entry ID %u\n", encoder->qpe_ins_count);
183
184    cmd[0] = 0x00;
185    val = encoder->qpe_ins_count - s_saved_ins_count;
186    s_saved_ins_count = encoder->qpe_ins_count;
187    end_cmd = lsqpack_enc_int(cmd, cmd + sizeof(cmd), val, 6);
188    assert(end_cmd > cmd);
189    return lsqpack_enc_decoder_in(encoder, cmd, end_cmd - cmd);
190}
191
192
193static int
194ack_stream (struct lsqpack_enc *encoder, uint64_t stream_id)
195{
196    unsigned char *end_cmd;
197    unsigned char cmd[80];
198
199    if (s_verbose)
200        fprintf(stderr, "ACK stream ID %"PRIu64"\n", stream_id);
201
202    cmd[0] = 0x80;
203    end_cmd = lsqpack_enc_int(cmd, cmd + sizeof(cmd), stream_id, 7);
204    assert(end_cmd > cmd);
205    return lsqpack_enc_decoder_in(encoder, cmd, end_cmd - cmd);
206}
207
208
209static int
210sync_table (struct lsqpack_enc *encoder, uint64_t num_inserts)
211{
212    unsigned char *end_cmd;
213    unsigned char cmd[80];
214
215    if (s_verbose)
216        fprintf(stderr, "Sync table num inserts %"PRIu64"\n", num_inserts);
217
218    cmd[0] = 0x00;
219    end_cmd = lsqpack_enc_int(cmd, cmd + sizeof(cmd), num_inserts, 6);
220    assert(end_cmd > cmd);
221    return lsqpack_enc_decoder_in(encoder, cmd, end_cmd - cmd);
222}
223
224
225static int
226cancel_stream (struct lsqpack_enc *encoder, uint64_t stream_id)
227{
228    unsigned char *end_cmd;
229    unsigned char cmd[80];
230
231    if (s_verbose)
232        fprintf(stderr, "Cancel stream ID %"PRIu64"\n", stream_id);
233
234    cmd[0] = 0x40;
235    end_cmd = lsqpack_enc_int(cmd, cmd + sizeof(cmd), stream_id, 6);
236    assert(end_cmd > cmd);
237    return lsqpack_enc_decoder_in(encoder, cmd, end_cmd - cmd);
238}
239
240
241int
242main (int argc, char **argv)
243{
244    FILE *in = stdin;
245    FILE *out = stdout;
246    int opt;
247    unsigned dyn_table_size     = LSQPACK_DEF_DYN_TABLE_SIZE,
248             max_risked_streams = LSQPACK_DEF_MAX_RISKED_STREAMS;
249    unsigned lineno, stream_id;
250    struct lsqpack_enc encoder;
251    char *line, *end, *tab;
252    ssize_t pref_sz;
253    enum lsqpack_enc_status st;
254    enum lsqpack_enc_opts enc_opts = 0;
255    size_t enc_sz, hea_sz, enc_off, hea_off;
256    int header_opened, r;
257    unsigned arg;
258    enum { ACK_NEVER, ACK_IMMEDIATE, } ack_mode = ACK_NEVER;
259    int process_annotations = 0;
260    char line_buf[0x1000];
261    unsigned char tsu_buf[LSQPACK_LONGEST_SDTC];
262    size_t tsu_buf_sz;
263    enum lsqpack_enc_header_flags hflags;
264    int fast = 0;
265    struct lsxpack_header xhdr;
266    unsigned char enc_buf[0x1000], hea_buf[0x1000], pref_buf[0x20];
267
268    while (-1 != (opt = getopt(argc, argv, "ADMSa:i:no:s:t:hvf")))
269    {
270        switch (opt)
271        {
272        case 'S':
273            enc_opts |= LSQPACK_ENC_OPT_SERVER;
274            break;
275        case 'D':
276            enc_opts |= LSQPACK_ENC_OPT_NO_DUP;
277            break;
278        case 'A':
279            enc_opts |= LSQPACK_ENC_OPT_IX_AGGR;
280            break;
281        case 'M':
282            enc_opts |= LSQPACK_ENC_OPT_NO_MEM_GUARD;
283            break;
284        case 'n':
285            ++process_annotations;
286            break;
287        case 'a':
288            ack_mode = atoi(optarg) ? ACK_IMMEDIATE : ACK_NEVER;
289            break;
290        case 'i':
291            if (0 != strcmp(optarg, "-"))
292            {
293                in = fopen(optarg, "r");
294                if (!in)
295                {
296                    fprintf(stderr, "cannot open `%s' for reading: %s\n",
297                                                optarg, strerror(errno));
298                    exit(EXIT_FAILURE);
299                }
300            }
301            break;
302        case 'o':
303            if (0 != strcmp(optarg, "-"))
304            {
305                out = fopen(optarg, "wb");
306                if (!out)
307                {
308                    fprintf(stderr, "cannot open `%s' for writing: %s\n",
309                                                optarg, strerror(errno));
310                    exit(EXIT_FAILURE);
311                }
312            }
313            break;
314        case 's':
315            max_risked_streams = atoi(optarg);
316            break;
317        case 't':
318            dyn_table_size = atoi(optarg);
319            break;
320        case 'h':
321            usage(argv[0]);
322            exit(EXIT_SUCCESS);
323        case 'f':
324            fast = 1;
325            break;
326        case 'v':
327            ++s_verbose;
328            break;
329        default:
330            exit(EXIT_FAILURE);
331        }
332    }
333
334    tsu_buf_sz = sizeof(tsu_buf);
335    if (0 != lsqpack_enc_init(&encoder, s_verbose ? stderr : NULL, dyn_table_size,
336                    dyn_table_size, max_risked_streams, enc_opts, tsu_buf,
337                    &tsu_buf_sz))
338    {
339        perror("lsqpack_enc_init");
340        exit(EXIT_FAILURE);
341    }
342
343    lineno = 0;
344    stream_id = 0;
345    enc_off = 0;
346    hea_off = 0;
347    header_opened = 0;
348
349    while (line = fgets(line_buf, sizeof(line_buf), in), line != NULL)
350    {
351        ++lineno;
352        end = strchr(line, '\n');
353        if (!end)
354        {
355            fprintf(stderr, "no newline on line %u\n", lineno);
356            exit(EXIT_FAILURE);
357        }
358        *end = '\0';
359
360        if (end == line)
361        {
362            if (header_opened)
363            {
364                size_t sz, pref_max = sizeof(pref_buf);
365                for (sz = (fast ? pref_max : 0); sz <= pref_max; sz++)
366                {
367                    pref_sz = lsqpack_enc_end_header(&encoder, pref_buf, sz, &hflags);
368                    if (pref_sz > 0)
369                    {
370                        if (max_risked_streams == 0)
371                            assert(!(hflags & LSQECH_REF_AT_RISK));
372                        break;
373                    }
374                }
375                assert(pref_sz <= lsqpack_enc_header_block_prefix_size(&encoder));
376                if (pref_sz < 0)
377                {
378                    fprintf(stderr, "end_header failed: %s", strerror(errno));
379                    exit(EXIT_FAILURE);
380                }
381                if (ack_mode == ACK_IMMEDIATE)
382                {
383                    if (!(2 == pref_sz && pref_buf[0] == 0 && pref_buf[1] == 0))
384                        r = ack_stream(&encoder, stream_id);
385                    else
386                        r = 0;
387                    if (r == 0 && encoder.qpe_ins_count > s_saved_ins_count)
388                        r = ack_last_entry_id(&encoder);
389                    else
390                        r = 0;
391                    if (r != 0)
392                    {
393                        fprintf(stderr, "acking stream %u failed: %s", stream_id,
394                                                                    strerror(errno));
395                        exit(EXIT_FAILURE);
396                    }
397                }
398                if (s_verbose)
399                    fprintf(stderr, "compression ratio: %.3f\n",
400                        lsqpack_enc_ratio(&encoder));
401                write_enc_and_header_streams(out, stream_id, enc_buf, enc_off,
402                                             pref_buf, pref_sz, hea_buf, hea_off);
403                enc_off = 0;
404                hea_off = 0;
405                header_opened = 0;
406            }
407            continue;
408        }
409
410        if (*line == '#')
411        {
412            if (!process_annotations)
413                continue;
414
415            /* Lines starting with ## are potential annotations */
416            if (ack_mode != ACK_IMMEDIATE
417                /* Ignore ACK annotations in immediate ACK mode, as we do
418                 * not tolerate duplicate ACKs.
419                 */
420                                && 1 == sscanf(line, "## %*[a] %u ", &arg))
421            {
422                if (0 != ack_stream(&encoder, arg))
423                {
424                    fprintf(stderr, "ACKing stream ID %u failed\n", arg);
425                    exit(EXIT_FAILURE);
426                }
427            }
428            else if (1 == sscanf(line, "## %*[s] %u", &arg))
429                sync_table(&encoder, arg);
430            else if (1 == sscanf(line, "## %*[c] %u", &arg))
431                cancel_stream(&encoder, arg);
432            else if (1 == sscanf(line, "## %*[t] %u", &arg))
433            {
434                tsu_buf_sz = sizeof(tsu_buf);
435                if (0 != lsqpack_enc_set_max_capacity(&encoder, arg, tsu_buf,
436                                                                &tsu_buf_sz))
437                {
438                    fprintf(stderr, "cannot set capacity to %u: %s\n", arg,
439                        strerror(errno));
440                    exit(EXIT_FAILURE);
441                }
442                write_enc_stream(out, tsu_buf, tsu_buf_sz);
443            }
444            continue;
445        }
446
447        tab = strchr(line, '\t');
448        if (!tab)
449        {
450            fprintf(stderr, "no TAB on line %u\n", lineno);
451            exit(EXIT_FAILURE);
452        }
453
454        if (!header_opened)
455        {
456            ++stream_id;
457            if (0 != lsqpack_enc_start_header(&encoder, stream_id, 0))
458            {
459                fprintf(stderr, "start_header failed: %s\n", strerror(errno));
460                exit(EXIT_FAILURE);
461            }
462            header_opened = 1;
463        }
464        if (fast)
465        {
466            enc_sz = sizeof(enc_buf) - enc_off;
467            hea_sz = sizeof(hea_buf) - hea_off;
468        }
469        else
470        {
471            /* Increase buffers one by one to exercise error conditions */
472            enc_sz = 0;
473            hea_sz = 0;
474        }
475        while (1)
476        {
477            lsxpack_header_set_offset2(&xhdr, line, 0, tab - line,
478                                            tab + 1 - line, end - tab - 1);
479            st = lsqpack_enc_encode(&encoder, enc_buf + enc_off, &enc_sz,
480                        hea_buf + hea_off, &hea_sz, &xhdr, 0);
481            switch (st)
482            {
483            case LQES_NOBUF_ENC:
484                if (enc_sz < sizeof(enc_buf) - enc_off)
485                    ++enc_sz;
486                else
487                    assert(0);
488                break;
489            case LQES_NOBUF_HEAD:
490                if (hea_sz < sizeof(hea_buf) - hea_off)
491                    ++hea_sz;
492                else
493                    assert(0);
494                break;
495            default:
496                assert(st == LQES_OK);
497                goto end_encode_one_header;
498            }
499        }
500        if (st != LQES_OK)
501        {
502            /* It could only run of of output space, so it's not really an
503             * error, but we make no provision in the interop encoder to
504             * grow the buffers.
505             */
506            fprintf(stderr, "Could not encode header on line %u: %u\n",
507                                                                lineno, st);
508            exit(EXIT_FAILURE);
509        }
510    end_encode_one_header:
511        enc_off += enc_sz;
512        hea_off += hea_sz;
513    }
514
515    if (s_verbose)
516        fprintf(stderr, "exited while loop\n");
517
518    (void) fclose(in);
519
520    if (header_opened)
521    {
522        if (s_verbose)
523            fprintf(stderr, "close opened header\n");
524        pref_sz = lsqpack_enc_end_header(&encoder, pref_buf, sizeof(pref_buf),
525                                                                    &hflags);
526        if (pref_sz < 0)
527        {
528            fprintf(stderr, "end_header failed: %s", strerror(errno));
529            exit(EXIT_FAILURE);
530        }
531        if (max_risked_streams == 0)
532            assert(!(hflags & LSQECH_REF_AT_RISK));
533        if (ack_mode == ACK_IMMEDIATE
534            && !(2 == pref_sz && pref_buf[0] == 0 && pref_buf[1] == 0)
535            && 0 != ack_stream(&encoder, stream_id))
536        {
537            fprintf(stderr, "acking stream %u failed: %s", stream_id,
538                                                                strerror(errno));
539            exit(EXIT_FAILURE);
540        }
541        if (s_verbose)
542            fprintf(stderr, "compression ratio: %.3f\n",
543                lsqpack_enc_ratio(&encoder));
544        write_enc_and_header_streams(out, stream_id, enc_buf, enc_off, pref_buf,
545                                     pref_sz, hea_buf, hea_off);
546    }
547
548    lsqpack_enc_cleanup(&encoder);
549
550    if (0 != fclose(out))
551    {
552        perror("fclose(out)");
553        exit(EXIT_FAILURE);
554    }
555
556    exit(EXIT_SUCCESS);
557}
558