From bde62adb5d1f50bfdc787c66dc9b2192260a1b79 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Fri, 21 Nov 2014 17:06:36 +0100 Subject: MAJOR: httpterm: implement adjustable response size Now the requested response size is honnored. It works both for chunks and for regular content-length. The code is not efficient and results in copies. On large responses and with 1 MB buffers, the data rate is about 1/3 of httpterm's with splicing, or half of httpterm's without splicing. It's expected that splicing will improve that once implemented. On chunked responses, the bandwidth is even lower because haproxy's body forwarding engine has to parse the response as well. --- src/proto_http.c | 99 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 54 insertions(+), 45 deletions(-) diff --git a/src/proto_http.c b/src/proto_http.c index 9fc9495..186ea54 100644 --- a/src/proto_http.c +++ b/src/proto_http.c @@ -3184,6 +3184,8 @@ struct hterm_ctx { int req_pieces; /* limit send() to random sizes without merging */ }; +static struct chunk hterm_chunk; + static int httpterm_send_http_headers(struct stream_interface *si) { struct session *s = session_from_task(si->owner); @@ -3269,9 +3271,9 @@ static void httpterm_io_handler(struct stream_interface *si) { struct appctx *appctx = __objt_appctx(si->end); struct session *s = session_from_task(si->owner); - struct channel *req = si->ob; struct channel *res = si->ib; struct hterm_ctx *hctx = (void *)&appctx->ctx; + struct chunk tmp_chunk; if (unlikely(si->state == SI_ST_DIS || si->state == SI_ST_CLO)) goto out; @@ -3291,53 +3293,50 @@ static void httpterm_io_handler(struct stream_interface *si) } if (appctx->st0 == HT_RES_BODY) { - unsigned int prev_len = si->ib->buf->i; - unsigned int data_len; - unsigned int last_len; - unsigned int last_fwd = 0; + int max; + int offset; - //if (hctx->req_chunked) { - // /* One difficulty we're facing is that we must prevent - // * the input data from being automatically forwarded to - // * the output area. For this, we temporarily disable - // * forwarding on the channel. - // */ - // last_fwd = si->ib->to_forward; - // si->ib->to_forward = 0; - // chunk_printf(&trash, "\r\n000000\r\n"); - // if (bi_putchk(si->ib, &trash) == -1) { - // si->ib->to_forward = last_fwd; - // goto fail; - // } - //} + /* we may loop on multiple small chunks */ + do { + max = bi_avail(si->ib); - data_len = si->ib->buf->i; - //if (stats_dump_stat_to_buffer(si, s->be->uri_auth)) - appctx->st0 = HT_RES_DONE; + /* reserve 12 chars for chunk size (CRLF 8hex CRLF) */ + if (hctx->req_chunked) + max -= 12; + + if (max < 0) + goto out; + + offset = 9 - (hctx->req_size % 10ULL); + if (max > hterm_chunk.size - offset) + max = hterm_chunk.size - offset; + + if ((long long)max > hctx->req_size) + max = hctx->req_size; - last_len = si->ib->buf->i; + if (hctx->req_chunked) { + if (max > hctx->req_chunked) + max = hctx->req_chunked; + if (max) + chunk_printf(&trash, "\r\n%x\r\n", max); + else + chunk_printf(&trash, "\r\n0\r\n\r\n"); + if (bi_putchk(si->ib, &trash) == -1) + goto fail; + } - /* Now we must either adjust or remove the chunk size. This is - * not easy because the chunk size might wrap at the end of the - * buffer, so we pretend we have nothing in the buffer, we write - * the size, then restore the buffer's contents. Note that we can - * only do that because no forwarding is scheduled on the stats - * applet. + tmp_chunk.str = hterm_chunk.str + offset; + tmp_chunk.len = max; + if (bi_putchk(si->ib, &tmp_chunk) == -1) + goto fail; + hctx->req_size -= max; + } while (hctx->req_chunked && max); /* this guarantees we write the final chunk */ + + /* if we end up here, it means we managed to write the whole + * response into the buffer, we're done. */ - //if (hctx->req_chunked) { - // si->ib->total -= (last_len - prev_len); - // si->ib->buf->i -= (last_len - prev_len); - // - // if (last_len != data_len) { - // chunk_printf(&trash, "\r\n%06x\r\n", (last_len - data_len)); - // bi_putchk(si->ib, &trash); - // - // si->ib->total += (last_len - data_len); - // si->ib->buf->i += (last_len - data_len); - // } - // /* now re-enable forwarding */ - // channel_forward(si->ib, last_fwd); - //} + if (hctx->req_size <= 0) + appctx->st0 = HT_RES_DONE; } // if (appctx->st0 == STAT_HTTP_POST) { @@ -3375,14 +3374,14 @@ static void httpterm_io_handler(struct stream_interface *si) // } fail: - /* update all other flags and resync with the other side */ - si_update(si); /* we don't want to expire timeouts while we're processing requests */ si->ib->rex = TICK_ETERNITY; si->ob->wex = TICK_ETERNITY; out: + /* update all other flags and resync with the other side */ + si_update(si); if (unlikely(si->state == SI_ST_DIS || si->state == SI_ST_CLO)) { /* check that we have released everything then unregister */ stream_int_unregister_handler(si); @@ -3409,6 +3408,16 @@ int http_handle_httpterm(struct session *s, struct channel *req) struct hterm_ctx *hctx; unsigned int mult, result; + /* allocate and initialize the stuf we want to return */ + if (unlikely(!hterm_chunk.size)) { + int index; + + chunk_init(&hterm_chunk, malloc(global.tune.bufsize), global.tune.bufsize); + for (index = 0; index < global.tune.bufsize; index++) + hterm_chunk.str[index] = ".123456789"[index % 10]; + hterm_chunk.len = hterm_chunk.size; + } + appctx = si_appctx(si); appctx->st0 = appctx->st1 = appctx->st2 = 0; -- 1.7.12.1