Allow headers size extend to maxRequestHeadersSize in http client

jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/HttpClientTransportOverHTTP.java

@@ -39,6 +39,8 @@ public class HttpClientTransportOverHTTP extends AbstractConnectorHttpClientTran
     private int headerCacheSize = 1024;
     private boolean headerCacheCaseSensitive;
 
+    private int maxRequestHeadersSize = 32 * 1024;
+
     public HttpClientTransportOverHTTP()
     {
         this(1);
@@ -127,4 +129,18 @@ public void setInitializeConnections(boolean initialize)
     {
         factory.setInitializeConnections(initialize);
     }
+
+    /**
+     * @return The maximum allowed size in bytes for the HTTP request headers
+     */
+    @ManagedAttribute("The maximum allowed size in bytes for the HTTP request headers")
+    public int getMaxRequestHeadersSize()
+    {
+        return maxRequestHeadersSize;
+    }
+
+    public void setMaxRequestHeadersSize(int maxRequestHeadersSize)
+    {
+        this.maxRequestHeadersSize = maxRequestHeadersSize;
+    }
 }

jetty-core/jetty-client/src/main/java/org/eclipse/jetty/client/transport/internal/HttpSenderOverHTTP.java

@@ -16,7 +16,9 @@
 import java.nio.ByteBuffer;
 
 import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.HttpClientTransport;
 import org.eclipse.jetty.client.HttpRequestException;
+import org.eclipse.jetty.client.transport.HttpClientTransportOverHTTP;
 import org.eclipse.jetty.client.transport.HttpExchange;
 import org.eclipse.jetty.client.transport.HttpRequest;
 import org.eclipse.jetty.client.transport.HttpSender;
@@ -179,9 +181,29 @@ protected Action process() throws Exception
                     }
                     case HEADER_OVERFLOW:
                     {
-                        headerBuffer.release();
-                        headerBuffer = null;
-                        throw new IllegalArgumentException("Request header too large");
+                        int maxRequestHeadersSize = -1;
+                        //For HTTP1.1 only
+                        HttpClientTransport transport = httpClient.getTransport();
+                        if (transport instanceof HttpClientTransportOverHTTP httpTransport)
+                        {
+                            maxRequestHeadersSize = httpTransport.getMaxRequestHeadersSize();
+                        }
+                        if (headerBuffer.capacity() < maxRequestHeadersSize)
+                        {
+                            RetainableByteBuffer newHeaderBuffer = bufferPool.acquire(maxRequestHeadersSize, useDirectByteBuffers);
+                            headerBuffer.getByteBuffer().flip();
+                            newHeaderBuffer.getByteBuffer().put(headerBuffer.getByteBuffer());
+                            RetainableByteBuffer toRelease = headerBuffer;
+                            headerBuffer  = newHeaderBuffer;
+                            toRelease.release();
+                            break;
+                        }
+                        else
+                        {
+                            headerBuffer.release();
+                            headerBuffer = null;
+                            throw new IllegalArgumentException("Request header too large");
+                        }
                     }
                     case NEED_CHUNK:
                     {

jetty-core/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientTest.java

@@ -29,6 +29,7 @@
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -56,6 +57,7 @@
 import org.eclipse.jetty.http.HttpScheme;
 import org.eclipse.jetty.http.HttpStatus;
 import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.io.ByteArrayEndPoint;
 import org.eclipse.jetty.io.ClientConnector;
 import org.eclipse.jetty.io.Content;
 import org.eclipse.jetty.io.EndPoint;
@@ -76,6 +78,7 @@
 import org.eclipse.jetty.util.component.LifeCycle;
 import org.hamcrest.Matchers;
 import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ArgumentsSource;
@@ -2012,4 +2015,208 @@ public void perform()
                 .send(this);
         }
     }
+
+    private static Random rnd = new Random();
+    private static final String CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890";
+
+    public static final int CHARS_LENGTH = CHARS.length();
+
+    protected static String getRandomString(int size)
+    {
+        StringBuilder sb = new StringBuilder(size);
+        while (sb.length() < size)
+        { // length of the random string.
+            int index = rnd.nextInt(CHARS_LENGTH);
+            sb.append(CHARS.charAt(index));
+        }
+        return sb.toString();
+    }
+
+    @ParameterizedTest
+    @ArgumentsSource(ScenarioProvider.class)
+    public void testSmallHeadersSize(Scenario scenario) throws Exception
+    {
+        startClient(scenario);
+        ByteArrayEndPoint endPoint = new ByteArrayEndPoint();
+        HttpDestination destination = new HttpDestination(client, new Origin("http", "localhost", 8080));
+        destination.start();
+        HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, destination, new Promise.Adapter<Connection>());
+        Request request = client.newRequest(URI.create("http://localhost/"));
+        request.agent(getRandomString(888)); //More than the request buffer size, but less than the default max request headers size
+        final CountDownLatch headersLatch = new CountDownLatch(1);
+        final CountDownLatch successLatch = new CountDownLatch(1);
+        final CountDownLatch failureLatch = new CountDownLatch(1);
+        request.listener(new Request.Listener()
+        {
+            @Override
+            public void onHeaders(Request request)
+            {
+                headersLatch.countDown();
+            }
+
+            @Override
+            public void onSuccess(Request request)
+            {
+                successLatch.countDown();
+            }
+
+            @Override
+            public void onFailure(Request request, Throwable failure)
+            {
+                failureLatch.countDown();
+            }
+        });
+        connection.send(request, null);
+
+        String requestString = endPoint.takeOutputString();
+        assertTrue(requestString.startsWith("GET / HTTP/1.1\r\nAccept-Encoding: gzip\r\n"));
+        assertTrue(headersLatch.await(5, TimeUnit.SECONDS));
+        assertTrue(successLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @ArgumentsSource(ScenarioProvider.class)
+    public void testMaxRequestHeadersSize(Scenario scenario) throws Exception
+    {
+        startClient(scenario);
+        byte[] buffer = new byte[32 * 1024];
+        ByteArrayEndPoint endPoint = new ByteArrayEndPoint(buffer, buffer.length);
+        HttpDestination destination = new HttpDestination(client, new Origin("http", "localhost", 8080));
+        destination.start();
+        HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, destination, new Promise.Adapter<Connection>());
+        Request request = client.newRequest(URI.create("http://localhost/"));
+        //More than the request buffer size, but less than the default max request headers size
+
+        int desiredHeadersSize = 20 * 1024;
+        int currentHeadersSize = 0;
+        int i = 0;
+        while (currentHeadersSize < desiredHeadersSize)
+        {
+            final int index = i++;
+            final String headerValue = getRandomString(800);
+            final int headerSize = headerValue.length();
+            currentHeadersSize += headerSize;
+            request.cookie(new HttpCookie()
+            {
+                @Override
+                public String getName()
+                {
+                    return "large" + index;
+                }
+
+                @Override
+                public String getValue()
+                {
+                    return headerValue;
+                }
+
+                @Override
+                public int getVersion()
+                {
+                    return 0;
+                }
+
+                @Override
+                public Map<String, String> getAttributes()
+                {
+                    return new HashMap<>();
+                }
+            });
+        }
+
+        final CountDownLatch headersLatch = new CountDownLatch(1);
+        final CountDownLatch successLatch = new CountDownLatch(1);
+        request.listener(new Request.Listener()
+        {
+            @Override
+            public void onHeaders(Request request)
+            {
+                headersLatch.countDown();
+            }
+
+            @Override
+            public void onSuccess(Request request)
+            {
+                successLatch.countDown();
+            }
+        });
+        connection.send(request, null);
+
+        String requestString = endPoint.takeOutputString();
+        assertTrue(requestString.startsWith("GET / HTTP/1.1\r\nAccept-Encoding: gzip\r\n"));
+        assertTrue(headersLatch.await(5, TimeUnit.SECONDS));
+        assertTrue(successLatch.await(5, TimeUnit.SECONDS));
+    }
+
+    @ParameterizedTest
+    @ArgumentsSource(ScenarioProvider.class)
+    public void testMaxRequestHeadersSizeOverflow(Scenario scenario) throws Exception
+    {
+        startClient(scenario);
+        byte[] buffer = new byte[32 * 1024];
+        ByteArrayEndPoint endPoint = new ByteArrayEndPoint(buffer, buffer.length);
+        HttpDestination destination = new HttpDestination(client, new Origin("http", "localhost", 8080));
+        destination.start();
+        HttpConnectionOverHTTP connection = new HttpConnectionOverHTTP(endPoint, destination, new Promise.Adapter<Connection>());
+        Request request = client.newRequest(URI.create("http://localhost/"));
+        //More than the request buffer size, but less than the default max request headers size
+
+        int desiredHeadersSize = 35 * 1024;
+        int currentHeadersSize = 0;
+        int i = 0;
+        while (currentHeadersSize < desiredHeadersSize)
+        {
+            final int index = i++;
+            final String headerValue = getRandomString(800);
+            final int headerSize = headerValue.length();
+            currentHeadersSize += headerSize;
+            request.cookie(new HttpCookie()
+            {
+                @Override
+                public String getName()
+                {
+                    return "large" + index;
+                }
+
+                @Override
+                public String getValue()
+                {
+                    return headerValue;
+                }
+
+                @Override
+                public int getVersion()
+                {
+                    return 0;
+                }
+
+                @Override
+                public Map<String, String> getAttributes()
+                {
+                    return new HashMap<>();
+                }
+            });
+        }
+
+        final CountDownLatch headersLatch = new CountDownLatch(1);
+        final CountDownLatch failureLatch = new CountDownLatch(1);
+        request.listener(new Request.Listener()
+        {
+            @Override
+            public void onHeaders(Request request)
+            {
+                headersLatch.countDown();
+            }
+
+            @Override
+            public void onFailure(Request request, Throwable failure)
+            {
+                failureLatch.countDown();
+            }
+        });
+        connection.send(request, null);
+
+        assertTrue(headersLatch.await(5, TimeUnit.SECONDS));
+        assertTrue(failureLatch.await(5, TimeUnit.SECONDS));
+    }
 }