test/jdk/java/net/httpclient/ShortResponseBody.java - master - githubmirror/jdk17u-dev

test/jdk/java/net/httpclient/ShortResponseBody.java
713 строк27 KB
Goetz Lindenmaier
8287593: ShortResponseBody could be made more resilient to rogue connections
02 фев 2023, 23:01
02 фев 2023, 23:017224cbe
/*
 * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */


import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import jdk.test.lib.net.SimpleSSLContext;
import org.testng.ITestContext;
import org.testng.ITestResult;
import org.testng.SkipException;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLHandshakeException;
import javax.net.ssl.SSLServerSocketFactory;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLSocket;
import static java.lang.System.out;
import static java.net.http.HttpClient.Builder.NO_PROXY;
import static java.net.http.HttpResponse.BodyHandlers.ofString;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static java.util.stream.Collectors.toList;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.fail;

public abstract class ShortResponseBody {

    Server closeImmediatelyServer;
    Server closeImmediatelyHttpsServer;
    Server variableLengthServer;
    Server variableLengthHttpsServer;
    Server fixedLengthServer;

    String httpURIClsImed;
    String httpsURIClsImed;
    String httpURIVarLen;
    String httpsURIVarLen;
    String httpURIFixLen;

    SSLContext sslContext;
    SSLParameters sslParameters;

    static final long PAUSE_FOR_GC = 5; // 5ms to let gc work
    static final long PAUSE_FOR_PEER = 5; // 5ms to let server react

    static final String EXPECTED_RESPONSE_BODY =
            "<html><body><h1>Heading</h1><p>Some Text</p></body></html>";

    // A request number used to replace %reqnb% in URLs with a unique
    // number for better log analysis
    static final AtomicLong reqnb = new AtomicLong();

    static final AtomicLong ids = new AtomicLong();
    final ThreadFactory factory = new ThreadFactory() {
        @Override
        public Thread newThread(Runnable r) {
            Thread thread = new Thread(r,  "HttpClient-Worker-" + ids.incrementAndGet());
            thread.setDaemon(true);
            return thread;
        }
    };
    final ExecutorService service = Executors.newCachedThreadPool(factory);

    final AtomicReference<SkipException> skiptests = new AtomicReference<>();
    void checkSkip() {
        var skip = skiptests.get();
        if (skip != null) throw skip;
    }
    static String name(ITestResult result) {
        var params = result.getParameters();
        return result.getName()
                + (params == null ? "()" : Arrays.toString(result.getParameters()));
    }

    @BeforeMethod
    void beforeMethod(ITestContext context) {
        System.gc();
        try {
            Thread.sleep(PAUSE_FOR_GC);
        } catch (InterruptedException x) {

        }
        if (context.getFailedTests().size() > 0) {
            if (skiptests.get() == null) {
                SkipException skip = new SkipException("some tests failed");
                skip.setStackTrace(new StackTraceElement[0]);
                skiptests.compareAndSet(null, skip);
            }
        }
    }

    @AfterClass
    static final void printFailedTests(ITestContext context) {
        out.println("\n=========================\n");
        try {
            var FAILURES = context.getFailedTests().getAllResults().stream()
                    .collect(Collectors.toMap(r -> name(r), ITestResult::getThrowable));

            if (FAILURES.isEmpty()) return;
            out.println("Failed tests: ");
            FAILURES.entrySet().forEach((e) -> {
                out.printf("\t%s: %s%n", e.getKey(), e.getValue());
                e.getValue().printStackTrace(out);
                e.getValue().printStackTrace();
            });
         } finally {
            out.println("\n=========================\n");
        }
    }

    @DataProvider(name = "sanity")
    public Object[][] sanity() {
        return new Object[][]{
            { httpURIVarLen  + "?length=all" },
            { httpsURIVarLen + "?length=all" },
            { httpURIFixLen  + "?length=all" },
        };
    }

    public static String uniqueURL(String url) {
        return url.replace("%reqnb%", String.valueOf(reqnb.incrementAndGet()));
    }

    @Test(dataProvider = "sanity")
    void sanity(String url) throws Exception {
        HttpClient client = newHttpClient();
        url = uniqueURL(url);
        HttpRequest request = HttpRequest.newBuilder(URI.create(url)).build();
        out.println("Request: " + request);
        HttpResponse<String> response = client.send(request, ofString());
        String body = response.body();
        assertEquals(body, EXPECTED_RESPONSE_BODY);
        client.sendAsync(request, ofString())
                .thenApply(resp -> resp.body())
                .thenAccept(b -> assertEquals(b, EXPECTED_RESPONSE_BODY))
                .join();
    }

    @DataProvider(name = "sanityBadRequest")
    public Object[][] sanityBadRequest() {
        return new Object[][]{
                { httpURIVarLen  }, // no query string
                { httpsURIVarLen },
                { httpURIFixLen  },
        };
    }

    @Test(dataProvider = "sanityBadRequest")
    void sanityBadRequest(String url) throws Exception {
        HttpClient client = newHttpClient();
        url = uniqueURL(url);
        HttpRequest request = HttpRequest.newBuilder(URI.create(url)).build();
        out.println("Request: " + request);
        HttpResponse<String> response = client.send(request, ofString());
        assertEquals(response.statusCode(), 400);
        assertEquals(response.body(), "");
    }

    @DataProvider(name = "uris")
    public Object[][] variants(ITestContext context) {
        String[][] cases = new String[][] {
            // The length query string is the total number of bytes in the reply,
            // including headers, before the server closes the connection. The
            // second arg is a partial-expected-detail message in the exception.
            { httpURIVarLen + "?length=0",   "no bytes"     }, // EOF without receiving anything
            { httpURIVarLen + "?length=1",   "status line"  }, // EOF during status-line
            { httpURIVarLen + "?length=2",   "status line"  },
            { httpURIVarLen + "?length=10",  "status line"  },
            { httpURIVarLen + "?length=19",  "header"       }, // EOF during Content-Type header
            { httpURIVarLen + "?length=30",  "header"       },
            { httpURIVarLen + "?length=45",  "header"       },
            { httpURIVarLen + "?length=48",  "header"       },
            { httpURIVarLen + "?length=51",  "header"       },
            { httpURIVarLen + "?length=98",  "header"       }, // EOF during Connection header
            { httpURIVarLen + "?length=100", "header"       },
            { httpURIVarLen + "?length=101", "header"       },
            { httpURIVarLen + "?length=104", "header"       },
            { httpURIVarLen + "?length=106", "chunked transfer encoding" }, // EOF during chunk header ( length )
            { httpURIVarLen + "?length=110", "chunked transfer encoding" }, // EOF during chunk response body data

            { httpsURIVarLen + "?length=0",   "no bytes"    },
            { httpsURIVarLen + "?length=1",   "status line" },
            { httpsURIVarLen + "?length=2",   "status line" },
            { httpsURIVarLen + "?length=10",  "status line" },
            { httpsURIVarLen + "?length=19",  "header"      },
            { httpsURIVarLen + "?length=30",  "header"      },
            { httpsURIVarLen + "?length=45",  "header"      },
            { httpsURIVarLen + "?length=48",  "header"      },
            { httpsURIVarLen + "?length=51",  "header"      },
            { httpsURIVarLen + "?length=98",  "header"      },
            { httpsURIVarLen + "?length=100", "header"      },
            { httpsURIVarLen + "?length=101", "header"      },
            { httpsURIVarLen + "?length=104", "header"      },
            { httpsURIVarLen + "?length=106", "chunked transfer encoding" },
            { httpsURIVarLen + "?length=110", "chunked transfer encoding" },

            { httpURIFixLen + "?length=0",   "no bytes"    }, // EOF without receiving anything
            { httpURIFixLen + "?length=1",   "status line" }, // EOF during status-line
            { httpURIFixLen + "?length=2",   "status line" },
            { httpURIFixLen + "?length=10",  "status line" },
            { httpURIFixLen + "?length=19",  "header"      }, // EOF during Content-Type header
            { httpURIFixLen + "?length=30",  "header"      },
            { httpURIFixLen + "?length=45",  "header"      },
            { httpURIFixLen + "?length=48",  "header"      },
            { httpURIFixLen + "?length=51",  "header"      },
            { httpURIFixLen + "?length=78",  "header"      }, // EOF during Connection header
            { httpURIFixLen + "?length=79",  "header"      },
            { httpURIFixLen + "?length=86",  "header"      },
            { httpURIFixLen + "?length=104", "fixed content-length" }, // EOF during body
            { httpURIFixLen + "?length=106", "fixed content-length" },
            { httpURIFixLen + "?length=110", "fixed content-length" },

            // ## ADD https fixed

            { httpURIClsImed,  "no bytes"},
            { httpsURIClsImed, "no bytes"},
        };

        if (context.getFailedTests().size() > 0) {
            // Shorten the log output by preventing useless
            // skip traces to be printed for subsequent methods
            // if one of the previous @Test method has failed.
            return new Object[0][];
        }

        List<Object[]> list = new ArrayList<>();
        Arrays.asList(cases).stream()
                .map(e -> new Object[] {e[0], e[1], true})  // reuse client
                .forEach(list::add);
        Arrays.asList(cases).stream()
                .map(e -> new Object[] {e[0], e[1], false}) // do not reuse client
                .forEach(list::add);
        return list.stream().toArray(Object[][]::new);
    }

    static final int ITERATION_COUNT = 3;

    HttpClient newHttpClient() {
        return HttpClient.newBuilder()
                .proxy(NO_PROXY)
                .sslContext(sslContext)
                .sslParameters(sslParameters)
                .executor(service)
                .build();
    }

    HttpClient sharedClient = null;
    HttpClient newHttpClient(boolean shared) {
        if (shared) {
            HttpClient sharedClient = this.sharedClient;
            if (sharedClient == null) {
                sharedClient = this.sharedClient = newHttpClient();
            }
            return sharedClient;
        }
        return newHttpClient();
    }

    // can be used to prolong request body publication
    static final class InfiniteInputStream extends InputStream {
        int count = 0;
        int k16 = 0;
        @Override
        public int read() throws IOException {
            if (++count == 1) {
                System.out.println("Start sending 1 byte");
            }
            if (count > 16 * 1024) {
                k16++;
                System.out.println("... 16K sent.");
                count = count % (16 * 1024);
                try {
                    Thread.sleep(PAUSE_FOR_PEER);
                } catch (InterruptedException x) {
                    // ignore
                }
            }
            if (k16 > 128) {
                System.out.println("WARNING: InfiniteInputStream: " +
                        "more than 128 16k buffers generated: returning EOF");
                return -1;
            }
            return 1;
        }

        @Override
        public int read(byte[] buf, int offset, int length) {
            //int count = offset;
            length = Math.max(0, Math.min(buf.length - offset, length));
            //for (; count < length; count++)
            //    buf[offset++] = 0x01;
            //return count;
            if (count == 0) {
                System.out.println("Start sending " + length);
            } else if (count > 16 * 1024) {
                k16++;
                System.out.println("... 16K sent.");
                count = count % (16 * 1024);
                try {
                    Thread.sleep(PAUSE_FOR_PEER);
                } catch (InterruptedException x) {
                    // ignore
                }
            }
            if (k16 > 128) {
                System.out.println("WARNING: InfiniteInputStream: " +
                        "more than 128 16k buffers generated: returning EOF");
                return -1;
            }
            count += length;
            return length;
        }
    }

    void assertExpectedMessage(HttpRequest request, Throwable t, String expected) {
        if (request.uri().getScheme().equalsIgnoreCase("https")
                && (t instanceof SSLHandshakeException)) {
            // OK
            out.println("Skipping expected " + t);
        } else {
            String msg = t.getMessage();
            assertTrue(msg.contains(expected),
                    "exception msg:[" + msg + "]");
        }
    }

    void assertExpectedMessage(HttpRequest request, Throwable t, List<String> expected) {
        if (request.uri().getScheme().equalsIgnoreCase("https")
                && (t instanceof SSLHandshakeException)) {
            // OK
            out.println("Skipping expected " + t);
        } else {
            String msg = t.getMessage();
            assertTrue(expected.stream().anyMatch(msg::contains),
                    "exception msg:[" + msg + "] not in " + Arrays.asList(expected));
        }
    }

    // Asserts that the "send" method appears in the stack of the given
    // exception. The synchronous API must contain the send method on the stack.
    static void assertSendMethodOnStack(IOException ioe) {
        final String cn = "jdk.internal.net.http.HttpClientImpl";
        List<StackTraceElement> list = Stream.of(ioe.getStackTrace())
                .filter(ste -> ste.getClassName().equals(cn)
                        && ste.getMethodName().equals("send"))
                .collect(toList());
        if (list.size() != 1) {
            ioe.printStackTrace(out);
            fail(cn + ".send method not found in stack.");
        }
    }

    // Asserts that the implementation-specific ConnectionExpiredException does
    // NOT appear anywhere in the exception or its causal chain.
    static void assertNoConnectionExpiredException(IOException ioe) {
        Throwable throwable = ioe;
        do {
            String cn = throwable.getClass().getSimpleName();
            if (cn.equals("ConnectionExpiredException")) {
                ioe.printStackTrace(out);
                fail("UNEXPECTED ConnectionExpiredException in:[" + ioe + "]");
            }
        } while ((throwable = throwable.getCause()) != null);
    }

    // -- infra

    /**
     * A server that, listens on a port, accepts new connections, and can be
     * closed.
     */
    static abstract class Server extends Thread implements AutoCloseable {
        protected final ServerSocket ss;
        protected volatile boolean closed;

        Server(String name) throws IOException {
            super(name);
            ss = newServerSocket();
            ss.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0));
            this.start();
        }

        protected ServerSocket newServerSocket() throws IOException {
            return new ServerSocket();
        }

        public int getPort() { return ss.getLocalPort(); }

        @Override
        public void close() {
            if (closed)
                return;
            closed = true;
            try {
                ss.close();
            } catch (IOException e) {
                out.println("Unexpected exception while closing server: " + e);
                e.printStackTrace(out);
                throw new UncheckedIOException("Unexpected: ", e);
            }
        }
    }

    /**
     * A server that closes the connection immediately, without reading or writing.
     */
    static class PlainCloseImmediatelyServer extends Server {
        PlainCloseImmediatelyServer() throws IOException {
            super("PlainCloseImmediatelyServer");
        }

        protected PlainCloseImmediatelyServer(String name) throws IOException {
            super(name);
        }

        @Override
        public void run() {
            while (!closed) {
                try (Socket s = ss.accept()) {
                    if (s instanceof SSLSocket) {
                        ((SSLSocket)s).startHandshake();
                    }
                    out.println("Server: got connection, closing immediately ");
                } catch (Throwable e) {
                    if (!closed) {
                        out.println("Unexpected exception in server: " + e);
                        e.printStackTrace(out);
                        throw new RuntimeException("Unexpected: ", e);
                    }
                }
            }
        }
    }

    /**
     * A server that closes the connection immediately, without reading or writing,
     * after completing the SSL handshake.
     */
    static final class SSLCloseImmediatelyServer extends PlainCloseImmediatelyServer {
        SSLCloseImmediatelyServer() throws IOException {
            super("SSLCloseImmediatelyServer");
        }
        @Override
        public ServerSocket newServerSocket() throws IOException {
            return SSLServerSocketFactory.getDefault().createServerSocket();
        }
    }

    /**
     * A server that replies with headers and a, possibly partial, reply, before
     * closing the connection. The number of bytes of written ( header + body),
     * is controllable through the "length" query string param in the requested
     * URI.
     */
    static abstract class ReplyingServer extends Server {

        private final String name;

        ReplyingServer(String name) throws IOException {
            super(name);
            this.name = name;
        }

        private static final String BAD_REQUEST_RESPONSE =
                "HTTP/1.1 400 Bad Request\r\n" +
                        "Content-Length: 0\r\n" +
                        "Connection: close\r\n\r\n";

        abstract String response();

        @Override
        public void run() {
            while (!closed) {
                try (Socket s = ss.accept()) {
                    out.print(name + ": got connection ");
                    InputStream is = s.getInputStream();
                    URI requestMethod = readRequestMethod(is);
                    out.print(requestMethod + " ");
                    URI uriPath = readRequestPath(is);
                    out.println(uriPath);
                    String headers = readRequestHeaders(is);

                    String query = uriPath.getRawQuery();
                    if (query == null) {
                        out.println("Unexpected request without query string received. Got headers: [" + headers + "]");
                        out.println("Replying with 400 Bad Request");
                        writeResponse(s, BAD_REQUEST_RESPONSE, BAD_REQUEST_RESPONSE.length());
                        continue;
                    }
                    String qv = query.split("=")[1];
                    int len;
                    if (qv.equals("all")) {
                        len = response().getBytes(US_ASCII).length;
                    } else {
                        len = Integer.parseInt(query.split("=")[1]);
                    }

                    out.println(name + ": writing " + len  + " bytes");
                    writeResponse(s, response(), len);
                } catch (Throwable e) {
                    if (!closed) {
                        out.println("Unexpected exception in server: " + e);
                        e.printStackTrace(out);
                        throw new RuntimeException("Unexpected: " + e, e);
                    }
                }
            }
        }

        private static void writeResponse(Socket socket, String response, int len) throws IOException {
            OutputStream os = socket.getOutputStream();
            byte[] responseBytes = response.getBytes(US_ASCII);
            for (int i = 0; i < len; ++i) {
                os.write(responseBytes[i]);
                os.flush();
            }
        }

        static final byte[] requestEnd = new byte[] { '\r', '\n', '\r', '\n' };

        // Read the request method
        static URI readRequestMethod(InputStream is) throws IOException {
            StringBuilder sb = new StringBuilder();
            int r;
            while ((r = is.read()) != -1 && r != 0x20) {
                sb.append((char)r);
            }
            return URI.create(sb.toString());
        }

        // Read the request URI path
        static URI readRequestPath(InputStream is) throws IOException {
            StringBuilder sb = new StringBuilder();
            int r;
            while ((r = is.read()) != -1 && r != 0x20) {
                sb.append((char)r);
            }
            return URI.create(sb.toString());
        }

        // Read until the end of a HTTP request headers
        static String readRequestHeaders(InputStream is) throws IOException {
            int requestEndCount = 0, r;
            StringBuilder sb = new StringBuilder();
            while ((r = is.read()) != -1) {
                sb.append((char) r);
                if (r == requestEnd[requestEndCount]) {
                    requestEndCount++;
                    if (requestEndCount == 4) {
                        break;
                    }
                } else {
                    requestEndCount = 0;
                }
            }
            return sb.toString();
        }
    }

    /** A server that issues a, possibly-partial, chunked reply. */
    static class PlainVariableLengthServer extends ReplyingServer {

        static final String CHUNKED_RESPONSE_BODY =
                "6\r\n"+ "<html>\r\n" +
                "6\r\n"+ "<body>\r\n" +
                "10\r\n"+ "<h1>Heading</h1>\r\n" +
                "10\r\n"+ "<p>Some Text</p>\r\n" +
                "7\r\n"+ "</body>\r\n" +
                "7\r\n"+ "</html>\r\n" +
                "0\r\n"+ "\r\n";

        static final String RESPONSE_HEADERS =
                "HTTP/1.1 200 OK\r\n" +
                "Content-Type: text/html; charset=utf-8\r\n" +
                "Transfer-Encoding: chunked\r\n" +
                "Connection: close\r\n\r\n";

        static final String RESPONSE = RESPONSE_HEADERS + CHUNKED_RESPONSE_BODY;

        PlainVariableLengthServer() throws IOException {
            super("PlainVariableLengthServer");
        }

        protected PlainVariableLengthServer(String name) throws IOException {
            super(name);
        }

        @Override
        String response( ) { return RESPONSE; }
    }

    /** A server that issues a, possibly-partial, chunked reply over SSL. */
    static final class SSLVariableLengthServer extends PlainVariableLengthServer {
        SSLVariableLengthServer() throws IOException {
            super("SSLVariableLengthServer");
        }
        @Override
        public ServerSocket newServerSocket() throws IOException {
            return SSLServerSocketFactory.getDefault().createServerSocket();
        }
    }

    /** A server that issues a fixed-length reply. */
    static final class FixedLengthServer extends ReplyingServer {

        static final String RESPONSE_BODY = EXPECTED_RESPONSE_BODY;

        static final String RESPONSE_HEADERS =
                "HTTP/1.1 200 OK\r\n" +
                "Content-Type: text/html; charset=utf-8\r\n" +
                "Content-Length: " + RESPONSE_BODY.length() + "\r\n" +
                "Connection: close\r\n\r\n";

        static final String RESPONSE = RESPONSE_HEADERS + RESPONSE_BODY;

        FixedLengthServer() throws IOException {
            super("FixedLengthServer");
        }

        @Override
        String response( ) { return RESPONSE; }
    }

    static String serverAuthority(Server server) {
        return InetAddress.getLoopbackAddress().getHostName() + ":"
                + server.getPort();
    }

    @BeforeTest
    public void setup() throws Exception {
        sslContext = new SimpleSSLContext().get();
        if (sslContext == null)
            throw new AssertionError("Unexpected null sslContext");
        SSLContext.setDefault(sslContext);

        sslParameters = new SSLParameters();

        closeImmediatelyServer = new PlainCloseImmediatelyServer();
        httpURIClsImed = "http://" + serverAuthority(closeImmediatelyServer)
                + "/http1/closeImmediately/req=%reqnb%/foo";

        closeImmediatelyHttpsServer = new SSLCloseImmediatelyServer();
        httpsURIClsImed = "https://" + serverAuthority(closeImmediatelyHttpsServer)
                + "/https1/closeImmediately/req=%reqnb%/foo";

        variableLengthServer = new PlainVariableLengthServer();
        httpURIVarLen = "http://" + serverAuthority(variableLengthServer)
                + "/http1/variable/req=%reqnb%/bar";

        variableLengthHttpsServer = new SSLVariableLengthServer();
        httpsURIVarLen = "https://" + serverAuthority(variableLengthHttpsServer)
                + "/https1/variable/req=%reqnb%/bar";

        fixedLengthServer = new FixedLengthServer();
        httpURIFixLen = "http://" + serverAuthority(fixedLengthServer)
                + "/http1/fixed/req=%reqnb%/baz";
    }

    @AfterTest
    public void teardown() throws Exception {
        if (sharedClient != null) sharedClient = null;
        closeImmediatelyServer.close();
        closeImmediatelyHttpsServer.close();
        variableLengthServer.close();
        variableLengthHttpsServer.close();
        fixedLengthServer.close();
    }
}