Apache HTTPClient 4. x vs多个JDK URL连接

Question

我正在试用ApacheHttpClient 4.3.6连接池管理器，以提高HTTP调用的吞吐量。我的假设是，HTTPClient实现通常使用持久性连接。然而，我的测试代码（包括在最后）的结果显示，使用JDKURLConnection的多个并发HTTP连接的性能更好。

1. 如何使HTTPClient快速

谢啦

import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;

public class FooTest {

public static void main(String[] args) throws Exception {
    runWithConnectionPool();
}

 private static String extract(BufferedReader reader) throws Exception {
     StringBuilder buffer = new StringBuilder();
     String line = null;
     while ((line = reader.readLine()) != null) {
         buffer.append(line);
     }
     return buffer.toString();
 }

 private static void runWithConnectionPool() throws Exception {
     PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
     cm.setMaxTotal(1);

     CloseableHttpClient httpClient = HttpClients.custom()
         .setConnectionManager(cm)
         .setMaxConnTotal(100)
         .setMaxConnPerRoute(100)
         .build();

     long start = System.currentTimeMillis();

     HttpGet getReq = new HttpGet("http://www.google.com");

     /*
       Option A: Using HTTP connection pool
       Option B: Individual JDK 8 URL connection
     */
 //    Thread[] workers = generateAndStart(10, httpClient, getReq, 0);                 // (A)
     Thread[] workers = generateAndStart(10, getReq.getURI().toURL(), 0);  // (B)

     for (int i = 0; i < workers.length; i++) {
         workers[i].join();
     }

     System.out.println("Elasped: " + (System.currentTimeMillis() - start));
 }

 private static Thread[] generateAndStart(int num, URL url, long delay) {
     Thread[] workers = new Thread[num];
     for (int i = 0; i < num; i++) {
         System.out.println("Starting worker: " + i);
         int j = i;
         workers[i] = new Thread(() -> connect(url, delay, j));
         workers[i].start();
     }
     return workers;
 }

 private static void connect(URL url, long delay, int ndx)  {
     try {
         System.out.println(url.toURI().toString() + " started.");
     } catch (Exception e) {
         e.printStackTrace();
     }

     try {

         URLConnection connection = url.openConnection();
         connection.addRequestProperty("Accept", "application/json");
         BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));

         String line;
         while ((line = reader.readLine()) != null) {
             ObjectMapper mapper = new ObjectMapper();
             System.out.println(line);
         }
         if (delay > 0) {
             System.out.println("Delayed.");
             sleep(delay);
         }
         reader.close();
     } catch (Exception e) {
         e.printStackTrace();
     }
 }

 private static Thread[] generateAndStart(int num, CloseableHttpClient httpClient, HttpGet getReq, long delay) {
     Thread[] workers = new Thread[num];
     for (int i = 0; i < num; i++) {
         System.out.println("Starting worker: " + i);
         final int j = i;
         workers[i] = new Thread(() -> connect(httpClient, getReq, delay, j));
         workers[i].start();
     }
     return workers;
 }

 private static void connect(CloseableHttpClient httpClient, HttpGet request, long delay, int ndx) {
     System.out.println(request.getURI().toString() + " started.");

     try(
         CloseableHttpResponse response = httpClient.execute(request, HttpClientContext.create());
         BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent()))) {

         String line;
         while ((line = reader.readLine()) != null) {
             ObjectMapper mapper = new ObjectMapper();
             System.out.println(line);
         }
         if (delay > 0) {
             System.out.println("Delayed.");
             sleep(delay);
         }
     } catch (Exception e) {
         e.printStackTrace();
     }
 }

 private static void sleep(long delay) {
     try {
         Thread.sleep(delay);
     } catch (Exception e) {
         e.printStackTrace();
     }
 }
 }

我做了一些观察和结论，

1. JDKjava。网URLConnection在URLConnection之前不会建立连接。调用getInputStream（）

我的观察来自我贴在这里的例子。它们是比上面粘贴的代码更好的示例。

Answer 1

在使用JDKURLConnection和Apache HTTPClient进行实验后，我找到了答案。

1. URLConnection速度很快，因为它为每个线程与服务器的每个连接打开新的套接字，而Apache HTTPClient根据其在多线程环境中的设置控制打开的套接字数量。当套接字被限制为很少时，两个HTTP库的总连接时间大致相同
2. Apache HTTPClient对不同的URL使用到同一服务器的持久连接

mitmproxy是一个很好且易于使用的HTTP连接验证工具。