1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.esigate.http;
17
18 import java.io.IOException;
19 import java.util.Arrays;
20 import java.util.Collections;
21 import java.util.HashSet;
22 import java.util.Properties;
23 import java.util.Set;
24
25 import org.apache.http.HttpHeaders;
26 import org.apache.http.HttpHost;
27 import org.apache.http.HttpResponse;
28 import org.apache.http.HttpStatus;
29 import org.apache.http.auth.AuthScope;
30 import org.apache.http.auth.Credentials;
31 import org.apache.http.auth.UsernamePasswordCredentials;
32 import org.apache.http.client.CookieStore;
33 import org.apache.http.client.CredentialsProvider;
34 import org.apache.http.client.HttpClient;
35 import org.apache.http.client.config.RequestConfig;
36 import org.apache.http.client.methods.CloseableHttpResponse;
37 import org.apache.http.config.Registry;
38 import org.apache.http.config.RegistryBuilder;
39 import org.apache.http.conn.HttpClientConnectionManager;
40 import org.apache.http.cookie.CookieSpecProvider;
41 import org.apache.http.impl.client.BasicCredentialsProvider;
42 import org.esigate.ConfigurationException;
43 import org.esigate.Driver;
44 import org.esigate.HttpErrorPage;
45 import org.esigate.Parameters;
46 import org.esigate.RequestExecutor;
47 import org.esigate.cache.CacheConfigHelper;
48 import org.esigate.cookie.CookieManager;
49 import org.esigate.events.EventManager;
50 import org.esigate.events.impl.FragmentEvent;
51 import org.esigate.events.impl.HttpClientBuilderEvent;
52 import org.esigate.extension.ExtensionFactory;
53 import org.esigate.http.cookie.CustomBrowserCompatSpecFactory;
54 import org.esigate.impl.DriverRequest;
55 import org.esigate.util.HttpRequestHelper;
56 import org.esigate.util.UriUtils;
57 import org.slf4j.Logger;
58 import org.slf4j.LoggerFactory;
59
60
61
62
63
64
65
66
67 public final class HttpClientRequestExecutor implements RequestExecutor {
68 private static final Logger LOG = LoggerFactory.getLogger(HttpClientRequestExecutor.class);
69 private static final Set<String> SIMPLE_METHODS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("GET",
70 "HEAD", "OPTIONS", "TRACE", "DELETE")));
71 private static final Set<String> ENTITY_METHODS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("POST",
72 "PUT", "PROPFIND", "PROPPATCH", "MKCOL", "COPY", "MOVE", "LOCK", "UNLOCK")));
73 private boolean preserveHost;
74 private CookieManager cookieManager;
75 private HttpClient httpClient;
76 private EventManager eventManager = null;
77 private int connectTimeout;
78 private int socketTimeout;
79 private HttpHost firstBaseUrlHost;
80
81
82
83
84
85
86 public static final class HttpClientRequestExecutorBuilder implements RequestExecutorBuilder {
87 private EventManager eventManager;
88 private Properties properties;
89 private Driver driver;
90 private HttpClientConnectionManager connectionManager;
91 private CookieManager cookieManager;
92
93 @Override
94 public HttpClientRequestExecutorBuilder setDriver(Driver pDriver) {
95 this.driver = pDriver;
96 return this;
97 }
98
99 @Override
100 public HttpClientRequestExecutorBuilder setProperties(Properties pProperties) {
101 this.properties = pProperties;
102 return this;
103 }
104
105 @Override
106 public HttpClientRequestExecutor build() {
107 if (eventManager == null) {
108 throw new ConfigurationException("eventManager is mandatory");
109 }
110 if (driver == null) {
111 throw new ConfigurationException("driver is mandatory");
112 }
113 if (properties == null) {
114 throw new ConfigurationException("properties is mandatory");
115 }
116 HttpClientRequestExecutor result = new HttpClientRequestExecutor();
117 result.eventManager = eventManager;
118 result.preserveHost = Parameters.PRESERVE_HOST.getValue(properties);
119 if (cookieManager == null) {
120 cookieManager = ExtensionFactory.getExtension(properties, Parameters.COOKIE_MANAGER, driver);
121 }
122 result.cookieManager = cookieManager;
123 result.connectTimeout = Parameters.CONNECT_TIMEOUT.getValue(properties);
124 result.socketTimeout = Parameters.SOCKET_TIMEOUT.getValue(properties);
125 result.httpClient = buildHttpClient();
126 String firstBaseURL = Parameters.REMOTE_URL_BASE.getValue(properties)[0];
127 result.firstBaseUrlHost = UriUtils.extractHost(firstBaseURL);
128 return result;
129 }
130
131 @Override
132 public HttpClientRequestExecutorBuilder setContentTypeHelper(ContentTypeHelper contentTypeHelper) {
133 return this;
134 }
135
136 public HttpClientRequestExecutorBuilder setConnectionManager(HttpClientConnectionManager pConnectionManager) {
137 this.connectionManager = pConnectionManager;
138 return this;
139 }
140
141 @Override
142 public HttpClientRequestExecutorBuilder setEventManager(EventManager pEventManager) {
143 this.eventManager = pEventManager;
144 return this;
145 }
146
147 public HttpClientRequestExecutorBuilder setCookieManager(CookieManager pCookieManager) {
148 this.cookieManager = pCookieManager;
149 return this;
150 }
151
152 private HttpClient buildHttpClient() {
153 HttpHost proxyHost = null;
154 Credentials proxyCredentials = null;
155
156 String proxyHostParameter = Parameters.PROXY_HOST.getValue(properties);
157 if (proxyHostParameter != null) {
158 int proxyPort = Parameters.PROXY_PORT.getValue(properties);
159 proxyHost = new HttpHost(proxyHostParameter, proxyPort);
160 String proxyUser = Parameters.PROXY_USER.getValue(properties);
161 if (proxyUser != null) {
162 String proxyPassword = Parameters.PROXY_PASSWORD.getValue(properties);
163 proxyCredentials = new UsernamePasswordCredentials(proxyUser, proxyPassword);
164 }
165 }
166
167 ProxyingHttpClientBuilder httpClientBuilder = new ProxyingHttpClientBuilder();
168 httpClientBuilder.disableContentCompression();
169 httpClientBuilder.setProperties(properties);
170
171 httpClientBuilder.setMaxConnPerRoute(Parameters.MAX_CONNECTIONS_PER_HOST.getValue(properties));
172 httpClientBuilder.setMaxConnTotal(Parameters.MAX_CONNECTIONS_PER_HOST.getValue(properties));
173
174
175 if (proxyHost != null) {
176 httpClientBuilder.setProxy(proxyHost);
177 if (proxyCredentials != null) {
178 CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
179 credentialsProvider.setCredentials(new AuthScope(proxyHost), proxyCredentials);
180 httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
181 }
182 }
183
184
185 boolean useCache = Parameters.USE_CACHE.getValue(properties);
186 httpClientBuilder.setUseCache(Parameters.USE_CACHE.getValue(properties));
187 if (useCache) {
188 httpClientBuilder.setHttpCacheStorage(CacheConfigHelper.createCacheStorage(properties));
189 httpClientBuilder.setCacheConfig(CacheConfigHelper.createCacheConfig(properties));
190 }
191
192
193 httpClientBuilder.setEventManager(eventManager);
194
195
196
197 if (connectionManager != null) {
198 httpClientBuilder.setConnectionManager(connectionManager);
199 }
200
201 Registry<CookieSpecProvider> cookieSpecRegistry =
202 RegistryBuilder
203 .<CookieSpecProvider>create()
204 .register(CustomBrowserCompatSpecFactory.CUSTOM_BROWSER_COMPATIBILITY,
205 new CustomBrowserCompatSpecFactory()).build();
206
207 RequestConfig config =
208 RequestConfig.custom().setCookieSpec(CustomBrowserCompatSpecFactory.CUSTOM_BROWSER_COMPATIBILITY)
209 .build();
210
211 httpClientBuilder.setDefaultCookieSpecRegistry(cookieSpecRegistry).setDefaultRequestConfig(config);
212
213 driver.getEventManager().fire(EventManager.EVENT_HTTP_BUILDER_INITIALIZATION,
214 new HttpClientBuilderEvent(httpClientBuilder));
215 return httpClientBuilder.build();
216 }
217 }
218
219 public static HttpClientRequestExecutorBuilder builder() {
220 return new HttpClientRequestExecutorBuilder();
221 }
222
223 private HttpClientRequestExecutor() {
224 }
225
226 @Override
227 public OutgoingRequest createOutgoingRequest(DriverRequest originalRequest, String uri, boolean proxy) {
228
229
230 HttpHost physicalHost = UriUtils.extractHost(uri);
231
232 if (!originalRequest.isExternal()) {
233 if (preserveHost) {
234
235 HttpHost virtualHost = HttpRequestHelper.getHost(originalRequest.getOriginalRequest());
236
237 uri = UriUtils.rewriteURI(uri, virtualHost);
238 } else {
239 uri = UriUtils.rewriteURI(uri, firstBaseUrlHost);
240 }
241 }
242
243 RequestConfig.Builder builder = RequestConfig.custom();
244 builder.setConnectTimeout(connectTimeout);
245 builder.setSocketTimeout(socketTimeout);
246
247
248
249 builder.setCookieSpec(CustomBrowserCompatSpecFactory.CUSTOM_BROWSER_COMPATIBILITY);
250
251 builder.setRedirectsEnabled(false);
252 RequestConfig config = builder.build();
253
254 OutgoingRequestContext context = new OutgoingRequestContext();
255
256 String method = "GET";
257 if (proxy) {
258 method = originalRequest.getOriginalRequest().getRequestLine().getMethod().toUpperCase();
259 }
260 OutgoingRequest outgoingRequest =
261 new OutgoingRequest(method, uri, originalRequest.getOriginalRequest().getProtocolVersion(),
262 originalRequest, config, context);
263 if (ENTITY_METHODS.contains(method)) {
264 outgoingRequest.setEntity(originalRequest.getOriginalRequest().getEntity());
265 } else if (!SIMPLE_METHODS.contains(method)) {
266 throw new UnsupportedHttpMethodException(method + " " + uri);
267 }
268
269 context.setPhysicalHost(physicalHost);
270 context.setOutgoingRequest(outgoingRequest);
271 context.setProxy(proxy);
272
273 return outgoingRequest;
274 }
275
276
277
278
279
280
281
282
283
284
285 @Override
286 public CloseableHttpResponse execute(OutgoingRequest httpRequest) throws HttpErrorPage {
287 OutgoingRequestContext context = httpRequest.getContext();
288 IncomingRequest originalRequest = httpRequest.getOriginalRequest().getOriginalRequest();
289
290 if (cookieManager != null) {
291 CookieStore cookieStore = new RequestCookieStore(cookieManager, httpRequest.getOriginalRequest());
292 context.setCookieStore(cookieStore);
293 }
294 HttpResponse result;
295
296 FragmentEvent event = new FragmentEvent(originalRequest, httpRequest, context);
297
298 eventManager.fire(EventManager.EVENT_FRAGMENT_PRE, event);
299
300 if (!event.isExit()) {
301
302 if (event.getHttpResponse() == null) {
303 if (httpRequest.containsHeader(HttpHeaders.EXPECT)) {
304 event.setHttpResponse(HttpErrorPage.generateHttpResponse(HttpStatus.SC_EXPECTATION_FAILED,
305 "'Expect' request header is not supported"));
306 } else {
307 try {
308 HttpHost physicalHost = context.getPhysicalHost();
309 result = httpClient.execute(physicalHost, httpRequest, context);
310 } catch (IOException e) {
311 result = HttpErrorPage.generateHttpResponse(e);
312 LOG.warn(httpRequest.getRequestLine() + " -> " + result.getStatusLine().toString());
313 }
314 event.setHttpResponse(BasicCloseableHttpResponse.adapt(result));
315 }
316 }
317
318 eventManager.fire(EventManager.EVENT_FRAGMENT_POST, event);
319 }
320 CloseableHttpResponse httpResponse = event.getHttpResponse();
321 if (httpResponse == null) {
322 throw new HttpErrorPage(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Request was cancelled by server",
323 "Request was cancelled by server");
324 }
325 if (HttpResponseUtils.isError(httpResponse)) {
326 throw new HttpErrorPage(httpResponse);
327 }
328 return httpResponse;
329 }
330
331 }