From b272509bf678138b04776e0206d6ac686a1f6517 Mon Sep 17 00:00:00 2001 From: datagear Date: Wed, 2 Sep 2020 21:40:00 +0800 Subject: [PATCH] =?UTF-8?q?[analysis]=E6=B7=BB=E5=8A=A0HTTP=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E9=9B=86=E5=AE=9E=E7=8E=B0=E7=B1=BBHttpDataSet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- datagear-analysis/pom.xml | 5 + .../analysis/support/AbstractCsvDataSet.java | 3 +- .../support/AbstractExcelDataSet.java | 3 +- .../analysis/support/AbstractJsonDataSet.java | 22 +- .../support/AbstractResolvableDataSet.java | 15 +- .../analysis/support/HttpDataSet.java | 590 ++++++++++++++++++ .../datagear/analysis/support/SqlDataSet.java | 4 +- 7 files changed, 626 insertions(+), 16 deletions(-) create mode 100644 datagear-analysis/src/main/java/org/datagear/analysis/support/HttpDataSet.java diff --git a/datagear-analysis/pom.xml b/datagear-analysis/pom.xml index 04d6cb8a..f41d50f4 100644 --- a/datagear-analysis/pom.xml +++ b/datagear-analysis/pom.xml @@ -59,6 +59,11 @@ poi-ooxml ${poi-ooxml.version} + + org.apache.httpcomponents.client5 + httpclient5 + 5.0.1 + diff --git a/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractCsvDataSet.java b/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractCsvDataSet.java index 2574b5ff..cec63bd9 100644 --- a/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractCsvDataSet.java +++ b/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractCsvDataSet.java @@ -40,10 +40,9 @@ public abstract class AbstractCsvDataSet extends AbstractResolvableDataSet imple super(); } - @SuppressWarnings("unchecked") public AbstractCsvDataSet(String id, String name) { - super(id, name, Collections.EMPTY_LIST); + super(id, name); } public AbstractCsvDataSet(String id, String name, List properties) diff --git a/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractExcelDataSet.java b/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractExcelDataSet.java index b98561a4..7fe6ec53 100644 --- a/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractExcelDataSet.java +++ b/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractExcelDataSet.java @@ -89,10 +89,9 @@ public abstract class AbstractExcelDataSet extends AbstractResolvableDataSet imp super(); } - @SuppressWarnings("unchecked") public AbstractExcelDataSet(String id, String name) { - super(id, name, Collections.EMPTY_LIST); + super(id, name); } public AbstractExcelDataSet(String id, String name, List properties) diff --git a/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractJsonDataSet.java b/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractJsonDataSet.java index 0df8be2b..ef038df1 100644 --- a/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractJsonDataSet.java +++ b/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractJsonDataSet.java @@ -40,10 +40,9 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl super(); } - @SuppressWarnings("unchecked") public AbstractJsonDataSet(String id, String name) { - super(id, name, Collections.EMPTY_LIST); + super(id, name); } public AbstractJsonDataSet(String id, String name, List properties) @@ -117,10 +116,13 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl JsonNode jsonNode = getObjectMapperNonStardand().readTree(jsonReader); - if (jsonNode == null || !isLegalResultDataJsonNode(jsonNode)) + if (!isLegalResultDataJsonNode(jsonNode)) throw new UnsupportedJsonResultDataException("Result data must be object or object array/list"); - Object data = getObjectMapperNonStardand().treeToValue(jsonNode, Object.class); + Object data = null; + + if (jsonNode != null) + data = getObjectMapperNonStardand().treeToValue(jsonNode, Object.class); if (resolveProperties) properties = resolveDataSetProperties(data); @@ -133,6 +135,15 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl return new ResolvedDataSetResult(result, properties); } + /** + * + * @param resultData + * 允许为{@code null} + * @param properties + * @param converter + * @return + * @throws Throwable + */ protected Object convertJsonResultData(Object resultData, List properties, DataSetPropertyValueConverter converter) throws Throwable { @@ -199,6 +210,7 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl *

* * @param jsonNode + * 允许为{@code null} * @return */ protected boolean isLegalResultDataJsonNode(JsonNode jsonNode) throws Throwable @@ -232,7 +244,7 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl * 解析JSON对象的{@linkplain DataSetProperty}。 * * @param resultData - * JSON对象、JSON对象数组、JSON对象列表 + * 允许为{@code null}，JSON对象、JSON对象数组、JSON对象列表 * @return * @throws Throwable */ diff --git a/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractResolvableDataSet.java b/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractResolvableDataSet.java index df424ae9..c1a6ec54 100644 --- a/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractResolvableDataSet.java +++ b/datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractResolvableDataSet.java @@ -7,6 +7,7 @@ */ package org.datagear.analysis.support; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -29,6 +30,12 @@ public abstract class AbstractResolvableDataSet extends AbstractDataSet implemen super(); } + @SuppressWarnings("unchecked") + public AbstractResolvableDataSet(String id, String name) + { + super(id, name, Collections.EMPTY_LIST); + } + public AbstractResolvableDataSet(String id, String name, List properties) { super(id, name, properties); @@ -57,10 +64,10 @@ public abstract class AbstractResolvableDataSet extends AbstractDataSet implemen * 解析结果。 * * @param paramValues - * @param properties 允许为{@code null}/空，此时，应自动解析并设置返回结果的{@linkplain ResolvedDataSetResult#setProperties(List)}； - * 如果不为{@code null}/空，直接将{@code properties}作为解析数据依据， - * 使用它处理结果数据， - * 并设置为返回结果的{@linkplain ResolvedDataSetResult#setProperties(List)} + * @param properties + * 允许为{@code null}/空，此时，应自动解析并设置返回结果的{@linkplain ResolvedDataSetResult#setProperties(List)}； + * 如果不为{@code null}/空，直接将{@code properties}作为解析数据依据， 使用它处理结果数据， + * 并设置为返回结果的{@linkplain ResolvedDataSetResult#setProperties(List)} * @return * @throws DataSetException */ diff --git a/datagear-analysis/src/main/java/org/datagear/analysis/support/HttpDataSet.java b/datagear-analysis/src/main/java/org/datagear/analysis/support/HttpDataSet.java new file mode 100644 index 00000000..8107ce9c --- /dev/null +++ b/datagear-analysis/src/main/java/org/datagear/analysis/support/HttpDataSet.java @@ -0,0 +1,590 @@ +/* + * Copyright (c) 2018 datagear.tech. All Rights Reserved. + */ + +/** + * + */ +package org.datagear.analysis.support; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.Serializable; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.apache.hc.client5.http.HttpResponseException; +import org.apache.hc.client5.http.classic.HttpClient; +import org.apache.hc.client5.http.classic.methods.HttpDelete; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.classic.methods.HttpHead; +import org.apache.hc.client5.http.classic.methods.HttpOptions; +import org.apache.hc.client5.http.classic.methods.HttpPatch; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.classic.methods.HttpPut; +import org.apache.hc.client5.http.classic.methods.HttpTrace; +import org.apache.hc.client5.http.entity.UrlEncodedFormEntity; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.core5.http.ClassicHttpRequest; +import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.NameValuePair; +import org.apache.hc.core5.http.io.HttpClientResponseHandler; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hc.core5.http.message.BasicNameValuePair; +import org.datagear.analysis.DataSetException; +import org.datagear.analysis.DataSetProperty; +import org.datagear.analysis.DataSetResult; +import org.datagear.analysis.ResolvedDataSetResult; +import org.datagear.util.IOUtil; +import org.datagear.util.StringUtil; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * HTTP数据集。 + *

+ * 此类的{@linkplain #getUri()}、{@linkplain #getRequestContent()}支持Freemarker模板语言。 + *

+ * + * @author datagear@163.com + * + */ +public class HttpDataSet extends AbstractResolvableDataSet +{ + public static final String HTTP_METHOD_GET = "GET"; + + public static final String HTTP_METHOD_POST = "POST"; + + public static final String HTTP_METHOD_PUT = "PUT"; + + public static final String HTTP_METHOD_HEAD = "HEAD"; + + public static final String HTTP_METHOD_PATCH = "PATCH"; + + public static final String HTTP_METHOD_DELETE = "DELETE"; + + public static final String HTTP_METHOD_OPTIONS = "OPTIONS"; + + public static final String HTTP_METHOD_TRACE = "TRACE"; + + /** + * 内容类型：表单式的参数名/值类型 + */ + public static final String CONTENT_TYPE_FORM = "application/x-www-form-urlencoded"; + + /** + * 内容类型：JSON + */ + public static final String CONTENT_TYPE_JSON = "application/json"; + + /** HTTP客户端 */ + private HttpClient httpClient; + + /** HTTP请求地址 */ + private String uri; + + /** 请求方法 */ + private String httpMethod = HTTP_METHOD_GET; + + /** 请求头 */ + @SuppressWarnings("unchecked") + private List headers = Collections.EMPTY_LIST; + + /** 请求内容类型 */ + private String requestContentType = CONTENT_TYPE_FORM; + + /** 请求内容JSON文本 */ + private String requestContent = ""; + + /** 响应类型 */ + private String responseContentType = CONTENT_TYPE_JSON; + + public HttpDataSet() + { + super(); + } + + public HttpDataSet(String id, String name, HttpClient httpClient, String uri) + { + super(id, name); + this.httpClient = httpClient; + this.uri = uri; + } + + public HttpDataSet(String id, String name, List properties, HttpClient httpClient, String uri) + { + super(id, name, properties); + this.httpClient = httpClient; + this.uri = uri; + } + + public HttpClient getHttpClient() + { + return httpClient; + } + + public void setHttpClient(HttpClient httpClient) + { + this.httpClient = httpClient; + } + + public String getUri() + { + return uri; + } + + /** + * 设置请求地址。 + *

+ * 请求地址支持Freemarker模板语言。 + *

+ * + * @param uri + */ + public void setUri(String uri) + { + this.uri = uri; + } + + public String getHttpMethod() + { + return httpMethod; + } + + /** + * 设置HTTP方法，参考{@code HTTP_METHOD_*}常量。 + * + * @param httpMethod + */ + public void setHttpMethod(String httpMethod) + { + this.httpMethod = httpMethod; + } + + public List getHeaders() + { + return headers; + } + + public void setHeaders(List headers) + { + this.headers = headers; + } + + public String getRequestContentType() + { + return requestContentType; + } + + /** + * 设置请求内容类型，参考{@code CONTENT_TYPE_*}。 + * + * @param requestContentType + */ + public void setRequestContentType(String requestContentType) + { + this.requestContentType = requestContentType; + } + + public String getRequestContent() + { + return requestContent; + } + + /** + * 设置请求内容JSON文本，为{@code null}或{@code ""}表示无请求内容。 + *

+ * 当{@linkplain #getRequestContentType()}为{@linkplain #CONTENT_TYPE_FORM}时，请求内容JSON文本格式应为： + *

+ * + * 
+	 * [
+	 *   {name: "...", value: "..."},
+	 *   {name: "...", value: "..."},
+	 *   ...
+	 * ]
+	 *
+ * + *

+ * 其中，{@code name}表示请求参数名，{@code value}表示请求参数值。 + *

+ *

+ * 当{@linkplain #getRequestContentType()}为{@linkplain #CONTENT_TYPE_JSON}时，请求内容JSON文本没有特殊格式要求。 + *

+ *

+ * 请求内容JSON文本支持Freemarker模板语言。 + *

+ * + * @param requestContent + */ + public void setRequestContent(String requestContent) + { + this.requestContent = requestContent; + } + + public String getResponseContentType() + { + return responseContentType; + } + + /** + * 设置相应类型。 + *

+ * 目前相应类型仅支持{@linkplain #CONTENT_TYPE_JSON}。 + *

+ * + * @param responseContentType + */ + public void setResponseContentType(String responseContentType) + { + this.responseContentType = responseContentType; + } + + @Override + protected ResolvedDataSetResult resolveResult(Map paramValues, List properties) + throws DataSetException + { + CloseableHttpResponse response = null; + + try + { + String uri = resolveTemplateUri(paramValues); + String requestContent = resolveTemplateRequestContent(paramValues); + + ClassicHttpRequest request = createHttpRequest(uri); + + setHttpHeaders(request, getHeaders()); + setHttpEntity(request, requestContent); + + JsonResponseHandler responseHandler = new JsonResponseHandler(); + responseHandler.setProperties(properties); + + ResolvedDataSetResult result = this.httpClient.execute(request, responseHandler); + + String templateResult = "URI:\n" + uri + "\n-----------------------------------------\n" + + "Request content:\n" + requestContent; + + return new TemplateResolvedDataSetResult(result.getResult(), result.getProperties(), templateResult); + } + catch (DataSetException e) + { + throw e; + } + catch (Throwable t) + { + throw new DataSetSourceParseException(t); + } + finally + { + IOUtil.close(response); + } + } + + protected void setHttpHeaders(ClassicHttpRequest request, List headers) throws Throwable + { + if (headers == null || headers.isEmpty()) + return; + + for (HttpHeader header : headers) + request.setHeader(header.getName(), header.getValue()); + } + + protected void setHttpEntity(ClassicHttpRequest request, String requestContent) throws Throwable + { + if (CONTENT_TYPE_FORM.equals(this.requestContentType)) + { + List params = toRequestParams(requestContent); + request.setEntity(new UrlEncodedFormEntity(params)); + } + else if (CONTENT_TYPE_JSON.equals(this.requestContentType)) + { + StringEntity entity = new StringEntity(requestContent, ContentType.APPLICATION_JSON); + request.setEntity(entity); + } + else + throw new DataSetException("HTTP request content-type [" + this.requestContentType + "] is not supported"); + } + + protected String resolveTemplateUri(Map paramValues) throws Throwable + { + return resolveAsFmkTemplate(this.uri, paramValues); + } + + protected String resolveTemplateRequestContent(Map paramValues) throws Throwable + { + return resolveAsFmkTemplate(this.requestContent, paramValues); + } + + protected ClassicHttpRequest createHttpRequest(String uri) throws Throwable + { + if (HTTP_METHOD_GET.equals(this.httpMethod)) + return new HttpGet(uri); + else if (HTTP_METHOD_POST.equals(this.httpMethod)) + return new HttpPost(uri); + else if (HTTP_METHOD_PUT.equals(this.httpMethod)) + return new HttpPut(uri); + else if (HTTP_METHOD_HEAD.equals(this.httpMethod)) + return new HttpHead(uri); + else if (HTTP_METHOD_PATCH.equals(this.httpMethod)) + return new HttpPatch(uri); + else if (HTTP_METHOD_DELETE.equals(this.httpMethod)) + return new HttpDelete(uri); + else if (HTTP_METHOD_OPTIONS.equals(this.httpMethod)) + return new HttpOptions(uri); + else if (HTTP_METHOD_TRACE.equals(this.httpMethod)) + return new HttpTrace(uri); + else + throw new DataSetException("HTTP method [" + this.httpMethod + "] is not supported"); + } + + /** + * 将指定JSON字符串转换为名/值列表。 + * + * @param requestContent + * 可能为{@code null}、{@code ""} + * @return 空列表表示无名/值参数 + * @throws Throwable + */ + @SuppressWarnings("unchecked") + protected List toRequestParams(String requestContent) throws Throwable + { + if (StringUtil.isEmpty(requestContent)) + return Collections.EMPTY_LIST; + + Object jsonObj = getObjectMapperNonStardand().readValue(requestContent, Object.class); + + if (jsonObj == null) + return Collections.EMPTY_LIST; + + if (!(jsonObj instanceof Collection)) + throw new DataSetException("The request content must be JSON array"); + + Collection collection = (Collection) jsonObj; + + List nameValuePairs = new ArrayList<>(collection.size()); + + int idx = 0; + for (Object ele : collection) + { + String name = null; + String value = null; + + if (ele instanceof Map) + { + Map eleMap = (Map) ele; + Object nameVal = eleMap.get("name"); + Object valueVal = eleMap.get("value"); + + if (nameVal instanceof String) + { + name = (String) nameVal; + if (valueVal != null) + value = (valueVal instanceof String ? (String) valueVal : valueVal.toString()); + } + } + + if (name == null) + throw new DataSetException("The request content " + idx + + "-th element must be JSON object : {name: \"...\", value: \"...\"}"); + + nameValuePairs.add(new BasicNameValuePair(name, value)); + + idx++; + } + + return nameValuePairs; + } + + protected ObjectMapper getObjectMapperNonStardand() + { + return JsonSupport.getObjectMapperNonStardand(); + } + + /** + * HTTP单个请求头。 + * + * @author datagear@163.com + * + */ + public static class HttpHeader implements Serializable + { + private static final long serialVersionUID = 1L; + + /** 名称 */ + private String name; + + /** 值 */ + private String value; + + public HttpHeader() + { + super(); + } + + public HttpHeader(String name, String value) + { + super(); + this.name = name; + this.value = value; + } + + public String getName() + { + return name; + } + + public void setName(String name) + { + this.name = name; + } + + public String getValue() + { + return value; + } + + public void setValue(String value) + { + this.value = value; + } + + @Override + public int hashCode() + { + final int prime = 31; + int result = 1; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + ((value == null) ? 0 : value.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + HttpHeader other = (HttpHeader) obj; + if (name == null) + { + if (other.name != null) + return false; + } + else if (!name.equals(other.name)) + return false; + if (value == null) + { + if (other.value != null) + return false; + } + else if (!value.equals(other.value)) + return false; + return true; + } + + @Override + public String toString() + { + return getClass().getSimpleName() + " [name=" + name + ", value=" + value + "]"; + } + } + + protected static class JsonResponseHandler implements HttpClientResponseHandler + { + private List properties; + + public JsonResponseHandler() + { + super(); + } + + public List getProperties() + { + return properties; + } + + /** + * 设置数据集属性。 + * + * @param properties + * 如果为{@code null}或空，则执行解析 + */ + public void setProperties(List properties) + { + this.properties = properties; + } + + @SuppressWarnings("unchecked") + @Override + public ResolvedDataSetResult handleResponse(ClassicHttpResponse response) throws HttpException, IOException + { + int code = response.getCode(); + HttpEntity entity = response.getEntity(); + + if (code < 200 || code >= 300) + throw new HttpResponseException(code, response.getReasonPhrase()); + + Reader reader = null; + + if (entity == null) + reader = IOUtil.getReader(""); + else + { + String contentTypeStr = entity.getContentType(); + + ContentType contentType = ContentType.APPLICATION_JSON; + if (!StringUtil.isEmpty(contentTypeStr)) + contentType = ContentType.parse(contentTypeStr); + Charset charset = contentType.getCharset(); + + reader = new InputStreamReader(entity.getContent(), charset); + } + + if (this.properties == null || this.properties.isEmpty()) + { + HttpResponseJsonDataSet jsonDataSet = new HttpResponseJsonDataSet(reader); + return jsonDataSet.resolve(Collections.EMPTY_MAP); + } + else + { + HttpResponseJsonDataSet jsonDataSet = new HttpResponseJsonDataSet(this.properties, reader); + DataSetResult result = jsonDataSet.getResult(Collections.EMPTY_MAP); + return new ResolvedDataSetResult(result, this.properties); + } + } + } + + protected static class HttpResponseJsonDataSet extends AbstractJsonDataSet + { + private Reader responseJsonReader; + + public HttpResponseJsonDataSet(Reader responseJsonReader) + { + super(HttpResponseJsonDataSet.class.getName(), HttpResponseJsonDataSet.class.getName()); + this.responseJsonReader = responseJsonReader; + } + + public HttpResponseJsonDataSet(List properties, Reader responseJsonReader) + { + super(HttpResponseJsonDataSet.class.getName(), HttpResponseJsonDataSet.class.getName(), properties); + this.responseJsonReader = responseJsonReader; + } + + @Override + protected TemplateResolvedSource getJsonReader(Map paramValues) throws Throwable + { + return new TemplateResolvedSource<>(this.responseJsonReader); + } + } +} diff --git a/datagear-analysis/src/main/java/org/datagear/analysis/support/SqlDataSet.java b/datagear-analysis/src/main/java/org/datagear/analysis/support/SqlDataSet.java index c88262ae..311dcdd6 100644 --- a/datagear-analysis/src/main/java/org/datagear/analysis/support/SqlDataSet.java +++ b/datagear-analysis/src/main/java/org/datagear/analysis/support/SqlDataSet.java @@ -13,7 +13,6 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,10 +54,9 @@ public class SqlDataSet extends AbstractResolvableDataSet implements ResolvableD super(); } - @SuppressWarnings("unchecked") public SqlDataSet(String id, String name, ConnectionFactory connectionFactory, String sql) { - super(id, name, Collections.EMPTY_LIST); + super(id, name); this.connectionFactory = connectionFactory; this.sql = sql; }