p42579631
/
datagear
forked from p85126437/datagear
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[analysis]重构JSON数据集类，添加单元测试类

This commit is contained in:
wodezy66 2020-08-30 21:40:33 +08:00
parent ccd1be157f
commit b40c0fd31c
14 changed files with 933 additions and 363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractCsvDataSet.java
View File

@ -101,6 +101,11 @@ public abstract class AbstractCsvDataSet extends AbstractFmkTemplateDataSet impl
}
/**
* 解析结果。
* <p>
* 如果{@linkplain #getCsvReader(Map)}返回的{@linkplain TemplateResolvedSource#hasResolvedTemplate()}，
* 此方法将返回{@linkplain TemplateResolvedDataSetResult}。
* </p>
*
* @param paramValues
* @param properties 允许为{@code null}，此时会自动解析
@ -110,12 +115,17 @@ public abstract class AbstractCsvDataSet extends AbstractFmkTemplateDataSet impl
protected ResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties)
throws DataSetException
{
Reader reader = null;
TemplateResolvedSource<Reader> reader = null;
try
{
reader = getCsvReader(paramValues);
ResolvedDataSetResult result = resolveResult(reader, properties);
ResolvedDataSetResult result = resolveResult(reader.getSource(), properties);
if (reader.hasResolvedTemplate())
result = new TemplateResolvedDataSetResult(result.getResult(), result.getProperties(),
reader.getResolvedTemplate());
return result;
}
@ -129,7 +139,7 @@ public abstract class AbstractCsvDataSet extends AbstractFmkTemplateDataSet impl
}
finally
{
IOUtil.close(reader);
IOUtil.close(reader.getSource());
}
}
@ -143,18 +153,19 @@ public abstract class AbstractCsvDataSet extends AbstractFmkTemplateDataSet impl
* @return
* @throws Throwable
*/
protected abstract Reader getCsvReader(Map<String, ?> paramValues) throws Throwable;
protected abstract TemplateResolvedSource<Reader> getCsvReader(Map<String, ?> paramValues) throws Throwable;
/**
* 解析结果。
*
* @param csvReader
* @param properties 允许为{@code null}，此时会自动解析
* @return
* @throws DataSetException
* @throws Throwable
*/
@SuppressWarnings({ "unchecked", "rawtypes" })
protected ResolvedDataSetResult resolveResult(Reader csvReader, List<DataSetProperty> properties)
throws DataSetException
throws Throwable
{
boolean resolveProperties = (properties == null || properties.isEmpty());
@ -204,8 +215,9 @@ public abstract class AbstractCsvDataSet extends AbstractFmkTemplateDataSet impl
* @param propertyNames 允许为{@code null}
* @param data 允许为{@code null}
* @return
* @throws Throwable
*/
protected ResolvedDataSetResult resolveResult(List<String> propertyNames, List<List<String>> data)
protected ResolvedDataSetResult resolveResult(List<String> propertyNames, List<List<String>> data) throws Throwable
{
List<DataSetProperty> properties = new ArrayList<DataSetProperty>(
(propertyNames == null ? 0 : propertyNames.size()));
@ -307,8 +319,9 @@ public abstract class AbstractCsvDataSet extends AbstractFmkTemplateDataSet impl
*
* @param s
* @return
* @throws Throwable
*/
protected Number parseNumberString(String s)
protected Number parseNumberString(String s) throws Throwable
{
return Double.parseDouble(s);
}
@ -318,10 +331,10 @@ public abstract class AbstractCsvDataSet extends AbstractFmkTemplateDataSet impl
* @param csvRecord
* @param forceIndex 是否强制使用索引号，将返回：{@code ["1"、"2"、....]}
* @return
* @throws DataSetException
* @throws Throwable
*/
protected List<String> resolveDataSetPropertyNames(CSVRecord csvRecord, boolean forceIndex)
throws DataSetException
throws Throwable
{
int size = csvRecord.size();
List<String> list = new ArrayList<>(size);
@ -346,8 +359,10 @@ public abstract class AbstractCsvDataSet extends AbstractFmkTemplateDataSet impl
* @param csvRecord
* @param properties 允许为{@code null}、元素为{@code null}
* @return
* @throws Throwable
*/
protected List<Object> resolveCSVRecordValues(CSVRecord csvRecord, List<DataSetProperty> properties)
throws Throwable
{
int size = csvRecord.size();
List<Object> list = new ArrayList<>(size);
@ -388,17 +403,10 @@ public abstract class AbstractCsvDataSet extends AbstractFmkTemplateDataSet impl
*
* @param reader
* @return
* @throws DataSetSourceParseException
* @throws Throwable
*/
protected CSVParser buildCSVParser(Reader reader) throws DataSetSourceParseException
protected CSVParser buildCSVParser(Reader reader) throws Throwable
{
try
{
return CSVFormat.DEFAULT.withIgnoreSurroundingSpaces().parse(reader);
}
catch(Throwable t)
{
throw new DataSetSourceParseException(t);
}
return CSVFormat.DEFAULT.withIgnoreSurroundingSpaces().parse(reader);
}
}

7
datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractCsvFileDataSet.java
View File

@ -17,6 +17,9 @@ import org.datagear.util.IOUtil;
/**
* 抽象CSV文件数据集。
* <p>
* 注意：此类不支持<code>Freemarker</code>模板语言。
* </p>
*
* @author datagear@163.com
*
@ -54,10 +57,10 @@ public abstract class AbstractCsvFileDataSet extends AbstractCsvDataSet
}
@Override
protected Reader getCsvReader(Map<String, ?> paramValues) throws Throwable
protected TemplateResolvedSource<Reader> getCsvReader(Map<String, ?> paramValues) throws Throwable
{
File file = getCsvFile(paramValues);
return IOUtil.getReader(file, this.encoding);
return new TemplateResolvedSource<Reader>(IOUtil.getReader(file, this.encoding));
}
/**

5
datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractDataSet.java
View File

@ -151,7 +151,10 @@ public abstract class AbstractDataSet extends AbstractIdentifiable implements Da
*/
protected Object convertToPropertyDataType(Object source, DataSetProperty property)
{
return convertToPropertyDataType(source, (property == null ? null : property.getType()));
if (property == null)
return source;
return convertToPropertyDataType(source, property.getType());
}
/**

285
datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractJsonDataSet.java
View File

@ -7,7 +7,10 @@
*/
package org.datagear.analysis.support;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -16,6 +19,13 @@ import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetResult;
import org.datagear.analysis.ResolvableDataSet;
import org.datagear.analysis.ResolvedDataSetResult;
import org.datagear.util.IOUtil;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.ValueNode;
/**
* 抽象JSON数据集。
@ -25,8 +35,6 @@ import org.datagear.analysis.ResolvedDataSetResult;
*/
public abstract class AbstractJsonDataSet extends AbstractFmkTemplateDataSet implements ResolvableDataSet
{
public static final JsonDataSetSupport JSON_DATA_SET_SUPPORT = new JsonDataSetSupport();
public AbstractJsonDataSet()
{
super();
@ -43,17 +51,282 @@ public abstract class AbstractJsonDataSet extends AbstractFmkTemplateDataSet imp
super(id, name, properties);
}
protected JsonDataSetSupport getJsonDataSetSupport()
@Override
public DataSetResult getResult(Map<String, ?> paramValues) throws DataSetException
{
return JSON_DATA_SET_SUPPORT;
List<DataSetProperty> properties = getProperties();
if (properties == null || properties.isEmpty())
throw new DataSetException("[getProperties()] must not be empty");
ResolvedDataSetResult result = resolveResult(paramValues, properties);
return result.getResult();
}
@Override
public ResolvedDataSetResult resolve(Map<String, ?> paramValues) throws DataSetException
{
DataSetResult result = getResult(paramValues);
List<DataSetProperty> properties = getJsonDataSetSupport().resolveDataSetProperties(result.getData());
return resolveResult(paramValues, null);
}
/**
* 解析结果。
* <p>
* 如果{@linkplain #getJsonReader(Map)}返回的{@linkplain TemplateResolvedSource#hasResolvedTemplate()}，
* 此方法将返回{@linkplain TemplateResolvedDataSetResult}。
* </p>
*
* @param paramValues
* @param properties 允许为{@code null}，此时会自动解析
* @return
* @throws DataSetException
*/
protected ResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties)
throws DataSetException
{
TemplateResolvedSource<Reader> reader = null;
try
{
reader = getJsonReader(paramValues);
ResolvedDataSetResult result = resolveResult(reader.getSource(), properties);
if(reader.hasResolvedTemplate())
result = new TemplateResolvedDataSetResult(result.getResult(), result.getProperties(),
reader.getResolvedTemplate());
return result;
}
catch(DataSetException e)
{
throw e;
}
catch(Throwable t)
{
throw new DataSetSourceParseException(t);
}
finally
{
IOUtil.close(reader.getSource());
}
}
/**
* 获取JSON输入流。
* <p>
* 实现方法应该返回实例级不变的输入流。
* </p>
*
* @param paramValues
* @return
* @throws Throwable
*/
protected abstract TemplateResolvedSource<Reader> getJsonReader(Map<String, ?> paramValues) throws Throwable;
/**
* 解析结果。
*
* @param jsonReader JSON输入流
* @param properties 允许为{@code null}，此时会自动解析
* @return
* @throws Throwable
*/
protected ResolvedDataSetResult resolveResult(Reader jsonReader, List<DataSetProperty> properties)
throws Throwable
{
boolean resolveProperties = (properties == null || properties.isEmpty());
JsonNode jsonNode = getObjectMapperNonStardand().readTree(jsonReader);
if (jsonNode == null || !isLegalResultDataJsonNode(jsonNode))
throw new UnsupportedJsonResultDataException("Result data must be object or object array/list");
Object data = getObjectMapperNonStardand().treeToValue(jsonNode, Object.class);
if (resolveProperties)
properties = resolveDataSetProperties(data);
if (!resolveProperties)
data = convertJsonResultData(data, properties);
DataSetResult result = new DataSetResult(data);
return new ResolvedDataSetResult(result, properties);
}
protected Object convertJsonResultData(Object resultData, List<DataSetProperty> properties) throws Throwable
{
Object re = null;
// JSON对象
if (resultData == null)
{
re = null;
}
else if (resultData instanceof Map<?, ?>)
{
Map<String, Object> reMap = new HashMap<String, Object>();
@SuppressWarnings("unchecked")
Map<String, Object> source = (Map<String, Object>) resultData;
for (Map.Entry<String, Object> entry : source.entrySet())
{
String name = entry.getKey();
Object value = entry.getValue();
DataSetProperty property = getDataNameTypeByName(properties, name);
value = convertToPropertyDataType(value, property);
reMap.put(name, value);
}
re = reMap;
}
else if (resultData instanceof List<?>)
{
List<?> list = (List<?>) resultData;
List<Object> reList = new ArrayList<Object>(list.size());
for (Object ele : list)
reList.add(convertJsonResultData(ele, properties));
re = reList;
}
else if (resultData instanceof Object[])
{
Object[] array = (Object[]) resultData;
Object[] reArray = new Object[array.length];
for (int i = 0; i < array.length; i++)
reArray[i] = convertJsonResultData(array[i], properties);
re = reArray;
}
else
throw new UnsupportedJsonResultDataException("Result data must be object or object array/list");
return re;
}
/**
* 是否是合法的数据集结果数据{@linkplain JsonNode}。
* <p>
* 参考{@linkplain DataSetResult#getData()}说明。
* </p>
*
* @param jsonNode
* @return
*/
protected boolean isLegalResultDataJsonNode(JsonNode jsonNode) throws Throwable
{
if (jsonNode == null || jsonNode.isNull())
return true;
if (jsonNode instanceof ValueNode)
return false;
if (jsonNode instanceof ArrayNode)
{
ArrayNode arrayNode = (ArrayNode) jsonNode;
for (int i = 0; i < arrayNode.size(); i++)
{
JsonNode eleNode = arrayNode.get(i);
if (eleNode == null || eleNode.isNull())
continue;
if (!(eleNode instanceof ObjectNode))
return false;
}
}
return true;
}
/**
* 解析JSON对象的{@linkplain DataSetProperty}。
*
* @param resultData JSON对象、JSON对象数组、JSON对象列表
* @return
* @throws Throwable
*/
@SuppressWarnings("unchecked")
protected List<DataSetProperty> resolveDataSetProperties(Object resultData) throws Throwable
{
if (resultData == null)
{
return Collections.EMPTY_LIST;
}
else if (resultData instanceof Map<?, ?>)
{
return resolveJsonObjDataSetProperties((Map<String, ?>) resultData);
}
else if (resultData instanceof List<?>)
{
List<?> list = (List<?>) resultData;
if (list.size() == 0)
return Collections.EMPTY_LIST;
else
return resolveJsonObjDataSetProperties((Map<String, ?>) list.get(0));
}
else if (resultData instanceof Object[])
{
Object[] array = (Object[]) resultData;
if (array.length == 0)
return Collections.EMPTY_LIST;
else
return resolveJsonObjDataSetProperties((Map<String, ?>) array[0]);
}
else
throw new UnsupportedJsonResultDataException("Result data must be object or object array/list");
}
/**
* 解析JSON对象的{@linkplain DataSetProperty}。
*
* @param jsonObj
* @return
* @throws Throwable
*/
protected List<DataSetProperty> resolveJsonObjDataSetProperties(Map<String, ?> jsonObj) throws Throwable
{
List<DataSetProperty> properties = new ArrayList<>();
if (jsonObj == null)
{
}
else
{
for (Map.Entry<String, ?> entry : jsonObj.entrySet())
{
Object value = entry.getValue();
String type = DataSetProperty.DataType.resolveDataType(value);
DataSetProperty property = new DataSetProperty(entry.getKey(), type);
// JSON数值只有NUMBER类型
if (DataSetProperty.DataType.isInteger(property.getType())
|| DataSetProperty.DataType.isDecimal(property.getType()))
property.setType(DataSetProperty.DataType.NUMBER);
properties.add(property);
}
}
return properties;
}
protected ObjectMapper getObjectMapperNonStardand()
{
return JsonSupport.getObjectMapperNonStardand();
}
}

13
datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractJsonFileDataSet.java
View File

@ -8,15 +8,19 @@
package org.datagear.analysis.support;
import java.io.File;
import java.io.Reader;
import java.util.List;
import java.util.Map;
import org.datagear.analysis.DataSetException;
import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetResult;
import org.datagear.util.IOUtil;
/**
* 抽象JSON文件数据集。
* <p>
* 注意：此类不支持<code>Freemarker</code>模板语言。
* </p>
*
* @author datagear@163.com
*
@ -54,11 +58,10 @@ public abstract class AbstractJsonFileDataSet extends AbstractJsonDataSet
}
@Override
public DataSetResult getResult(Map<String, ?> paramValues) throws DataSetException
protected TemplateResolvedSource<Reader> getJsonReader(Map<String, ?> paramValues) throws Throwable
{
File jsonFile = getJsonFile(paramValues);
Object data = getJsonDataSetSupport().resolveResultData(jsonFile, getEncoding());
return new DataSetResult(data);
File file = getJsonFile(paramValues);
return new TemplateResolvedSource<Reader>(IOUtil.getReader(file, this.encoding));
}
/**

15
datagear-analysis/src/main/java/org/datagear/analysis/support/CsvValueDataSet.java
View File

@ -8,11 +8,15 @@ import java.io.Reader;
import java.util.List;
import java.util.Map;
import org.datagear.analysis.DataSetException;
import org.datagear.analysis.DataSetProperty;
import org.datagear.util.IOUtil;
/**
* CSV值数据集。
* <p>
* 此类的{@linkplain #getValue()}支持<code>Freemarker</code>模板语言。
* </p>
*
* @author datagear@163.com
*
@ -61,8 +65,15 @@ public class CsvValueDataSet extends AbstractCsvDataSet
}
@Override
protected Reader getCsvReader(Map<String, ?> paramValues) throws Throwable
public TemplateResolvedDataSetResult resolve(Map<String, ?> paramValues) throws DataSetException
{
return IOUtil.getReader(this.value);
return (TemplateResolvedDataSetResult) resolveResult(paramValues, null);
}
@Override
protected TemplateResolvedSource<Reader> getCsvReader(Map<String, ?> paramValues) throws Throwable
{
String csv = resolveTemplate(this.value, paramValues);
return new TemplateResolvedSource<Reader>(IOUtil.getReader(csv), csv);
}
}

274
datagear-analysis/src/main/java/org/datagear/analysis/support/JsonDataSetSupport.java
View File

@ -1,274 +0,0 @@
/*
* Copyright (c) 2018 datagear.tech. All Rights Reserved.
*/
package org.datagear.analysis.support;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.datagear.analysis.DataSet;
import org.datagear.analysis.DataSetException;
import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetResult;
import org.datagear.util.IOUtil;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.ValueNode;
/**
* JSON {@linkplain DataSet}支持类。
*
* @author datagear@163.com
*
*/
public class JsonDataSetSupport extends JsonSupport
{
public JsonDataSetSupport()
{
super();
}
/**
* 解析JSON数据。
*
* @param jsonValue
* @return
* @throws DataSetSourceParseException
* @throws DataSetException
*/
public Object resolveValue(String jsonValue) throws DataSetSourceParseException, DataSetException
{
StringReader reader = new StringReader(jsonValue);
return resolveValue(reader);
}
/**
* 解析JSON数据。
*
* @param jsonReader
* @return
* @throws DataSetSourceParseException
* @throws DataSetException
*/
public Object resolveValue(Reader jsonReader) throws DataSetSourceParseException, DataSetException
{
try
{
return parseNonStardand(jsonReader, Object.class);
}
catch (Throwable t)
{
throw new DataSetException(t);
}
}
/**
* 解析数据集结果数据。
*
* @param jsonValue
* @return
* @throws DataSetSourceParseException
* @throws UnsupportedJsonResultDataException
* @throws DataSetException
*/
public Object resolveResultData(String jsonValue)
throws DataSetSourceParseException, UnsupportedJsonResultDataException, DataSetException
{
StringReader reader = new StringReader(jsonValue);
return resolveResultData(reader);
}
/**
* 解析数据集结果数据。
*
* @param file
* @param encoding
* @return
* @throws DataSetSourceParseException
* @throws UnsupportedJsonResultDataException
* @throws DataSetException
*/
public Object resolveResultData(File file, String encoding)
throws DataSetSourceParseException, UnsupportedJsonResultDataException, DataSetException
{
Reader reader = null;
try
{
reader = IOUtil.getReader(file, encoding);
return resolveResultData(reader);
}
catch (IOException e)
{
throw new DataSetException(e);
}
finally
{
IOUtil.close(reader);
}
}
/**
* 解析数据集结果数据。
*
* @param reader
* @return
* @throws DataSetSourceParseException
* @throws UnsupportedJsonResultDataException
* @throws DataSetException
*/
public Object resolveResultData(Reader reader)
throws DataSetSourceParseException, UnsupportedJsonResultDataException, DataSetException
{
JsonNode jsonNode = null;
try
{
jsonNode = getObjectMapperNonStardand().readTree(reader);
}
catch (Throwable t)
{
throw new DataSetSourceParseException(t);
}
if (!isLegalResultDataJsonNode(jsonNode))
throw new UnsupportedJsonResultDataException("Result data must be object or object array/list");
if (jsonNode == null)
return null;
Object data = null;
try
{
data = getObjectMapperNonStardand().treeToValue(jsonNode, Object.class);
}
catch (Throwable t)
{
throw new DataSetException(t);
}
return data;
}
/**
* 是否是合法的数据集结果数据{@linkplain JsonNode}。
* <p>
* 参考{@linkplain DataSetResult#getData()}说明。
* </p>
*
* @param jsonNode
* @return
*/
public boolean isLegalResultDataJsonNode(JsonNode jsonNode)
{
if (jsonNode == null || jsonNode.isNull())
return true;
if (jsonNode instanceof ValueNode)
return false;
if (jsonNode instanceof ArrayNode)
{
ArrayNode arrayNode = (ArrayNode) jsonNode;
for (int i = 0; i < arrayNode.size(); i++)
{
JsonNode eleNode = arrayNode.get(i);
if (eleNode == null || eleNode.isNull())
continue;
if (!(eleNode instanceof ObjectNode))
return false;
}
}
return true;
}
/**
* 解析JSON对象的{@linkplain DataSetProperty}。
*
* @param resultData
* JSON对象、JSON对象数组、JSON对象列表
* @return
* @throws UnsupportedJsonResultDataException
*/
@SuppressWarnings("unchecked")
public List<DataSetProperty> resolveDataSetProperties(Object resultData) throws UnsupportedJsonResultDataException
{
if (resultData == null)
{
return Collections.EMPTY_LIST;
}
else if (resultData instanceof Map<?, ?>)
{
return resolveJsonObjDataSetProperties((Map<String, ?>) resultData);
}
else if (resultData instanceof List<?>)
{
List<?> list = (List<?>) resultData;
if (list.size() == 0)
return Collections.EMPTY_LIST;
else
return resolveJsonObjDataSetProperties((Map<String, ?>) list.get(0));
}
else if (resultData instanceof Object[])
{
Object[] array = (Object[]) resultData;
if (array.length == 0)
return Collections.EMPTY_LIST;
else
return resolveJsonObjDataSetProperties((Map<String, ?>) array[0]);
}
else
throw new UnsupportedJsonResultDataException("Result data must be object or object array/list");
}
/**
* 解析JSON对象的{@linkplain DataSetProperty}。
*
* @param jsonObj
* @return
*/
public List<DataSetProperty> resolveJsonObjDataSetProperties(Map<String, ?> jsonObj)
{
List<DataSetProperty> properties = new ArrayList<>();
if (jsonObj == null)
{
}
else
{
for (Map.Entry<String, ?> entry : jsonObj.entrySet())
{
Object value = entry.getValue();
String type = DataSetProperty.DataType.resolveDataType(value);
DataSetProperty property = new DataSetProperty(entry.getKey(), type);
// JSON数值只有NUMBER类型
if (DataSetProperty.DataType.isInteger(property.getType())
|| DataSetProperty.DataType.isDecimal(property.getType()))
property.setType(DataSetProperty.DataType.NUMBER);
properties.add(property);
}
}
return properties;
}
}

21
datagear-analysis/src/main/java/org/datagear/analysis/support/JsonValueDataSet.java
View File

@ -7,12 +7,13 @@
*/
package org.datagear.analysis.support;
import java.io.Reader;
import java.util.List;
import java.util.Map;
import org.datagear.analysis.DataSetException;
import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetResult;
import org.datagear.util.IOUtil;
/**
* JSON字符串值数据集。
@ -55,25 +56,15 @@ public class JsonValueDataSet extends AbstractJsonDataSet
}
@Override
public DataSetResult getResult(Map<String, ?> paramValues) throws DataSetException
public TemplateResolvedDataSetResult resolve(Map<String, ?> paramValues) throws DataSetException
{
String json = resolveTemplate(this.value, paramValues);
Object data = getJsonDataSetSupport().resolveResultData(json);
return new DataSetResult(data);
return (TemplateResolvedDataSetResult) resolveResult(paramValues, null);
}
@Override
public TemplateResolvedDataSetResult resolve(Map<String, ?> paramValues) throws DataSetException
protected TemplateResolvedSource<Reader> getJsonReader(Map<String, ?> paramValues) throws Throwable
{
String json = resolveTemplate(this.value, paramValues);
Object data = getJsonDataSetSupport().resolveResultData(json);
DataSetResult result = new DataSetResult(data);
List<DataSetProperty> properties = getJsonDataSetSupport().resolveDataSetProperties(result.getData());
return new TemplateResolvedDataSetResult(result, properties, json);
return new TemplateResolvedSource<Reader>(IOUtil.getReader(json), json);
}
}

62
datagear-analysis/src/main/java/org/datagear/analysis/support/TemplateResolvedSource.java Normal file
View File

@ -0,0 +1,62 @@
/*
* Copyright 2018 datagear.tech. All Rights Reserved.
*/
package org.datagear.analysis.support;
/**
* 模板已解析的源。
*
* @author datagear@163.com
*
*/
public class TemplateResolvedSource<T>
{
private T source;
/** 已解析的模板内容 */
private String resolvedTemplate = null;
public TemplateResolvedSource()
{
super();
}
public TemplateResolvedSource(T source)
{
super();
this.source = source;
}
public TemplateResolvedSource(T source, String resolvedTemplate)
{
super();
this.source = source;
this.resolvedTemplate = resolvedTemplate;
}
public T getSource()
{
return source;
}
public void setSource(T source)
{
this.source = source;
}
public boolean hasResolvedTemplate()
{
return (this.resolvedTemplate != null && !this.resolvedTemplate.isEmpty());
}
public String getResolvedTemplate()
{
return resolvedTemplate;
}
public void setResolvedTemplate(String resolvedTemplate)
{
this.resolvedTemplate = resolvedTemplate;
}
}

154
datagear-analysis/src/test/java/org/datagear/analysis/support/CsvValueDataSetTest.java Normal file
View File

@ -0,0 +1,154 @@
/*
* Copyright 2018 datagear.tech. All Rights Reserved.
*/
package org.datagear.analysis.support;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.datagear.analysis.DataSetParam;
import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetResult;
import org.junit.Test;
/**
* {@linkplain CsvValueDataSet}单元测试类。
*
* @author datagear@163.com
*
*/
public class CsvValueDataSetTest
{
@Test
public void getResultTest_hasParam()
{
List<DataSetProperty> properties = new ArrayList<>();
properties.add(new DataSetProperty("name", DataSetProperty.DataType.STRING));
properties.add(new DataSetProperty("value", DataSetProperty.DataType.NUMBER));
properties.add(new DataSetProperty("size", DataSetProperty.DataType.NUMBER));
List<DataSetParam> params = new ArrayList<DataSetParam>();
params.add(new DataSetParam("size", DataSetParam.DataType.NUMBER, true));
CsvValueDataSet dataSet = new CsvValueDataSet("a", "a", properties,
"name, value, size \n aaa, 11, ${size}");
dataSet.setParams(params);
dataSet.setNameRow(1);
Map<String, Object> paramValues = new HashMap<>();
paramValues.put("size", 12);
DataSetResult result = dataSet.getResult(paramValues);
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getData();
{
assertEquals(1, data.size());
{
Map<String, Object> row = data.get(0);
assertEquals("aaa", row.get("name"));
assertEquals(11, ((Number) row.get("value")).intValue());
assertEquals(12, ((Number) row.get("size")).intValue());
}
}
}
@Test
public void getResultTest_hasParam_convertPropertyValue()
{
List<DataSetProperty> properties = new ArrayList<>();
properties.add(new DataSetProperty("name", DataSetProperty.DataType.STRING));
properties.add(new DataSetProperty("value", DataSetProperty.DataType.NUMBER));
properties.add(new DataSetProperty("size", DataSetProperty.DataType.STRING));
List<DataSetParam> params = new ArrayList<DataSetParam>();
params.add(new DataSetParam("size", DataSetParam.DataType.NUMBER, true));
CsvValueDataSet dataSet = new CsvValueDataSet("a", "a", properties,
"name, value, size \n aaa, 11, ${size}");
dataSet.setParams(params);
dataSet.setNameRow(1);
Map<String, Object> paramValues = new HashMap<>();
paramValues.put("size", 12);
DataSetResult result = dataSet.getResult(paramValues);
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getData();
{
assertEquals(1, data.size());
{
Map<String, Object> row = data.get(0);
assertEquals("aaa", row.get("name"));
assertEquals(11, ((Number) row.get("value")).intValue());
assertEquals("12", row.get("size"));
}
}
}
@Test
public void resolveTest_hasParam()
{
List<DataSetParam> params = new ArrayList<DataSetParam>();
params.add(new DataSetParam("size", DataSetParam.DataType.NUMBER, true));
CsvValueDataSet dataSet = new CsvValueDataSet("a", "a",
"name, value, size \n aaa, 11, ${size}");
dataSet.setParams(params);
dataSet.setNameRow(1);
Map<String, Object> paramValues = new HashMap<>();
paramValues.put("size", 12);
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();
assertEquals("name, value, size \n aaa, 11, 12", result.getTemplateResult());
{
assertEquals(3, properties.size());
{
DataSetProperty property = properties.get(0);
assertEquals("name", property.getName());
assertEquals(DataSetProperty.DataType.STRING, property.getType());
}
{
DataSetProperty property = properties.get(1);
assertEquals("value", property.getName());
assertEquals(DataSetProperty.DataType.NUMBER, property.getType());
}
{
DataSetProperty property = properties.get(2);
assertEquals("size", property.getName());
assertEquals(DataSetProperty.DataType.NUMBER, property.getType());
}
}
{
assertEquals(1, data.size());
{
Map<String, Object> row = data.get(0);
assertEquals("aaa", row.get("name"));
assertEquals(11, ((Number) row.get("value")).intValue());
assertEquals(12, ((Number) row.get("size")).intValue());
}
}
}
}

38
datagear-analysis/src/test/java/org/datagear/analysis/support/JsonDataSetSupportTest.java
View File

@ -1,38 +0,0 @@
/*
* Copyright (c) 2018 datagear.tech. All Rights Reserved.
*/
/**
*
*/
package org.datagear.analysis.support;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
/**
* {@linkplain JsonDataSetSupport}单元测试类。
*
* @author datagear@163.com
*
*/
public class JsonDataSetSupportTest
{
private JsonDataSetSupport jsonDataSetSupport = new JsonDataSetSupport();
@Test
public void resolveResultDataTest_String()
{
Object data = jsonDataSetSupport.resolveResultData("{name:'a', value: 3}");
Assert.assertTrue(data instanceof Map);
@SuppressWarnings("unchecked")
Map<String, Object> map = (Map<String, Object>) data;
Assert.assertEquals("a", map.get("name"));
Assert.assertEquals(3, ((Number) map.get("value")).intValue());
}
}

203
datagear-analysis/src/test/java/org/datagear/analysis/support/JsonDirectoryFileDataSetTest.java Normal file
View File

@ -0,0 +1,203 @@
/*
* Copyright 2018 datagear.tech. All Rights Reserved.
*/
package org.datagear.analysis.support;
import static org.junit.Assert.assertEquals;
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetResult;
import org.datagear.analysis.ResolvedDataSetResult;
import org.junit.Test;
/**
* {@linkplain JsonDirectoryFileDataSet}单元测试类。
*
* @author datagear@163.com
*
*/
public class JsonDirectoryFileDataSetTest
{
private static final File DIRECTORY = new File("src/test/resources/org/datagear/analysis/support/");
@Test
public void getResultTest()
{
List<DataSetProperty> properties = new ArrayList<>();
properties.add(new DataSetProperty("name", DataSetProperty.DataType.STRING));
properties.add(new DataSetProperty("value", DataSetProperty.DataType.NUMBER));
properties.add(new DataSetProperty("尺寸", DataSetProperty.DataType.NUMBER));
properties.add(new DataSetProperty("date", DataSetProperty.DataType.STRING));
JsonDirectoryFileDataSet dataSet = new JsonDirectoryFileDataSet("a", "a", properties, DIRECTORY,
"JsonDirectoryFileDataSetTest-0.json");
@SuppressWarnings("unchecked")
DataSetResult result = dataSet.getResult(Collections.EMPTY_MAP);
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getData();
{
assertEquals(3, data.size());
{
Map<String, Object> row = data.get(0);
assertEquals("aaa", row.get("name"));
assertEquals(11, ((Number) row.get("value")).intValue());
assertEquals(12, ((Number) row.get("尺寸")).intValue());
assertEquals("2020-08-01", row.get("date"));
}
{
Map<String, Object> row = data.get(1);
assertEquals("bbb", row.get("name"));
assertEquals(21, ((Number) row.get("value")).intValue());
assertEquals(22, ((Number) row.get("尺寸")).intValue());
assertEquals("2020-08-02", row.get("date"));
}
{
Map<String, Object> row = data.get(2);
assertEquals("ccc", row.get("name"));
assertEquals(31, ((Number) row.get("value")).intValue());
assertEquals(32, ((Number) row.get("尺寸")).intValue());
assertEquals("2020-08-03", row.get("date"));
}
}
}
@Test
public void getResultTest_convertPropertyValue()
{
List<DataSetProperty> properties = new ArrayList<>();
properties.add(new DataSetProperty("name", DataSetProperty.DataType.STRING));
properties.add(new DataSetProperty("value", DataSetProperty.DataType.NUMBER));
properties.add(new DataSetProperty("尺寸", DataSetProperty.DataType.NUMBER));
properties.add(new DataSetProperty("date", DataSetProperty.DataType.DATE));
JsonDirectoryFileDataSet dataSet = new JsonDirectoryFileDataSet("a", "a", properties, DIRECTORY,
"JsonDirectoryFileDataSetTest-0.json");
@SuppressWarnings("unchecked")
DataSetResult result = dataSet.getResult(Collections.EMPTY_MAP);
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getData();
{
assertEquals(3, data.size());
SimpleDateFormat dateFormat = new SimpleDateFormat(DataFormat.DEFAULT_DATE_FORMAT);
{
Map<String, Object> row = data.get(0);
assertEquals("aaa", row.get("name"));
assertEquals(11, ((Number) row.get("value")).intValue());
assertEquals(12, ((Number) row.get("尺寸")).intValue());
assertEquals("2020-08-01", dateFormat.format(((Date) row.get("date"))));
}
{
Map<String, Object> row = data.get(1);
assertEquals("bbb", row.get("name"));
assertEquals(21, ((Number) row.get("value")).intValue());
assertEquals(22, ((Number) row.get("尺寸")).intValue());
assertEquals("2020-08-02", dateFormat.format(((Date) row.get("date"))));
}
{
Map<String, Object> row = data.get(2);
assertEquals("ccc", row.get("name"));
assertEquals(31, ((Number) row.get("value")).intValue());
assertEquals(32, ((Number) row.get("尺寸")).intValue());
assertEquals("2020-08-03", dateFormat.format(((Date) row.get("date"))));
}
}
}
@Test
public void resolveTest()
{
JsonDirectoryFileDataSet dataSet = new JsonDirectoryFileDataSet("a", "a", DIRECTORY,
"JsonDirectoryFileDataSetTest-0.json");
@SuppressWarnings("unchecked")
ResolvedDataSetResult result = dataSet.resolve(Collections.EMPTY_MAP);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();
{
assertEquals(4, properties.size());
{
DataSetProperty property = properties.get(0);
assertEquals("name", property.getName());
assertEquals(DataSetProperty.DataType.STRING, property.getType());
}
{
DataSetProperty property = properties.get(1);
assertEquals("value", property.getName());
assertEquals(DataSetProperty.DataType.NUMBER, property.getType());
}
{
DataSetProperty property = properties.get(2);
assertEquals("尺寸", property.getName());
assertEquals(DataSetProperty.DataType.NUMBER, property.getType());
}
{
DataSetProperty property = properties.get(3);
assertEquals("date", property.getName());
assertEquals(DataSetProperty.DataType.STRING, property.getType());
}
}
{
assertEquals(3, data.size());
{
Map<String, Object> row = data.get(0);
assertEquals("aaa", row.get("name"));
assertEquals(11, ((Number) row.get("value")).intValue());
assertEquals(12, ((Number) row.get("尺寸")).intValue());
assertEquals("2020-08-01", row.get("date"));
}
{
Map<String, Object> row = data.get(1);
assertEquals("bbb", row.get("name"));
assertEquals(21, ((Number) row.get("value")).intValue());
assertEquals(22, ((Number) row.get("尺寸")).intValue());
assertEquals("2020-08-02", row.get("date"));
}
{
Map<String, Object> row = data.get(2);
assertEquals("ccc", row.get("name"));
assertEquals(31, ((Number) row.get("value")).intValue());
assertEquals(32, ((Number) row.get("尺寸")).intValue());
assertEquals("2020-08-03", row.get("date"));
}
}
}
}

151
datagear-analysis/src/test/java/org/datagear/analysis/support/JsonValueDataSetTest.java Normal file
View File

@ -0,0 +1,151 @@
/*
* Copyright 2018 datagear.tech. All Rights Reserved.
*/
package org.datagear.analysis.support;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.datagear.analysis.DataSetParam;
import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetResult;
import org.junit.Test;
/**
* {@linkplain JsonValueDataSet}单元测试类。
*
* @author datagear@163.com
*
*/
public class JsonValueDataSetTest
{
@Test
public void getResultTest_hasParam()
{
List<DataSetProperty> properties = new ArrayList<>();
properties.add(new DataSetProperty("name", DataSetProperty.DataType.STRING));
properties.add(new DataSetProperty("value", DataSetProperty.DataType.NUMBER));
properties.add(new DataSetProperty("size", DataSetProperty.DataType.NUMBER));
List<DataSetParam> params = new ArrayList<DataSetParam>();
params.add(new DataSetParam("size", DataSetParam.DataType.NUMBER, true));
JsonValueDataSet dataSet = new JsonValueDataSet("a", "a", properties,
"[ { name:'aaa', value: 11, size: ${size} } ]");
dataSet.setParams(params);
Map<String, Object> paramValues = new HashMap<>();
paramValues.put("size", 12);
DataSetResult result = dataSet.getResult(paramValues);
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getData();
{
assertEquals(1, data.size());
{
Map<String, Object> row = data.get(0);
assertEquals("aaa", row.get("name"));
assertEquals(11, ((Number) row.get("value")).intValue());
assertEquals(12, ((Number) row.get("size")).intValue());
}
}
}
@Test
public void getResultTest_hasParam_convertPropertyValue()
{
List<DataSetProperty> properties = new ArrayList<>();
properties.add(new DataSetProperty("name", DataSetProperty.DataType.STRING));
properties.add(new DataSetProperty("value", DataSetProperty.DataType.NUMBER));
properties.add(new DataSetProperty("size", DataSetProperty.DataType.STRING));
List<DataSetParam> params = new ArrayList<DataSetParam>();
params.add(new DataSetParam("size", DataSetParam.DataType.NUMBER, true));
JsonValueDataSet dataSet = new JsonValueDataSet("a", "a", properties,
"[ { name:'aaa', value: 11, size: ${size} } ]");
dataSet.setParams(params);
Map<String, Object> paramValues = new HashMap<>();
paramValues.put("size", 12);
DataSetResult result = dataSet.getResult(paramValues);
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getData();
{
assertEquals(1, data.size());
{
Map<String, Object> row = data.get(0);
assertEquals("aaa", row.get("name"));
assertEquals(11, ((Number) row.get("value")).intValue());
assertEquals("12", row.get("size"));
}
}
}
@Test
public void resolveTest_hasParam()
{
List<DataSetParam> params = new ArrayList<DataSetParam>();
params.add(new DataSetParam("size", DataSetParam.DataType.NUMBER, true));
JsonValueDataSet dataSet = new JsonValueDataSet("a", "a",
"[ { name:'aaa', value: 11, size: ${size} } ]");
dataSet.setParams(params);
Map<String, Object> paramValues = new HashMap<>();
paramValues.put("size", 12);
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();
assertEquals("[ { name:'aaa', value: 11, size: 12 } ]", result.getTemplateResult());
{
assertEquals(3, properties.size());
{
DataSetProperty property = properties.get(0);
assertEquals("name", property.getName());
assertEquals(DataSetProperty.DataType.STRING, property.getType());
}
{
DataSetProperty property = properties.get(1);
assertEquals("value", property.getName());
assertEquals(DataSetProperty.DataType.NUMBER, property.getType());
}
{
DataSetProperty property = properties.get(2);
assertEquals("size", property.getName());
assertEquals(DataSetProperty.DataType.NUMBER, property.getType());
}
}
{
assertEquals(1, data.size());
{
Map<String, Object> row = data.get(0);
assertEquals("aaa", row.get("name"));
assertEquals(11, ((Number) row.get("value")).intValue());
assertEquals(12, ((Number) row.get("size")).intValue());
}
}
}
}

20
datagear-analysis/src/test/resources/org/datagear/analysis/support/JsonDirectoryFileDataSetTest-0.json Normal file
View File

@ -0,0 +1,20 @@
[
{
name: "aaa",
value: 11,
"尺寸": 12,
date: "2020-08-01"
},
{
name: "bbb",
value: 21,
"尺寸": 22,
date: "2020-08-02"
},
{
name: "ccc",
value: 31,
"尺寸": 32,
date: "2020-08-03"
}
]