p85126437
/
datagear
forked from p81075629/datagear
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

重构数据集实现类，支持属性定义与结果数据无需一一对应

This commit is contained in:
datagear 2021-03-27 18:48:48 +08:00
parent 5b56b12945
commit 516134efe8
4 changed files with 301 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractDataSet.java
View File

@ -9,6 +9,7 @@ package org.datagear.analysis.support;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -147,33 +148,77 @@ public abstract class AbstractDataSet extends AbstractIdentifiable implements Da
*
* @param cn
* @param rs
* @param rawData {@code Collection<Map<String, ?>>}、{@code Map<String, ?>[]}、{@code Map<String, ?>}、{@code null}
* @param properties
* @param dataSetOption 允许为{@code null}
* @return {@code List<Map<String, ?>>}、{@code Map<String, ?>[]}、{@code Map<String, ?>}、{@code null}
* @throws Throwable
*/
@SuppressWarnings({ "unchecked", "rawtypes" })
protected ResolvedDataSetResult resolveResult(Object rawData, List<DataSetProperty> properties) throws Throwable
{
Object data = null;
if (rawData == null)
{
}
else if (rawData instanceof Collection<?>)
{
Collection<Map<String, ?>> rawCollection = (Collection<Map<String, ?>>) rawData;
data = convertRawData(rawCollection, properties);
}
else if (rawData instanceof Map<?, ?>[])
{
Map<?, ?>[] rawArray = (Map<?, ?>[]) rawData;
List<Map<String, ?>> rawCollection = (List) Arrays.asList(rawArray);
List<Map<String, Object>> dataList = convertRawData(rawCollection, properties);
data = dataList.toArray(new Map<?, ?>[dataList.size()]);
}
else if (rawData instanceof Map<?, ?>)
{
Map<?, ?> rawMap = (Map<?, ?>) rawData;
List<Map<String, ?>> rawCollection = (List) Arrays.asList(rawMap);
List<Map<String, Object>> dataList = convertRawData(rawCollection, properties);
data = dataList.get(0);
}
else
throw new UnsupportedOperationException(
"Unsupported raw data type : " + rawData.getClass().getSimpleName());
return new ResolvedDataSetResult(new DataSetResult(data), properties);
}
/**
* 转换原始数据。
*
* @param rawData
* @param properties
* @param dataSetOption
* 允许为{@code null}
* @return
* @throws Throwable
*/
protected ResolvedDataSetResult resolveResult(List<? extends Map<String, ?>> rawData,
List<DataSetProperty> properties)
throws Throwable
protected List<Map<String, Object>> convertRawData(Collection<? extends Map<String, ?>> rawData,
List<DataSetProperty> properties) throws Throwable
{
DataSetPropertyValueConverter converter = createDataSetPropertyValueConverter();
List<Map<String, ?>> data = new ArrayList<>(rawData.size());
List<Map<String, Object>> data = new ArrayList<>(rawData.size());
for (int i = 0, len = rawData.size(), plen = properties.size(); i < len; i++)
int plen = properties.size();
for (Map<String, ?> rowRaw : rawData)
{
// 应当仅保留数据集属性对应的数据
Map<String, Object> row = new HashMap<>();
Map<String, ?> rowRaw = rawData.get(i);
for (int j = 0; j < plen; j++)
{
DataSetProperty property = properties.get(j);
String name = property.getName();
String name = property.getName();
Object value = rowRaw.get(name);
value = convertToPropertyDataType(converter, value, property);
@ -183,7 +228,7 @@ public abstract class AbstractDataSet extends AbstractIdentifiable implements Da
data.add(row);
}
return new ResolvedDataSetResult(new DataSetResult(data), properties);
return data;
}
/**

342
datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractExcelDataSet.java
View File

@ -10,6 +10,7 @@ package org.datagear.analysis.support;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -27,7 +28,6 @@ import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.datagear.analysis.DataSetException;
import org.datagear.analysis.DataSetOption;
import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetResult;
import org.datagear.analysis.ResolvableDataSet;
import org.datagear.analysis.ResolvedDataSetResult;
import org.datagear.analysis.support.RangeExpResolver.IndexRange;
@ -341,218 +341,208 @@ public abstract class AbstractExcelDataSet extends AbstractResolvableDataSet imp
*
* @param paramValues
* @param sheet
* @param properties
* 允许为{@code null}，此时会自动解析
* @param dataSetOption
* 允许为{@code null}
* @param properties 允许为{@code null}，此时会自动解析
* @param dataSetOption 允许为{@code null}
* @return
* @throws DataSetException
* @throws Throwable
*/
protected ResolvedDataSetResult resolveResultForSheet(Map<String, ?> paramValues, Sheet sheet,
List<DataSetProperty> properties, DataSetOption dataSetOption) throws DataSetException
List<DataSetProperty> properties, DataSetOption dataSetOption) throws Throwable
{
boolean resolveProperties = (properties == null || properties.isEmpty());
List<Row> excelRows = new ArrayList<Row>();
if (resolveProperties)
properties = new ArrayList<>();
for (Row row : sheet)
excelRows.add(row);
List<List<Object>> data = new ArrayList<>();
List<String> rawDataPropertyNames = resolvePropertyNames(excelRows);
List<Map<String, Object>> rawData = resolveRawData(rawDataPropertyNames, excelRows, dataSetOption);
if (properties == null || properties.isEmpty())
properties = resolveProperties(rawDataPropertyNames, rawData);
return resolveResult(rawData, properties);
}
/**
* 解析数据属性名列表。
*
* @param excelRows
* @return
* @throws Throwable
*/
@SuppressWarnings("unused")
protected List<String> resolvePropertyNames(List<Row> excelRows) throws Throwable
{
List<String> propertyNames = null;
DataSetPropertyValueConverter converter = createDataSetPropertyValueConverter();
try
for (int i = 0, len = excelRows.size(); i < len; i++)
{
int rowIdx = 0;
int dataRowIdx = 0;
Row row = excelRows.get(i);
for (Row row : sheet)
if (isNameRow(i))
{
if (isNameRow(rowIdx))
{
if (resolveProperties)
propertyNames = resolveDataSetPropertyNames(row, false);
}
else if (isDataRow(rowIdx))
{
if (resolveProperties && dataRowIdx == 0 && propertyNames == null)
propertyNames = resolveDataSetPropertyNames(row, true);
propertyNames = new ArrayList<String>();
// 名称行不一定在数据行之前，此时可能还无法确定属性名，所以暂时采用列表存储
List<Object> rowObj = new ArrayList<>();
int colIdx = 0;
for (Cell cell : row)
{
if (isDataColumn(colIdx))
{
String name = null;
try
{
name = cell.getStringCellValue();
}
catch(Throwable t)
{
}
if (StringUtil.isEmpty(name))
name = CellReference.convertNumToColString(colIdx);
propertyNames.add(name);
}
colIdx++;
}
break;
}
else if (isDataRow(i))
{
if (propertyNames == null)
{
propertyNames = new ArrayList<String>();
int colIdx = 0;
int dataColIdx = 0;
for (Cell cell : row)
{
if (isDataColumn(colIdx))
{
DataSetProperty property = null;
if (!resolveProperties)
{
if (dataColIdx >= properties.size())
throw new DataSetSourceParseException(
"No property defined for column index " + dataColIdx);
property = properties.get(dataColIdx);
}
Object value = resolvePropertyValue(cell, property, converter);
if (resolveProperties)
{
property = resolveDataSetProperty(row, rowIdx, dataRowIdx, cell, colIdx, dataColIdx,
value, properties);
}
rowObj.add(value);
dataColIdx++;
String name = CellReference.convertNumToColString(colIdx);
propertyNames.add(name);
}
colIdx++;
}
boolean reachMaxCount = isReachResultDataMaxCount(dataSetOption, data.size());
boolean breakLoop = (reachMaxCount && (!resolveProperties || isAfterNameRow(rowIdx)));
if (!reachMaxCount)
data.add(rowObj);
dataRowIdx++;
if (breakLoop)
break;
}
rowIdx++;
if (isAfterNameRow(i))
break;
}
}
catch (DataSetException e)
{
throw e;
}
catch (Throwable t)
{
throw new DataSetSourceParseException(t);
}
if (resolveProperties)
inflateDataSetProperties(properties, propertyNames);
DataSetResult result = new DataSetResult(listRowsToMapRows(data, properties));
return new ResolvedDataSetResult(result, properties);
}
protected void inflateDataSetProperties(List<DataSetProperty> properties, List<String> propertyNames)
{
for (int i = 0; i < properties.size(); i++)
{
DataSetProperty property = properties.get(i);
property.setName(propertyNames.get(i));
if (StringUtil.isEmpty(property.getType()))
property.setType(DataSetProperty.DataType.UNKNOWN);
}
}
/**
* 解析{@linkplain DataSetProperty}并写入{@code properties}。
*
* @param row
* @param rowIdx
* @param dataRowIdx
* @param cell
* @param colIdx
* @param dataColIdx
* @param cellValue
* @param properties
* @return
*/
protected DataSetProperty resolveDataSetProperty(Row row, int rowIdx, int dataRowIdx, Cell cell, int colIdx,
int dataColIdx, Object cellValue, List<DataSetProperty> properties)
{
DataSetProperty property = null;
if (dataRowIdx == 0)
{
// 空单元格先不处理数据类型，等待后续有非空单元格再判断
String dataType = (cellValue == null ? "" : resolvePropertyDataType(cellValue));
// 名称行不一定在数据行之前，所以此时可能无法确定属性名
property = new DataSetProperty("should-be-set-later", dataType);
properties.add(property);
}
else
{
property = properties.get(dataColIdx);
if (StringUtil.isEmpty(property.getType()) && cellValue != null)
property.setType(resolvePropertyDataType(cellValue));
}
return property;
}
/**
* 解析属性名。
*
* @param nameRow
* @return
*/
protected List<String> resolveDataSetPropertyNames(Row nameRow, boolean forceColumnString)
{
List<String> propertyNames = new ArrayList<>();
int colIdx = 0;
for (Cell cell : nameRow)
{
if (isDataColumn(colIdx))
{
String name = null;
if (forceColumnString)
name = CellReference.convertNumToColString(colIdx);
else
{
try
{
name = cell.getStringCellValue();
}
catch (Throwable t)
{
}
if (StringUtil.isEmpty(name))
name = CellReference.convertNumToColString(colIdx);
}
propertyNames.add(name);
}
colIdx++;
}
if (propertyNames == null)
propertyNames = Collections.emptyList();
return propertyNames;
}
/**
* 解析{@linkplain DataSetProperty}。
*
* @param rawDataPropertyNames
* @param rawData 允许为{@code null}
* @return
* @throws Throwable
*/
protected List<DataSetProperty> resolveProperties(List<String> rawDataPropertyNames,
List<Map<String, Object>> rawData) throws Throwable
{
int propertyLen = rawDataPropertyNames.size();
List<DataSetProperty> properties = new ArrayList<>(propertyLen);
for (String name : rawDataPropertyNames)
properties.add(new DataSetProperty(name, DataSetProperty.DataType.UNKNOWN));
if (rawData != null && rawData.size() > 0)
{
for (Map<String, Object> row : rawData)
{
int resolvedPropertyTypeCount = 0;
for (int i = 0; i < propertyLen; i++)
{
DataSetProperty property = properties.get(i);
if (!DataSetProperty.DataType.UNKNOWN.equals(property.getType()))
{
resolvedPropertyTypeCount++;
continue;
}
Object value = row.get(rawDataPropertyNames.get(i));
if (value != null)
property.setType(resolvePropertyDataType(value));
}
if (resolvedPropertyTypeCount == propertyLen)
break;
}
}
return properties;
}
/**
* 解析原始数据。
*
* @param propertyNames
* @param excelRows
* @param dataSetOption
* @return
* @throws Throwable
*/
protected List<Map<String, Object>> resolveRawData(List<String> propertyNames, List<Row> excelRows,
DataSetOption dataSetOption) throws Throwable
{
List<Map<String, Object>> data = new ArrayList<>();
for (int i = 0, len = excelRows.size(); i < len; i++)
{
if (isNameRow(i) || !isDataRow(i))
continue;
if (isReachResultDataMaxCount(dataSetOption, data.size()))
break;
Map<String, Object> row = new HashMap<>();
Row excelRow = excelRows.get(i);
int colIdx = 0;
int dataColIdx = 0;
for (Cell cell : excelRow)
{
if (isDataColumn(colIdx))
{
String name = propertyNames.get(dataColIdx);
Object value = resolveCellValue(cell);
row.put(name, value);
dataColIdx++;
}
colIdx++;
}
data.add(row);
}
return data;
}
/**
* 解析单元格属性值。
*
* @param cell
* @param property
* 允许为{@code null}
* @param converter
* @return
* @throws DataSetSourceParseException
* @throws DataSetException
*/
protected Object resolvePropertyValue(Cell cell, DataSetProperty property, DataSetPropertyValueConverter converter)
throws DataSetSourceParseException, DataSetException
protected Object resolveCellValue(Cell cell) throws DataSetSourceParseException, DataSetException
{
CellType cellType = cell.getCellTypeEnum();
@ -588,17 +578,15 @@ public abstract class AbstractExcelDataSet extends AbstractResolvableDataSet imp
cellValue = cell.getStringCellValue();
}
}
catch (DataSetException e)
catch(DataSetException e)
{
throw e;
}
catch (Throwable t)
catch(Throwable t)
{
throw new DataSetSourceParseException(t);
}
cellValue = convertToPropertyDataType(converter, cellValue, property);
return cellValue;
}

184
datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractJsonDataSet.java
View File

@ -9,8 +9,8 @@ package org.datagear.analysis.support;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -154,40 +154,30 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl
protected ResolvedDataSetResult resolveResult(Reader jsonReader, List<DataSetProperty> properties,
DataSetOption dataSetOption) throws Throwable
{
boolean resolveProperties = (properties == null || properties.isEmpty());
JsonNode jsonNode = getObjectMapperNonStardand().readTree(jsonReader);
if (!isLegalResultDataJsonNode(jsonNode))
throw new UnsupportedJsonResultDataException("Result data must be JSON object or array");
Object data = null;
Object rawData = resolveRawData(jsonNode, getDataJsonPath(), dataSetOption);
if (jsonNode != null)
data = readDataByJsonPath(jsonNode, getDataJsonPath());
if (properties == null || properties.isEmpty())
properties = resolveProperties(rawData);
if (resolveProperties)
properties = resolveDataSetProperties(data);
data = convertJsonResultData(data, properties, dataSetOption, createDataSetPropertyValueConverter());
DataSetResult result = new DataSetResult(data);
return new ResolvedDataSetResult(result, properties);
return resolveResult(rawData, properties);
}
/**
* 读取指定JSON路径的数据。
* 解析原始数据数据。
*
* @param jsonNode
* 允许为{@code null}
* @param dataJsonPath
* 允许为{@code null}
* @param jsonNode 允许为{@code null}
* @param dataJsonPath 允许为{@code null}
* @param dataSetOption 允许为{@code null}
* @return
* @throws ReadJsonDataPathException
* @throws Throwable
*/
protected Object readDataByJsonPath(JsonNode jsonNode, String dataJsonPath)
protected Object resolveRawData(JsonNode jsonNode, String dataJsonPath, DataSetOption dataSetOption)
throws ReadJsonDataPathException, Throwable
{
if (jsonNode == null)
@ -195,115 +185,64 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl
Object data = getObjectMapperNonStardand().treeToValue(jsonNode, Object.class);
if (data == null)
return null;
if (StringUtil.isEmpty(dataJsonPath))
return data;
String stdDataJsonPath = dataJsonPath.trim();
if (StringUtil.isEmpty(stdDataJsonPath))
return data;
// 转换"stores[0].books"、"[1].stores"简化模式为规范的JSONPath
if (!stdDataJsonPath.startsWith("$"))
if (data != null && !StringUtil.isEmpty(dataJsonPath))
{
if (stdDataJsonPath.startsWith("["))
stdDataJsonPath = "$" + stdDataJsonPath;
else
stdDataJsonPath = "$." + stdDataJsonPath;
}
String stdDataJsonPath = dataJsonPath.trim();
try
{
return JsonPath.compile(stdDataJsonPath).read(data, JACKSON_JSON_PATH_CONFIGURATION);
}
catch (Throwable t)
{
throw new ReadJsonDataPathException(dataJsonPath, t);
}
}
/**
*
* @param resultData
* 允许为{@code null}
* @param properties
* @param dataSetOption
* 允许为{@code null}
* @param converter
* @return
* @throws Throwable
*/
protected Object convertJsonResultData(Object resultData, List<DataSetProperty> properties,
DataSetOption dataSetOption, DataSetPropertyValueConverter converter) throws Throwable
{
Object re = null;
// JSON对象
if (resultData == null)
{
re = null;
}
else if (resultData instanceof Map<?, ?>)
{
Map<String, Object> reMap = new HashMap<>();
@SuppressWarnings("unchecked")
Map<String, Object> source = (Map<String, Object>) resultData;
for (Map.Entry<String, Object> entry : source.entrySet())
if (!StringUtil.isEmpty(stdDataJsonPath))
{
String name = entry.getKey();
Object value = entry.getValue();
// 转换"stores[0].books"、"[1].stores"简化模式为规范的JSONPath
if (!stdDataJsonPath.startsWith("$"))
{
if (stdDataJsonPath.startsWith("["))
stdDataJsonPath = "$" + stdDataJsonPath;
else
stdDataJsonPath = "$." + stdDataJsonPath;
}
DataSetProperty property = getDataNameTypeByName(properties, name);
value = convertToPropertyDataType(converter, value, property);
reMap.put(name, value);
try
{
data = JsonPath.compile(stdDataJsonPath).read(data, JACKSON_JSON_PATH_CONFIGURATION);
}
catch(Throwable t)
{
throw new ReadJsonDataPathException(dataJsonPath, t);
}
}
re = reMap;
}
else if (resultData instanceof List<?>)
if (data != null && hasResultDataMaxCount(dataSetOption))
{
List<?> list = (List<?>) resultData;
List<Object> reList = new ArrayList<>(list.size());
for (int i = 0; i < list.size(); i++)
if (data instanceof Collection<?>)
{
if (isReachResultDataMaxCount(dataSetOption, reList.size()))
break;
Collection<?> collection = (List<?>) data;
List<Object> dataList = new ArrayList<>();
Object ele = list.get(i);
reList.add(convertJsonResultData(ele, properties, null, converter));
for (Object ele : collection)
{
if (isReachResultDataMaxCount(dataSetOption, dataList.size()))
break;
dataList.add(ele);
}
data = dataList;
}
re = reList;
}
else if (resultData instanceof Object[])
{
Object[] array = (Object[]) resultData;
Object[] reArray = new Object[evalResultDataCount(dataSetOption, array.length)];
for (int i = 0; i < array.length; i++)
else if (data instanceof Object[])
{
if (isReachResultDataMaxCount(dataSetOption, i + 1))
break;
Object[] array = (Object[]) data;
Object[] dataArray = new Object[evalResultDataCount(dataSetOption, array.length)];
reArray[i] = convertJsonResultData(array[i], properties, null, converter);
for (int i = 0; i < dataArray.length; i++)
{
dataArray[i] = array[i];
}
data = dataArray;
}
re = reArray;
}
else
throw new UnsupportedJsonResultDataException("Result data must be object or object array/list");
return re;
return data;
}
/**
@ -344,15 +283,14 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl
}
/**
* 解析JSON对象的{@linkplain DataSetProperty}。
* 解析{@linkplain DataSetProperty}。
*
* @param resultData
* 允许为{@code null}，JSON对象、JSON对象数组、JSON对象列表
* @param resultData 允许为{@code null}，JSON对象、JSON对象数组、JSON对象列表
* @return
* @throws Throwable
*/
@SuppressWarnings("unchecked")
protected List<DataSetProperty> resolveDataSetProperties(Object resultData) throws Throwable
protected List<DataSetProperty> resolveProperties(Object resultData) throws Throwable
{
if (resultData == null)
{
@ -360,7 +298,7 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl
}
else if (resultData instanceof Map<?, ?>)
{
return resolveJsonObjDataSetProperties((Map<String, ?>) resultData);
return resolveJsonObjProperties((Map<String, ?>) resultData);
}
else if (resultData instanceof List<?>)
{
@ -369,7 +307,7 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl
if (list.size() == 0)
return Collections.EMPTY_LIST;
else
return resolveJsonObjDataSetProperties((Map<String, ?>) list.get(0));
return resolveJsonObjProperties((Map<String, ?>) list.get(0));
}
else if (resultData instanceof Object[])
{
@ -378,20 +316,20 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl
if (array.length == 0)
return Collections.EMPTY_LIST;
else
return resolveJsonObjDataSetProperties((Map<String, ?>) array[0]);
return resolveJsonObjProperties((Map<String, ?>) array[0]);
}
else
throw new UnsupportedJsonResultDataException("Result data must be object or object array/list");
}
/**
* 解析JSON对象的{@linkplain DataSetProperty}。
* 解析{@linkplain DataSetProperty}。
*
* @param jsonObj
* @return
* @throws Throwable
*/
protected List<DataSetProperty> resolveJsonObjDataSetProperties(Map<String, ?> jsonObj) throws Throwable
protected List<DataSetProperty> resolveJsonObjProperties(Map<String, ?> jsonObj) throws Throwable
{
List<DataSetProperty> properties = new ArrayList<>();

19
datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractResolvableDataSet.java
View File

@ -78,6 +78,25 @@ public abstract class AbstractResolvableDataSet extends AbstractDataSet implemen
protected abstract ResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties,
DataSetOption dataSetOption) throws DataSetException;
/**
* 是否有{@linkplain DataSetOption#getResultDataMaxCount()}。
*
* @param dataSetOption 允许为{@code null}
* @return
*/
protected boolean hasResultDataMaxCount(DataSetOption dataSetOption)
{
if (dataSetOption == null)
return false;
int maxCount = dataSetOption.getResultDataMaxCount();
if (maxCount < 0)
return false;
return true;
}
/**
* 给定数目是否已到达{@linkplain DataSetOption#getResultDataMaxCount()}。
*