p42579631
/
datagear
forked from p85126437/datagear
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

数据集预览功能限定最大返回数据条目数，避免数据量较大时影响页面响应；

This commit is contained in:
datagear 2020-11-17 21:35:10 +08:00
parent 00e3b48c50
commit 14f7d5a123
23 changed files with 358 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Roadmap.txt
View File

@ -4,7 +4,7 @@
ok springboot迁移；
ok 修复看板表单对于留空的输入框未能正确设置图表数据集参数值的BUG；
ok 数据导入/导出参数改为JSON，不然单条时报错；
数据集预览对于大数据量仅显示靠前部分数据；
ok 数据集预览功能限定最大返回数据条目数，避免数据量较大时影响页面响应；
添加更多内置图表插件：极坐标柱状图、嵌套饼图、盒须图、地图热力图、路径图；
共享看板支持设置密码；

40
datagear-analysis/src/main/java/org/datagear/analysis/DataSetOption.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright (c) 2018 datagear.tech. All Rights Reserved.
*/
/**
*
*/
package org.datagear.analysis;
/**
* 数据集选项。
*
* @author datagear@163.com
*
*/
public class DataSetOption
{
/** 结果数据最大返回数目 */
private int resultDataMaxCount = -1;
public DataSetOption()
{
super();
}
/**
* 获取结果数据最大返回数目。
*
* @return {@code <0} 表示不限定数目
*/
public int getResultDataMaxCount()
{
return resultDataMaxCount;
}
public void setResultDataMaxCount(int resultDataMaxCount)
{
this.resultDataMaxCount = resultDataMaxCount;
}
}

4
datagear-analysis/src/main/java/org/datagear/analysis/ResolvableDataSet.java
View File

@ -24,8 +24,10 @@ public interface ResolvableDataSet extends DataSet
* 解析{@linkplain ResolvedDataSetResult}。
*
* @param paramValues
* @param dataSetOption
* 设置选项，允许为{@code null}
* @return
* @throws DataSetException
*/
ResolvedDataSetResult resolve(Map<String, ?> paramValues) throws DataSetException;
ResolvedDataSetResult resolve(Map<String, ?> paramValues, DataSetOption dataSetOption) throws DataSetException;
}

40
datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractCsvDataSet.java
View File

@ -18,6 +18,7 @@ import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.datagear.analysis.DataSetException;
import org.datagear.analysis.DataSetOption;
import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetResult;
import org.datagear.analysis.ResolvableDataSet;
@ -89,8 +90,8 @@ public abstract class AbstractCsvDataSet extends AbstractResolvableDataSet imple
* </p>
*/
@Override
protected ResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties)
throws DataSetException
protected ResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties,
DataSetOption dataSetOption) throws DataSetException
{
TemplateResolvedSource<Reader> reader = null;
@ -98,7 +99,7 @@ public abstract class AbstractCsvDataSet extends AbstractResolvableDataSet imple
{
reader = getCsvReader(paramValues);
ResolvedDataSetResult result = resolveResult(reader.getSource(), properties);
ResolvedDataSetResult result = resolveResult(reader.getSource(), properties, dataSetOption);
if (reader.hasResolvedTemplate())
result = new TemplateResolvedDataSetResult(result.getResult(), result.getProperties(),
@ -134,7 +135,8 @@ public abstract class AbstractCsvDataSet extends AbstractResolvableDataSet imple
protected abstract TemplateResolvedSource<Reader> getCsvReader(Map<String, ?> paramValues) throws Throwable;
@SuppressWarnings({ "unchecked", "rawtypes" })
protected ResolvedDataSetResult resolveResult(Reader csvReader, List<DataSetProperty> properties) throws Throwable
protected ResolvedDataSetResult resolveResult(Reader csvReader, List<DataSetProperty> properties,
DataSetOption dataSetOption) throws Throwable
{
boolean resolveProperties = (properties == null || properties.isEmpty());
@ -160,12 +162,19 @@ public abstract class AbstractCsvDataSet extends AbstractResolvableDataSet imple
if (resolveProperties && dataRowIdx == 0 && propertyNames == null)
propertyNames = resolveDataSetPropertyNames(csvRecord, true);
if (resolveProperties)
data.add(resolveCSVRecordValues(csvRecord, null, converter));
else
data.add(resolveCSVRecordValues(csvRecord, properties, converter));
List<Object> rowObj = (resolveProperties ? resolveCSVRecordValues(csvRecord, null, converter)
: resolveCSVRecordValues(csvRecord, properties, converter));
boolean reachMaxCount = isReachResultDataMaxCount(dataSetOption, data.size());
boolean breakLoop = (reachMaxCount && (!resolveProperties || isAfterNameRow(rowIdx)));
if (!reachMaxCount)
data.add(rowObj);
dataRowIdx++;
if (breakLoop)
break;
}
rowIdx++;
@ -373,6 +382,21 @@ public abstract class AbstractCsvDataSet extends AbstractResolvableDataSet imple
return ((rowIndex + 1) == this.nameRow);
}
/**
* 是否在名称行之后。
* <p>
* 如果没有名称行，应返回{@code true}。
* </p>
*
* @param rowIndex
* 行索引（以{@code 0}计数）
* @return
*/
protected boolean isAfterNameRow(int rowIndex)
{
return ((rowIndex + 1) > this.nameRow);
}
/**
* 构建{@linkplain CSVParser}。
*

49
datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractExcelDataSet.java
View File

@ -25,6 +25,7 @@ import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.datagear.analysis.DataSetException;
import org.datagear.analysis.DataSetOption;
import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetResult;
import org.datagear.analysis.ResolvableDataSet;
@ -223,8 +224,8 @@ public abstract class AbstractExcelDataSet extends AbstractResolvableDataSet imp
}
@Override
protected ResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties)
throws DataSetException
protected ResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties,
DataSetOption dataSetOption) throws DataSetException
{
File file = null;
@ -244,9 +245,9 @@ public abstract class AbstractExcelDataSet extends AbstractResolvableDataSet imp
ResolvedDataSetResult result = null;
if (isXls(file))
result = resolveResultForXls(paramValues, file, properties);
result = resolveResultForXls(paramValues, file, properties, dataSetOption);
else
result = resolveResultForXlsx(paramValues, file, properties);
result = resolveResultForXlsx(paramValues, file, properties, dataSetOption);
return result;
}
@ -258,11 +259,13 @@ public abstract class AbstractExcelDataSet extends AbstractResolvableDataSet imp
* @param file
* @param properties
* 允许为{@code null}，此时会自动解析
* @param dataSetOption
* 允许为{@code null}
* @return
* @throws DataSetException
*/
protected ResolvedDataSetResult resolveResultForXls(Map<String, ?> paramValues, File file,
List<DataSetProperty> properties) throws DataSetException
List<DataSetProperty> properties, DataSetOption dataSetOption) throws DataSetException
{
POIFSFileSystem poifs = null;
HSSFWorkbook wb = null;
@ -274,7 +277,7 @@ public abstract class AbstractExcelDataSet extends AbstractResolvableDataSet imp
Sheet sheet = wb.getSheetAt(getSheetIndex() - 1);
return resolveResultForSheet(paramValues, sheet, properties);
return resolveResultForSheet(paramValues, sheet, properties, dataSetOption);
}
catch (DataSetException e)
{
@ -298,11 +301,13 @@ public abstract class AbstractExcelDataSet extends AbstractResolvableDataSet imp
* @param file
* @param properties
* 允许为{@code null}，此时会自动解析
* @param dataSetOption
* 允许为{@code null}
* @return
* @throws DataSetException
*/
protected ResolvedDataSetResult resolveResultForXlsx(Map<String, ?> paramValues, File file,
List<DataSetProperty> properties) throws DataSetException
List<DataSetProperty> properties, DataSetOption dataSetOption) throws DataSetException
{
OPCPackage pkg = null;
XSSFWorkbook wb = null;
@ -314,7 +319,7 @@ public abstract class AbstractExcelDataSet extends AbstractResolvableDataSet imp
Sheet sheet = wb.getSheetAt(getSheetIndex() - 1);
return resolveResultForSheet(paramValues, sheet, properties);
return resolveResultForSheet(paramValues, sheet, properties, dataSetOption);
}
catch (DataSetException e)
{
@ -338,11 +343,13 @@ public abstract class AbstractExcelDataSet extends AbstractResolvableDataSet imp
* @param sheet
* @param properties
* 允许为{@code null}，此时会自动解析
* @param dataSetOption
* 允许为{@code null}
* @return
* @throws DataSetException
*/
protected ResolvedDataSetResult resolveResultForSheet(Map<String, ?> paramValues, Sheet sheet,
List<DataSetProperty> properties) throws DataSetException
List<DataSetProperty> properties, DataSetOption dataSetOption) throws DataSetException
{
boolean resolveProperties = (properties == null || properties.isEmpty());
@ -409,9 +416,16 @@ public abstract class AbstractExcelDataSet extends AbstractResolvableDataSet imp
colIdx++;
}
data.add(rowObj);
boolean reachMaxCount = isReachResultDataMaxCount(dataSetOption, data.size());
boolean breakLoop = (reachMaxCount && (!resolveProperties || isAfterNameRow(rowIdx)));
if (!reachMaxCount)
data.add(rowObj);
dataRowIdx++;
if (breakLoop)
break;
}
rowIdx++;
@ -600,6 +614,21 @@ public abstract class AbstractExcelDataSet extends AbstractResolvableDataSet imp
return ((rowIndex + 1) == this.nameRow);
}
/**
* 是否在名称行之后。
* <p>
* 如果没有名称行，应返回{@code true}。
* </p>
*
* @param rowIndex
* 行索引（以{@code 0}计数）
* @return
*/
protected boolean isAfterNameRow(int rowIndex)
{
return ((rowIndex + 1) > this.nameRow);
}
/**
* 是否数据行。
*

38
datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractJsonDataSet.java
View File

@ -15,6 +15,7 @@ import java.util.List;
import java.util.Map;
import org.datagear.analysis.DataSetException;
import org.datagear.analysis.DataSetOption;
import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetResult;
import org.datagear.analysis.ResolvableDataSet;
@ -95,15 +96,15 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl
* </p>
*/
@Override
protected ResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties)
throws DataSetException
protected ResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties,
DataSetOption dataSetOption) throws DataSetException
{
TemplateResolvedSource<Reader> reader = null;
try
{
reader = getJsonReader(paramValues);
ResolvedDataSetResult result = resolveResult(reader.getSource(), properties);
ResolvedDataSetResult result = resolveResult(reader.getSource(), properties, dataSetOption);
if (reader.hasResolvedTemplate())
result = new TemplateResolvedDataSetResult(result.getResult(), result.getProperties(),
@ -145,10 +146,13 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl
* JSON输入流
* @param properties
* 允许为{@code null}，此时会自动解析
* @param dataSetOption
* 允许为{@code null}
* @return
* @throws Throwable
*/
protected ResolvedDataSetResult resolveResult(Reader jsonReader, List<DataSetProperty> properties) throws Throwable
protected ResolvedDataSetResult resolveResult(Reader jsonReader, List<DataSetProperty> properties,
DataSetOption dataSetOption) throws Throwable
{
boolean resolveProperties = (properties == null || properties.isEmpty());
@ -165,8 +169,7 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl
if (resolveProperties)
properties = resolveDataSetProperties(data);
if (!resolveProperties)
data = convertJsonResultData(data, properties, createDataSetPropertyValueConverter());
data = convertJsonResultData(data, properties, dataSetOption, createDataSetPropertyValueConverter());
DataSetResult result = new DataSetResult(data);
@ -227,12 +230,14 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl
* @param resultData
* 允许为{@code null}
* @param properties
* @param dataSetOption
* 允许为{@code null}
* @param converter
* @return
* @throws Throwable
*/
protected Object convertJsonResultData(Object resultData, List<DataSetProperty> properties,
DataSetPropertyValueConverter converter) throws Throwable
DataSetOption dataSetOption, DataSetPropertyValueConverter converter) throws Throwable
{
Object re = null;
@ -268,8 +273,14 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl
List<Object> reList = new ArrayList<>(list.size());
for (Object ele : list)
reList.add(convertJsonResultData(ele, properties, converter));
for (int i = 0; i < list.size(); i++)
{
if (isReachResultDataMaxCount(dataSetOption, reList.size()))
break;
Object ele = list.get(i);
reList.add(convertJsonResultData(ele, properties, null, converter));
}
re = reList;
}
@ -277,10 +288,15 @@ public abstract class AbstractJsonDataSet extends AbstractResolvableDataSet impl
{
Object[] array = (Object[]) resultData;
Object[] reArray = new Object[array.length];
Object[] reArray = new Object[evalResultDataCount(dataSetOption, array.length)];
for (int i = 0; i < array.length; i++)
reArray[i] = convertJsonResultData(array[i], properties, converter);
{
if (isReachResultDataMaxCount(dataSetOption, i + 1))
break;
reArray[i] = convertJsonResultData(array[i], properties, null, converter);
}
re = reArray;
}

53
datagear-analysis/src/main/java/org/datagear/analysis/support/AbstractResolvableDataSet.java
View File

@ -12,6 +12,7 @@ import java.util.List;
import java.util.Map;
import org.datagear.analysis.DataSetException;
import org.datagear.analysis.DataSetOption;
import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetResult;
import org.datagear.analysis.ResolvableDataSet;
@ -49,15 +50,16 @@ public abstract class AbstractResolvableDataSet extends AbstractDataSet implemen
if (properties == null || properties.isEmpty())
throw new DataSetException("[getProperties()] must not be empty");
ResolvedDataSetResult result = resolveResult(paramValues, properties);
ResolvedDataSetResult result = resolveResult(paramValues, properties, null);
return result.getResult();
}
@Override
public ResolvedDataSetResult resolve(Map<String, ?> paramValues) throws DataSetException
public ResolvedDataSetResult resolve(Map<String, ?> paramValues, DataSetOption dataSetOption)
throws DataSetException
{
return resolveResult(paramValues, null);
return resolveResult(paramValues, null, dataSetOption);
}
/**
@ -68,9 +70,50 @@ public abstract class AbstractResolvableDataSet extends AbstractDataSet implemen
* 允许为{@code null}/空，此时，应自动解析并设置返回结果的{@linkplain ResolvedDataSetResult#setProperties(List)}；
* 如果不为{@code null}/空，直接将{@code properties}作为解析数据依据， 使用它处理结果数据，
* 并设置为返回结果的{@linkplain ResolvedDataSetResult#setProperties(List)}
* @param dataSetOption
* 设置选项，允许为{@code null}
* @return
* @throws DataSetException
*/
protected abstract ResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties)
throws DataSetException;
protected abstract ResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties,
DataSetOption dataSetOption) throws DataSetException;
/**
* 给定数目是否已到达{@linkplain DataSetOption#getResultDataMaxCount()}。
*
* @param dataSetOption
* 允许为{@code null}
* @param count
* @return
*/
protected boolean isReachResultDataMaxCount(DataSetOption dataSetOption, int count)
{
if (dataSetOption == null)
return false;
int maxCount = dataSetOption.getResultDataMaxCount();
if (maxCount < 0)
return false;
return count >= maxCount;
}
/**
* 计算结果数据数目。
*
* @param dataSetOption
* 允许为{@code null}
* @param defaultCount
* @return
*/
protected int evalResultDataCount(DataSetOption dataSetOption, int defaultCount)
{
if (dataSetOption == null)
return defaultCount;
int maxCount = dataSetOption.getResultDataMaxCount();
return (maxCount < 0 ? defaultCount : Math.min(maxCount, defaultCount));
}
}

6
datagear-analysis/src/main/java/org/datagear/analysis/support/CsvValueDataSet.java
View File

@ -9,6 +9,7 @@ import java.util.List;
import java.util.Map;
import org.datagear.analysis.DataSetException;
import org.datagear.analysis.DataSetOption;
import org.datagear.analysis.DataSetProperty;
import org.datagear.util.IOUtil;
@ -65,9 +66,10 @@ public class CsvValueDataSet extends AbstractCsvDataSet
}
@Override
public TemplateResolvedDataSetResult resolve(Map<String, ?> paramValues) throws DataSetException
public TemplateResolvedDataSetResult resolve(Map<String, ?> paramValues, DataSetOption dataSetOption)
throws DataSetException
{
return (TemplateResolvedDataSetResult) resolveResult(paramValues, null);
return (TemplateResolvedDataSetResult) resolveResult(paramValues, null, dataSetOption);
}
@Override

25
datagear-analysis/src/main/java/org/datagear/analysis/support/HttpDataSet.java
View File

@ -34,6 +34,7 @@ import org.apache.hc.core5.http.io.HttpClientResponseHandler;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.datagear.analysis.DataSetException;
import org.datagear.analysis.DataSetOption;
import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetResult;
import org.datagear.analysis.ResolvedDataSetResult;
@ -322,14 +323,15 @@ public class HttpDataSet extends AbstractResolvableDataSet
}
@Override
public TemplateResolvedDataSetResult resolve(Map<String, ?> paramValues) throws DataSetException
public TemplateResolvedDataSetResult resolve(Map<String, ?> paramValues, DataSetOption dataSetOption)
throws DataSetException
{
return resolveResult(paramValues, null);
return resolveResult(paramValues, null, dataSetOption);
}
@Override
protected TemplateResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties)
throws DataSetException
protected TemplateResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties,
DataSetOption dataSetOption) throws DataSetException
{
try
{
@ -345,6 +347,7 @@ public class HttpDataSet extends AbstractResolvableDataSet
JsonResponseHandler responseHandler = new JsonResponseHandler();
responseHandler.setProperties(properties);
responseHandler.setResponseDataJsonPath(getResponseDataJsonPath());
responseHandler.setDataSetOption(dataSetOption);
ResolvedDataSetResult result = this.httpClient.execute(request, responseHandler);
@ -504,6 +507,8 @@ public class HttpDataSet extends AbstractResolvableDataSet
private String responseDataJsonPath = "";
private DataSetOption dataSetOption = null;
public JsonResponseHandler()
{
super();
@ -535,6 +540,16 @@ public class HttpDataSet extends AbstractResolvableDataSet
this.responseDataJsonPath = responseDataJsonPath;
}
public DataSetOption getDataSetOption()
{
return dataSetOption;
}
public void setDataSetOption(DataSetOption dataSetOption)
{
this.dataSetOption = dataSetOption;
}
@SuppressWarnings("unchecked")
@Override
public ResolvedDataSetResult handleResponse(ClassicHttpResponse response) throws HttpException, IOException
@ -560,7 +575,7 @@ public class HttpDataSet extends AbstractResolvableDataSet
HttpResponseJsonDataSet jsonDataSet = new HttpResponseJsonDataSet(reader);
jsonDataSet.setDataJsonPath(this.responseDataJsonPath);
return jsonDataSet.resolve(Collections.EMPTY_MAP);
return jsonDataSet.resolve(Collections.EMPTY_MAP, this.dataSetOption);
}
else
{

6
datagear-analysis/src/main/java/org/datagear/analysis/support/JsonValueDataSet.java
View File

@ -12,6 +12,7 @@ import java.util.List;
import java.util.Map;
import org.datagear.analysis.DataSetException;
import org.datagear.analysis.DataSetOption;
import org.datagear.analysis.DataSetProperty;
import org.datagear.util.IOUtil;
@ -56,9 +57,10 @@ public class JsonValueDataSet extends AbstractJsonDataSet
}
@Override
public TemplateResolvedDataSetResult resolve(Map<String, ?> paramValues) throws DataSetException
public TemplateResolvedDataSetResult resolve(Map<String, ?> paramValues, DataSetOption dataSetOption)
throws DataSetException
{
return (TemplateResolvedDataSetResult) resolveResult(paramValues, null);
return (TemplateResolvedDataSetResult) resolveResult(paramValues, null, dataSetOption);
}
@Override

32
datagear-analysis/src/main/java/org/datagear/analysis/support/SqlDataSet.java
View File

@ -19,6 +19,7 @@ import java.util.Map;
import org.datagear.analysis.DataSet;
import org.datagear.analysis.DataSetException;
import org.datagear.analysis.DataSetOption;
import org.datagear.analysis.DataSetProperty;
import org.datagear.analysis.DataSetProperty.DataType;
import org.datagear.analysis.DataSetResult;
@ -90,14 +91,15 @@ public class SqlDataSet extends AbstractResolvableDataSet implements ResolvableD
}
@Override
public TemplateResolvedDataSetResult resolve(Map<String, ?> paramValues) throws DataSetException
public TemplateResolvedDataSetResult resolve(Map<String, ?> paramValues, DataSetOption dataSetOption)
throws DataSetException
{
return resolveResult(paramValues, null);
return resolveResult(paramValues, null, dataSetOption);
}
@Override
protected TemplateResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties)
throws DataSetException
protected TemplateResolvedDataSetResult resolveResult(Map<String, ?> paramValues, List<DataSetProperty> properties,
DataSetOption dataSetOption) throws DataSetException
{
String sql = resolveAsFmkTemplate(getSql(), paramValues);
@ -131,8 +133,7 @@ public class SqlDataSet extends AbstractResolvableDataSet implements ResolvableD
try
{
ResultSet rs = qrs.getResultSet();
ResolvedDataSetResult result = resolveResult(cn, rs, properties);
ResolvedDataSetResult result = resolveResult(cn, rs, properties, dataSetOption);
return new TemplateResolvedDataSetResult(result.getResult(), result.getProperties(), sql);
}
@ -170,15 +171,17 @@ public class SqlDataSet extends AbstractResolvableDataSet implements ResolvableD
* @param rs
* @param properties
* 允许为{@code null}，此时会自动解析
* @param dataSetOption
* 允许为{@code null}
* @return
* @throws Throwable
*/
protected ResolvedDataSetResult resolveResult(Connection cn, ResultSet rs, List<DataSetProperty> properties)
throws Throwable
protected ResolvedDataSetResult resolveResult(Connection cn, ResultSet rs, List<DataSetProperty> properties,
DataSetOption dataSetOption) throws Throwable
{
boolean resolveProperties = (properties == null || properties.isEmpty());
List<Map<String, ?>> datas = new ArrayList<>();
List<Map<String, ?>> data = new ArrayList<>();
JdbcSupport jdbcSupport = getJdbcSupport();
DataSetPropertyValueConverter converter = createDataSetPropertyValueConverter();
@ -223,12 +226,19 @@ public class SqlDataSet extends AbstractResolvableDataSet implements ResolvableD
row.put(property.getName(), value);
}
datas.add(row);
boolean reachMaxCount = isReachResultDataMaxCount(dataSetOption, data.size());
boolean breakLoop = reachMaxCount;
if (!reachMaxCount)
data.add(row);
rowIdx++;
if (breakLoop)
break;
}
DataSetResult result = new DataSetResult(datas);
DataSetResult result = new DataSetResult(data);
return new ResolvedDataSetResult(result, properties);
}

2
datagear-analysis/src/test/java/org/datagear/analysis/support/AbstractJsonDataSetTest.java
View File

@ -34,7 +34,7 @@ public class AbstractJsonDataSetTest
dataSet.setDataJsonPath("path0.path1[0].path2");
TemplateResolvedDataSetResult result = dataSet.resolve(Collections.emptyMap());
TemplateResolvedDataSetResult result = dataSet.resolve(Collections.emptyMap(), null);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();

4
datagear-analysis/src/test/java/org/datagear/analysis/support/CsvDirectoryFileDataSetTest.java
View File

@ -81,7 +81,7 @@ public class CsvDirectoryFileDataSetTest
dataSet.setNameRow(1);
@SuppressWarnings("unchecked")
ResolvedDataSetResult result = dataSet.resolve(Collections.EMPTY_MAP);
ResolvedDataSetResult result = dataSet.resolve(Collections.EMPTY_MAP, null);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();
@ -144,7 +144,7 @@ public class CsvDirectoryFileDataSetTest
"CsvDirectoryFileDataSetTest-0.csv");
@SuppressWarnings("unchecked")
ResolvedDataSetResult result = dataSet.resolve(Collections.EMPTY_MAP);
ResolvedDataSetResult result = dataSet.resolve(Collections.EMPTY_MAP, null);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();

17
datagear-analysis/src/test/java/org/datagear/analysis/support/CsvValueDataSetTest.java
View File

@ -32,11 +32,10 @@ public class CsvValueDataSetTest
properties.add(new DataSetProperty("value", DataSetProperty.DataType.NUMBER));
properties.add(new DataSetProperty("size", DataSetProperty.DataType.NUMBER));
List<DataSetParam> params = new ArrayList<DataSetParam>();
List<DataSetParam> params = new ArrayList<>();
params.add(new DataSetParam("size", DataSetParam.DataType.NUMBER, true));
CsvValueDataSet dataSet = new CsvValueDataSet("a", "a", properties,
"name, value, size \n aaa, 11, ${size}");
CsvValueDataSet dataSet = new CsvValueDataSet("a", "a", properties, "name, value, size \n aaa, 11, ${size}");
dataSet.setParams(params);
dataSet.setNameRow(1);
@ -68,11 +67,10 @@ public class CsvValueDataSetTest
properties.add(new DataSetProperty("value", DataSetProperty.DataType.NUMBER));
properties.add(new DataSetProperty("size", DataSetProperty.DataType.STRING));
List<DataSetParam> params = new ArrayList<DataSetParam>();
List<DataSetParam> params = new ArrayList<>();
params.add(new DataSetParam("size", DataSetParam.DataType.NUMBER, true));
CsvValueDataSet dataSet = new CsvValueDataSet("a", "a", properties,
"name, value, size \n aaa, 11, ${size}");
CsvValueDataSet dataSet = new CsvValueDataSet("a", "a", properties, "name, value, size \n aaa, 11, ${size}");
dataSet.setParams(params);
dataSet.setNameRow(1);
@ -99,18 +97,17 @@ public class CsvValueDataSetTest
@Test
public void resolveTest_hasParam()
{
List<DataSetParam> params = new ArrayList<DataSetParam>();
List<DataSetParam> params = new ArrayList<>();
params.add(new DataSetParam("size", DataSetParam.DataType.NUMBER, true));
CsvValueDataSet dataSet = new CsvValueDataSet("a", "a",
"name, value, size \n aaa, 11, ${size}");
CsvValueDataSet dataSet = new CsvValueDataSet("a", "a", "name, value, size \n aaa, 11, ${size}");
dataSet.setParams(params);
dataSet.setNameRow(1);
Map<String, Object> paramValues = new HashMap<>();
paramValues.put("size", 12);
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues);
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues, null);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();

6
datagear-analysis/src/test/java/org/datagear/analysis/support/ExcelDirectoryFileDataSetTest.java
View File

@ -142,7 +142,7 @@ public class ExcelDirectoryFileDataSetTest
"ExcelDirectoryFileDataSetTest-0.xlsx");
dataSet.setNameRow(1);
ResolvedDataSetResult resolvedResult = dataSet.resolve(new HashMap<>());
ResolvedDataSetResult resolvedResult = dataSet.resolve(new HashMap<>(), null);
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) resolvedResult.getResult().getData();
@ -217,7 +217,7 @@ public class ExcelDirectoryFileDataSetTest
"ExcelDirectoryFileDataSetTest-1.xls");
dataSet.setNameRow(1);
ResolvedDataSetResult resolvedResult = dataSet.resolve(new HashMap<>());
ResolvedDataSetResult resolvedResult = dataSet.resolve(new HashMap<>(), null);
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) resolvedResult.getResult().getData();
@ -294,7 +294,7 @@ public class ExcelDirectoryFileDataSetTest
dataSet.setDataRowExp("2,3-");
dataSet.setDataColumnExp("A,C-");
ResolvedDataSetResult resolvedResult = dataSet.resolve(new HashMap<>());
ResolvedDataSetResult resolvedResult = dataSet.resolve(new HashMap<>(), null);
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) resolvedResult.getResult().getData();

14
datagear-analysis/src/test/java/org/datagear/analysis/support/HttpDataSetTest.java
View File

@ -161,7 +161,7 @@ public class HttpDataSetTest
Map<String, Object> paramValues = new HashMap<>();
paramValues.put("param", "pv");
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues);
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues, null);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();
@ -212,7 +212,7 @@ public class HttpDataSetTest
dataSet.setRequestMethod(HttpDataSet.REQUEST_METHOD_GET);
TemplateResolvedDataSetResult result = dataSet.resolve(Collections.emptyMap());
TemplateResolvedDataSetResult result = dataSet.resolve(Collections.emptyMap(), null);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();
@ -229,7 +229,7 @@ public class HttpDataSetTest
dataSet.setRequestMethod(HttpDataSet.REQUEST_METHOD_POST);
TemplateResolvedDataSetResult result = dataSet.resolve(Collections.emptyMap());
TemplateResolvedDataSetResult result = dataSet.resolve(Collections.emptyMap(), null);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();
@ -255,7 +255,7 @@ public class HttpDataSetTest
Map<String, Object> paramValues = new HashMap<>();
paramValues.put("param", pv1);
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues);
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues, null);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();
@ -316,7 +316,7 @@ public class HttpDataSetTest
Map<String, Object> paramValues = new HashMap<>();
paramValues.put("param", pv1);
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues);
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues, null);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();
@ -376,7 +376,7 @@ public class HttpDataSetTest
Map<String, Object> paramValues = new HashMap<>();
paramValues.put("param", pv1);
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues);
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues, null);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();
@ -426,7 +426,7 @@ public class HttpDataSetTest
SERVER + "/testResponseJsonPath");
dataSet.setResponseDataJsonPath("path0.path1[0].path2");
TemplateResolvedDataSetResult result = dataSet.resolve(Collections.emptyMap());
TemplateResolvedDataSetResult result = dataSet.resolve(Collections.emptyMap(), null);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();

2
datagear-analysis/src/test/java/org/datagear/analysis/support/JsonDirectoryFileDataSetTest.java
View File

@ -136,7 +136,7 @@ public class JsonDirectoryFileDataSetTest
"JsonDirectoryFileDataSetTest-0.json");
@SuppressWarnings("unchecked")
ResolvedDataSetResult result = dataSet.resolve(Collections.EMPTY_MAP);
ResolvedDataSetResult result = dataSet.resolve(Collections.EMPTY_MAP, null);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();

2
datagear-analysis/src/test/java/org/datagear/analysis/support/JsonValueDataSetTest.java
View File

@ -107,7 +107,7 @@ public class JsonValueDataSetTest
Map<String, Object> paramValues = new HashMap<>();
paramValues.put("size", 12);
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues);
TemplateResolvedDataSetResult result = dataSet.resolve(paramValues, null);
List<DataSetProperty> properties = result.getProperties();
@SuppressWarnings("unchecked")
List<Map<String, Object>> data = (List<Map<String, Object>>) result.getResult().getData();

27
datagear-web/src/main/java/org/datagear/web/controller/DataSetController.java
View File

@ -19,6 +19,7 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.datagear.analysis.DataSet;
import org.datagear.analysis.DataSetOption;
import org.datagear.analysis.DataSetParam;
import org.datagear.analysis.ResolvedDataSetResult;
import org.datagear.analysis.support.AbstractDataSet;
@ -716,7 +717,7 @@ public class DataSetController extends AbstractSchemaConnController
Map<String, Object> convertedParamValues = getDataSetParamValueConverter().convert(preview.getParamValues(),
dataSet.getParams());
TemplateResolvedDataSetResult result = dataSet.resolve(convertedParamValues);
TemplateResolvedDataSetResult result = dataSet.resolve(convertedParamValues, preview.getDataSetOption());
return result;
}
@ -744,7 +745,7 @@ public class DataSetController extends AbstractSchemaConnController
Map<String, Object> convertedParamValues = getDataSetParamValueConverter().convert(preview.getParamValues(),
dataSet.getParams());
TemplateResolvedDataSetResult result = dataSet.resolve(convertedParamValues);
TemplateResolvedDataSetResult result = dataSet.resolve(convertedParamValues, preview.getDataSetOption());
return result;
}
@ -766,7 +767,7 @@ public class DataSetController extends AbstractSchemaConnController
Map<String, Object> convertedParamValues = getDataSetParamValueConverter().convert(preview.getParamValues(),
dataSet.getParams());
ResolvedDataSetResult result = dataSet.resolve(convertedParamValues);
ResolvedDataSetResult result = dataSet.resolve(convertedParamValues, preview.getDataSetOption());
return result;
}
@ -787,7 +788,7 @@ public class DataSetController extends AbstractSchemaConnController
Map<String, Object> convertedParamValues = getDataSetParamValueConverter().convert(preview.getParamValues(),
dataSet.getParams());
ResolvedDataSetResult result = dataSet.resolve(convertedParamValues);
ResolvedDataSetResult result = dataSet.resolve(convertedParamValues, preview.getDataSetOption());
return result;
}
@ -806,7 +807,7 @@ public class DataSetController extends AbstractSchemaConnController
Map<String, Object> convertedParamValues = getDataSetParamValueConverter().convert(preview.getParamValues(),
dataSet.getParams());
TemplateResolvedDataSetResult result = dataSet.resolve(convertedParamValues);
TemplateResolvedDataSetResult result = dataSet.resolve(convertedParamValues, preview.getDataSetOption());
return result;
}
@ -827,7 +828,7 @@ public class DataSetController extends AbstractSchemaConnController
Map<String, Object> convertedParamValues = getDataSetParamValueConverter().convert(preview.getParamValues(),
dataSet.getParams());
ResolvedDataSetResult result = dataSet.resolve(convertedParamValues);
ResolvedDataSetResult result = dataSet.resolve(convertedParamValues, preview.getDataSetOption());
return result;
}
@ -848,7 +849,7 @@ public class DataSetController extends AbstractSchemaConnController
Map<String, Object> convertedParamValues = getDataSetParamValueConverter().convert(preview.getParamValues(),
dataSet.getParams());
TemplateResolvedDataSetResult result = dataSet.resolve(convertedParamValues);
TemplateResolvedDataSetResult result = dataSet.resolve(convertedParamValues, preview.getDataSetOption());
return result;
}
@ -1018,6 +1019,8 @@ public class DataSetController extends AbstractSchemaConnController
@SuppressWarnings("unchecked")
private Map<String, Object> paramValues = Collections.EMPTY_MAP;
private DataSetOption dataSetOption = null;
public AbstractDataSetPreview()
{
super();
@ -1042,6 +1045,16 @@ public class DataSetController extends AbstractSchemaConnController
{
this.paramValues = paramValues;
}
public DataSetOption getDataSetOption()
{
return dataSetOption;
}
public void setDataSetOption(DataSetOption dataSetOption)
{
this.dataSetOption = dataSetOption;
}
}
public static class SqlDataSetPreview extends AbstractDataSetPreview<SqlDataSet>

1
datagear-web/src/main/resources/org/datagear/web/messages/message.properties
View File

@ -686,6 +686,7 @@ dataSet.setParamValue=设置参数值
dataSet.refreshSqlResult=刷新查询结果
dataSet.previewButtonTip=预览结果数据（编辑区：Ctrl+Enter）
dataSet.showResolvedSource=显示参数化语句解析结果
dataSet.previewResultDataMaxCount=预览返回结果数据的最大条目数
dataSet.DataCategory.DIMENSION=维度
dataSet.DataCategory.SCALAR=量度
dataSet.validation.previewRequired=请先执行预览确保数据正确

25
datagear-web/src/main/resources/org/datagear/web/static/css/common.css
View File

@ -2398,11 +2398,16 @@ table.dataTable tbody tr td select{
vertical-align: top;
border: 0px;
}
.page-form-dataSet .workspace .workspace-operation-wrapper .preview-result-table-wrapper .result-resolved-source{
.page-form-dataSet .workspace .workspace-operation-wrapper .preview-result-table-wrapper .preview-result-foot{
position: relative;
top: 0.1em;
top: 0.2em;
display: none;
height: 1.2em;
}
.page-form-dataSet .workspace .workspace-operation-wrapper .preview-result-table-wrapper .result-resolved-source{
float: right;
width: 85%;
display: none;
width: 100%;
text-align: right;
}
.page-form-dataSet .workspace .workspace-operation-wrapper .preview-result-table-wrapper .result-resolved-source-panel{
@ -2426,6 +2431,20 @@ table.dataTable tbody tr td select{
-moz-box-sizing: border-box;
-webkit-box-sizing: border-box;
}
.page-form-dataSet .workspace .workspace-operation-wrapper .preview-result-table-wrapper .result-data-max-count{
float: left;
width: 10%;
height: 100%;
position: relative;
}
.page-form-dataSet .workspace .workspace-operation-wrapper .preview-result-table-wrapper .result-data-max-count input{
position: absolute;
left: 0;
top: 0;
bottom: 0;
width: 100%;
font-size: 0.8em;
}
.page-form-dataSet .workspace .workspace-operation-wrapper .input-in-table{
width: 90%!important;
}

15
datagear-web/src/main/resources/org/datagear/web/templates/analysis/dataSet/include/dataSet_form_html_wow.ftl
View File

@ -13,13 +13,18 @@
<button type="button" class="refresh-button ui-button ui-corner-all ui-widget ui-button-icon-only" title="<@spring.message code='dataSet.refreshSqlResult' />"><span class="ui-button-icon ui-icon ui-icon-refresh"></span><span class="ui-button-icon-space"> </span><@spring.message code='dataSet.refreshSqlResult' /></button>
</div>
<table id="${pageId}-previewResultTable" width='100%' class='hover stripe'></table>
<div class="result-resolved-source">
<button type="button" class="show-resolved-source-button ui-button ui-corner-all ui-widget ui-button-icon-only" title="<@spring.message code='dataSet.showResolvedSource' />"><span class="ui-button-icon ui-icon ui-icon-comment"></span><span class="ui-button-icon-space"> </span><@spring.message code='dataSet.showResolvedSource' /></button>
<div class="result-resolved-source-panel ui-widget ui-widget-content ui-corner-all ui-widget-shadow ui-front">
<div class="result-resolved-source-panel-content">
<textarea class="ui-widget ui-widget-content"></textarea>
<div class="preview-result-foot">
<div class="result-resolved-source">
<button type="button" class="show-resolved-source-button ui-button ui-corner-all ui-widget ui-button-icon-only" title="<@spring.message code='dataSet.showResolvedSource' />"><span class="ui-button-icon ui-icon ui-icon-comment"></span><span class="ui-button-icon-space"> </span><@spring.message code='dataSet.showResolvedSource' /></button>
<div class="result-resolved-source-panel ui-widget ui-widget-content ui-corner-all ui-widget-shadow ui-front">
<div class="result-resolved-source-panel-content">
<textarea class="ui-widget ui-widget-content"></textarea>
</div>
</div>
</div>
<div class="result-data-max-count" title="<@spring.message code='dataSet.previewResultDataMaxCount' />">
<input type="text" class="resultDataMaxCountInput ui-widget ui-widget-content ui-corner-all" />
</div>
</div>
</div>
<div id="${pageId}-dataSetParams" class="params-table-wrapper minor-dataTable">

45
datagear-web/src/main/resources/org/datagear/web/templates/analysis/dataSet/include/dataSet_form_js.ftl
View File

@ -592,6 +592,8 @@ po.previewOptions.url = "...";
$panel.position({ my : "right top", at : "left+5 top", of : po.element(".workspace-operation-wrapper")});
};
po.resultDataMaxCountDefault = 100;
//预览设置项
po.previewOptions =
{
@ -601,7 +603,8 @@ po.previewOptions.url = "...";
data:
{
dataSet: {},
paramValues: {}
paramValues: {},
dataSetOption: { resultDataMaxCount: po.resultDataMaxCountDefault }
},
//预览操作前置回调函数，返回false阻止
beforePreview: function(){},
@ -618,6 +621,32 @@ po.previewOptions.url = "...";
success: function(previewResponse){}
};
po.resultDataMaxCountVal = function(val)
{
var $input = po.element(".resultDataMaxCountInput");
if(val === undefined)
{
val = parseInt($input.val());
var validVal = val;
if(isNaN(validVal))
validVal = po.resultDataMaxCountDefault;
else if(validVal < 1)
validVal = 1;
if(validVal != val)
{
val = validVal;
$input.val(val);
}
return val;
}
else
$input.val(val);
};
//获取、设置上一次预览是否成功
po.previewSuccess = function(success)
{
@ -699,6 +728,14 @@ po.previewOptions.url = "...";
$panel.hide();
});
po.resultDataMaxCountVal(po.resultDataMaxCountDefault);
po.element(".resultDataMaxCountInput").on("keydown", function(e)
{
//防止提交数据集表单
if(e.keyCode == $.ui.keyCode.ENTER)
return false;
});
$(po.element()).on("click", function(event)
{
var $target = $(event.target);
@ -722,10 +759,14 @@ po.previewOptions.url = "...";
{
$(this).button("disable");
});
po.element(".preview-result-foot").hide();
var table = po.previewResultTableElement();
var initDataTable = !$.isDatatTable(table);
po.previewOptions.data.dataSetOption.resultDataMaxCount = po.resultDataMaxCountVal();
$.ajaxJson(
{
url : po.previewOptions.url,
@ -777,6 +818,8 @@ po.previewOptions.url = "...";
table.addClass("preview-result-table-inited");
table.dataTable(settings);
po.element(".preview-result-foot").show();
if(previewResponse.templateResult)
{
po.element(".result-resolved-source textarea").val(previewResponse.templateResult);