p78543092
/
haflow
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add log4j. change module upload dir from /classes to /lib.

This commit is contained in:
zhaowei8188127 2013-09-06 10:01:56 +08:00
parent 08c7320763
commit 668914b9cb
16 changed files with 48 additions and 704 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
haflow.module.init.jar Normal file
View File

Binary file not shown.

2
src/main/java/haflow/engine/oozie/FlowDeployService.java
View File

@ -24,7 +24,7 @@ public class FlowDeployService {
if (jarPath.endsWith(".jar")) { if (jarPath.endsWith(".jar")) {
File jarFile = new File(jarPath); File jarFile = new File(jarPath);
File dstPath = new File(deployPath + "/lib"); File dstPath = new File(deployPath + "/lib");
// System.out.println(jarPath); System.out.println(jarPath);
this.copyJarFile(jarFile, dstPath, jarFile.getName()); this.copyJarFile(jarFile, dstPath, jarFile.getName());
} }
} }

5
src/main/java/haflow/engine/oozie/OozieEngine.java
View File

@ -39,6 +39,7 @@ import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource; import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamResult;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.w3c.dom.Document; import org.w3c.dom.Document;
@ -47,6 +48,7 @@ import org.w3c.dom.Document;
@Component @Component
public class OozieEngine extends AbstractEngine { public class OozieEngine extends AbstractEngine {
private Logger logger = Logger.getLogger(this.getClass().getName());
private ModuleUtil moduleUtil; private ModuleUtil moduleUtil;
private NodeConfigurationService nodeConfigurationService; private NodeConfigurationService nodeConfigurationService;
private ClusterConfiguration clusterConfiguration; private ClusterConfiguration clusterConfiguration;
@ -151,7 +153,7 @@ public class OozieEngine extends AbstractEngine {
List<Integer> sorted = new TopologicalSort(graph).getOrder(); List<Integer> sorted = new TopologicalSort(graph).getOrder();
if (sorted == null) { if (sorted == null) {
messageBuilder.append("Error: Flow is has Circles!"); messageBuilder.append("Error: Flow has Circles!");
} else { } else {
String flowName = flow.getName(); String flowName = flow.getName();
String workflowXml = genWorkflowXml(flowName, sorted, String workflowXml = genWorkflowXml(flowName, sorted,
@ -212,6 +214,7 @@ public class OozieEngine extends AbstractEngine {
System.out.println(messageBuilder.toString()); System.out.println(messageBuilder.toString());
model.setMessage(messageBuilder.toString()); model.setMessage(messageBuilder.toString());
System.out.println(messageBuilder.toString()); System.out.println(messageBuilder.toString());
logger.info(messageBuilder);
return model; return model;
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();

39
src/main/java/haflow/module/basic/DemoJavaModule.java
View File

@ -1,39 +0,0 @@
package haflow.module.basic;
import haflow.module.AbstractJavaModule;
import haflow.module.DataType;
import haflow.module.Module;
import haflow.module.ModuleConfiguration;
import haflow.module.ModuleConfigurationType;
import haflow.module.ModuleEndpoint;
import haflow.module.ModuleType;
import java.util.Map;
@Module(id = "ada600a8-aa63-968a-ca46-9085a0e0bd2f", name = "DemoJava", category = "Basic", type = ModuleType.JAVA, configurations = { @ModuleConfiguration(key = "arg_test", displayName = "Arguments Test", pattern = "^(.*)$", type = ModuleConfigurationType.PLAIN_TEXT), }, inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = {
@ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText),
@ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) })
public class DemoJavaModule extends AbstractJavaModule {
@Override
public boolean validate(Map<String, String> configurations,
Map<String, String> inputs, Map<String, String> outputs) {
// TODO Auto-generated method stub
return false;
}
@Override
public String getMainClass() {
return DemoJavaModule.class.getName();
}
public static void main(String[] args) {
System.out.println("Demo Java Main");
System.out.println("# Arguments: " + args.length);
for (int i = 0; i < args.length; i++) {
System.out.println("Argument[" + i + "]: " + args[i]);
}
}
}

92
src/main/java/haflow/module/datamining/BuildForestModule.java
View File

@ -1,92 +0,0 @@
package haflow.module.datamining;
import haflow.module.AbstractJavaModule;
import haflow.module.DataType;
import haflow.module.Module;
import haflow.module.ModuleConfiguration;
import haflow.module.ModuleConfigurationType;
import haflow.module.ModuleEndpoint;
import haflow.module.ModuleType;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Module(id = "ada600a8-aa63-968a-ca46-4356a0e0bdac", name = "BuildForest", category = "DataMining-Mahout", type = ModuleType.JAVA,
configurations = {
@ModuleConfiguration(key = "data", displayName = "data: Data path", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "dataset", displayName = "dataset: Dataset path",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "selection", displayName = "selection: Number of variables to select randomly at each tree-node.", pattern = "^(.*)$",type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "no-complete", displayName = "no-complete: The tree is not complemented", pattern = "^(.*)$",type=ModuleConfigurationType.BOOLEAN),
@ModuleConfiguration(key = "minsplit", displayName = "minsplit: Minimum Split", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "minprop", displayName = "minprop: Minimum Porportion", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "seed", displayName = "seed: Seed Value", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "partial", displayName = "partial: use the Partial Data Implementation", pattern = "^(.*)$", type=ModuleConfigurationType.BOOLEAN),
@ModuleConfiguration(key = "nbtrees", displayName = "nbtrees: Number of trees to grow.", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "output", displayName = "output: Output path", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT)},
inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = {
@ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText),
@ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) })
public class BuildForestModule extends AbstractJavaModule {
@Override
public boolean validate(Map<String, String> configurations,
Map<String, String> inputs, Map<String, String> outputs) {
// TODO Auto-generated method stub
return false;
}
@Override
public String getMainClass() {
return "org.apache.mahout.classifier.df.mapreduce.BuildForest";
}
@Override
public List<String> getArguments(Map<String, String> configurations) {
Module module= this.getClass().getAnnotation(Module.class);
ModuleConfiguration[] confs = module.configurations();
List<String> result = new ArrayList<String>();
for (String key : configurations.keySet()) {
ModuleConfigurationType confType = getConfigurationType(key, confs);
switch(confType){
case BOOLEAN:
String boolValue = configurations.get(key);
if( boolValue.equals("true")){
result.add("--" + key );
}
break;
case PLAIN_TEXT:
String textValue = configurations.get(key).trim();
if( textValue.length() > 0){
result.add("--" + key);
result.add(configurations.get(key));
}
break;
case OTHER:
default:
System.out.println("Invalid Parameters!");
break;
}
}
return result;
}
private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){
for( ModuleConfiguration conf : confs){
if( key.equals(conf.key()))
return conf.type();
}
return ModuleConfigurationType.OTHER;
}
public static void main(String[] args) {
System.out.println("Demo Java Main");
System.out.println("# Arguments: " + args.length);
for (int i = 0; i < args.length; i++) {
System.out.println("Argument[" + i + "]: " + args[i]);
}
}
}

83
src/main/java/haflow/module/datamining/CanopyClusterModule.java
View File

@ -1,83 +0,0 @@
package haflow.module.datamining;
import haflow.module.AbstractJavaModule;
import haflow.module.DataType;
import haflow.module.Module;
import haflow.module.ModuleConfiguration;
import haflow.module.ModuleConfigurationType;
import haflow.module.ModuleEndpoint;
import haflow.module.ModuleType;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Module(id = "ada600a8-aa63-968a-ca46-4356a0e0bd00", name = "CanopyCluster", category = "DataMining-Mahout", type = ModuleType.JAVA,
configurations = {
@ModuleConfiguration(key = "input", displayName = "input", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "output", displayName = "output",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "distanceMeasure", displayName = "distanceMeasure", pattern = "^(.*)$",type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "t1", displayName = "t1", pattern = "^(.*)$",type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "t2", displayName = "t2", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "t3", displayName = "t3", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "t4", displayName = "t4", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "clusterFilter", displayName = "clusterFilter", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "overwrite", displayName = "overwrite", pattern = "^(.*)$", type=ModuleConfigurationType.BOOLEAN),
@ModuleConfiguration(key = "outlierThreshold", displayName = "outlierThreshold", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT)},
inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = {
@ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText),
@ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) })
public class CanopyClusterModule extends AbstractJavaModule {
@Override
public boolean validate(Map<String, String> configurations,
Map<String, String> inputs, Map<String, String> outputs) {
// TODO Auto-generated method stub
return false;
}
@Override
public String getMainClass() {
return "org.apache.mahout.clustering.canopy.CanopyDriver";
}
@Override
public List<String> getArguments(Map<String, String> configurations) {
Module module= this.getClass().getAnnotation(Module.class);
ModuleConfiguration[] confs = module.configurations();
List<String> result = new ArrayList<String>();
for (String key : configurations.keySet()) {
ModuleConfigurationType confType = getConfigurationType(key, confs);
switch(confType){
case BOOLEAN:
String boolValue = configurations.get(key);
if( boolValue.equals("true")){
result.add("--" + key );
}
break;
case PLAIN_TEXT:
String textValue = configurations.get(key).trim();
if( textValue.length() > 0){
result.add("--" + key);
result.add(configurations.get(key));
}
break;
case OTHER:
default:
System.out.println("Invalid Parameters!");
break;
}
}
return result;
}
private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){
for( ModuleConfiguration conf : confs){
if( key.equals(conf.key()))
return conf.type();
}
return ModuleConfigurationType.OTHER;
}
}

93
src/main/java/haflow/module/datamining/DescribeModule.java
View File

@ -1,93 +0,0 @@
package haflow.module.datamining;
import haflow.module.AbstractJavaModule;
import haflow.module.DataType;
import haflow.module.Module;
import haflow.module.ModuleConfiguration;
import haflow.module.ModuleConfigurationType;
import haflow.module.ModuleEndpoint;
import haflow.module.ModuleType;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Module(id = "ada600a8-aa63-968a-ca46-4356a0e0bdab", name = "Describe", category = "DataMining-Mahout", type = ModuleType.JAVA, configurations = {
@ModuleConfiguration(key = "path", displayName = "path: Data path", pattern = "^(.*)$", type = ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "descriptor", displayName = "descriptor: Data descriptor", pattern = "^(.*)$", type = ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "file", displayName = "file: Path to generated descriptor file", pattern = "^(.*)$", type = ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "regression", displayName = "regression: Regression Problem", pattern = "^(.*)$", type = ModuleConfigurationType.BOOLEAN) }, inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = {
@ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText),
@ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) })
public class DescribeModule extends AbstractJavaModule {
@Override
public boolean validate(Map<String, String> configurations,
Map<String, String> inputs, Map<String, String> outputs) {
// TODO Auto-generated method stub
return false;
}
@Override
public String getMainClass() {
return "org.apache.mahout.classifier.df.tools.Describe";
}
@Override
public List<String> getArguments(Map<String, String> configurations) {
Module module = this.getClass().getAnnotation(Module.class);
ModuleConfiguration[] confs = module.configurations();
List<String> result = new ArrayList<String>();
for (String key : configurations.keySet()) {
ModuleConfigurationType confType = getConfigurationType(key, confs);
switch (confType) {
case BOOLEAN:
String boolValue = configurations.get(key);
if( boolValue.equals("true")){
result.add("--" + key);
}
break;
case PLAIN_TEXT:
String textValue = configurations.get(key).trim();
if( textValue.length() > 0){
if(key.equals("descriptor")){
result.add("--" + key);
String[] descriptors = textValue.split(" ");
for(String desp : descriptors){
result.add(desp);
}
}else{
result.add("--" + key);
result.add(configurations.get(key));
}
}
break;
case OTHER:
default:
System.out.println("Invalid Parameters!");
break;
}
}
return result;
}
private ModuleConfigurationType getConfigurationType(String key,
ModuleConfiguration[] confs) {
for (ModuleConfiguration conf : confs) {
if (key.equals(conf.key()))
return conf.type();
}
return ModuleConfigurationType.OTHER;
}
public static void main(String[] args) {
System.out.println("Demo Java Main");
System.out.println("# Arguments: " + args.length);
for (int i = 0; i < args.length; i++) {
System.out.println("Argument[" + i + "]: " + args[i]);
}
}
}

88
src/main/java/haflow/module/datamining/TestForestModule.java
View File

@ -1,88 +0,0 @@
package haflow.module.datamining;
import haflow.module.AbstractJavaModule;
import haflow.module.DataType;
import haflow.module.Module;
import haflow.module.ModuleConfiguration;
import haflow.module.ModuleConfigurationType;
import haflow.module.ModuleEndpoint;
import haflow.module.ModuleType;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Module(id = "ada600a8-aa63-968a-ca46-4356a0e0bdad", name = "TestForest", category = "DataMining-Mahout", type = ModuleType.JAVA,
configurations = {
@ModuleConfiguration(key = "input", displayName = "input : Path to job input directory.", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "dataset", displayName = "dataset : Dataset path",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "model", displayName = "model: Path to the Decision Forest", pattern = "^(.*)$",type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "output", displayName = "output: The directory pathname for output.", pattern = "^(.*)$",type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "analyze", displayName = "analyze: ", pattern = "^(.*)$", type=ModuleConfigurationType.BOOLEAN),
@ModuleConfiguration(key = "mapreduce", displayName = "mapreduce: ", pattern = "^(.*)$", type=ModuleConfigurationType.BOOLEAN)},
inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = {
@ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText),
@ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) })
public class TestForestModule extends AbstractJavaModule {
@Override
public boolean validate(Map<String, String> configurations,
Map<String, String> inputs, Map<String, String> outputs) {
// TODO Auto-generated method stub
return false;
}
@Override
public String getMainClass() {
return "org.apache.mahout.classifier.df.mapreduce.TestForest";
}
@Override
public List<String> getArguments(Map<String, String> configurations) {
Module module= this.getClass().getAnnotation(Module.class);
ModuleConfiguration[] confs = module.configurations();
List<String> result = new ArrayList<String>();
for (String key : configurations.keySet()) {
ModuleConfigurationType confType = getConfigurationType(key, confs);
switch(confType){
case BOOLEAN:
String boolValue = configurations.get(key);
if( boolValue.equals("true")){
result.add("--" + key );
}
break;
case PLAIN_TEXT:
String textValue = configurations.get(key).trim();
if( textValue.length() > 0){
result.add("--" + key);
result.add(configurations.get(key));
}
break;
case OTHER:
default:
System.out.println("Invalid Parameters!");
break;
}
}
return result;
}
private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){
for( ModuleConfiguration conf : confs){
if( key.equals(conf.key()))
return conf.type();
}
return ModuleConfigurationType.OTHER;
}
public static void main(String[] args) {
System.out.println("Demo Java Main");
System.out.println("# Arguments: " + args.length);
for (int i = 0; i < args.length; i++) {
System.out.println("Argument[" + i + "]: " + args[i]);
}
}
}

75
src/main/java/haflow/module/dm/weka/WekaAssociationModule.java
View File

@ -1,75 +0,0 @@
package haflow.module.dm.weka;
import haflow.module.AbstractJavaModule;
import haflow.module.DataType;
import haflow.module.Module;
import haflow.module.ModuleConfiguration;
import haflow.module.ModuleConfigurationType;
import haflow.module.ModuleEndpoint;
import haflow.module.ModuleType;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Module(id = "ada600a8-aa63-968a-ca46-0156a0e0bdff", name = "Apriori", category = "DataMining-Weka", type = ModuleType.JAVA,
configurations = {
@ModuleConfiguration(key = "input", displayName = "input", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "output", displayName = "output",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT)},
inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = {
@ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText),
@ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) })
public class WekaAssociationModule extends AbstractJavaModule {
@Override
public boolean validate(Map<String, String> configurations,
Map<String, String> inputs, Map<String, String> outputs) {
// TODO Auto-generated method stub
return false;
}
@Override
public String getMainClass() {
return "org.apache.mahout.clustering.canopy.CanopyDriver";
}
@Override
public List<String> getArguments(Map<String, String> configurations) {
Module module= this.getClass().getAnnotation(Module.class);
ModuleConfiguration[] confs = module.configurations();
List<String> result = new ArrayList<String>();
for (String key : configurations.keySet()) {
ModuleConfigurationType confType = getConfigurationType(key, confs);
switch(confType){
case BOOLEAN:
String boolValue = configurations.get(key);
if( boolValue.equals("true")){
result.add("--" + key );
}
break;
case PLAIN_TEXT:
String textValue = configurations.get(key).trim();
if( textValue.length() > 0){
result.add("--" + key);
result.add(configurations.get(key));
}
break;
case OTHER:
default:
System.out.println("Invalid Parameters!");
break;
}
}
return result;
}
private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){
for( ModuleConfiguration conf : confs){
if( key.equals(conf.key()))
return conf.type();
}
return ModuleConfigurationType.OTHER;
}
}

75
src/main/java/haflow/module/dm/weka/WekaClassifierModule.java
View File

@ -1,75 +0,0 @@
package haflow.module.dm.weka;
import haflow.module.AbstractJavaModule;
import haflow.module.DataType;
import haflow.module.Module;
import haflow.module.ModuleConfiguration;
import haflow.module.ModuleConfigurationType;
import haflow.module.ModuleEndpoint;
import haflow.module.ModuleType;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Module(id = "ada600a8-aa63-968a-ca46-0056a0e0bdff", name = "KMeans", category = "DataMining-Weka", type = ModuleType.JAVA,
configurations = {
@ModuleConfiguration(key = "input", displayName = "input", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "output", displayName = "output",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT)},
inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = {
@ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText),
@ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) })
public class WekaClassifierModule extends AbstractJavaModule {
@Override
public boolean validate(Map<String, String> configurations,
Map<String, String> inputs, Map<String, String> outputs) {
// TODO Auto-generated method stub
return false;
}
@Override
public String getMainClass() {
return "org.apache.mahout.clustering.canopy.CanopyDriver";
}
@Override
public List<String> getArguments(Map<String, String> configurations) {
Module module= this.getClass().getAnnotation(Module.class);
ModuleConfiguration[] confs = module.configurations();
List<String> result = new ArrayList<String>();
for (String key : configurations.keySet()) {
ModuleConfigurationType confType = getConfigurationType(key, confs);
switch(confType){
case BOOLEAN:
String boolValue = configurations.get(key);
if( boolValue.equals("true")){
result.add("--" + key );
}
break;
case PLAIN_TEXT:
String textValue = configurations.get(key).trim();
if( textValue.length() > 0){
result.add("--" + key);
result.add(configurations.get(key));
}
break;
case OTHER:
default:
System.out.println("Invalid Parameters!");
break;
}
}
return result;
}
private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){
for( ModuleConfiguration conf : confs){
if( key.equals(conf.key()))
return conf.type();
}
return ModuleConfigurationType.OTHER;
}
}

75
src/main/java/haflow/module/dm/weka/WekaClusterModule.java
View File

@ -1,75 +0,0 @@
package haflow.module.dm.weka;
import haflow.module.AbstractJavaModule;
import haflow.module.DataType;
import haflow.module.Module;
import haflow.module.ModuleConfiguration;
import haflow.module.ModuleConfigurationType;
import haflow.module.ModuleEndpoint;
import haflow.module.ModuleType;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Module(id = "ada600a8-aa63-968a-ca46-0256a0e0bdff", name = "ID3", category = "DataMining-Weka", type = ModuleType.JAVA,
configurations = {
@ModuleConfiguration(key = "input", displayName = "input", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "output", displayName = "output",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT)},
inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = {
@ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText),
@ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) })
public class WekaClusterModule extends AbstractJavaModule {
@Override
public boolean validate(Map<String, String> configurations,
Map<String, String> inputs, Map<String, String> outputs) {
// TODO Auto-generated method stub
return false;
}
@Override
public String getMainClass() {
return "org.apache.mahout.clustering.canopy.CanopyDriver";
}
@Override
public List<String> getArguments(Map<String, String> configurations) {
Module module= this.getClass().getAnnotation(Module.class);
ModuleConfiguration[] confs = module.configurations();
List<String> result = new ArrayList<String>();
for (String key : configurations.keySet()) {
ModuleConfigurationType confType = getConfigurationType(key, confs);
switch(confType){
case BOOLEAN:
String boolValue = configurations.get(key);
if( boolValue.equals("true")){
result.add("--" + key );
}
break;
case PLAIN_TEXT:
String textValue = configurations.get(key).trim();
if( textValue.length() > 0){
result.add("--" + key);
result.add(configurations.get(key));
}
break;
case OTHER:
default:
System.out.println("Invalid Parameters!");
break;
}
}
return result;
}
private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){
for( ModuleConfiguration conf : confs){
if( key.equals(conf.key()))
return conf.type();
}
return ModuleConfigurationType.OTHER;
}
}

75
src/main/java/haflow/module/statistics/R/RSVDModule.java
View File

@ -1,75 +0,0 @@
package haflow.module.statistics.R;
import haflow.module.AbstractJavaModule;
import haflow.module.DataType;
import haflow.module.Module;
import haflow.module.ModuleConfiguration;
import haflow.module.ModuleConfigurationType;
import haflow.module.ModuleEndpoint;
import haflow.module.ModuleType;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@Module(id = "ada600a8-aa63-968a-ca46-ff00a0e0bdff", name = "SVD", category = "Statistics-R", type = ModuleType.JAVA,
configurations = {
@ModuleConfiguration(key = "input", displayName = "input", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT),
@ModuleConfiguration(key = "output", displayName = "output",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT)},
inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = {
@ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText),
@ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) })
public class RSVDModule extends AbstractJavaModule {
@Override
public boolean validate(Map<String, String> configurations,
Map<String, String> inputs, Map<String, String> outputs) {
// TODO Auto-generated method stub
return false;
}
@Override
public String getMainClass() {
return "org.apache.mahout.clustering.canopy.CanopyDriver";
}
@Override
public List<String> getArguments(Map<String, String> configurations) {
Module module= this.getClass().getAnnotation(Module.class);
ModuleConfiguration[] confs = module.configurations();
List<String> result = new ArrayList<String>();
for (String key : configurations.keySet()) {
ModuleConfigurationType confType = getConfigurationType(key, confs);
switch(confType){
case BOOLEAN:
String boolValue = configurations.get(key);
if( boolValue.equals("true")){
result.add("--" + key );
}
break;
case PLAIN_TEXT:
String textValue = configurations.get(key).trim();
if( textValue.length() > 0){
result.add("--" + key);
result.add(configurations.get(key));
}
break;
case OTHER:
default:
System.out.println("Invalid Parameters!");
break;
}
}
return result;
}
private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){
for( ModuleConfiguration conf : confs){
if( key.equals(conf.key()))
return conf.type();
}
return ModuleConfigurationType.OTHER;
}
}

16
src/main/java/haflow/ui/controller/ModuleController.java
View File

@ -1,11 +1,12 @@
package haflow.ui.controller; package haflow.ui.controller;
import java.io.File;
import java.util.UUID;
import haflow.ui.helper.ModuleHelper; import haflow.ui.helper.ModuleHelper;
import haflow.ui.model.ModuleListModel; import haflow.ui.model.ModuleListModel;
import java.io.File;
import java.util.UUID;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller; import org.springframework.stereotype.Controller;
import org.springframework.util.FileCopyUtils; import org.springframework.util.FileCopyUtils;
@ -20,6 +21,7 @@ import org.springframework.web.servlet.ModelAndView;
@Controller @Controller
@RequestMapping("/module") @RequestMapping("/module")
public class ModuleController { public class ModuleController {
private Logger logger = Logger.getLogger(ModuleController.class.getName());
private ModuleHelper moduleHelper; private ModuleHelper moduleHelper;
private ModuleHelper getModuleHelper() { private ModuleHelper getModuleHelper() {
@ -42,15 +44,15 @@ public class ModuleController {
if (!file.isEmpty()) { if (!file.isEmpty()) {
try { try {
String fileName = file.getOriginalFilename(); String fileName = file.getOriginalFilename();
String[] suffixs = fileName.split("\\."); if (fileName.endsWith(".jar")) {
String suffix = "." + suffixs[suffixs.length - 1];
if ((".jar".indexOf(suffix.toLowerCase()) != -1)) {
byte[] bytes = file.getBytes(); byte[] bytes = file.getBytes();
String uploadDir = Thread.currentThread() String clazzPath = Thread.currentThread()
.getContextClassLoader().getResource("/").getFile(); .getContextClassLoader().getResource("/").getFile();
String uploadDir = clazzPath.substring(0, clazzPath.length()-8) + "lib/";
String filePath = uploadDir + fileName; String filePath = uploadDir + fileName;
File toUpload = new File(filePath); File toUpload = new File(filePath);
FileCopyUtils.copy(bytes, toUpload); FileCopyUtils.copy(bytes, toUpload);
logger.info(fileName + " uploaded to : " + filePath);
return new ModelAndView("upload-success"); return new ModelAndView("upload-success");
} else { } else {
return new ModelAndView("upload-error"); return new ModelAndView("upload-error");

2
src/main/java/haflow/util/ClassHelper.java
View File

@ -68,6 +68,8 @@ public class ClassHelper {
.replace("/", "."); .replace("/", ".");
classNames.add(className); classNames.add(className);
} }
}else if(filePath.endsWith(".jar")){
classNames.addAll(getClassNameFromJar(packageName, true, new File(filePath)));
} }
} }
} }

27
src/main/java/haflow/util/ModuleJarClassLoader.java Normal file
View File

@ -0,0 +1,27 @@
package haflow.util;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import org.apache.log4j.Logger;
public class ModuleJarClassLoader{
private static Logger logger = Logger.getLogger(ModuleJarClassLoader.class.getName());
public static ClassLoader loadJarClasses(String jarFilePath){
logger.info("Start loading jar " + jarFilePath);
try {
URL url = new URL("file:/" + jarFilePath);
System.out.println(url.toExternalForm());
URL[] urls = {url};
ClassLoader loader = URLClassLoader.newInstance(urls, ModuleJarClassLoader.class.getClassLoader());
return loader;
} catch (MalformedURLException e) {
e.printStackTrace();
logger.error(e.getMessage());
}
return null;
}
}

5
src/main/resources/log4j.properties Normal file
View File

@ -0,0 +1,5 @@
log4j.rootLogger=INFO, Console
log4j.appender.Console=org.apache.log4j.ConsoleAppender
log4j.appender.Console.layout=org.apache.log4j.PatternLayout
log4j.appender.Console.layout.ConversionPattern=%d [%t] %-5p [%c] - %m%n