diff --git a/haflow.module.init.jar b/haflow.module.init.jar new file mode 100644 index 0000000..cf42b59 Binary files /dev/null and b/haflow.module.init.jar differ diff --git a/src/main/java/haflow/engine/oozie/FlowDeployService.java b/src/main/java/haflow/engine/oozie/FlowDeployService.java index b429aa5..a7e2228 100644 --- a/src/main/java/haflow/engine/oozie/FlowDeployService.java +++ b/src/main/java/haflow/engine/oozie/FlowDeployService.java @@ -24,7 +24,7 @@ public class FlowDeployService { if (jarPath.endsWith(".jar")) { File jarFile = new File(jarPath); File dstPath = new File(deployPath + "/lib"); - // System.out.println(jarPath); + System.out.println(jarPath); this.copyJarFile(jarFile, dstPath, jarFile.getName()); } } diff --git a/src/main/java/haflow/engine/oozie/OozieEngine.java b/src/main/java/haflow/engine/oozie/OozieEngine.java index 48ff40d..bfbf949 100644 --- a/src/main/java/haflow/engine/oozie/OozieEngine.java +++ b/src/main/java/haflow/engine/oozie/OozieEngine.java @@ -39,6 +39,7 @@ import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; +import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.w3c.dom.Document; @@ -47,6 +48,7 @@ import org.w3c.dom.Document; @Component public class OozieEngine extends AbstractEngine { + private Logger logger = Logger.getLogger(this.getClass().getName()); private ModuleUtil moduleUtil; private NodeConfigurationService nodeConfigurationService; private ClusterConfiguration clusterConfiguration; @@ -151,7 +153,7 @@ public class OozieEngine extends AbstractEngine { List sorted = new TopologicalSort(graph).getOrder(); if (sorted == null) { - messageBuilder.append("Error: Flow is has Circles!"); + messageBuilder.append("Error: Flow has Circles!"); } else { String flowName = flow.getName(); String workflowXml = genWorkflowXml(flowName, sorted, @@ -212,6 +214,7 @@ public class OozieEngine extends AbstractEngine { System.out.println(messageBuilder.toString()); model.setMessage(messageBuilder.toString()); System.out.println(messageBuilder.toString()); + logger.info(messageBuilder); return model; } catch (Exception e) { e.printStackTrace(); diff --git a/src/main/java/haflow/module/basic/DemoJavaModule.java b/src/main/java/haflow/module/basic/DemoJavaModule.java deleted file mode 100644 index e6ab6b8..0000000 --- a/src/main/java/haflow/module/basic/DemoJavaModule.java +++ /dev/null @@ -1,39 +0,0 @@ -package haflow.module.basic; - -import haflow.module.AbstractJavaModule; -import haflow.module.DataType; -import haflow.module.Module; -import haflow.module.ModuleConfiguration; -import haflow.module.ModuleConfigurationType; -import haflow.module.ModuleEndpoint; -import haflow.module.ModuleType; - -import java.util.Map; - -@Module(id = "ada600a8-aa63-968a-ca46-9085a0e0bd2f", name = "DemoJava", category = "Basic", type = ModuleType.JAVA, configurations = { @ModuleConfiguration(key = "arg_test", displayName = "Arguments Test", pattern = "^(.*)$", type = ModuleConfigurationType.PLAIN_TEXT), }, inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = { - @ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText), - @ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }) -public class DemoJavaModule extends AbstractJavaModule { - - @Override - public boolean validate(Map configurations, - Map inputs, Map outputs) { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getMainClass() { - return DemoJavaModule.class.getName(); - } - - public static void main(String[] args) { - System.out.println("Demo Java Main"); - - System.out.println("# Arguments: " + args.length); - for (int i = 0; i < args.length; i++) { - System.out.println("Argument[" + i + "]: " + args[i]); - } - } - -} diff --git a/src/main/java/haflow/module/datamining/BuildForestModule.java b/src/main/java/haflow/module/datamining/BuildForestModule.java deleted file mode 100644 index 2b90723..0000000 --- a/src/main/java/haflow/module/datamining/BuildForestModule.java +++ /dev/null @@ -1,92 +0,0 @@ -package haflow.module.datamining; - -import haflow.module.AbstractJavaModule; -import haflow.module.DataType; -import haflow.module.Module; -import haflow.module.ModuleConfiguration; -import haflow.module.ModuleConfigurationType; -import haflow.module.ModuleEndpoint; -import haflow.module.ModuleType; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Module(id = "ada600a8-aa63-968a-ca46-4356a0e0bdac", name = "BuildForest", category = "DataMining-Mahout", type = ModuleType.JAVA, - configurations = { - @ModuleConfiguration(key = "data", displayName = "data: Data path", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "dataset", displayName = "dataset: Dataset path",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "selection", displayName = "selection: Number of variables to select randomly at each tree-node.", pattern = "^(.*)$",type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "no-complete", displayName = "no-complete: The tree is not complemented", pattern = "^(.*)$",type=ModuleConfigurationType.BOOLEAN), - @ModuleConfiguration(key = "minsplit", displayName = "minsplit: Minimum Split", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "minprop", displayName = "minprop: Minimum Porportion", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "seed", displayName = "seed: Seed Value", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "partial", displayName = "partial: use the Partial Data Implementation", pattern = "^(.*)$", type=ModuleConfigurationType.BOOLEAN), - @ModuleConfiguration(key = "nbtrees", displayName = "nbtrees: Number of trees to grow.", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "output", displayName = "output: Output path", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT)}, - inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = { - @ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText), - @ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }) -public class BuildForestModule extends AbstractJavaModule { - - @Override - public boolean validate(Map configurations, - Map inputs, Map outputs) { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getMainClass() { - return "org.apache.mahout.classifier.df.mapreduce.BuildForest"; - } - - @Override - public List getArguments(Map configurations) { - Module module= this.getClass().getAnnotation(Module.class); - ModuleConfiguration[] confs = module.configurations(); - - List result = new ArrayList(); - for (String key : configurations.keySet()) { - ModuleConfigurationType confType = getConfigurationType(key, confs); - switch(confType){ - case BOOLEAN: - String boolValue = configurations.get(key); - if( boolValue.equals("true")){ - result.add("--" + key ); - } - break; - case PLAIN_TEXT: - String textValue = configurations.get(key).trim(); - if( textValue.length() > 0){ - result.add("--" + key); - result.add(configurations.get(key)); - } - break; - case OTHER: - default: - System.out.println("Invalid Parameters!"); - break; - } - } - return result; - } - - private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){ - for( ModuleConfiguration conf : confs){ - if( key.equals(conf.key())) - return conf.type(); - } - return ModuleConfigurationType.OTHER; - } - - public static void main(String[] args) { - System.out.println("Demo Java Main"); - - System.out.println("# Arguments: " + args.length); - for (int i = 0; i < args.length; i++) { - System.out.println("Argument[" + i + "]: " + args[i]); - } - } - -} diff --git a/src/main/java/haflow/module/datamining/CanopyClusterModule.java b/src/main/java/haflow/module/datamining/CanopyClusterModule.java deleted file mode 100644 index b166107..0000000 --- a/src/main/java/haflow/module/datamining/CanopyClusterModule.java +++ /dev/null @@ -1,83 +0,0 @@ -package haflow.module.datamining; - -import haflow.module.AbstractJavaModule; -import haflow.module.DataType; -import haflow.module.Module; -import haflow.module.ModuleConfiguration; -import haflow.module.ModuleConfigurationType; -import haflow.module.ModuleEndpoint; -import haflow.module.ModuleType; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Module(id = "ada600a8-aa63-968a-ca46-4356a0e0bd00", name = "CanopyCluster", category = "DataMining-Mahout", type = ModuleType.JAVA, - configurations = { - @ModuleConfiguration(key = "input", displayName = "input", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "output", displayName = "output",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "distanceMeasure", displayName = "distanceMeasure", pattern = "^(.*)$",type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "t1", displayName = "t1", pattern = "^(.*)$",type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "t2", displayName = "t2", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "t3", displayName = "t3", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "t4", displayName = "t4", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "clusterFilter", displayName = "clusterFilter", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "overwrite", displayName = "overwrite", pattern = "^(.*)$", type=ModuleConfigurationType.BOOLEAN), - @ModuleConfiguration(key = "outlierThreshold", displayName = "outlierThreshold", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT)}, - inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = { - @ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText), - @ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }) -public class CanopyClusterModule extends AbstractJavaModule { - - @Override - public boolean validate(Map configurations, - Map inputs, Map outputs) { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getMainClass() { - return "org.apache.mahout.clustering.canopy.CanopyDriver"; - } - - @Override - public List getArguments(Map configurations) { - Module module= this.getClass().getAnnotation(Module.class); - ModuleConfiguration[] confs = module.configurations(); - - List result = new ArrayList(); - for (String key : configurations.keySet()) { - ModuleConfigurationType confType = getConfigurationType(key, confs); - switch(confType){ - case BOOLEAN: - String boolValue = configurations.get(key); - if( boolValue.equals("true")){ - result.add("--" + key ); - } - break; - case PLAIN_TEXT: - String textValue = configurations.get(key).trim(); - if( textValue.length() > 0){ - result.add("--" + key); - result.add(configurations.get(key)); - } - break; - case OTHER: - default: - System.out.println("Invalid Parameters!"); - break; - } - } - return result; - } - - private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){ - for( ModuleConfiguration conf : confs){ - if( key.equals(conf.key())) - return conf.type(); - } - return ModuleConfigurationType.OTHER; - } - -} diff --git a/src/main/java/haflow/module/datamining/DescribeModule.java b/src/main/java/haflow/module/datamining/DescribeModule.java deleted file mode 100644 index a594467..0000000 --- a/src/main/java/haflow/module/datamining/DescribeModule.java +++ /dev/null @@ -1,93 +0,0 @@ -package haflow.module.datamining; - -import haflow.module.AbstractJavaModule; -import haflow.module.DataType; -import haflow.module.Module; -import haflow.module.ModuleConfiguration; -import haflow.module.ModuleConfigurationType; -import haflow.module.ModuleEndpoint; -import haflow.module.ModuleType; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Module(id = "ada600a8-aa63-968a-ca46-4356a0e0bdab", name = "Describe", category = "DataMining-Mahout", type = ModuleType.JAVA, configurations = { - @ModuleConfiguration(key = "path", displayName = "path: Data path", pattern = "^(.*)$", type = ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "descriptor", displayName = "descriptor: Data descriptor", pattern = "^(.*)$", type = ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "file", displayName = "file: Path to generated descriptor file", pattern = "^(.*)$", type = ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "regression", displayName = "regression: Regression Problem", pattern = "^(.*)$", type = ModuleConfigurationType.BOOLEAN) }, inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = { - @ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText), - @ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }) -public class DescribeModule extends AbstractJavaModule { - - @Override - public boolean validate(Map configurations, - Map inputs, Map outputs) { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getMainClass() { - return "org.apache.mahout.classifier.df.tools.Describe"; - } - - @Override - public List getArguments(Map configurations) { - Module module = this.getClass().getAnnotation(Module.class); - ModuleConfiguration[] confs = module.configurations(); - - List result = new ArrayList(); - for (String key : configurations.keySet()) { - ModuleConfigurationType confType = getConfigurationType(key, confs); - switch (confType) { - case BOOLEAN: - String boolValue = configurations.get(key); - if( boolValue.equals("true")){ - result.add("--" + key); - } - break; - case PLAIN_TEXT: - String textValue = configurations.get(key).trim(); - if( textValue.length() > 0){ - if(key.equals("descriptor")){ - result.add("--" + key); - String[] descriptors = textValue.split(" "); - for(String desp : descriptors){ - result.add(desp); - } - }else{ - result.add("--" + key); - result.add(configurations.get(key)); - } - } - break; - case OTHER: - default: - System.out.println("Invalid Parameters!"); - break; - } - } - return result; - } - - private ModuleConfigurationType getConfigurationType(String key, - ModuleConfiguration[] confs) { - for (ModuleConfiguration conf : confs) { - if (key.equals(conf.key())) - return conf.type(); - } - return ModuleConfigurationType.OTHER; - } - - public static void main(String[] args) { - System.out.println("Demo Java Main"); - - System.out.println("# Arguments: " + args.length); - for (int i = 0; i < args.length; i++) { - System.out.println("Argument[" + i + "]: " + args[i]); - } - } - -} diff --git a/src/main/java/haflow/module/datamining/TestForestModule.java b/src/main/java/haflow/module/datamining/TestForestModule.java deleted file mode 100644 index 00e2d1d..0000000 --- a/src/main/java/haflow/module/datamining/TestForestModule.java +++ /dev/null @@ -1,88 +0,0 @@ -package haflow.module.datamining; - -import haflow.module.AbstractJavaModule; -import haflow.module.DataType; -import haflow.module.Module; -import haflow.module.ModuleConfiguration; -import haflow.module.ModuleConfigurationType; -import haflow.module.ModuleEndpoint; -import haflow.module.ModuleType; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Module(id = "ada600a8-aa63-968a-ca46-4356a0e0bdad", name = "TestForest", category = "DataMining-Mahout", type = ModuleType.JAVA, - configurations = { - @ModuleConfiguration(key = "input", displayName = "input : Path to job input directory.", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "dataset", displayName = "dataset : Dataset path",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "model", displayName = "model: Path to the Decision Forest", pattern = "^(.*)$",type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "output", displayName = "output: The directory pathname for output.", pattern = "^(.*)$",type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "analyze", displayName = "analyze: ", pattern = "^(.*)$", type=ModuleConfigurationType.BOOLEAN), - @ModuleConfiguration(key = "mapreduce", displayName = "mapreduce: ", pattern = "^(.*)$", type=ModuleConfigurationType.BOOLEAN)}, - inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = { - @ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText), - @ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }) -public class TestForestModule extends AbstractJavaModule { - - @Override - public boolean validate(Map configurations, - Map inputs, Map outputs) { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getMainClass() { - return "org.apache.mahout.classifier.df.mapreduce.TestForest"; - } - - @Override - public List getArguments(Map configurations) { - Module module= this.getClass().getAnnotation(Module.class); - ModuleConfiguration[] confs = module.configurations(); - - List result = new ArrayList(); - for (String key : configurations.keySet()) { - ModuleConfigurationType confType = getConfigurationType(key, confs); - switch(confType){ - case BOOLEAN: - String boolValue = configurations.get(key); - if( boolValue.equals("true")){ - result.add("--" + key ); - } - break; - case PLAIN_TEXT: - String textValue = configurations.get(key).trim(); - if( textValue.length() > 0){ - result.add("--" + key); - result.add(configurations.get(key)); - } - break; - case OTHER: - default: - System.out.println("Invalid Parameters!"); - break; - } - } - return result; - } - - private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){ - for( ModuleConfiguration conf : confs){ - if( key.equals(conf.key())) - return conf.type(); - } - return ModuleConfigurationType.OTHER; - } - - public static void main(String[] args) { - System.out.println("Demo Java Main"); - - System.out.println("# Arguments: " + args.length); - for (int i = 0; i < args.length; i++) { - System.out.println("Argument[" + i + "]: " + args[i]); - } - } - -} diff --git a/src/main/java/haflow/module/dm/weka/WekaAssociationModule.java b/src/main/java/haflow/module/dm/weka/WekaAssociationModule.java deleted file mode 100644 index c73c481..0000000 --- a/src/main/java/haflow/module/dm/weka/WekaAssociationModule.java +++ /dev/null @@ -1,75 +0,0 @@ -package haflow.module.dm.weka; - -import haflow.module.AbstractJavaModule; -import haflow.module.DataType; -import haflow.module.Module; -import haflow.module.ModuleConfiguration; -import haflow.module.ModuleConfigurationType; -import haflow.module.ModuleEndpoint; -import haflow.module.ModuleType; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Module(id = "ada600a8-aa63-968a-ca46-0156a0e0bdff", name = "Apriori", category = "DataMining-Weka", type = ModuleType.JAVA, - configurations = { - @ModuleConfiguration(key = "input", displayName = "input", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "output", displayName = "output",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT)}, - inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = { - @ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText), - @ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }) -public class WekaAssociationModule extends AbstractJavaModule { - - @Override - public boolean validate(Map configurations, - Map inputs, Map outputs) { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getMainClass() { - return "org.apache.mahout.clustering.canopy.CanopyDriver"; - } - - @Override - public List getArguments(Map configurations) { - Module module= this.getClass().getAnnotation(Module.class); - ModuleConfiguration[] confs = module.configurations(); - - List result = new ArrayList(); - for (String key : configurations.keySet()) { - ModuleConfigurationType confType = getConfigurationType(key, confs); - switch(confType){ - case BOOLEAN: - String boolValue = configurations.get(key); - if( boolValue.equals("true")){ - result.add("--" + key ); - } - break; - case PLAIN_TEXT: - String textValue = configurations.get(key).trim(); - if( textValue.length() > 0){ - result.add("--" + key); - result.add(configurations.get(key)); - } - break; - case OTHER: - default: - System.out.println("Invalid Parameters!"); - break; - } - } - return result; - } - - private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){ - for( ModuleConfiguration conf : confs){ - if( key.equals(conf.key())) - return conf.type(); - } - return ModuleConfigurationType.OTHER; - } - -} diff --git a/src/main/java/haflow/module/dm/weka/WekaClassifierModule.java b/src/main/java/haflow/module/dm/weka/WekaClassifierModule.java deleted file mode 100644 index bd3d51e..0000000 --- a/src/main/java/haflow/module/dm/weka/WekaClassifierModule.java +++ /dev/null @@ -1,75 +0,0 @@ -package haflow.module.dm.weka; - -import haflow.module.AbstractJavaModule; -import haflow.module.DataType; -import haflow.module.Module; -import haflow.module.ModuleConfiguration; -import haflow.module.ModuleConfigurationType; -import haflow.module.ModuleEndpoint; -import haflow.module.ModuleType; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Module(id = "ada600a8-aa63-968a-ca46-0056a0e0bdff", name = "KMeans", category = "DataMining-Weka", type = ModuleType.JAVA, - configurations = { - @ModuleConfiguration(key = "input", displayName = "input", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "output", displayName = "output",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT)}, - inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = { - @ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText), - @ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }) -public class WekaClassifierModule extends AbstractJavaModule { - - @Override - public boolean validate(Map configurations, - Map inputs, Map outputs) { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getMainClass() { - return "org.apache.mahout.clustering.canopy.CanopyDriver"; - } - - @Override - public List getArguments(Map configurations) { - Module module= this.getClass().getAnnotation(Module.class); - ModuleConfiguration[] confs = module.configurations(); - - List result = new ArrayList(); - for (String key : configurations.keySet()) { - ModuleConfigurationType confType = getConfigurationType(key, confs); - switch(confType){ - case BOOLEAN: - String boolValue = configurations.get(key); - if( boolValue.equals("true")){ - result.add("--" + key ); - } - break; - case PLAIN_TEXT: - String textValue = configurations.get(key).trim(); - if( textValue.length() > 0){ - result.add("--" + key); - result.add(configurations.get(key)); - } - break; - case OTHER: - default: - System.out.println("Invalid Parameters!"); - break; - } - } - return result; - } - - private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){ - for( ModuleConfiguration conf : confs){ - if( key.equals(conf.key())) - return conf.type(); - } - return ModuleConfigurationType.OTHER; - } - -} diff --git a/src/main/java/haflow/module/dm/weka/WekaClusterModule.java b/src/main/java/haflow/module/dm/weka/WekaClusterModule.java deleted file mode 100644 index 506a049..0000000 --- a/src/main/java/haflow/module/dm/weka/WekaClusterModule.java +++ /dev/null @@ -1,75 +0,0 @@ -package haflow.module.dm.weka; - -import haflow.module.AbstractJavaModule; -import haflow.module.DataType; -import haflow.module.Module; -import haflow.module.ModuleConfiguration; -import haflow.module.ModuleConfigurationType; -import haflow.module.ModuleEndpoint; -import haflow.module.ModuleType; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Module(id = "ada600a8-aa63-968a-ca46-0256a0e0bdff", name = "ID3", category = "DataMining-Weka", type = ModuleType.JAVA, - configurations = { - @ModuleConfiguration(key = "input", displayName = "input", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "output", displayName = "output",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT)}, - inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = { - @ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText), - @ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }) -public class WekaClusterModule extends AbstractJavaModule { - - @Override - public boolean validate(Map configurations, - Map inputs, Map outputs) { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getMainClass() { - return "org.apache.mahout.clustering.canopy.CanopyDriver"; - } - - @Override - public List getArguments(Map configurations) { - Module module= this.getClass().getAnnotation(Module.class); - ModuleConfiguration[] confs = module.configurations(); - - List result = new ArrayList(); - for (String key : configurations.keySet()) { - ModuleConfigurationType confType = getConfigurationType(key, confs); - switch(confType){ - case BOOLEAN: - String boolValue = configurations.get(key); - if( boolValue.equals("true")){ - result.add("--" + key ); - } - break; - case PLAIN_TEXT: - String textValue = configurations.get(key).trim(); - if( textValue.length() > 0){ - result.add("--" + key); - result.add(configurations.get(key)); - } - break; - case OTHER: - default: - System.out.println("Invalid Parameters!"); - break; - } - } - return result; - } - - private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){ - for( ModuleConfiguration conf : confs){ - if( key.equals(conf.key())) - return conf.type(); - } - return ModuleConfigurationType.OTHER; - } - -} diff --git a/src/main/java/haflow/module/statistics/R/RSVDModule.java b/src/main/java/haflow/module/statistics/R/RSVDModule.java deleted file mode 100644 index 515b2fb..0000000 --- a/src/main/java/haflow/module/statistics/R/RSVDModule.java +++ /dev/null @@ -1,75 +0,0 @@ -package haflow.module.statistics.R; - -import haflow.module.AbstractJavaModule; -import haflow.module.DataType; -import haflow.module.Module; -import haflow.module.ModuleConfiguration; -import haflow.module.ModuleConfigurationType; -import haflow.module.ModuleEndpoint; -import haflow.module.ModuleType; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -@Module(id = "ada600a8-aa63-968a-ca46-ff00a0e0bdff", name = "SVD", category = "Statistics-R", type = ModuleType.JAVA, - configurations = { - @ModuleConfiguration(key = "input", displayName = "input", pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT), - @ModuleConfiguration(key = "output", displayName = "output",pattern = "^(.*)$", type=ModuleConfigurationType.PLAIN_TEXT)}, - inputs = { @ModuleEndpoint(name = "from", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }, outputs = { - @ModuleEndpoint(name = "ok", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText), - @ModuleEndpoint(name = "error", minNumber = 1, maxNumber = 1, dataType = DataType.PlainText) }) -public class RSVDModule extends AbstractJavaModule { - - @Override - public boolean validate(Map configurations, - Map inputs, Map outputs) { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getMainClass() { - return "org.apache.mahout.clustering.canopy.CanopyDriver"; - } - - @Override - public List getArguments(Map configurations) { - Module module= this.getClass().getAnnotation(Module.class); - ModuleConfiguration[] confs = module.configurations(); - - List result = new ArrayList(); - for (String key : configurations.keySet()) { - ModuleConfigurationType confType = getConfigurationType(key, confs); - switch(confType){ - case BOOLEAN: - String boolValue = configurations.get(key); - if( boolValue.equals("true")){ - result.add("--" + key ); - } - break; - case PLAIN_TEXT: - String textValue = configurations.get(key).trim(); - if( textValue.length() > 0){ - result.add("--" + key); - result.add(configurations.get(key)); - } - break; - case OTHER: - default: - System.out.println("Invalid Parameters!"); - break; - } - } - return result; - } - - private ModuleConfigurationType getConfigurationType(String key, ModuleConfiguration[] confs){ - for( ModuleConfiguration conf : confs){ - if( key.equals(conf.key())) - return conf.type(); - } - return ModuleConfigurationType.OTHER; - } - -} diff --git a/src/main/java/haflow/ui/controller/ModuleController.java b/src/main/java/haflow/ui/controller/ModuleController.java index 6bcc74b..c7912a2 100644 --- a/src/main/java/haflow/ui/controller/ModuleController.java +++ b/src/main/java/haflow/ui/controller/ModuleController.java @@ -1,11 +1,12 @@ package haflow.ui.controller; -import java.io.File; -import java.util.UUID; - import haflow.ui.helper.ModuleHelper; import haflow.ui.model.ModuleListModel; +import java.io.File; +import java.util.UUID; + +import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.util.FileCopyUtils; @@ -20,6 +21,7 @@ import org.springframework.web.servlet.ModelAndView; @Controller @RequestMapping("/module") public class ModuleController { + private Logger logger = Logger.getLogger(ModuleController.class.getName()); private ModuleHelper moduleHelper; private ModuleHelper getModuleHelper() { @@ -42,15 +44,15 @@ public class ModuleController { if (!file.isEmpty()) { try { String fileName = file.getOriginalFilename(); - String[] suffixs = fileName.split("\\."); - String suffix = "." + suffixs[suffixs.length - 1]; - if ((".jar".indexOf(suffix.toLowerCase()) != -1)) { + if (fileName.endsWith(".jar")) { byte[] bytes = file.getBytes(); - String uploadDir = Thread.currentThread() + String clazzPath = Thread.currentThread() .getContextClassLoader().getResource("/").getFile(); + String uploadDir = clazzPath.substring(0, clazzPath.length()-8) + "lib/"; String filePath = uploadDir + fileName; File toUpload = new File(filePath); FileCopyUtils.copy(bytes, toUpload); + logger.info(fileName + " uploaded to : " + filePath); return new ModelAndView("upload-success"); } else { return new ModelAndView("upload-error"); diff --git a/src/main/java/haflow/util/ClassHelper.java b/src/main/java/haflow/util/ClassHelper.java index 69d2367..e718c04 100644 --- a/src/main/java/haflow/util/ClassHelper.java +++ b/src/main/java/haflow/util/ClassHelper.java @@ -68,6 +68,8 @@ public class ClassHelper { .replace("/", "."); classNames.add(className); } + }else if(filePath.endsWith(".jar")){ + classNames.addAll(getClassNameFromJar(packageName, true, new File(filePath))); } } } diff --git a/src/main/java/haflow/util/ModuleJarClassLoader.java b/src/main/java/haflow/util/ModuleJarClassLoader.java new file mode 100644 index 0000000..aec14c0 --- /dev/null +++ b/src/main/java/haflow/util/ModuleJarClassLoader.java @@ -0,0 +1,27 @@ +package haflow.util; + +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; + +import org.apache.log4j.Logger; + +public class ModuleJarClassLoader{ + private static Logger logger = Logger.getLogger(ModuleJarClassLoader.class.getName()); + + public static ClassLoader loadJarClasses(String jarFilePath){ + logger.info("Start loading jar " + jarFilePath); + try { + URL url = new URL("file:/" + jarFilePath); + System.out.println(url.toExternalForm()); + URL[] urls = {url}; + ClassLoader loader = URLClassLoader.newInstance(urls, ModuleJarClassLoader.class.getClassLoader()); + return loader; + } catch (MalformedURLException e) { + e.printStackTrace(); + logger.error(e.getMessage()); + } + return null; + } + +} diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties new file mode 100644 index 0000000..01de8f3 --- /dev/null +++ b/src/main/resources/log4j.properties @@ -0,0 +1,5 @@ +log4j.rootLogger=INFO, Console + +log4j.appender.Console=org.apache.log4j.ConsoleAppender +log4j.appender.Console.layout=org.apache.log4j.PatternLayout +log4j.appender.Console.layout.ConversionPattern=%d [%t] %-5p [%c] - %m%n \ No newline at end of file