move all the configurations to the src/main/resources folder.
This commit is contained in:
parent
b59ba985c4
commit
c65a8b2764
|
@ -4,13 +4,13 @@ import haflow.dto.entity.Flow;
|
|||
import haflow.dto.entity.Node;
|
||||
import haflow.dto.profile.NodeConfiguration;
|
||||
import haflow.engine.model.DirectedGraph;
|
||||
import haflow.engine.model.GlobalConfiguration;
|
||||
import haflow.engine.model.TopologicalSort;
|
||||
import haflow.module.AbstractHiveModule;
|
||||
import haflow.module.AbstractJavaModule;
|
||||
import haflow.module.Module;
|
||||
import haflow.service.NodeConfigurationService;
|
||||
import haflow.util.DocumentTransformer;
|
||||
import haflow.util.OozieJobGlobalConfiguration;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
@ -143,7 +143,7 @@ public class OozieFlowXmlGenerator {
|
|||
}
|
||||
|
||||
private NodeConfigurationService nodeConfigurationService;
|
||||
private GlobalConfiguration globalConfiguration;
|
||||
private OozieJobGlobalConfiguration globalConfiguration;
|
||||
private GraphTransformer graphTransformer;
|
||||
|
||||
@Autowired
|
||||
|
@ -153,7 +153,7 @@ public class OozieFlowXmlGenerator {
|
|||
}
|
||||
|
||||
@Autowired
|
||||
private void setGlobalConfiguration(GlobalConfiguration globalConfiguration) {
|
||||
private void setGlobalConfiguration(OozieJobGlobalConfiguration globalConfiguration) {
|
||||
this.globalConfiguration = globalConfiguration;
|
||||
}
|
||||
|
||||
|
|
|
@ -1,35 +1,56 @@
|
|||
package haflow.engine.oozie;
|
||||
|
||||
import haflow.util.ClusterConfiguration;
|
||||
import haflow.util.OozieJobConfiguration;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.oozie.client.OozieClient;
|
||||
import org.apache.oozie.client.OozieClientException;
|
||||
import org.apache.oozie.client.WorkflowJob;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class OozieService {
|
||||
|
||||
private final String APP_PATH;
|
||||
private final String OOZIE_URL;
|
||||
private OozieClient oozieClient;
|
||||
|
||||
private OozieJobConfiguration oozieJobConfiguration;
|
||||
|
||||
@Autowired
|
||||
private void setOozieJobConfiguration(
|
||||
OozieJobConfiguration oozieJobConfiguration) {
|
||||
this.oozieJobConfiguration = oozieJobConfiguration;
|
||||
}
|
||||
|
||||
private final String APP_PATH = "hdfs://m150:9000/user/root/examples/apps/";
|
||||
private OozieClient wc;
|
||||
|
||||
public OozieService() {
|
||||
wc = new OozieClient("http://m150:11000/oozie/");
|
||||
@Autowired
|
||||
public OozieService(ClusterConfiguration clusterConfiguration) {
|
||||
this.APP_PATH = clusterConfiguration.getProperty(ClusterConfiguration.WORKSPACE_HDFS);
|
||||
this.OOZIE_URL = clusterConfiguration.getProperty(ClusterConfiguration.OOZIE_URL);
|
||||
oozieClient = new OozieClient(OOZIE_URL);
|
||||
}
|
||||
|
||||
public String runJob(String flowName) {
|
||||
Properties conf = wc.createConfiguration();
|
||||
Properties conf = oozieClient.createConfiguration();
|
||||
conf.setProperty(OozieClient.APP_PATH, APP_PATH + flowName);
|
||||
conf.setProperty("nameNode", "hdfs://m150:9000");
|
||||
conf.setProperty("jobTracker", "m150:9001");
|
||||
conf.setProperty("queueName", "default");
|
||||
conf.setProperty("examplesRoot", "examples");
|
||||
conf.setProperty("oozie.use.system.libpath", "true");
|
||||
conf.setProperty("oozie.libpath", "share/lib/hive");
|
||||
|
||||
Properties oozieJobConfsFromFiles = this.oozieJobConfiguration.getProperties();
|
||||
for( String key : oozieJobConfsFromFiles.stringPropertyNames()){
|
||||
conf.setProperty(key, oozieJobConfsFromFiles.getProperty(key));
|
||||
}
|
||||
// conf.setProperty("nameNode", "hdfs://m150:9000");
|
||||
// conf.setProperty("jobTracker", "m150:9001");
|
||||
// conf.setProperty("queueName", "default");
|
||||
// conf.setProperty("examplesRoot", "examples");
|
||||
// conf.setProperty("oozie.use.system.libpath", "true");
|
||||
// conf.setProperty("oozie.libpath", "share/lib/hive");
|
||||
|
||||
String jobId = null;
|
||||
try {
|
||||
jobId = wc.run(conf);
|
||||
jobId = oozieClient.run(conf);
|
||||
// System.out.println("Workflow job submitted! Id:" + jobId);
|
||||
} catch (OozieClientException e) {
|
||||
e.printStackTrace();
|
||||
|
|
|
@ -9,28 +9,24 @@ import org.springframework.stereotype.Component;
|
|||
@Component
|
||||
public class ClusterConfiguration {
|
||||
private static final String PROERTIES_FILE_NAME = "cluster.properties";
|
||||
|
||||
public static final String OOZIE_URL = "oozie_url";
|
||||
public static final String WORKSPACE_LOCAL = "workspace_local";
|
||||
public static final String WORKSPACE_HDFS = "workspace_hdfs";
|
||||
public static final String MIDDLE_DATA_HDFS = "middle_data_hdfs";
|
||||
public static final String WORKSPACE_LOCAL = "workspace_local";
|
||||
|
||||
public static final String FS_DEFAULT_NAME = "fs.default.name";
|
||||
|
||||
|
||||
private Properties properties;
|
||||
|
||||
private Properties getProperties() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
private void setProperties(Properties properties) {
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
public ClusterConfiguration() {
|
||||
ClassLoader loader = ClusterConfiguration.class.getClassLoader();
|
||||
URL url = loader.getResource(ClusterConfiguration.PROERTIES_FILE_NAME);
|
||||
this.setProperties(new Properties());
|
||||
this.properties = new Properties();
|
||||
try {
|
||||
FileInputStream inputFile = new FileInputStream(url.getFile());
|
||||
this.getProperties().load(inputFile);
|
||||
this.properties.load(inputFile);
|
||||
inputFile.close();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
|
@ -38,6 +34,6 @@ public class ClusterConfiguration {
|
|||
}
|
||||
|
||||
public String getProperty(String key) {
|
||||
return this.getProperties().getProperty(key);
|
||||
return this.properties.getProperty(key);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,35 @@
|
|||
package haflow.util;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.net.URL;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class OozieJobConfiguration {
|
||||
private static final String GLOBAL_CONFIGURATION_FILE_NAME = "oozie.job.properties";
|
||||
private Properties properties;
|
||||
|
||||
public OozieJobConfiguration(){
|
||||
this.loadProperties();
|
||||
}
|
||||
private Properties loadProperties() {
|
||||
// Properties properties;
|
||||
ClassLoader loader = ClusterConfiguration.class.getClassLoader();
|
||||
URL url = loader.getResource(GLOBAL_CONFIGURATION_FILE_NAME);
|
||||
properties = new Properties();
|
||||
try {
|
||||
FileInputStream inputFile = new FileInputStream(url.getFile());
|
||||
properties.load(inputFile);
|
||||
inputFile.close();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
|
||||
public Properties getProperties(){
|
||||
return this.properties;
|
||||
}
|
||||
}
|
|
@ -1,27 +1,48 @@
|
|||
package haflow.engine.model;
|
||||
package haflow.util;
|
||||
|
||||
import haflow.util.ClusterConfiguration;
|
||||
import java.io.FileInputStream;
|
||||
import java.net.URL;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@Component
|
||||
public class GlobalConfiguration {
|
||||
public class OozieJobGlobalConfiguration {
|
||||
|
||||
private static final String GLOBAL_CONFIGURATION_FILE_NAME = "oozie.job.global.properties";
|
||||
|
||||
public static final String JOB_TRACKER = "job-tracker";
|
||||
|
||||
public static final String NAME_NODE = "name-node";
|
||||
public static final String MAPRED_JOB_QUEUE_NAME = "mapred.job.queue.name";
|
||||
|
||||
public static final String OOZIE_USER_SYSTEM_LIBPATH = "oozie.use.system.libpath";
|
||||
public static final String OOZIE_HIVE_CONNECTION_URL = "oozie.hive.connection.url";
|
||||
public static final String OOZIE_HIVE_CONNECTION_NAME = "oozie.hive.connection.name";
|
||||
public static final String OOZIE_HIVE_CONNECTION_PWD = "oozie.hive.connection.password";
|
||||
|
||||
private Map<String, String> map;
|
||||
|
||||
@Autowired
|
||||
public GlobalConfiguration(ClusterConfiguration cc) {
|
||||
|
||||
private Properties loadProperties() {
|
||||
Properties properties;
|
||||
ClassLoader loader = ClusterConfiguration.class.getClassLoader();
|
||||
URL url = loader.getResource(GLOBAL_CONFIGURATION_FILE_NAME);
|
||||
properties = new Properties();
|
||||
try {
|
||||
FileInputStream inputFile = new FileInputStream(url.getFile());
|
||||
properties.load(inputFile);
|
||||
inputFile.close();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
|
||||
// @Autowired
|
||||
public OozieJobGlobalConfiguration() {
|
||||
Properties cc = this.loadProperties();
|
||||
this.map = new HashMap<String, String>();
|
||||
map.put(JOB_TRACKER, cc.getProperty(JOB_TRACKER));
|
||||
map.put(NAME_NODE, cc.getProperty(NAME_NODE));
|
|
@ -22,6 +22,9 @@ public final class SessionHelper {
|
|||
public SessionHelper() {
|
||||
try {
|
||||
Configuration cfg = new Configuration().configure();
|
||||
// cfg.setProperty("hibernate.connection.url", "jdbc:mysql://localhost:3306/haflow");
|
||||
// cfg.setProperty("hibernate.connection.username", "root");
|
||||
// cfg.setProperty("hibernate.connection.password", "123456");
|
||||
ServiceRegistry serviceRegistry = new StandardServiceRegistryBuilder()
|
||||
.applySettings(cfg.getProperties()).build();
|
||||
this.setSessionFactory(cfg.buildSessionFactory(serviceRegistry));
|
||||
|
|
|
@ -1,17 +1,6 @@
|
|||
oozie_url = http://m150:11000/oozie/
|
||||
workspace_local = D:/haflow/flows/
|
||||
workspace_hdfs = hdfs://m150:9000/user/root/examples/apps/
|
||||
middle_data_hdfs = hdfs://m150:9000/haflow/runtime/
|
||||
|
||||
fs.default.name = hdfs://m150:9000
|
||||
|
||||
name-node=hdfs://m150:9000
|
||||
job-tracker=m150:9001
|
||||
mapred.job.queue.name=default
|
||||
oozie.use.system.libpath=true
|
||||
|
||||
oozie.hive.host=m150
|
||||
oozie.hive.port=10000
|
||||
|
||||
oozie.hive.connection.url=jdbc:hive://m150:10000/default
|
||||
oozie.hive.connection.name=""
|
||||
oozie.hive.connection.password=""
|
||||
fs.default.name = hdfs://m150:9000
|
|
@ -3,7 +3,7 @@
|
|||
<hibernate-configuration>
|
||||
<session-factory>
|
||||
<property name="hibernate.connection.driver_class">com.mysql.jdbc.Driver </property>
|
||||
<property name="hibernate.connection.url"> jdbc:mysql://localhost:3306/haflow
|
||||
<property name="hibernate.connection.url"> jdbc:mysql://localhost:3306/haflow
|
||||
</property>
|
||||
<property name="hibernate.connection.username">root </property>
|
||||
<property name="hibernate.connection.password">123456 </property>
|
||||
|
|
|
@ -0,0 +1,12 @@
|
|||
job-tracker=m150:9001
|
||||
|
||||
name-node=hdfs://m150:9000
|
||||
mapred.job.queue.name=default
|
||||
|
||||
oozie.use.system.libpath=true
|
||||
oozie.hive.connection.url=jdbc:hive://m150:10000/default
|
||||
oozie.hive.connection.name=""
|
||||
oozie.hive.connection.password=""
|
||||
|
||||
#oozie.hive.host=m150
|
||||
#oozie.hive.port=10000
|
|
@ -0,0 +1,6 @@
|
|||
nameNode = "hdfs://m150:9000"
|
||||
jobTracker = "m150:9001"
|
||||
queueName = "default"
|
||||
examplesRoot = "examples"
|
||||
oozie.use.system.libpath = "true"
|
||||
oozie.libpath = "share/lib/hive"
|
|
@ -294,7 +294,7 @@ HAFlow.Main.prototype.paintReportList = function() {
|
|||
this.simpleReportListPaneId = this.reportListContainerId + "_" + "test";
|
||||
var reportListPane = new dijit.layout.ContentPane({
|
||||
id : this.simpleReportListPaneId,
|
||||
title : "Simple",
|
||||
title : myfile.charts,
|
||||
});
|
||||
this.ui.secondTrailingContainer.addChild(reportListPane);
|
||||
text = "<div class=\"reportmodule\" id=\"reportmodule_" + "text"
|
||||
|
|
|
@ -80,5 +80,6 @@ myfile={
|
|||
"size":"Size",
|
||||
"flowInfo":"Flow Info",
|
||||
"data":"Data",
|
||||
"charts":"Charts"
|
||||
};
|
||||
|
||||
|
|
|
@ -82,5 +82,6 @@
|
|||
"size":"大小",
|
||||
"flowInfo":"流程信息",
|
||||
"data":"数据",
|
||||
"charts":"图表",
|
||||
};
|
||||
|
Loading…
Reference in New Issue