diff --git a/Bench4Q-Agent/src/main/java/org/bench4q/agent/plugin/basic/HBasePlugin/HBasePlugin.java b/Bench4Q-Agent/src/main/java/org/bench4q/agent/plugin/basic/HBasePlugin/HBasePlugin.java index 699c17fa..1f468205 100644 --- a/Bench4Q-Agent/src/main/java/org/bench4q/agent/plugin/basic/HBasePlugin/HBasePlugin.java +++ b/Bench4Q-Agent/src/main/java/org/bench4q/agent/plugin/basic/HBasePlugin/HBasePlugin.java @@ -1,89 +1,158 @@ -package org.bench4q.agent.plugin.basic.HBasePlugin; - -import java.util.UUID; - -import org.bench4q.agent.plugin.Constructor; -import org.bench4q.agent.plugin.Parameter; -import org.bench4q.agent.plugin.Plugin; -import org.bench4q.agent.plugin.behavior.Behavior; -import org.bench4q.agent.plugin.behavior.BehaviorType; -import org.bench4q.agent.utils.Type.SupportTypes; -import org.bench4q.share.exception.Bench4QRunTimeException; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; -import org.apache.hadoop.hbase.filter.Filter; -import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.log4j.Logger; - -@Plugin("HBase") -public class HBasePlugin { - private final Configuration conf; - private final HTable tableUnderTest; - - @Constructor - public HBasePlugin( - @Parameter(value = "dnsServer", type = SupportTypes.Field) String dnsServer, - @Parameter(value = "zookeeperQuorum", type = SupportTypes.Field) String zookeeperQuorum, - @Parameter(value = "zookeeperDataDir", type = SupportTypes.Field) String zookeeperDataDir, - @Parameter(value = "masterAdrress", type = SupportTypes.Field) String masterAdrress, - @Parameter(value = "tableName", type = SupportTypes.Field) String tableName) { - conf = HBaseConfiguration.create(); - try { - Configuration config = new Configuration(); - config.set("hbase.master.dns.nameserver", dnsServer); - config.setBoolean("hbase.cluster.distributed", true); - config.set("hbase.zookeeper.quorum", zookeeperQuorum); - config.set("hbase.zookeeper.property.dataDir", zookeeperDataDir); - config.set("hbase.master.info.bindAddress", masterAdrress); - @SuppressWarnings({ "unused", "resource" }) - HBaseAdmin admin = new HBaseAdmin(config); - tableUnderTest = new HTable(conf, tableName); - } catch (Exception e) { - e.printStackTrace(); - throw new Bench4QRunTimeException("Construct HBasePlugin fails!", e); - } - } - - @Behavior(value = "Insert", type = BehaviorType.USER_BEHAVIOR) - public HBaseReturn insert( - @Parameter(value = "key", type = SupportTypes.Field) String key, - @Parameter(value = "value", type = SupportTypes.Field) String value) { - try { - Put put = new Put(UUID.randomUUID().toString().getBytes()); - put.add("key1".getBytes(), "key1".getBytes(), key.getBytes()); - put.add("value1".getBytes(), "value1".getBytes(), value.getBytes()); - this.tableUnderTest.put(put); - } catch (Exception ex) { - Logger.getLogger(HBasePlugin.class).info(ex, ex); - return new HBaseReturn(false); - } - return new HBaseReturn(true); - } - - @Behavior(value = "Query", type = BehaviorType.USER_BEHAVIOR) - public HBaseReturn query( - @Parameter(value = "key", type = SupportTypes.Field) String key) { - try { - Filter filter = new SingleColumnValueFilter(Bytes.toBytes("key1"), - Bytes.toBytes("key1"), CompareOp.EQUAL, Bytes.toBytes(key)); // 当列column1的值为aaa时进行查询 - Scan s = new Scan(); - s.setFilter(filter); - ResultScanner resultScanner = this.tableUnderTest.getScanner(s); - for (Result r : resultScanner) { - System.out.println("get the row key:" + new String(r.getRow())); - } - } catch (Exception ex) { - Logger.getLogger(HBasePlugin.class).info(ex, ex); - return new HBaseReturn(false); - } - return new HBaseReturn(true); - } -} +package org.bench4q.agent.plugin.basic.HBasePlugin; + +import java.io.IOException; +import java.util.UUID; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; +import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.log4j.Logger; +import org.bench4q.agent.plugin.Constructor; +import org.bench4q.agent.plugin.Parameter; +import org.bench4q.agent.plugin.Plugin; +import org.bench4q.agent.plugin.behavior.Behavior; +import org.bench4q.agent.plugin.behavior.BehaviorType; +import org.bench4q.agent.utils.Type.SupportTypes; +import org.bench4q.share.exception.Bench4QRunTimeException; + +@Plugin("HBase") +public class HBasePlugin { + private final Configuration conf; + private final HTable tableUnderTest; + + @Constructor + public HBasePlugin( + @Parameter(value = "dnsServer", type = SupportTypes.Field) String dnsServer, + @Parameter(value = "zookeeperQuorum", type = SupportTypes.Field) String zookeeperQuorum, + @Parameter(value = "zookeeperDataDir", type = SupportTypes.Field) String zookeeperDataDir, + @Parameter(value = "masterAdrress", type = SupportTypes.Field) String masterAdrress, + @Parameter(value = "tableName", type = SupportTypes.Field) String tableName) { + conf = HBaseConfiguration.create(); + try { + Configuration config = new Configuration(); + config.set("hbase.master.dns.nameserver", dnsServer); + config.setBoolean("hbase.cluster.distributed", true); + config.set("hbase.zookeeper.quorum", zookeeperQuorum); + config.set("hbase.zookeeper.property.dataDir", zookeeperDataDir); + config.set("hbase.master.info.bindAddress", masterAdrress); + @SuppressWarnings({ "unused", "resource" }) + HBaseAdmin admin = new HBaseAdmin(config); + tableUnderTest = new HTable(conf, tableName); + } catch (Exception e) { + Logger.getLogger(HBasePlugin.class).error(e, e); + throw new Bench4QRunTimeException("Construct HBasePlugin fails!", e); + } + } + + @Behavior(value = "Insert", type = BehaviorType.USER_BEHAVIOR) + public HBaseReturn insert( + @Parameter(value = "key", type = SupportTypes.Field) String key, + @Parameter(value = "value", type = SupportTypes.Field) String value) { + try { + Put put = new Put(UUID.randomUUID().toString().getBytes()); + put.add("key1".getBytes(), "key1".getBytes(), key.getBytes()); + put.add("value1".getBytes(), "value1".getBytes(), value.getBytes()); + this.tableUnderTest.put(put); + return new HBaseReturn(true); + } catch (Exception ex) { + Logger.getLogger(HBasePlugin.class).info(ex, ex); + return new HBaseReturn(false); + } + } + + @Behavior(value = "Query", type = BehaviorType.USER_BEHAVIOR) + public HBaseReturn query( + @Parameter(value = "key", type = SupportTypes.Field) String key) { + try { + Filter filter = new SingleColumnValueFilter(Bytes.toBytes("key1"), + Bytes.toBytes("key1"), CompareOp.EQUAL, Bytes.toBytes(key)); + Scan s = new Scan(); + s.setFilter(filter); + ResultScanner resultScanner = this.tableUnderTest.getScanner(s); + for (Result r : resultScanner) { + System.out.println("get the row key:" + new String(r.getRow())); + } + return new HBaseReturn(true); + } catch (Exception ex) { + Logger.getLogger(HBasePlugin.class).info(ex, ex); + return new HBaseReturn(false); + } + } + + /** + * get style, read according to rowKey + * @param rowkey + * @throws IOException + * @return + */ + @Behavior(value = "Get", type = BehaviorType.USER_BEHAVIOR) + public HBaseReturn get(@Parameter(value = "rowkey", type = SupportTypes.Field) String rowkey) { + try { + Get g = new Get(Bytes.toBytes(rowkey)); + Result r = tableUnderTest.get(g); + for(KeyValue kv : r.raw()){ + System.out.println("column: " + new String(kv.getRow())); + System.out.println("value: " + new String(kv.getValue())); + } + return new HBaseReturn(true); + } catch (IOException ex) { + Logger.getLogger(HBasePlugin.class).info(ex, ex); + return new HBaseReturn(false); + } + } + + + /** + * Perform a range scan for a set of records in the database. + * @param startkey The record key of the first record to read. + * @param recordcount The number of records to read + * @return + */ + + @Behavior(value = "Scan", type = BehaviorType.USER_BEHAVIOR) + public HBaseReturn scan(@Parameter(value = "startkey", type = SupportTypes.Field) String startkey, + @Parameter(value = "recordcount", type = SupportTypes.Field) String recordcount) + { + Scan s = new Scan(Bytes.toBytes(startkey)); + Integer count = Integer.getInteger(recordcount); + s.setCaching(count); + ResultScanner scanner = null; + try { + scanner = tableUnderTest.getScanner(s); + int numResults = 0; + for (Result rr = scanner.next(); rr != null; rr = scanner.next()) + { + + String key = Bytes.toString(rr.getRow()); + for (KeyValue kv : rr.raw()) { + System.out.println("result: key =" + kv.getQualifier() + " value = " + kv.getValue()); + } + numResults++; + if (numResults >= count) + { + break; + } + } + + }catch (IOException e) { + Logger.getLogger(HBasePlugin.class).info(e, e); + return new HBaseReturn(false); + }finally { + scanner.close(); + } + + return new HBaseReturn(true); + } + +} \ No newline at end of file diff --git a/Bench4Q-Agent/src/main/java/org/bench4q/agent/plugin/basic/MongoDB/MongoDBPlugin.java b/Bench4Q-Agent/src/main/java/org/bench4q/agent/plugin/basic/MongoDB/MongoDBPlugin.java index 62a01a7f..4de0603b 100644 --- a/Bench4Q-Agent/src/main/java/org/bench4q/agent/plugin/basic/MongoDB/MongoDBPlugin.java +++ b/Bench4Q-Agent/src/main/java/org/bench4q/agent/plugin/basic/MongoDB/MongoDBPlugin.java @@ -1,6 +1,8 @@ package org.bench4q.agent.plugin.basic.MongoDB; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import org.apache.log4j.Logger; import org.bench4q.agent.plugin.Constructor; @@ -46,9 +48,20 @@ public class MongoDBPlugin { @Parameter(value = "value", type = SupportTypes.Field) String value) { Mongo mongo = null; try { - mongo = new Mongo(new ServerAddress(hostName, port), + List hostsList = new ArrayList(); + String[] hosts = hostName.split(","); + for(String host : hosts) + { + hostsList.add(new ServerAddress(host)); + } + + mongo = new Mongo(hostsList , + new MongoOptions(new MongoClientOptions.Builder() + .maxWaitTime(1000 * 40).build())); + /*mongo = new Mongo(new ServerAddress(hostName, port), new MongoOptions(new MongoClientOptions.Builder() .maxWaitTime(1000 * 40).build())); + */ DB db = mongo.getDB(dbName); DBCollection table = db.getCollection(this.tableUnderTest); DBObject valueToInsert = new BasicDBObject(); @@ -72,7 +85,16 @@ public class MongoDBPlugin { @Parameter(value = "properties", type = SupportTypes.Table) String properties) { Mongo mongo = null; try { - mongo = new Mongo(hostName, port); + List hostsList = new ArrayList(); + String[] hosts = hostName.split(","); + for(String host : hosts) + { + hostsList.add(new ServerAddress(host)); + } + + mongo = new Mongo(hostsList , + new MongoOptions(new MongoClientOptions.Builder() + .maxWaitTime(1000 * 40).build())); DB db = mongo.getDB(dbName); DBCollection table = db.getCollection(this.tableUnderTest); Table propertiesTable = Table.buildTable(properties, @@ -99,4 +121,4 @@ public class MongoDBPlugin { } } -} +} \ No newline at end of file diff --git a/Bench4Q-Agent/src/main/java/org/bench4q/agent/plugin/basic/Oracle/OracleDBReturn.java b/Bench4Q-Agent/src/main/java/org/bench4q/agent/plugin/basic/Oracle/OracleDBReturn.java new file mode 100644 index 00000000..9ac3cf19 --- /dev/null +++ b/Bench4Q-Agent/src/main/java/org/bench4q/agent/plugin/basic/Oracle/OracleDBReturn.java @@ -0,0 +1,10 @@ +package org.bench4q.agent.plugin.basic.Oracle; + +import org.bench4q.agent.plugin.basic.PluginReturn; + +public class OracleDBReturn extends PluginReturn { + + public OracleDBReturn(boolean success){ + super(success); + } +} diff --git a/Bench4Q-Master/src/main/resources/org/bench4q/master/config/httpCaptureConfig/bench4q.png b/Bench4Q-Master/src/main/resources/org/bench4q/master/config/httpCaptureConfig/bench4q.png new file mode 100644 index 00000000..3522a5af Binary files /dev/null and b/Bench4Q-Master/src/main/resources/org/bench4q/master/config/httpCaptureConfig/bench4q.png differ diff --git a/Bench4Q-Master/src/main/resources/org/bench4q/master/config/httpCaptureConfig/maxq.properties b/Bench4Q-Master/src/main/resources/org/bench4q/master/config/httpCaptureConfig/maxq.properties index 8b23e25a..6f37461a 100644 --- a/Bench4Q-Master/src/main/resources/org/bench4q/master/config/httpCaptureConfig/maxq.properties +++ b/Bench4Q-Master/src/main/resources/org/bench4q/master/config/httpCaptureConfig/maxq.properties @@ -51,4 +51,8 @@ generator.isac.timer=ConstantTimer #extra properties for a negative exponential law distribution #generator.isac.timer.random.dist=negexpo -#generator.isac.timer.random.delta=10 \ No newline at end of file +#generator.isac.timer.random.delta=10 +########################################## +#file append to record html +########################################## +record.flag.file=recordFlag.html \ No newline at end of file diff --git a/Bench4Q-Master/src/main/resources/org/bench4q/master/config/httpCaptureConfig/recordFlag.html b/Bench4Q-Master/src/main/resources/org/bench4q/master/config/httpCaptureConfig/recordFlag.html new file mode 100644 index 00000000..6705e6e1 --- /dev/null +++ b/Bench4Q-Master/src/main/resources/org/bench4q/master/config/httpCaptureConfig/recordFlag.html @@ -0,0 +1,6 @@ + \ No newline at end of file diff --git a/Bench4Q-Master/src/main/resources/org/bench4q/master/config/httpCaptureConfig/recordProxy.png b/Bench4Q-Master/src/main/resources/org/bench4q/master/config/httpCaptureConfig/recordProxy.png new file mode 100644 index 00000000..f96f5c49 Binary files /dev/null and b/Bench4Q-Master/src/main/resources/org/bench4q/master/config/httpCaptureConfig/recordProxy.png differ diff --git a/Bench4Q-Recorder/.classpath b/Bench4Q-Recorder/.classpath index 3ada2efa..7810c17b 100644 --- a/Bench4Q-Recorder/.classpath +++ b/Bench4Q-Recorder/.classpath @@ -18,7 +18,7 @@ - + diff --git a/Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/Config.java b/Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/Config.java index 6c426e6f..55f3b182 100644 --- a/Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/Config.java +++ b/Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/Config.java @@ -54,10 +54,9 @@ public class Config { String maxqDir = System.getProperty("maxq.dir"); String pathSep = System.getProperty("file.separator"); // "\\src\\main\\resources\\org\\bench4q\\master\\config\\httpCaptureConfig\\maxq.properties"; - String propertiesFileName = System.getProperty("user.dir") + "src" - + pathSep + "main" + pathSep + "resources" + pathSep + "org" - + pathSep + "bench4q" + pathSep + "master" + pathSep + "config" - + pathSep + "httpCaptureConfig" + pathSep + "maxq.properties"; + String propertiesFileName = "org" + pathSep + "bench4q" + pathSep + + "master" + pathSep + "config" + pathSep + "httpCaptureConfig" + + pathSep + "maxq.properties"; InputStream propertiesStream = Config.class.getClassLoader() .getResourceAsStream(propertiesFileName); diff --git a/Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/RequestHandler.java b/Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/RequestHandler.java index 4de87b2d..75c73d19 100644 --- a/Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/RequestHandler.java +++ b/Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/RequestHandler.java @@ -41,8 +41,9 @@ public class RequestHandler implements Runnable { try { this.header = new HttpRequestHeader(this.clientIn); } catch (IOException e) { - if(this.header != null){ - log.error("truncated request from browser: " + this.header.url, e); + if (this.header != null) { + log.error("truncated request from browser: " + this.header.url, + e); System.out.println(this.header.url); } throw new Utils.SilentException(); @@ -51,7 +52,7 @@ public class RequestHandler implements Runnable { throw new Utils.UserException( "Bench4Q only supports the HTTP protocol."); URL url = new URL(this.header.url); - + System.out.println(this.header.url); Config.ProxySettings proxy = this.config.getProxySettings(); int port; String host; @@ -161,18 +162,19 @@ public class RequestHandler implements Runnable { int len; while ((len = this.serverIn.read(buf, 0, buf.length)) >= 0) { log.trace("read " + Integer.toString(len)); - + this.buffer.write(buf, 0, len); } log.trace("transferred response len:" + len); - ResponseModel responseModel = new ResponseModel(this.buffer.toByteArray()); + ResponseModel responseModel = new ResponseModel( + this.buffer.toByteArray()); this.proxyServer.processResponse(this.header, requestBody, responseModel); try { this.clientOut.write(responseModel.getResponse()); } catch (SocketException e) { - log.info("browser stopped listening: " - + e.getMessage() + this.buffer.toString()); + log.info("browser stopped listening: " + e.getMessage() + + this.buffer.toString()); throw new Utils.SilentException(); } log.trace("processed response"); diff --git a/Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/HtmlDocumentParser.java b/Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/HtmlDocumentParser.java index e6bef172..83a529de 100644 --- a/Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/HtmlDocumentParser.java +++ b/Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/HtmlDocumentParser.java @@ -1,9 +1,12 @@ package org.bench4q.recorder.httpcapture.generator; +import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; import org.apache.log4j.Logger; +import org.bench4q.recorder.httpcapture.Config; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -12,24 +15,39 @@ import org.jsoup.select.Elements; public class HtmlDocumentParser { private Document document; private static Logger logger = Logger.getLogger(HtmlDocumentParser.class); - private static String recordFlagHtml = "
"; + private static String recordFlagHtml = null; + + private static void initRecordFlagHtml() { + String pathSep = System.getProperty("file.separator"); + String recordFlagFilePath = "org" + pathSep + "bench4q" + pathSep + + "master" + pathSep + "config" + pathSep + "httpCaptureConfig" + + pathSep + Config.getConfig().getProperty("record.flag.file"); + + InputStream is = HtmlDocumentParser.class.getClassLoader() + .getResourceAsStream(recordFlagFilePath); + if (is == null) + return; + StringBuffer out = new StringBuffer(); + byte[] b = new byte[4096]; + int n; + try { + while ((n = is.read(b)) != -1) { + out.append(new String(b, 0, n)); + } + recordFlagHtml = out.toString(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + + private static String getRecordFlagHtml() { + if (recordFlagHtml == null) { + initRecordFlagHtml(); + } + return recordFlagHtml; + } private Document getDocument() { return document; @@ -43,18 +61,23 @@ public class HtmlDocumentParser { this.setDocument(Jsoup.parse(htmlContent, baseUrl)); } - public String toString(){ + public String toString() { return this.getDocument().toString(); } + public void beginParse() { } + /** * */ - public void appendRecordFlag(){ - this.getDocument().body().append(recordFlagHtml); + public void appendRecordFlag() { + String recordFlagHtml = getRecordFlagHtml(); + if (recordFlagHtml != null) + this.getDocument().body().append(recordFlagHtml); } + public List buildParentChildrenRelationship(int parentBatchId) { List result = new ArrayList(); if (parentBatchId == -1) { diff --git a/Bench4Q-Recorder/src/test/java/org/bench4q/recorder/Test_RequestHandler.java b/Bench4Q-Recorder/src/test/java/org/bench4q/recorder/Test_RequestHandler.java new file mode 100644 index 00000000..64206bdc --- /dev/null +++ b/Bench4Q-Recorder/src/test/java/org/bench4q/recorder/Test_RequestHandler.java @@ -0,0 +1,60 @@ +package org.bench4q.recorder; + +import java.io.IOException; +import java.net.ServerSocket; + +import org.apache.commons.httpclient.HostConfiguration; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpException; +import org.apache.commons.httpclient.URI; +import org.apache.commons.httpclient.URIException; +import org.apache.commons.httpclient.methods.GetMethod; +import org.bench4q.recorder.httpcapture.ProxyServer; +import org.bench4q.recorder.httpcapture.RequestHandler; +import org.junit.Test; + +public class Test_RequestHandler { + private static final int PORT = 8910; + @Test + public void test() { + try { + ProxyServer proxyServer = new ProxyServer(PORT); + RequestHandler requestHandler = new RequestHandler(proxyServer, + proxyServer.getServerSocket().accept()); + requestHandler.run(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void test2() { + new SendRequest().run(); + } + + private static class SendRequest implements Runnable { + private HttpClient httpClient = new HttpClient(); + public void run() { + HostConfiguration hostConfiguration = new HostConfiguration(); + hostConfiguration.setProxy("127.0.0.1", PORT); + httpClient.setHostConfiguration(hostConfiguration); + GetMethod get = new GetMethod(); + try { + get.setURI(new URI("http://www.baidu.com")); + int statusCode = this.httpClient.executeMethod(get); + System.out.println(statusCode); + } catch (URIException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (HttpException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + } + + } +}