p05617423
/
bench4q
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

success to record baidu and show record flag

This commit is contained in:
hmm 2014-08-29 16:35:15 +08:00
parent 0fb144f0df
commit 17178a2191
9 changed files with 156 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Bench4Q-Recorder/RecordScriptTestCase/baidu.html
View File

File diff suppressed because one or more lines are too long

76
Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/HttpRequestHeader.java
View File

@ -1,11 +1,13 @@
package org.bench4q.recorder.httpcapture;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.ProtocolException;
import java.util.StringTokenizer;
import org.apache.log4j.Logger;
/**
* Parses and stores a http server request. Originally posted to comp.lang.java
* in 1996.
@ -77,7 +79,7 @@ public class HttpRequestHeader {
* Anything in the header that was unrecognized by this class.
*/
public String unrecognized = new String();
/**
* indicate the request go through proxy
*/
@ -196,6 +198,66 @@ public class HttpRequestHeader {
return sb.toString();
}
public void buildRequestHeader(HttpURLConnection urlConnection) {
if (0 == method.length())
method = "GET";
try {
urlConnection.setRequestMethod(method);
} catch (ProtocolException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
if (0 < userAgent.length())
urlConnection.addRequestProperty("User-Agent", userAgent);
if (0 < referer.length())
urlConnection.addRequestProperty("Referer", referer);
if (pragmaNoCache)
urlConnection.addRequestProperty("Pragma", "no-cache");
if (0 < ifModifiedSince.length())
urlConnection.addRequestProperty("If-Modified-Since",
ifModifiedSince);
// ACCEPT TYPES //
if (0 < accept.length())
urlConnection.addRequestProperty("Accept", accept);
else
urlConnection.addRequestProperty("Accept", "*/" + "* ");
if (0 < acceptLanguage.length()) {
urlConnection.addRequestProperty(
HeaderValue.REQUEST_ACCEPT_LANGUAGE, acceptLanguage);
}
if (0 < contentType.length())
urlConnection.addRequestProperty("Content-Type", contentType);
if (0 < contentLength)
urlConnection.addRequestProperty("Content-Length", "contentLength");
if (0 != authorization.length())
urlConnection.addRequestProperty("Authorization", "authorization");
// add flag show the request from proxy
urlConnection.addRequestProperty("FromProxy",
Boolean.toString(fromProxy));
if (0 != unrecognized.length()){
String[] unknowHeaders = unrecognized.split(CR);
if(unknowHeaders != null){
for(int i=0;i<unknowHeaders.length;i++){
int keyPos = unknowHeaders[i].indexOf(":");
if(keyPos == -1)continue;
urlConnection.addRequestProperty(unknowHeaders[i].substring(0,keyPos), unknowHeaders[i].substring(keyPos+1));
}
}
}
}
/*
* Rebuilds the header in a string
*
@ -241,10 +303,10 @@ public class HttpRequestHeader {
if (0 != authorization.length())
Request += "Authorization: " + authorization + CR;
//add flag show the request from proxy
Request += "FromProxy:"+fromProxy + CR;
// add flag show the request from proxy
Request += "FromProxy:" + fromProxy + CR;
if (sendUnknowen) {
if (0 != unrecognized.length())
Request += unrecognized;

54
Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/RequestHandler.java
View File

@ -7,6 +7,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.ProtocolException;
import java.net.Socket;
import java.net.SocketException;
import java.net.URL;
@ -23,6 +24,7 @@ import javax.net.ssl.TrustManager;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.log4j.Logger;
import org.bench4q.recorder.httpcapture.Utils.SilentException;
public class RequestHandler implements Runnable {
private static final Logger log = Logger.getLogger(RequestHandler.class);
@ -46,7 +48,6 @@ public class RequestHandler implements Runnable {
private HttpRequestHeader header;
private ProxyServer proxyServer;
private Socket clientSocket;
// private Socket serverSocket;
private HttpURLConnection serverConnection;
private ByteArrayOutputStream buffer;
private static Object mutex = new Object();
@ -89,13 +90,10 @@ public class RequestHandler implements Runnable {
if (port < 1)
port = 80;
serverConnection = (HttpURLConnection) url.openConnection();
// serverConnection.addRequestProperty("host", host);
// serverConnection.addRequestProperty("port", Integer.toString(port));
// serverConnection.setDoOutput(true);
// this.serverOut = serverConnection.getOutputStream();
serverConnection.addRequestProperty("host", host);
serverConnection.addRequestProperty("port", Integer.toString(port));
}
private String stripProxyInfoFromRequestHeader()
@ -117,21 +115,18 @@ public class RequestHandler implements Runnable {
int BUF_SIZE = rs < ss ? ss : rs;
byte[] buf = new byte[BUF_SIZE];
try{
initClientServerConnections(this.clientSocket);
String headerStr;
// if (this.config.getProxySettings() == null)
// headerStr = stripProxyInfoFromRequestHeader();
// else
headerStr = this.header.toString();
log.trace("read request header");
byte[] bytes = headerStr.getBytes();
// this.serverOut.write(bytes, 0, bytes.length);
log.trace("wrote request header");
}catch(SilentException e){
return;
}
this.header.buildRequestHeader(serverConnection);
log.trace("set request header");
byte[] requestBody;
if (this.header.contentLength > 0) {
serverConnection.setDoOutput(true);
this.serverOut = serverConnection.getOutputStream();
this.buffer.reset();
int len = 0;
int num = 0;
@ -154,6 +149,9 @@ public class RequestHandler implements Runnable {
log.trace("wrote " + Integer.toString(len) + " bytes");
num += len;
}
//close send request to remote server
serverOut.flush();
serverOut.close();
requestBody = this.buffer.toByteArray();
log.trace("transferred rest of request body");
} else {
@ -162,9 +160,7 @@ public class RequestHandler implements Runnable {
}
this.clientSocket.shutdownInput();
// close send request to remote server
// serverOut.flush();
// serverOut.close();
this.serverIn = serverConnection.getInputStream();
synchronized (mutex) {
this.proxyServer.processRequest(this.header, requestBody);
@ -172,23 +168,22 @@ public class RequestHandler implements Runnable {
this.buffer.reset();
int len;
int num = 0;
while ((len = this.serverIn.read(buf, 0, buf.length)) >= 0) {
log.trace("read " + Integer.toString(len));
// this.clientOut.write(buf, 0, len);
this.buffer.write(buf, 0, len);
num += len;
}
log.trace("transferred response len:" + len);
ResponseModel responseModel = new ResponseModel(
this.buffer.toByteArray(), serverConnection);
this.proxyServer.processResponse(this.header, requestBody,
responseModel);
try {
this.clientOut.write(changeHeaderToByte(
serverConnection.getHeaderFields(), responseModel.getResponse().length));
serverConnection.getHeaderFields(),
responseModel.getResponse().length));
this.clientOut.write(responseModel.getResponse());
} catch (SocketException e) {
log.info("browser stopped listening: " + e.getMessage()
@ -201,10 +196,6 @@ public class RequestHandler implements Runnable {
} catch (Exception e) {
log.error(e, e);
} finally {
// if (this.serverSocket != null) {
// this.serverSocket.close();
// log.trace("closed server socket");
// }
this.clientSocket.close();
log.trace("closed client socket");
}
@ -225,7 +216,7 @@ public class RequestHandler implements Runnable {
sb.append(changeList2String(headerMap.get(key)) + "\r\n");
continue;
}
if (key.equals("Transfer-Encoding")) {
if (key.equals("Transfer-Encoding") || key.equals("Content-Encoding")) {
continue;
}
sb.append(key + ": " + changeList2String(headerMap.get(key))
@ -236,7 +227,6 @@ public class RequestHandler implements Runnable {
sb.append("Content-Length: " + responseLen).append("\r\n");
}
sb.append("\r\n");
System.out.println(sb.toString());
return sb.toString().getBytes();
}

1
Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/AbstractCodeGenerator.java
View File

@ -329,7 +329,6 @@ public abstract class AbstractCodeGenerator implements IScriptGenerator,
public void processResponse(HttpRequestHeader header, byte[] requestBody,
ResponseModel response) throws Exception {
System.out.println("enter processResponse");
if (this.ignoreNextResponse) {
log.debug("Ignoring response");
this.ignoreNextResponse = false;

14
Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/Bench4qCodeGenerator.java
View File

@ -202,10 +202,20 @@ public class Bench4qCodeGenerator extends AbstractCodeGenerator {
@Override
public byte[] doParseHtmlContent(String responseBody, String rootUrl) {
int htmlStart = responseBody.indexOf("<html");
int htmlEnd = responseBody.indexOf("</html>");
if (htmlStart == -1 || htmlEnd == -1) {
int bodyStart = responseBody.indexOf("<body");
if (htmlStart == -1 || bodyStart == -1) {
return responseBody.getBytes();
}
int htmlEnd = responseBody.indexOf("</html>");
int bodyEnd = responseBody.indexOf("</body>");
if(bodyEnd == -1){
responseBody += "</body>";
}
if(htmlEnd == -1){
responseBody += "</html>";
htmlEnd = responseBody.indexOf("</html>");
}
return parseDocument(rootUrl, responseBody, htmlStart, htmlEnd).getBytes();
}

4
Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/ContentDecoder.java
View File

@ -31,8 +31,8 @@ public class ContentDecoder {
while ((len = inputStream.read(buf)) > 0) {
outputStream.write(buf, 0, len);
}
System.out.println(outputStream.toString());
System.out.println("ouputStream's size is" + outputStream.size());
// System.out.println(outputStream.toString());
// System.out.println("ouputStream's size is" + outputStream.size());
return outputStream.toByteArray();
} catch (Exception e) {
logger.error(e, e);

25
Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/GzipDecoder.java
View File

@ -3,6 +3,8 @@ package org.bench4q.recorder.httpcapture.generator;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.zip.GZIPInputStream;
public class GzipDecoder extends ContentDecoder {
@ -23,4 +25,27 @@ public class GzipDecoder extends ContentDecoder {
return null;
}
}
public String decodeContent(InputStream inputStream, Charset charset){
InputStream is = inputStream;
GZIPInputStream gzin;
try {
gzin = new GZIPInputStream(is);
InputStreamReader isr = new InputStreamReader(gzin, charset);
java.io.BufferedReader br = new java.io.BufferedReader(isr);
StringBuffer sb = new StringBuffer();
String tempbf;
while ((tempbf = br.readLine()) != null) {
sb.append(tempbf);
sb.append("\r\n");
}
isr.close();
gzin.close();
return sb.toString();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return null;
}
}
}

120
Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/ResponseParser.java
View File

@ -2,12 +2,10 @@ package org.bench4q.recorder.httpcapture.generator;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.nio.charset.Charset;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import org.apache.commons.httpclient.ChunkedInputStream;
import org.apache.commons.io.output.ByteArrayOutputStream;
@ -49,16 +47,16 @@ public class ResponseParser {
this.setResponseHeader(parseResponseHeader(response.getConn()));
if (this.getResponseHeader().isValidResponseHeader()
&& this.getResponseHeader().isHtmlContentType()) {
// this.parseResponseBody();
this.setResponseBody(new String(response.getResponse(),Charset.forName("utf-8")));
this.parseResponseBody();
}
}
private ResponseHeader parseResponseHeader(
HttpURLConnection httpURLConnection) {
ResponseHeader result = new ResponseHeader();
if(httpURLConnection == null) return result;
if (httpURLConnection == null)
return result;
result.setContentEncoding(httpURLConnection.getContentEncoding());
result.setContentLength(httpURLConnection.getContentLength());
try {
@ -78,17 +76,7 @@ public class ResponseParser {
}
private String parseContentLength(Map<String, List<String>> map) {
// String respStr = preprocess(new String(this.getResponse()));
// int pos = respStr.indexOf("content-length:");
// if (pos > -1) {
// pos += 15;
// int end = respStr.indexOf("\r\n", pos);
// if (end == -1) {
// return "-1";
// }
// return respStr.substring(pos, end).trim();
// }
// return "-1";
List<String> values = map.get("Content-Length:");
if (values != null) {
return values.get(0);
@ -97,13 +85,7 @@ public class ResponseParser {
}
private String parseContentEncoding(Map<String, List<String>> map) {
// String respStr = preprocess(new String(this.getResponse()));
// int pos = respStr.indexOf("content-encoding:");
// if (pos > -1) {
// pos += 18;
// int end = respStr.indexOf("\r\n", pos);
// return respStr.substring(pos, end);
// }
String ret = "";
List<String> values = map.get("Content-Encoding:");
if (values != null) {
@ -113,21 +95,7 @@ public class ResponseParser {
}
private String parseCharset(HttpURLConnection httpURLConnection) {
// String response = preprocess(new String(this.getResponse()));
// String ret = null;
// int pos = response.indexOf("content-type:");
// if (pos > -1) {
// pos += 14;
// int end = response.indexOf("\r\n", pos);
// int middle = response.indexOf(";", pos);
// if (middle > -1 && middle < end) {
// ret = response.substring(middle + 1, end);
// }
// }
// if (ret != null) {
// int begin = ret.indexOf("charset=");
// ret = ret.substring(begin + 8);
// }
String ret = null;
String value = httpURLConnection.getHeaderField("Content-Type");
int pos;
@ -139,29 +107,7 @@ public class ResponseParser {
}
private String parseContentType(HttpURLConnection httpURLConnection) {
// String response = preprocess(new String(this.getResponse()));
// String contentType = null;
// int pos = response.indexOf("content-type:");
// if (pos > -1) {
// pos += 14;
// int end = response.indexOf("\r\n", pos);
// int end2 = response.indexOf(";", pos);
// if ((end2 > -1) && (end2 < end))
// end = end2;
// if (end > -1)
// contentType = response.substring(pos, end).trim();
//
// logger.debug(" Content-Type: " + contentType);
// } else {
// logger.debug(" No content-type header! First few lines:");
// StringTokenizer st = new StringTokenizer(response, "\n");
// int i = 0;
// while ((st.hasMoreTokens()) && (i < 5)) {
// logger.debug(st.nextToken());
// ++i;
// }
// }
// return contentType;
String ret = null;
String value = httpURLConnection.getHeaderField("Content-Type");
int pos;
@ -177,42 +123,16 @@ public class ResponseParser {
return ret;
}
// private String parseResponseCode() {
// String response = preprocess(new String(this.getResponse()));
// String respCode = null;
// int pos = response.indexOf(" ");
// if (pos != -1) {
// int end = response.indexOf(" ", pos + 1);
// int end2 = response.indexOf("\n", pos + 1);
// if ((end2 != -1) && (end2 < end))
// end = end2;
// if (end != -1)
// respCode = response.substring(pos + 1, end).trim();
// }
// logger.debug("HTTP response code: " + respCode);
// return respCode;
// }
private String parseTransferEncoding(HttpURLConnection httpURLConnection) {
// String response = preprocess(new String(this.getResponse()));
// int pos = response.indexOf("transfer-encoding:");
// if (pos > -1) {
// pos += 19;
// int end = response.indexOf("\r\n", pos);
// return response.substring(pos, end);
// }
// return null;
return httpURLConnection.getHeaderField("Transfer-Encoding:");
return httpURLConnection.getHeaderField("Transfer-Encoding");
}
private byte[] buildResponseBody() {
String response = preprocess(new String(this.getResponse()));
int pos = response.indexOf("\r\n\r\n");
pos += 4;
String testRes = response.substring(pos);
System.out.println(testRes);
ByteArrayInputStream in = new ByteArrayInputStream(this.getResponse(),
pos, this.getResponse().length - pos);
0, this.getResponse().length);
ChunkedInputStream chunkedIS = null;
ByteArrayOutputStream out = new ByteArrayOutputStream();
try {
@ -247,20 +167,10 @@ public class ResponseParser {
ContentDecoder contentDecoder = ContentDecoder.createDecoder(this
.getResponseHeader().getContentEncoding());
byte[] contentBodyAfterDecoded = null;
if (this.getResponseHeader().getTransferEncoding() != null
&& this.getResponseHeader().getTransferEncoding()
.equals("chunked")) {
contentBodyAfterDecoded = contentDecoder
.decodeContent(new ByteArrayInputStream(buildResponseBody()));
} else {
contentBodyAfterDecoded = contentDecoder
.decodeContent(new ByteArrayInputStream(this.getResponse(),
this.getResponse().length
- this.getResponseHeader()
.getContentLength(), this
.getResponseHeader().getContentLength()));
}
contentBodyAfterDecoded = contentDecoder
.decodeContent(new ByteArrayInputStream(this.getResponse(), 0,
this.getResponse().length));
if (contentBodyAfterDecoded == null) {
return;
}

20
Bench4Q-Recorder/src/test/java/org/bench4q/recorder/Test_RequestHandler.java
View File

@ -1,23 +1,17 @@
package org.bench4q.recorder;
import java.io.IOException;
import java.nio.charset.Charset;
import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.bench4q.recorder.httpcapture.Bench4qTestScriptAdapter;
import org.bench4q.recorder.httpcapture.IScriptAdapter;
import org.bench4q.recorder.httpcapture.ProxyServer;
import org.bench4q.recorder.httpcapture.RequestHandler;
import org.bench4q.recorder.httpcapture.generator.Bench4qCodeGenerator;
import org.bench4q.recorder.httpcapture.generator.ChildrenUrl;
import org.bench4q.share.models.agent.RunScenarioModel;
import org.bench4q.share.models.agent.scriptrecord.BehaviorModel;
import org.bench4q.share.models.agent.scriptrecord.UsePluginModel;
import org.junit.Test;
public class Test_RequestHandler {
@ -51,13 +45,19 @@ public class Test_RequestHandler {
HostConfiguration hostConfiguration = new HostConfiguration();
hostConfiguration.setProxy("127.0.0.1", PORT);
httpClient.setHostConfiguration(hostConfiguration);
GetMethod get = new GetMethod();
// CustomGetMethod get = new CustomGetMethod("http://www.baidu.com");
GetMethod get = new GetMethod("http://www.baidu.com");
try {
get.setURI(new URI("http://www.baidu.com"));
get.addRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
get.addRequestHeader("Accept-Language", "zh-CN,zh;q=0.8");
get.addRequestHeader("Cache-Control", "max-age=0");
// get.addRequestHeader("Accept-Encoding", "gzip,deflate,sdch");
// get.addRequestHeader("Cookie","__zpspc=188.1.1409195668.1409195668.1%234%7C%7C%7C%7C%7C; BDUSS=E45eXUtamxkY2JUbS12ZjFUSmJPbWdmUU13Ri1ZVHlzWWx2S1B3YXRxRVNUU1pVQVFBQUFBJCQAAAAAAAAAAAEAAAAOfOsxd29uYW5ndW8xMQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABLA~lMSwP5Tc; cflag=65535%3A1; BD_UPN=123143; BD_HOME=1; BAIDUID=12B697CB05FD662D2CA4416FF118FCD8:FG=1; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; BD_CK_SAM=1; H_PS_PSSID=8307_5228_1466_7800_8235_8488_8057_6506_6017_8251_7607_7799_8483_8457_8167_8509_8435_8382_8114; BD_HOME=1");
int statusCode = this.httpClient.executeMethod(get);
System.out.println(statusCode);
System.out.println(new String(get.getResponseBody(), Charset
.forName("utf-8")));
System.out.println(get.getResponseBodyAsString());
} catch (URIException e) {
// TODO Auto-generated catch block
e.printStackTrace();