success to record baidu and show record flag

Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/HttpRequestHeader.java

@@ -1,11 +1,13 @@
 package org.bench4q.recorder.httpcapture;
 
-import org.apache.log4j.Logger;
-
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.ProtocolException;
 import java.util.StringTokenizer;
 
+import org.apache.log4j.Logger;
+
 /**
  * Parses and stores a http server request. Originally posted to comp.lang.java
  * in 1996.
@@ -77,7 +79,7 @@ public class HttpRequestHeader {
 	 * Anything in the header that was unrecognized by this class.
 	 */
 	public String unrecognized = new String();
-	
+
 	/**
 	 * indicate the request go through proxy
 	 */
@@ -196,6 +198,66 @@ public class HttpRequestHeader {
 		return sb.toString();
 	}
 
+	public void buildRequestHeader(HttpURLConnection urlConnection) {
+		if (0 == method.length())
+			method = "GET";
+
+		try {
+			urlConnection.setRequestMethod(method);
+		} catch (ProtocolException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+
+		if (0 < userAgent.length())
+			urlConnection.addRequestProperty("User-Agent", userAgent);
+
+		if (0 < referer.length())
+			urlConnection.addRequestProperty("Referer", referer);
+
+		if (pragmaNoCache)
+			urlConnection.addRequestProperty("Pragma", "no-cache");
+
+		if (0 < ifModifiedSince.length())
+			urlConnection.addRequestProperty("If-Modified-Since",
+					ifModifiedSince);
+
+		// ACCEPT TYPES //
+		if (0 < accept.length())
+			urlConnection.addRequestProperty("Accept", accept);
+		else
+			urlConnection.addRequestProperty("Accept", "*/" + "* ");
+		if (0 < acceptLanguage.length()) {
+			urlConnection.addRequestProperty(
+					HeaderValue.REQUEST_ACCEPT_LANGUAGE, acceptLanguage);
+		}
+		if (0 < contentType.length())
+			urlConnection.addRequestProperty("Content-Type", contentType);
+
+		if (0 < contentLength)
+			urlConnection.addRequestProperty("Content-Length", "contentLength");
+
+		if (0 != authorization.length())
+			urlConnection.addRequestProperty("Authorization", "authorization");
+
+		// add flag show the request from proxy
+		urlConnection.addRequestProperty("FromProxy",
+				Boolean.toString(fromProxy));
+
+		if (0 != unrecognized.length()){
+			String[] unknowHeaders = unrecognized.split(CR);
+			if(unknowHeaders != null){
+				for(int i=0;i<unknowHeaders.length;i++){
+					int keyPos = unknowHeaders[i].indexOf(":");
+					if(keyPos == -1)continue;
+					
+					urlConnection.addRequestProperty(unknowHeaders[i].substring(0,keyPos),  unknowHeaders[i].substring(keyPos+1));
+				}
+			}
+		}
+				
+	}
+
 	/*
 	 * Rebuilds the header in a string
 	 * 
@@ -241,10 +303,10 @@ public class HttpRequestHeader {
 
 		if (0 != authorization.length())
 			Request += "Authorization: " + authorization + CR;
-		
+
-		//add flag show the request from proxy
+		// add flag show the request from proxy
-		Request += "FromProxy:"+fromProxy + CR;
+		Request += "FromProxy:" + fromProxy + CR;
-		
+
 		if (sendUnknowen) {
 			if (0 != unrecognized.length())
 				Request += unrecognized;

Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/RequestHandler.java

@@ -7,6 +7,7 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
+import java.net.ProtocolException;
 import java.net.Socket;
 import java.net.SocketException;
 import java.net.URL;
@@ -23,6 +24,7 @@ import javax.net.ssl.TrustManager;
 import org.apache.commons.httpclient.HttpMethod;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.log4j.Logger;
+import org.bench4q.recorder.httpcapture.Utils.SilentException;
 
 public class RequestHandler implements Runnable {
 	private static final Logger log = Logger.getLogger(RequestHandler.class);
@@ -46,7 +48,6 @@ public class RequestHandler implements Runnable {
 	private HttpRequestHeader header;
 	private ProxyServer proxyServer;
 	private Socket clientSocket;
-	// private Socket serverSocket;
 	private HttpURLConnection serverConnection;
 	private ByteArrayOutputStream buffer;
 	private static Object mutex = new Object();
@@ -89,13 +90,10 @@ public class RequestHandler implements Runnable {
 
 		if (port < 1)
 			port = 80;
-
 		serverConnection = (HttpURLConnection) url.openConnection();
 
-		// serverConnection.addRequestProperty("host", host);
+		serverConnection.addRequestProperty("host", host);
-		// serverConnection.addRequestProperty("port", Integer.toString(port));
+		serverConnection.addRequestProperty("port", Integer.toString(port));
-//		serverConnection.setDoOutput(true);
-//		this.serverOut = serverConnection.getOutputStream();
 	}
 
 	private String stripProxyInfoFromRequestHeader()
@@ -117,21 +115,18 @@ public class RequestHandler implements Runnable {
 				int BUF_SIZE = rs < ss ? ss : rs;
 
 				byte[] buf = new byte[BUF_SIZE];
-
+				try{
 				initClientServerConnections(this.clientSocket);
-				String headerStr;
+				}catch(SilentException e){
-				// if (this.config.getProxySettings() == null)
+					return;
-				// headerStr = stripProxyInfoFromRequestHeader();
+				}
-				// else
+			
-				headerStr = this.header.toString();
+				this.header.buildRequestHeader(serverConnection);
-				log.trace("read request header");
+				log.trace("set request header");
-
-				byte[] bytes = headerStr.getBytes();
-				// this.serverOut.write(bytes, 0, bytes.length);
-
-				log.trace("wrote request header");
 				byte[] requestBody;
 				if (this.header.contentLength > 0) {
+					serverConnection.setDoOutput(true);
+					this.serverOut = serverConnection.getOutputStream();
 					this.buffer.reset();
 					int len = 0;
 					int num = 0;
@@ -154,6 +149,9 @@ public class RequestHandler implements Runnable {
 						log.trace("wrote " + Integer.toString(len) + " bytes");
 						num += len;
 					}
+					 //close send request to remote server
+					 serverOut.flush();
+					 serverOut.close();
 					requestBody = this.buffer.toByteArray();
 					log.trace("transferred rest of request body");
 				} else {
@@ -162,9 +160,7 @@ public class RequestHandler implements Runnable {
 				}
 
 				this.clientSocket.shutdownInput();
-				// close send request to remote server
+			
-//				serverOut.flush();
-//				serverOut.close();
 				this.serverIn = serverConnection.getInputStream();
 				synchronized (mutex) {
 					this.proxyServer.processRequest(this.header, requestBody);
@@ -172,23 +168,22 @@ public class RequestHandler implements Runnable {
 
 					this.buffer.reset();
 					int len;
-					int num = 0;
 					while ((len = this.serverIn.read(buf, 0, buf.length)) >= 0) {
 						log.trace("read " + Integer.toString(len));
 						// this.clientOut.write(buf, 0, len);
 						this.buffer.write(buf, 0, len);
-						num += len;
 					}
 					log.trace("transferred response len:" + len);
-					
+
 					ResponseModel responseModel = new ResponseModel(
 							this.buffer.toByteArray(), serverConnection);
 					this.proxyServer.processResponse(this.header, requestBody,
 							responseModel);
 					try {
 						this.clientOut.write(changeHeaderToByte(
-								serverConnection.getHeaderFields(), responseModel.getResponse().length));
+								serverConnection.getHeaderFields(),
-						
+								responseModel.getResponse().length));
+
 						this.clientOut.write(responseModel.getResponse());
 					} catch (SocketException e) {
 						log.info("browser stopped listening: " + e.getMessage()
@@ -201,10 +196,6 @@ public class RequestHandler implements Runnable {
 			} catch (Exception e) {
 				log.error(e, e);
 			} finally {
-				// if (this.serverSocket != null) {
-				// this.serverSocket.close();
-				// log.trace("closed server socket");
-				// }
 				this.clientSocket.close();
 				log.trace("closed client socket");
 			}
@@ -225,7 +216,7 @@ public class RequestHandler implements Runnable {
 				sb.append(changeList2String(headerMap.get(key)) + "\r\n");
 				continue;
 			}
-			if (key.equals("Transfer-Encoding")) {
+			if (key.equals("Transfer-Encoding") || key.equals("Content-Encoding")) {
 				continue;
 			}
 			sb.append(key + ": " + changeList2String(headerMap.get(key))
@@ -236,7 +227,6 @@ public class RequestHandler implements Runnable {
 			sb.append("Content-Length: " + responseLen).append("\r\n");
 		}
 		sb.append("\r\n");
-		System.out.println(sb.toString());
 		return sb.toString().getBytes();
 	}
 

Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/AbstractCodeGenerator.java

@@ -329,7 +329,6 @@ public abstract class AbstractCodeGenerator implements IScriptGenerator,
 
 	public void processResponse(HttpRequestHeader header, byte[] requestBody,
 			ResponseModel response) throws Exception {
-		System.out.println("enter processResponse");
 		if (this.ignoreNextResponse) {
 			log.debug("Ignoring response");
 			this.ignoreNextResponse = false;

Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/Bench4qCodeGenerator.java

@@ -202,10 +202,20 @@ public class Bench4qCodeGenerator extends AbstractCodeGenerator {
 	@Override
 	public byte[] doParseHtmlContent(String responseBody, String rootUrl) {
 		int htmlStart = responseBody.indexOf("<html");
-		int htmlEnd = responseBody.indexOf("</html>");
+		int bodyStart = responseBody.indexOf("<body");
-		if (htmlStart == -1 || htmlEnd == -1) {
+		if (htmlStart == -1 || bodyStart == -1) {
 			return responseBody.getBytes();
 		}
+		int htmlEnd = responseBody.indexOf("</html>");
+		int bodyEnd = responseBody.indexOf("</body>");
+		if(bodyEnd == -1){
+			responseBody += "</body>";
+		
+		}
+		if(htmlEnd == -1){
+			responseBody += "</html>";
+			htmlEnd = responseBody.indexOf("</html>");
+		}
 		return parseDocument(rootUrl, responseBody, htmlStart, htmlEnd).getBytes();
 	}
 

Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/ContentDecoder.java

@@ -31,8 +31,8 @@ public class ContentDecoder {
 			while ((len = inputStream.read(buf)) > 0) {
 				outputStream.write(buf, 0, len);
 			}
-			System.out.println(outputStream.toString());
+//			System.out.println(outputStream.toString());
-			System.out.println("ouputStream's size is" + outputStream.size());
+//			System.out.println("ouputStream's size is" + outputStream.size());
 			return outputStream.toByteArray();
 		} catch (Exception e) {
 			logger.error(e, e);

Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/GzipDecoder.java

@@ -3,6 +3,8 @@ package org.bench4q.recorder.httpcapture.generator;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
 import java.util.zip.GZIPInputStream;
 
 public class GzipDecoder extends ContentDecoder {
@@ -23,4 +25,27 @@ public class GzipDecoder extends ContentDecoder {
 			return null;
 		}
 	}
+	
+	public String decodeContent(InputStream inputStream, Charset charset){
+		InputStream is = inputStream;
+		GZIPInputStream gzin;
+		try {
+			gzin = new GZIPInputStream(is);
+			InputStreamReader isr = new InputStreamReader(gzin, charset); 
+	        java.io.BufferedReader br = new java.io.BufferedReader(isr);
+	        StringBuffer sb = new StringBuffer();
+	        String tempbf;
+	        while ((tempbf = br.readLine()) != null) {
+	            sb.append(tempbf);
+	            sb.append("\r\n");
+	        }
+	        isr.close();
+	        gzin.close();
+	        return sb.toString();
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+			return null;
+		}
+	}
 }

Bench4Q-Recorder/src/main/java/org/bench4q/recorder/httpcapture/generator/ResponseParser.java

@@ -2,12 +2,10 @@ package org.bench4q.recorder.httpcapture.generator;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.net.HttpURLConnection;
 import java.nio.charset.Charset;
 import java.util.List;
 import java.util.Map;
-import java.util.StringTokenizer;
 
 import org.apache.commons.httpclient.ChunkedInputStream;
 import org.apache.commons.io.output.ByteArrayOutputStream;
@@ -49,16 +47,16 @@ public class ResponseParser {
 		this.setResponseHeader(parseResponseHeader(response.getConn()));
 		if (this.getResponseHeader().isValidResponseHeader()
 				&& this.getResponseHeader().isHtmlContentType()) {
-//			this.parseResponseBody();
+			this.parseResponseBody();
-			this.setResponseBody(new String(response.getResponse(),Charset.forName("utf-8")));
+
-		
 		}
 	}
 
 	private ResponseHeader parseResponseHeader(
 			HttpURLConnection httpURLConnection) {
 		ResponseHeader result = new ResponseHeader();
-		if(httpURLConnection == null) return result;
+		if (httpURLConnection == null)
+			return result;
 		result.setContentEncoding(httpURLConnection.getContentEncoding());
 		result.setContentLength(httpURLConnection.getContentLength());
 		try {
@@ -78,17 +76,7 @@ public class ResponseParser {
 	}
 
 	private String parseContentLength(Map<String, List<String>> map) {
-		// String respStr = preprocess(new String(this.getResponse()));
+	
-		// int pos = respStr.indexOf("content-length:");
-		// if (pos > -1) {
-		// pos += 15;
-		// int end = respStr.indexOf("\r\n", pos);
-		// if (end == -1) {
-		// return "-1";
-		// }
-		// return respStr.substring(pos, end).trim();
-		// }
-		// return "-1";
 		List<String> values = map.get("Content-Length:");
 		if (values != null) {
 			return values.get(0);
@@ -97,13 +85,7 @@ public class ResponseParser {
 	}
 
 	private String parseContentEncoding(Map<String, List<String>> map) {
-		// String respStr = preprocess(new String(this.getResponse()));
+	
-		// int pos = respStr.indexOf("content-encoding:");
-		// if (pos > -1) {
-		// pos += 18;
-		// int end = respStr.indexOf("\r\n", pos);
-		// return respStr.substring(pos, end);
-		// }
 		String ret = "";
 		List<String> values = map.get("Content-Encoding:");
 		if (values != null) {
@@ -113,21 +95,7 @@ public class ResponseParser {
 	}
 
 	private String parseCharset(HttpURLConnection httpURLConnection) {
-		// String response = preprocess(new String(this.getResponse()));
+
-		// String ret = null;
-		// int pos = response.indexOf("content-type:");
-		// if (pos > -1) {
-		// pos += 14;
-		// int end = response.indexOf("\r\n", pos);
-		// int middle = response.indexOf(";", pos);
-		// if (middle > -1 && middle < end) {
-		// ret = response.substring(middle + 1, end);
-		// }
-		// }
-		// if (ret != null) {
-		// int begin = ret.indexOf("charset=");
-		// ret = ret.substring(begin + 8);
-		// }
 		String ret = null;
 		String value = httpURLConnection.getHeaderField("Content-Type");
 		int pos;
@@ -139,29 +107,7 @@ public class ResponseParser {
 	}
 
 	private String parseContentType(HttpURLConnection httpURLConnection) {
-		// String response = preprocess(new String(this.getResponse()));
+	
-		// String contentType = null;
-		// int pos = response.indexOf("content-type:");
-		// if (pos > -1) {
-		// pos += 14;
-		// int end = response.indexOf("\r\n", pos);
-		// int end2 = response.indexOf(";", pos);
-		// if ((end2 > -1) && (end2 < end))
-		// end = end2;
-		// if (end > -1)
-		// contentType = response.substring(pos, end).trim();
-		//
-		// logger.debug("  Content-Type: " + contentType);
-		// } else {
-		// logger.debug("  No content-type header!  First few lines:");
-		// StringTokenizer st = new StringTokenizer(response, "\n");
-		// int i = 0;
-		// while ((st.hasMoreTokens()) && (i < 5)) {
-		// logger.debug(st.nextToken());
-		// ++i;
-		// }
-		// }
-		// return contentType;
 		String ret = null;
 		String value = httpURLConnection.getHeaderField("Content-Type");
 		int pos;
@@ -177,42 +123,16 @@ public class ResponseParser {
 		return ret;
 	}
 
-	// private String parseResponseCode() {
-	// String response = preprocess(new String(this.getResponse()));
-	// String respCode = null;
-	// int pos = response.indexOf(" ");
-	// if (pos != -1) {
-	// int end = response.indexOf(" ", pos + 1);
-	// int end2 = response.indexOf("\n", pos + 1);
-	// if ((end2 != -1) && (end2 < end))
-	// end = end2;
-	// if (end != -1)
-	// respCode = response.substring(pos + 1, end).trim();
-	// }
-	// logger.debug("HTTP response code: " + respCode);
-	// return respCode;
-	// }
 
 	private String parseTransferEncoding(HttpURLConnection httpURLConnection) {
-		// String response = preprocess(new String(this.getResponse()));
+	
-		// int pos = response.indexOf("transfer-encoding:");
+		return httpURLConnection.getHeaderField("Transfer-Encoding");
-		// if (pos > -1) {
-		// pos += 19;
-		// int end = response.indexOf("\r\n", pos);
-		// return response.substring(pos, end);
-		// }
-		// return null;
-		return httpURLConnection.getHeaderField("Transfer-Encoding:");
 	}
 
 	private byte[] buildResponseBody() {
-		String response = preprocess(new String(this.getResponse()));
+	
-		int pos = response.indexOf("\r\n\r\n");
-		pos += 4;
-		String testRes = response.substring(pos);
-		System.out.println(testRes);
 		ByteArrayInputStream in = new ByteArrayInputStream(this.getResponse(),
-				pos, this.getResponse().length - pos);
+				0, this.getResponse().length);
 		ChunkedInputStream chunkedIS = null;
 		ByteArrayOutputStream out = new ByteArrayOutputStream();
 		try {
@@ -247,20 +167,10 @@ public class ResponseParser {
 		ContentDecoder contentDecoder = ContentDecoder.createDecoder(this
 				.getResponseHeader().getContentEncoding());
 		byte[] contentBodyAfterDecoded = null;
-		if (this.getResponseHeader().getTransferEncoding() != null
-				&& this.getResponseHeader().getTransferEncoding()
-						.equals("chunked")) {
 
-			contentBodyAfterDecoded = contentDecoder
+		contentBodyAfterDecoded = contentDecoder
-					.decodeContent(new ByteArrayInputStream(buildResponseBody()));
+				.decodeContent(new ByteArrayInputStream(this.getResponse(), 0,
-		} else {
+						this.getResponse().length));
-			contentBodyAfterDecoded = contentDecoder
-					.decodeContent(new ByteArrayInputStream(this.getResponse(),
-							this.getResponse().length
-									- this.getResponseHeader()
-											.getContentLength(), this
-									.getResponseHeader().getContentLength()));
-		}
 		if (contentBodyAfterDecoded == null) {
 			return;
 		}

Bench4Q-Recorder/src/test/java/org/bench4q/recorder/Test_RequestHandler.java

@@ -1,23 +1,17 @@
 package org.bench4q.recorder;
 
 import java.io.IOException;
-import java.nio.charset.Charset;
 
 import org.apache.commons.httpclient.HostConfiguration;
 import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.URI;
 import org.apache.commons.httpclient.URIException;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.bench4q.recorder.httpcapture.Bench4qTestScriptAdapter;
-import org.bench4q.recorder.httpcapture.IScriptAdapter;
 import org.bench4q.recorder.httpcapture.ProxyServer;
 import org.bench4q.recorder.httpcapture.RequestHandler;
 import org.bench4q.recorder.httpcapture.generator.Bench4qCodeGenerator;
-import org.bench4q.recorder.httpcapture.generator.ChildrenUrl;
 import org.bench4q.share.models.agent.RunScenarioModel;
-import org.bench4q.share.models.agent.scriptrecord.BehaviorModel;
-import org.bench4q.share.models.agent.scriptrecord.UsePluginModel;
 import org.junit.Test;
 
 public class Test_RequestHandler {
@@ -51,13 +45,19 @@ public class Test_RequestHandler {
 			HostConfiguration hostConfiguration = new HostConfiguration();
 			hostConfiguration.setProxy("127.0.0.1", PORT);
 			httpClient.setHostConfiguration(hostConfiguration);
-			GetMethod get = new GetMethod();
+//			CustomGetMethod get = new CustomGetMethod("http://www.baidu.com");
+			GetMethod get = new GetMethod("http://www.baidu.com");
+			
 			try {
-				get.setURI(new URI("http://www.baidu.com"));
+				get.addRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
+				get.addRequestHeader("Accept-Language", "zh-CN,zh;q=0.8");
+				get.addRequestHeader("Cache-Control", "max-age=0");
+//				get.addRequestHeader("Accept-Encoding", "gzip,deflate,sdch");
+//				get.addRequestHeader("Cookie","__zpspc=188.1.1409195668.1409195668.1%234%7C%7C%7C%7C%7C; BDUSS=E45eXUtamxkY2JUbS12ZjFUSmJPbWdmUU13Ri1ZVHlzWWx2S1B3YXRxRVNUU1pVQVFBQUFBJCQAAAAAAAAAAAEAAAAOfOsxd29uYW5ndW8xMQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABLA~lMSwP5Tc; cflag=65535%3A1; BD_UPN=123143; BD_HOME=1; BAIDUID=12B697CB05FD662D2CA4416FF118FCD8:FG=1; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; BD_CK_SAM=1; H_PS_PSSID=8307_5228_1466_7800_8235_8488_8057_6506_6017_8251_7607_7799_8483_8457_8167_8509_8435_8382_8114; BD_HOME=1");
+				
 				int statusCode = this.httpClient.executeMethod(get);
 				System.out.println(statusCode);
-				System.out.println(new String(get.getResponseBody(), Charset
+				System.out.println(get.getResponseBodyAsString());
-						.forName("utf-8")));
 			} catch (URIException e) {
 				// TODO Auto-generated catch block
 				e.printStackTrace();