get the real cause of the distortion, It's a kind of content encoding,

as a gzip format, so i add a test to decode this kind of http response

src/main/java/org/bench4q/master/scriptrecord/httpcapture/Bench4qTestScriptAdapter.java

@@ -87,6 +87,15 @@ public class Bench4qTestScriptAdapter implements IScriptAdapter {
 		return parentBatchId < 0;
 	}
 
+	public boolean isInChildrenUrl(String url) {
+		for (ChildrenUrl childrenUrl : this.getChildrenUrls()) {
+			if (childrenUrl.getUrl().equals(url)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
 	private int getUrlParentBatch(BehaviorBaseModel model) {
 		int parentBatchId = -1;
 		for (ParameterModel parameter : model.getParameters()) {

src/main/java/org/bench4q/master/scriptrecord/httpcapture/IScriptAdapter.java

@@ -24,4 +24,6 @@ public interface IScriptAdapter {
 	public abstract List<ChildrenUrl> getChildrenUrls();
 
 	public abstract int getParentBatchIdWithParentUrl(String url);
+
+	public abstract boolean isInChildrenUrl(String url);
 }

src/main/java/org/bench4q/master/scriptrecord/httpcapture/generator/AbstractCodeGenerator.java

@@ -28,6 +28,7 @@ import org.bench4q.master.scriptrecord.httpcapture.IScriptAdapter;
 import org.bench4q.master.scriptrecord.httpcapture.Param;
 import org.bench4q.master.scriptrecord.httpcapture.ProxyServer;
 import org.bench4q.master.scriptrecord.httpcapture.Utils;
+import org.bench4q.master.scriptrecord.httpcapture.Utils.UserException;
 import org.bench4q.share.models.agent.RunScenarioModel;
 import org.bench4q.share.models.agent.scriptrecord.BehaviorBaseModel;
 
@@ -46,7 +47,7 @@ public abstract class AbstractCodeGenerator implements IScriptGenerator,
 	protected boolean headersExist = false;
 	private IScriptAdapter scriptAdapter;
 	private String testName;
-	private String defaultTestName;
+	@SuppressWarnings("unused")
 	private String testPath;
 	private boolean ignoreNextResponse;
 	private Pattern[] namePatterns;
@@ -61,11 +62,11 @@ public abstract class AbstractCodeGenerator implements IScriptGenerator,
 	static final boolean $assertionsDisabled = !(AbstractCodeGenerator.class
 			.desiredAssertionStatus());
 
-	public static boolean isCpRspToStdout() {
+	private static boolean isCpRspToStdout() {
 		return cpRspToStdout;
 	}
 
-	public static boolean isCpRspToFile() {
+	private static boolean isCpRspToFile() {
 		return cpRspToFile;
 	}
 
@@ -81,37 +82,19 @@ public abstract class AbstractCodeGenerator implements IScriptGenerator,
 
 		this.testName = this.config.getProperty("test.default_testname",
 				"Bench4QTest");
-		this.defaultTestName = this.testName;
-
 		this.insertThread = new Thread(this);
 		this.insertThread.start();
 	}
 
-	public String getDefaultTestName() {
-		return this.defaultTestName;
-	}
-
-	public void setDefaultTestName(String defaultTestName) {
-		this.defaultTestName = defaultTestName;
-	}
-
 	public IScriptAdapter getScriptAdapter() {
 		return this.scriptAdapter;
 	}
 
-	public String getTestName() {
-		return this.testName;
-	}
-
-	public void setTestName(String testName) {
+	private void setTestName(String testName) {
 		this.testName = testName;
 	}
 
-	public String getTestPath() {
-		return this.testPath;
-	}
-
-	public void setTestPath(String testPath) {
+	private void setTestPath(String testPath) {
 		this.testPath = testPath;
 	}
 
@@ -195,17 +178,7 @@ public abstract class AbstractCodeGenerator implements IScriptGenerator,
 		Action action = new Action();
 
 		String newCharset = null;
-		Matcher charsetParser = Pattern.compile("charset=([a-z0-9_\\-]+)", 2)
-				.matcher(header.contentType);
-		if (charsetParser.matches()) {
-			newCharset = charsetParser.group();
-			if (!(newCharset.equals(this.charset))) {
-				doSetCharset(newCharset);
-			} else if (this.charset != null) {
-				doSetCharset(null);
-			}
-
-		}
+		newCharset = doForCharset(header.contentType);
 
 		doHeaders(createHeaders(header));
 
@@ -308,6 +281,22 @@ public abstract class AbstractCodeGenerator implements IScriptGenerator,
 		doCallUrl(action.getUrl(), method, data_str, cont_len_str);
 	}
 
+	private String doForCharset(String contentType) throws UserException {
+		Matcher charsetParser = Pattern.compile("charset=([a-z0-9_\\-]+)", 2)
+				.matcher(contentType);
+		String newCharset = "";
+		if (charsetParser.matches()) {
+			newCharset = charsetParser.group();
+			if (!(newCharset.equals(this.charset))) {
+				doSetCharset(newCharset);
+			} else if (this.charset != null) {
+				doSetCharset(this.charset);
+			}
+
+		}
+		return newCharset;
+	}
+
 	private HeaderValue[] createHeaders(HttpRequestHeader header) {
 		Vector<HeaderValue> v = new Vector<HeaderValue>();
 		if (header.contentType.length() > 0)
@@ -329,13 +318,13 @@ public abstract class AbstractCodeGenerator implements IScriptGenerator,
 			this.ignoreNextResponse = false;
 			return;
 		}
-
-		String responseWithProperCase = new String(response);
-		String respStr = responseWithProperCase.toLowerCase();
-
+		// TODO:Get charset from responseHeader
+		String respStr = new String(response).toLowerCase();
 		String respCode = parseResponseCode(respStr);
 		String contentType = parseContentType(respStr);
-
+		String charset = parseCharset(respStr);
+		@SuppressWarnings("unused")
+		String contentEncoding = parseContentEncoding(respStr);
 		doAssertResponse(respCode);
 
 		if ((contentType != null) && (mimeTypes.get(contentType) != null)
@@ -348,11 +337,11 @@ public abstract class AbstractCodeGenerator implements IScriptGenerator,
 
 				if (contentType.toLowerCase().compareTo("text/html") == 0) {
 					doTidyCode(HttpTestCase.staticUrlDecode(header.url));
-					doParseHtmlContent(responseWithProperCase, header.url);
-					if (cpRspToStdout)
+					doParseHtmlContent(response, charset, header.url);
+					if (isCpRspToStdout())
 						doResponseForStdOut(HttpTestCase.staticUrlDecode(
 								header.url).trim());
-					else if (cpRspToFile)
+					else if (isCpRspToFile())
 						doResponseForFile();
 				}
 			}
@@ -367,6 +356,82 @@ public abstract class AbstractCodeGenerator implements IScriptGenerator,
 			this.isFirstRequest = false;
 	}
 
+	private String parseContentEncoding(String respStr) {
+		int pos = respStr.indexOf("content-encoding:");
+		if (pos != -1) {
+			pos += 18;
+			int end = respStr.indexOf("/r/n", pos);
+			return respStr.substring(pos, end);
+		}
+		return null;
+	}
+
+	private String parseCharset(String response) {
+		String ret = null;
+		int pos = response.indexOf("content-type:");
+		if (pos > -1) {
+			pos += 14;
+			int end = response.indexOf("\r\n", pos);
+			int middle = response.indexOf(";", pos);
+			if (middle > -1 && middle < end) {
+				ret = response.substring(middle + 1, end);
+			}
+		}
+		if (ret != null) {
+			int begin = ret.indexOf("charset=");
+			ret = ret.substring(begin + 8);
+		}
+		return ret;
+	}
+
+	private String parseContentType(String response) {
+		String contentType = null;
+
+		int pos = response.indexOf("content-type:");
+		if (pos > -1) {
+			pos += 14;
+			int end = response.indexOf("\r\n", pos);
+			int end2 = response.indexOf(";", pos);
+			if ((end2 > -1) && (end2 < end))
+				end = end2;
+			if (end > -1)
+				contentType = response.substring(pos, end).trim();
+
+			log.debug("  Content-Type: " + contentType);
+		} else {
+			log.debug("  No content-type header!  First few lines:");
+			StringTokenizer st = new StringTokenizer(response, "\n");
+			int i = 0;
+			while ((st.hasMoreTokens()) && (i < 5)) {
+				log.debug(st.nextToken());
+				++i;
+			}
+		}
+		return contentType;
+	}
+
+	private String parseResponseCode(String response) {
+		String respCode = null;
+		int pos = response.indexOf(" ");
+		if (pos != -1) {
+			int end = response.indexOf(" ", pos + 1);
+			int end2 = response.indexOf("\n", pos + 1);
+			if ((end2 != -1) && (end2 < end))
+				end = end2;
+			if (end != -1)
+				respCode = response.substring(pos + 1, end).trim();
+		}
+
+		log.debug("HTTP response code: " + respCode);
+		return respCode;
+	}
+
+	@SuppressWarnings("unused")
+	private String decodeContent(String response, String encode) {
+		// TODO:
+		return null;
+	}
+
 	public void run() {
 		this.outstandingInserts = new LinkedList<BehaviorBaseModel>();
 		try {
@@ -426,7 +491,8 @@ public abstract class AbstractCodeGenerator implements IScriptGenerator,
 	public abstract void doTidyCode(String paramString)
 			throws Utils.UserException;
 
-	public abstract void doParseHtmlContent(String responseContent, String url);
+	public abstract void doParseHtmlContent(byte[] response, String charset,
+			String rootUrl);
 
 	public abstract void doHeaders(HeaderValue[] paramArrayOfHeaderValue);
 
@@ -438,49 +504,6 @@ public abstract class AbstractCodeGenerator implements IScriptGenerator,
 		}
 	}
 
-	private String parseContentType(String response) {
-		String contentType = null;
-
-		int pos = response.indexOf("content-type:");
-		if (pos > -1) {
-			pos += 14;
-			int end = response.indexOf("\r\n", pos);
-			int end2 = response.indexOf(";", pos);
-			if ((end2 > -1) && (end2 < end))
-				end = end2;
-			if (end > -1)
-				contentType = response.substring(pos, end).trim();
-
-			log.debug("  Content-Type: " + contentType);
-		} else {
-			log.debug("  No content-type header!  First few lines:");
-			StringTokenizer st = new StringTokenizer(response, "\n");
-			int i = 0;
-			while ((st.hasMoreTokens()) && (i < 5)) {
-				log.debug(st.nextToken());
-				++i;
-			}
-		}
-		return contentType;
-	}
-
-	private String parseResponseCode(String response) {
-		String respCode = null;
-		int pos = response.indexOf(" ");
-		if (pos != -1) {
-			int end = response.indexOf(" ", pos + 1);
-
-			int end2 = response.indexOf("\n", pos + 1);
-			if ((end2 != -1) && (end2 < end))
-				end = end2;
-			if (end != -1)
-				respCode = response.substring(pos + 1, end).trim();
-		}
-
-		log.debug("HTTP response code: " + respCode);
-		return respCode;
-	}
-
 	private void setMultiPartData(Action action, HttpRequestHeader header,
 			byte[] strarray) {
 		String str = new String(strarray);

src/main/java/org/bench4q/master/scriptrecord/httpcapture/generator/Bench4qCodeGenerator.java

@@ -1,8 +1,12 @@
 package org.bench4q.master.scriptrecord.httpcapture.generator;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.log4j.Logger;
 import org.bench4q.master.scriptrecord.httpcapture.Config;
 import org.bench4q.master.scriptrecord.httpcapture.HeaderValue;
@@ -71,7 +75,7 @@ public class Bench4qCodeGenerator extends AbstractCodeGenerator {
 	}
 
 	public static String getGeneratorDescription() {
-		return "ISAC scenario for CLIF.ow2.org";
+		return "Bench4Q Scenario";
 	}
 
 	public void doNew() {
@@ -152,7 +156,8 @@ public class Bench4qCodeGenerator extends AbstractCodeGenerator {
 
 	public void doCallUrl(String url, String method, String data,
 			String contentLength) throws Utils.UserException {
-		if (!(isFirstRequest()))
+		if ((!isFirstRequest())
+				&& (!this.getScriptAdapter().isInChildrenUrl(url)))
 			doInsertDelay(getTimeElapsedSinceLastestRequest());
 		UserBehaviorModel userBehavior = createHttpBehavior(
 				this.getRequestCount(), url, method);
@@ -199,13 +204,20 @@ public class Bench4qCodeGenerator extends AbstractCodeGenerator {
 	}
 
 	@Override
-	public void doParseHtmlContent(String responseContent, String rootUrl) {
+	public void doParseHtmlContent(byte[] response, String charset,
+			String rootUrl) {
+		if (charset == null) {
+			charset = "ANSI";
+		}
+		String responseContent = new String(response, Charset.forName(charset));
 		int htmlStart = responseContent.indexOf("<html>");
 		int htmlEnd = responseContent.indexOf("</html>");
 		if (htmlStart == -1 || htmlEnd == -1) {
 			return;
 		}
 		String htmlContent = responseContent.substring(htmlStart, htmlEnd + 8);
+
+		saveToFile(htmlContent);
 		Document document = Jsoup.parse(htmlContent);
 		document.setBaseUri(rootUrl);
 		Elements allChildrenLinks = new Elements();
@@ -231,6 +243,17 @@ public class Bench4qCodeGenerator extends AbstractCodeGenerator {
 
 	}
 
+	private void saveToFile(String htmlContent) {
+		// For Test
+		try {
+			File storeFile = new File("Script/baidu.html");
+			FileUtils.writeStringToFile(storeFile, htmlContent);
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+
 	public void doResponseForStdOut(String url) {
 	}
 

src/main/java/org/bench4q/master/scriptrecord/httpcapture/generator/ContentDecompress.java

@@ -0,0 +1,5 @@
+package org.bench4q.master.scriptrecord.httpcapture.generator;
+
+public class ContentDecompress {
+
+}

src/test/java/org/bench4q/master/test/recordscript/TestDomGenerator.java

@@ -24,7 +24,8 @@ public class TestDomGenerator extends TestRecordBase {
 				.insertUserBehaviorsToScenario(
 						createUserBehaviorModel(parentUrl));
 		this.getCodeGenerator().doParseHtmlContent(
-				FileUtils.readFileToString(new File(url.getPath())), parentUrl);
+				FileUtils.readFileToString(new File(url.getPath())).getBytes(),
+				"utf-8", parentUrl);
 		assertTrue(this.getCodeGenerator().getScriptAdapter().getChildrenUrls()
 				.size() == 8);
 		for (ChildrenUrl childrenUrl : this.getCodeGenerator()
@@ -40,8 +41,8 @@ public class TestDomGenerator extends TestRecordBase {
 		this.getAdpater().insertUserBehaviorsToScenario(
 				createUserBehaviorModel(parentUrl));
 		this.getCodeGenerator().doParseHtmlContent(
-				FileUtils.readFileToString(new File(this.url.getPath())),
-				parentUrl);
+				FileUtils.readFileToString(new File(this.url.getPath()))
+						.getBytes(), "utf-8", parentUrl);
 		System.out.println(this.getCodeGenerator().getScriptAdapter()
 				.getChildrenUrls().size());
 		assertTrue(this.getCodeGenerator().getScriptAdapter().getChildrenUrls()
@@ -59,13 +60,18 @@ public class TestDomGenerator extends TestRecordBase {
 		this.getAdpater().insertUserBehaviorsToScenario(
 				createUserBehaviorModel(independentUrl));
 		this.getCodeGenerator().doParseHtmlContent(
-				FileUtils.readFileToString(new File(this.url.getPath())),
-				parentUrl + "error");
+				FileUtils.readFileToString(new File(this.url.getPath()))
+						.getBytes(), "utf-8", parentUrl + "error");
 		assertTrue(this.getCodeGenerator().getScriptAdapter().getChildrenUrls()
 				.size() == 0);
 	}
 
 	public void testChildUrlWithNotProperCase() {
+
+	}
+
+	@Test
+	public void testUncompressGzipContent() {
 		
 	}
 }