410 lines
10 KiB
C++
410 lines
10 KiB
C++
/*-------------------------------------------------------------------------
|
|
* drawElements Quality Program Test Executor
|
|
* ------------------------------------------
|
|
*
|
|
* Copyright 2014 The Android Open Source Project
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*
|
|
*//*!
|
|
* \file
|
|
* \brief Test log compare utility.
|
|
*//*--------------------------------------------------------------------*/
|
|
|
|
#include "xeTestLogParser.hpp"
|
|
#include "xeTestResultParser.hpp"
|
|
#include "deFilePath.hpp"
|
|
#include "deString.h"
|
|
#include "deThread.hpp"
|
|
#include "deCommandLine.hpp"
|
|
|
|
#include <vector>
|
|
#include <string>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <set>
|
|
#include <map>
|
|
|
|
using std::vector;
|
|
using std::string;
|
|
using std::set;
|
|
using std::map;
|
|
|
|
enum OutputMode
|
|
{
|
|
OUTPUTMODE_ALL = 0,
|
|
OUTPUTMODE_DIFF,
|
|
|
|
OUTPUTMODE_LAST
|
|
};
|
|
|
|
enum OutputFormat
|
|
{
|
|
OUTPUTFORMAT_TEXT = 0,
|
|
OUTPUTFORMAT_CSV,
|
|
|
|
OUTPUTFORMAT_LAST
|
|
};
|
|
|
|
enum OutputValue
|
|
{
|
|
OUTPUTVALUE_STATUS_CODE = 0,
|
|
OUTPUTVALUE_STATUS_DETAILS,
|
|
|
|
OUTPUTVALUE_LAST
|
|
};
|
|
|
|
namespace opt
|
|
{
|
|
|
|
DE_DECLARE_COMMAND_LINE_OPT(OutMode, OutputMode);
|
|
DE_DECLARE_COMMAND_LINE_OPT(OutFormat, OutputFormat);
|
|
DE_DECLARE_COMMAND_LINE_OPT(OutValue, OutputValue);
|
|
|
|
static void registerOptions (de::cmdline::Parser& parser)
|
|
{
|
|
using de::cmdline::Option;
|
|
using de::cmdline::NamedValue;
|
|
|
|
static const NamedValue<OutputMode> s_outputModes[] =
|
|
{
|
|
{ "all", OUTPUTMODE_ALL },
|
|
{ "diff", OUTPUTMODE_DIFF }
|
|
};
|
|
static const NamedValue<OutputFormat> s_outputFormats[] =
|
|
{
|
|
{ "text", OUTPUTFORMAT_TEXT },
|
|
{ "csv", OUTPUTFORMAT_CSV }
|
|
};
|
|
static const NamedValue<OutputValue> s_outputValues[] =
|
|
{
|
|
{ "code", OUTPUTVALUE_STATUS_CODE },
|
|
{ "details", OUTPUTVALUE_STATUS_DETAILS }
|
|
};
|
|
|
|
parser << Option<OutFormat> ("f", "format", "Output format", s_outputFormats, "csv")
|
|
<< Option<OutMode> ("m", "mode", "Output mode", s_outputModes, "all")
|
|
<< Option<OutValue> ("v", "value", "Value to extract", s_outputValues, "code");
|
|
}
|
|
|
|
} // opt
|
|
|
|
struct CommandLine
|
|
{
|
|
CommandLine (void)
|
|
: outMode (OUTPUTMODE_ALL)
|
|
, outFormat (OUTPUTFORMAT_CSV)
|
|
, outValue (OUTPUTVALUE_STATUS_CODE)
|
|
{
|
|
}
|
|
|
|
OutputMode outMode;
|
|
OutputFormat outFormat;
|
|
OutputValue outValue;
|
|
vector<string> filenames;
|
|
};
|
|
|
|
struct ShortBatchResult
|
|
{
|
|
vector<xe::TestCaseResultHeader> resultHeaders;
|
|
map<string, int> resultMap;
|
|
};
|
|
|
|
class ShortResultHandler : public xe::TestLogHandler
|
|
{
|
|
public:
|
|
ShortResultHandler (ShortBatchResult& result)
|
|
: m_result(result)
|
|
{
|
|
}
|
|
|
|
void setSessionInfo (const xe::SessionInfo&)
|
|
{
|
|
// Ignored.
|
|
}
|
|
|
|
xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
|
|
{
|
|
return xe::TestCaseResultPtr(new xe::TestCaseResultData(casePath));
|
|
}
|
|
|
|
void testCaseResultUpdated (const xe::TestCaseResultPtr&)
|
|
{
|
|
// Ignored.
|
|
}
|
|
|
|
void testCaseResultComplete (const xe::TestCaseResultPtr& caseData)
|
|
{
|
|
xe::TestCaseResultHeader header;
|
|
int caseNdx = (int)m_result.resultHeaders.size();
|
|
|
|
header.casePath = caseData->getTestCasePath();
|
|
header.caseType = xe::TESTCASETYPE_SELF_VALIDATE;
|
|
header.statusCode = caseData->getStatusCode();
|
|
header.statusDetails = caseData->getStatusDetails();
|
|
|
|
if (header.statusCode == xe::TESTSTATUSCODE_LAST)
|
|
{
|
|
xe::TestCaseResult fullResult;
|
|
|
|
xe::parseTestCaseResultFromData(&m_testResultParser, &fullResult, *caseData.get());
|
|
|
|
header = xe::TestCaseResultHeader(fullResult);
|
|
}
|
|
|
|
// Insert into result list & map.
|
|
m_result.resultHeaders.push_back(header);
|
|
m_result.resultMap[header.casePath] = caseNdx;
|
|
}
|
|
|
|
private:
|
|
ShortBatchResult& m_result;
|
|
xe::TestResultParser m_testResultParser;
|
|
};
|
|
|
|
static void readLogFile (ShortBatchResult& batchResult, const char* filename)
|
|
{
|
|
std::ifstream in (filename, std::ifstream::binary|std::ifstream::in);
|
|
ShortResultHandler resultHandler (batchResult);
|
|
xe::TestLogParser parser (&resultHandler);
|
|
deUint8 buf [1024];
|
|
int numRead = 0;
|
|
|
|
for (;;)
|
|
{
|
|
in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
|
|
numRead = (int)in.gcount();
|
|
|
|
if (numRead <= 0)
|
|
break;
|
|
|
|
parser.parse(&buf[0], numRead);
|
|
}
|
|
|
|
in.close();
|
|
}
|
|
|
|
class LogFileReader : public de::Thread
|
|
{
|
|
public:
|
|
LogFileReader (ShortBatchResult& batchResult, const char* filename)
|
|
: m_batchResult (batchResult)
|
|
, m_filename (filename)
|
|
{
|
|
}
|
|
|
|
void run (void)
|
|
{
|
|
readLogFile(m_batchResult, m_filename.c_str());
|
|
}
|
|
|
|
private:
|
|
ShortBatchResult& m_batchResult;
|
|
std::string m_filename;
|
|
};
|
|
|
|
static void computeCaseList (vector<string>& cases, const vector<ShortBatchResult>& batchResults)
|
|
{
|
|
// \todo [2012-07-10 pyry] Do proper case ordering (eg. handle missing cases nicely).
|
|
set<string> addedCases;
|
|
|
|
for (vector<ShortBatchResult>::const_iterator batchIter = batchResults.begin(); batchIter != batchResults.end(); batchIter++)
|
|
{
|
|
for (vector<xe::TestCaseResultHeader>::const_iterator caseIter = batchIter->resultHeaders.begin(); caseIter != batchIter->resultHeaders.end(); caseIter++)
|
|
{
|
|
if (addedCases.find(caseIter->casePath) == addedCases.end())
|
|
{
|
|
cases.push_back(caseIter->casePath);
|
|
addedCases.insert(caseIter->casePath);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static void getTestResultHeaders (vector<xe::TestCaseResultHeader>& headers, const vector<ShortBatchResult>& batchResults, const char* casePath)
|
|
{
|
|
headers.resize(batchResults.size());
|
|
|
|
for (int ndx = 0; ndx < (int)batchResults.size(); ndx++)
|
|
{
|
|
const ShortBatchResult& batchResult = batchResults[ndx];
|
|
map<string, int>::const_iterator resultPos = batchResult.resultMap.find(casePath);
|
|
|
|
if (resultPos != batchResult.resultMap.end())
|
|
headers[ndx] = batchResult.resultHeaders[resultPos->second];
|
|
else
|
|
{
|
|
headers[ndx].casePath = casePath;
|
|
headers[ndx].caseType = xe::TESTCASETYPE_SELF_VALIDATE;
|
|
headers[ndx].statusCode = xe::TESTSTATUSCODE_LAST;
|
|
}
|
|
}
|
|
}
|
|
|
|
static const char* getStatusCodeName (xe::TestStatusCode code)
|
|
{
|
|
if (code == xe::TESTSTATUSCODE_LAST)
|
|
return "Missing";
|
|
else
|
|
return xe::getTestStatusCodeName(code);
|
|
}
|
|
|
|
static bool runCompare (const CommandLine& cmdLine, std::ostream& dst)
|
|
{
|
|
vector<ShortBatchResult> results;
|
|
vector<string> batchNames;
|
|
bool compareOk = true;
|
|
|
|
XE_CHECK(!cmdLine.filenames.empty());
|
|
|
|
try
|
|
{
|
|
// Read in batch results
|
|
results.resize(cmdLine.filenames.size());
|
|
{
|
|
std::vector<de::SharedPtr<LogFileReader> > readers;
|
|
|
|
for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
|
|
{
|
|
readers.push_back(de::SharedPtr<LogFileReader>(new LogFileReader(results[ndx], cmdLine.filenames[ndx].c_str())));
|
|
readers.back()->start();
|
|
}
|
|
|
|
for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
|
|
{
|
|
readers[ndx]->join();
|
|
|
|
// Use file name as batch name.
|
|
batchNames.push_back(de::FilePath(cmdLine.filenames[ndx].c_str()).getBaseName());
|
|
}
|
|
}
|
|
|
|
// Compute unified case list.
|
|
vector<string> caseList;
|
|
computeCaseList(caseList, results);
|
|
|
|
// Stats.
|
|
int numCases = (int)caseList.size();
|
|
int numEqual = 0;
|
|
|
|
if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
|
|
{
|
|
dst << "TestCasePath";
|
|
for (vector<string>::const_iterator nameIter = batchNames.begin(); nameIter != batchNames.end(); nameIter++)
|
|
dst << "," << *nameIter;
|
|
dst << "\n";
|
|
}
|
|
|
|
// Compare cases.
|
|
for (vector<string>::const_iterator caseIter = caseList.begin(); caseIter != caseList.end(); caseIter++)
|
|
{
|
|
const string& caseName = *caseIter;
|
|
vector<xe::TestCaseResultHeader> headers;
|
|
bool allEqual = true;
|
|
|
|
getTestResultHeaders(headers, results, caseName.c_str());
|
|
|
|
for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin()+1; iter != headers.end(); iter++)
|
|
{
|
|
if (iter->statusCode != headers[0].statusCode)
|
|
{
|
|
allEqual = false;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (allEqual)
|
|
numEqual += 1;
|
|
|
|
if (cmdLine.outMode == OUTPUTMODE_ALL || !allEqual)
|
|
{
|
|
if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
|
|
{
|
|
dst << caseName << "\n";
|
|
for (int ndx = 0; ndx < (int)headers.size(); ndx++)
|
|
dst << " " << batchNames[ndx] << ": " << getStatusCodeName(headers[ndx].statusCode) << " (" << headers[ndx].statusDetails << ")\n";
|
|
dst << "\n";
|
|
}
|
|
else if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
|
|
{
|
|
dst << caseName;
|
|
for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin(); iter != headers.end(); iter++)
|
|
dst << "," << (cmdLine.outValue == OUTPUTVALUE_STATUS_CODE ? getStatusCodeName(iter->statusCode) : iter->statusDetails.c_str());
|
|
dst << "\n";
|
|
}
|
|
}
|
|
}
|
|
|
|
compareOk = numEqual == numCases;
|
|
|
|
if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
|
|
{
|
|
dst << " " << numEqual << " / " << numCases << " test case results match.\n";
|
|
dst << " Comparison " << (compareOk ? "passed" : "FAILED") << "!\n";
|
|
}
|
|
}
|
|
catch (const std::exception& e)
|
|
{
|
|
printf("%s\n", e.what());
|
|
compareOk = false;
|
|
}
|
|
|
|
return compareOk;
|
|
}
|
|
|
|
static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
|
|
{
|
|
de::cmdline::Parser parser;
|
|
de::cmdline::CommandLine opts;
|
|
|
|
XE_CHECK(argc >= 1);
|
|
|
|
opt::registerOptions(parser);
|
|
|
|
if (!parser.parse(argc-1, &argv[1], &opts, std::cerr) ||
|
|
opts.getArgs().empty())
|
|
{
|
|
std::cout << argv[0] << ": [options] [filenames]\n";
|
|
parser.help(std::cout);
|
|
return false;
|
|
}
|
|
|
|
cmdLine.outFormat = opts.getOption<opt::OutFormat>();
|
|
cmdLine.outMode = opts.getOption<opt::OutMode>();
|
|
cmdLine.outValue = opts.getOption<opt::OutValue>();
|
|
cmdLine.filenames = opts.getArgs();
|
|
|
|
return true;
|
|
}
|
|
|
|
int main (int argc, const char* const* argv)
|
|
{
|
|
CommandLine cmdLine;
|
|
|
|
if (!parseCommandLine(cmdLine, argc, argv))
|
|
return -1;
|
|
|
|
try
|
|
{
|
|
bool compareOk = runCompare(cmdLine, std::cout);
|
|
return compareOk ? 0 : -1;
|
|
}
|
|
catch (const std::exception& e)
|
|
{
|
|
printf("FATAL ERROR: %s\n", e.what());
|
|
return -1;
|
|
}
|
|
}
|