diff --git a/metrics/libmetrics.gypi b/metrics/libmetrics.gypi index 65de6f57b..5b90a550c 100644 --- a/metrics/libmetrics.gypi +++ b/metrics/libmetrics.gypi @@ -23,9 +23,9 @@ 'sources': [ 'c_metrics_library.cc', 'metrics_library.cc', + 'serialization/metric_sample.cc', + 'serialization/serialization_utils.cc', 'timer.cc', - 'components/metrics/chromeos/metric_sample.cc', - 'components/metrics/chromeos/serialization_utils.cc', ], 'include_dirs': ['.'], }, diff --git a/metrics/metrics.gyp b/metrics/metrics.gyp index cd0768244..9614826e0 100644 --- a/metrics/metrics.gyp +++ b/metrics/metrics.gyp @@ -75,12 +75,11 @@ }, 'sources': [ 'uploader/upload_service.cc', + 'uploader/metrics_hashes.cc', 'uploader/metrics_log.cc', + 'uploader/metrics_log_base.cc', 'uploader/system_profile_cache.cc', 'uploader/sender_http.cc', - 'components/metrics/metrics_log_base.cc', - 'components/metrics/metrics_log_manager.cc', - 'components/metrics/metrics_hashes.cc', ], 'include_dirs': ['.'] }, @@ -136,6 +135,7 @@ 'includes': ['../common-mk/common_test.gypi'], 'sources': [ 'metrics_library_test.cc', + 'serialization/serialization_utils_unittest.cc', ], 'link_settings': { 'libraries': [ @@ -157,6 +157,8 @@ 'type': 'executable', 'sources': [ 'persistent_integer.cc', + 'uploader/metrics_hashes_unittest.cc', + 'uploader/metrics_log_base_unittest.cc', 'uploader/mock/sender_mock.cc', 'uploader/upload_service_test.cc', ], diff --git a/metrics/metrics_library.cc b/metrics/metrics_library.cc index c352bcf51..5088caee3 100644 --- a/metrics/metrics_library.cc +++ b/metrics/metrics_library.cc @@ -13,13 +13,11 @@ #include #include -#include "components/metrics/chromeos/metric_sample.h" -#include "components/metrics/chromeos/serialization_utils.h" +#include "metrics/serialization/metric_sample.h" +#include "metrics/serialization/serialization_utils.h" #include "policy/device_policy.h" -#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) - static const char kAutotestPath[] = "/var/log/metrics/autotest-events"; static const char kUMAEventsPath[] = "/var/run/metrics/uma-events"; static const char kConsentFile[] = "/home/chronos/Consent To Send Stats"; @@ -206,7 +204,7 @@ void MetricsLibrary::SetPolicyProvider(policy::PolicyProvider* provider) { } bool MetricsLibrary::SendCrosEventToUMA(const std::string& event) { - for (size_t i = 0; i < ARRAY_SIZE(kCrosEventNames); i++) { + for (size_t i = 0; i < arraysize(kCrosEventNames); i++) { if (strcmp(event.c_str(), kCrosEventNames[i]) == 0) { return SendEnumToUMA(kCrosEventHistogramName, i, kCrosEventHistogramMax); } diff --git a/metrics/serialization/metric_sample.cc b/metrics/serialization/metric_sample.cc new file mode 100644 index 000000000..5447497ce --- /dev/null +++ b/metrics/serialization/metric_sample.cc @@ -0,0 +1,197 @@ +// Copyright 2014 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "metrics/serialization/metric_sample.h" + +#include +#include + +#include "base/logging.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/string_split.h" +#include "base/strings/stringprintf.h" + +namespace metrics { + +MetricSample::MetricSample(MetricSample::SampleType sample_type, + const std::string& metric_name, + int sample, + int min, + int max, + int bucket_count) + : type_(sample_type), + name_(metric_name), + sample_(sample), + min_(min), + max_(max), + bucket_count_(bucket_count) { +} + +MetricSample::~MetricSample() { +} + +bool MetricSample::IsValid() const { + return name().find(' ') == std::string::npos && + name().find('\0') == std::string::npos && !name().empty(); +} + +std::string MetricSample::ToString() const { + if (type_ == CRASH) { + return base::StringPrintf("crash%c%s%c", + '\0', + name().c_str(), + '\0'); + } else if (type_ == SPARSE_HISTOGRAM) { + return base::StringPrintf("sparsehistogram%c%s %d%c", + '\0', + name().c_str(), + sample_, + '\0'); + } else if (type_ == LINEAR_HISTOGRAM) { + return base::StringPrintf("linearhistogram%c%s %d %d%c", + '\0', + name().c_str(), + sample_, + max_, + '\0'); + } else if (type_ == HISTOGRAM) { + return base::StringPrintf("histogram%c%s %d %d %d %d%c", + '\0', + name().c_str(), + sample_, + min_, + max_, + bucket_count_, + '\0'); + } else { + // The type can only be USER_ACTION. + CHECK_EQ(type_, USER_ACTION); + return base::StringPrintf("useraction%c%s%c", + '\0', + name().c_str(), + '\0'); + } +} + +int MetricSample::sample() const { + CHECK_NE(type_, USER_ACTION); + CHECK_NE(type_, CRASH); + return sample_; +} + +int MetricSample::min() const { + CHECK_EQ(type_, HISTOGRAM); + return min_; +} + +int MetricSample::max() const { + CHECK_NE(type_, CRASH); + CHECK_NE(type_, USER_ACTION); + CHECK_NE(type_, SPARSE_HISTOGRAM); + return max_; +} + +int MetricSample::bucket_count() const { + CHECK_EQ(type_, HISTOGRAM); + return bucket_count_; +} + +// static +scoped_ptr MetricSample::CrashSample( + const std::string& crash_name) { + return scoped_ptr( + new MetricSample(CRASH, crash_name, 0, 0, 0, 0)); +} + +// static +scoped_ptr MetricSample::HistogramSample( + const std::string& histogram_name, + int sample, + int min, + int max, + int bucket_count) { + return scoped_ptr(new MetricSample( + HISTOGRAM, histogram_name, sample, min, max, bucket_count)); +} + +// static +scoped_ptr MetricSample::ParseHistogram( + const std::string& serialized_histogram) { + std::vector parts; + base::SplitString(serialized_histogram, ' ', &parts); + + if (parts.size() != 5) + return scoped_ptr(); + int sample, min, max, bucket_count; + if (parts[0].empty() || !base::StringToInt(parts[1], &sample) || + !base::StringToInt(parts[2], &min) || + !base::StringToInt(parts[3], &max) || + !base::StringToInt(parts[4], &bucket_count)) { + return scoped_ptr(); + } + + return HistogramSample(parts[0], sample, min, max, bucket_count); +} + +// static +scoped_ptr MetricSample::SparseHistogramSample( + const std::string& histogram_name, + int sample) { + return scoped_ptr( + new MetricSample(SPARSE_HISTOGRAM, histogram_name, sample, 0, 0, 0)); +} + +// static +scoped_ptr MetricSample::ParseSparseHistogram( + const std::string& serialized_histogram) { + std::vector parts; + base::SplitString(serialized_histogram, ' ', &parts); + if (parts.size() != 2) + return scoped_ptr(); + int sample; + if (parts[0].empty() || !base::StringToInt(parts[1], &sample)) + return scoped_ptr(); + + return SparseHistogramSample(parts[0], sample); +} + +// static +scoped_ptr MetricSample::LinearHistogramSample( + const std::string& histogram_name, + int sample, + int max) { + return scoped_ptr( + new MetricSample(LINEAR_HISTOGRAM, histogram_name, sample, 0, max, 0)); +} + +// static +scoped_ptr MetricSample::ParseLinearHistogram( + const std::string& serialized_histogram) { + std::vector parts; + int sample, max; + base::SplitString(serialized_histogram, ' ', &parts); + if (parts.size() != 3) + return scoped_ptr(); + if (parts[0].empty() || !base::StringToInt(parts[1], &sample) || + !base::StringToInt(parts[2], &max)) { + return scoped_ptr(); + } + + return LinearHistogramSample(parts[0], sample, max); +} + +// static +scoped_ptr MetricSample::UserActionSample( + const std::string& action_name) { + return scoped_ptr( + new MetricSample(USER_ACTION, action_name, 0, 0, 0, 0)); +} + +bool MetricSample::IsEqual(const MetricSample& metric) { + return type_ == metric.type_ && name_ == metric.name_ && + sample_ == metric.sample_ && min_ == metric.min_ && + max_ == metric.max_ && bucket_count_ == metric.bucket_count_; +} + +} // namespace metrics diff --git a/metrics/serialization/metric_sample.h b/metrics/serialization/metric_sample.h new file mode 100644 index 000000000..877114d0a --- /dev/null +++ b/metrics/serialization/metric_sample.h @@ -0,0 +1,119 @@ +// Copyright 2014 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef METRICS_SERIALIZATION_METRIC_SAMPLE_H_ +#define METRICS_SERIALIZATION_METRIC_SAMPLE_H_ + +#include + +#include "base/gtest_prod_util.h" +#include "base/macros.h" +#include "base/memory/scoped_ptr.h" + +namespace metrics { + +// This class is used by libmetrics (ChromeOS) to serialize +// and deserialize measurements to send them to a metrics sending service. +// It is meant to be a simple container with serialization functions. +class MetricSample { + public: + // Types of metric sample used. + enum SampleType { + CRASH, + HISTOGRAM, + LINEAR_HISTOGRAM, + SPARSE_HISTOGRAM, + USER_ACTION + }; + + ~MetricSample(); + + // Returns true if the sample is valid (can be serialized without ambiguity). + // + // This function should be used to filter bad samples before serializing them. + bool IsValid() const; + + // Getters for type and name. All types of metrics have these so we do not + // need to check the type. + SampleType type() const { return type_; } + const std::string& name() const { return name_; } + + // Getters for sample, min, max, bucket_count. + // Check the metric type to make sure the request make sense. (ex: a crash + // sample does not have a bucket_count so we crash if we call bucket_count() + // on it.) + int sample() const; + int min() const; + int max() const; + int bucket_count() const; + + // Returns a serialized version of the sample. + // + // The serialized message for each type is: + // crash: crash\0|name_|\0 + // user action: useraction\0|name_|\0 + // histogram: histogram\0|name_| |sample_| |min_| |max_| |bucket_count_|\0 + // sparsehistogram: sparsehistogram\0|name_| |sample_|\0 + // linearhistogram: linearhistogram\0|name_| |sample_| |max_|\0 + std::string ToString() const; + + // Builds a crash sample. + static scoped_ptr CrashSample(const std::string& crash_name); + + // Builds a histogram sample. + static scoped_ptr HistogramSample( + const std::string& histogram_name, + int sample, + int min, + int max, + int bucket_count); + // Deserializes a histogram sample. + static scoped_ptr ParseHistogram(const std::string& serialized); + + // Builds a sparse histogram sample. + static scoped_ptr SparseHistogramSample( + const std::string& histogram_name, + int sample); + // Deserializes a sparse histogram sample. + static scoped_ptr ParseSparseHistogram( + const std::string& serialized); + + // Builds a linear histogram sample. + static scoped_ptr LinearHistogramSample( + const std::string& histogram_name, + int sample, + int max); + // Deserializes a linear histogram sample. + static scoped_ptr ParseLinearHistogram( + const std::string& serialized); + + // Builds a user action sample. + static scoped_ptr UserActionSample( + const std::string& action_name); + + // Returns true if sample and this object represent the same sample (type, + // name, sample, min, max, bucket_count match). + bool IsEqual(const MetricSample& sample); + + private: + MetricSample(SampleType sample_type, + const std::string& metric_name, + const int sample, + const int min, + const int max, + const int bucket_count); + + const SampleType type_; + const std::string name_; + const int sample_; + const int min_; + const int max_; + const int bucket_count_; + + DISALLOW_COPY_AND_ASSIGN(MetricSample); +}; + +} // namespace metrics + +#endif // METRICS_SERIALIZATION_METRIC_SAMPLE_H_ diff --git a/metrics/serialization/serialization_utils.cc b/metrics/serialization/serialization_utils.cc new file mode 100644 index 000000000..fea493e6a --- /dev/null +++ b/metrics/serialization/serialization_utils.cc @@ -0,0 +1,216 @@ +// Copyright 2014 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "metrics/serialization/serialization_utils.h" + +#include + +#include +#include + +#include "base/file_util.h" +#include "base/files/file_path.h" +#include "base/files/scoped_file.h" +#include "base/logging.h" +#include "base/memory/scoped_ptr.h" +#include "base/memory/scoped_vector.h" +#include "base/strings/string_split.h" +#include "base/strings/string_util.h" +#include "metrics/serialization/metric_sample.h" + +#define READ_WRITE_ALL_FILE_FLAGS \ + (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH) + +namespace metrics { +namespace { + +// Reads the next message from |file_descriptor| into |message|. +// +// |message| will be set to the empty string if no message could be read (EOF) +// or the message was badly constructed. +// +// Returns false if no message can be read from this file anymore (EOF or +// unrecoverable error). +bool ReadMessage(int fd, std::string* message) { + CHECK(message); + + int result; + int32 message_size; + // The file containing the metrics do not leave the device so the writer and + // the reader will always have the same endianness. + result = HANDLE_EINTR(read(fd, &message_size, sizeof(message_size))); + if (result < 0) { + DPLOG(ERROR) << "reading metrics message header"; + return false; + } + if (result == 0) { + // This indicates a normal EOF. + return false; + } + if (result < static_cast(sizeof(message_size))) { + DLOG(ERROR) << "bad read size " << result << ", expecting " + << sizeof(message_size); + return false; + } + + // kMessageMaxLength applies to the entire message: the 4-byte + // length field and the content. + if (message_size > SerializationUtils::kMessageMaxLength) { + DLOG(ERROR) << "message too long : " << message_size; + if (HANDLE_EINTR(lseek(fd, message_size - 4, SEEK_CUR)) == -1) { + DLOG(ERROR) << "error while skipping message. abort"; + return false; + } + // Badly formatted message was skipped. Treat the badly formatted sample as + // an empty sample. + message->clear(); + return true; + } + + message_size -= sizeof(message_size); // The message size includes itself. + char buffer[SerializationUtils::kMessageMaxLength]; + if (!base::ReadFromFD(fd, buffer, message_size)) { + DPLOG(ERROR) << "reading metrics message body"; + return false; + } + *message = std::string(buffer, message_size); + return true; +} + +} // namespace + +scoped_ptr SerializationUtils::ParseSample( + const std::string& sample) { + if (sample.empty()) + return scoped_ptr(); + + std::vector parts; + base::SplitString(sample, '\0', &parts); + // We should have two null terminated strings so split should produce + // three chunks. + if (parts.size() != 3) { + DLOG(ERROR) << "splitting message on \\0 produced " << parts.size() + << " parts (expected 3)"; + return scoped_ptr(); + } + const std::string& name = parts[0]; + const std::string& value = parts[1]; + + if (LowerCaseEqualsASCII(name, "crash")) { + return MetricSample::CrashSample(value); + } else if (LowerCaseEqualsASCII(name, "histogram")) { + return MetricSample::ParseHistogram(value); + } else if (LowerCaseEqualsASCII(name, "linearhistogram")) { + return MetricSample::ParseLinearHistogram(value); + } else if (LowerCaseEqualsASCII(name, "sparsehistogram")) { + return MetricSample::ParseSparseHistogram(value); + } else if (LowerCaseEqualsASCII(name, "useraction")) { + return MetricSample::UserActionSample(value); + } else { + DLOG(ERROR) << "invalid event type: " << name << ", value: " << value; + } + return scoped_ptr(); +} + +void SerializationUtils::ReadAndTruncateMetricsFromFile( + const std::string& filename, + ScopedVector* metrics) { + struct stat stat_buf; + int result; + + result = stat(filename.c_str(), &stat_buf); + if (result < 0) { + if (errno != ENOENT) + DPLOG(ERROR) << filename << ": bad metrics file stat"; + + // Nothing to collect---try later. + return; + } + if (stat_buf.st_size == 0) { + // Also nothing to collect. + return; + } + base::ScopedFD fd(open(filename.c_str(), O_RDWR)); + if (fd.get() < 0) { + DPLOG(ERROR) << filename << ": cannot open"; + return; + } + result = flock(fd.get(), LOCK_EX); + if (result < 0) { + DPLOG(ERROR) << filename << ": cannot lock"; + return; + } + + // This processes all messages in the log. When all messages are + // read and processed, or an error occurs, truncate the file to zero size. + for (;;) { + std::string message; + + if (!ReadMessage(fd.get(), &message)) + break; + + scoped_ptr sample = ParseSample(message); + if (sample) + metrics->push_back(sample.release()); + } + + result = ftruncate(fd.get(), 0); + if (result < 0) + DPLOG(ERROR) << "truncate metrics log"; + + result = flock(fd.get(), LOCK_UN); + if (result < 0) + DPLOG(ERROR) << "unlock metrics log"; +} + +bool SerializationUtils::WriteMetricToFile(const MetricSample& sample, + const std::string& filename) { + if (!sample.IsValid()) + return false; + + base::ScopedFD file_descriptor(open(filename.c_str(), + O_WRONLY | O_APPEND | O_CREAT, + READ_WRITE_ALL_FILE_FLAGS)); + + if (file_descriptor.get() < 0) { + DLOG(ERROR) << "error openning the file"; + return false; + } + + fchmod(file_descriptor.get(), READ_WRITE_ALL_FILE_FLAGS); + // Grab a lock to avoid chrome truncating the file + // underneath us. Keep the file locked as briefly as possible. + // Freeing file_descriptor will close the file and and remove the lock. + if (HANDLE_EINTR(flock(file_descriptor.get(), LOCK_EX)) < 0) { + DLOG(ERROR) << "error locking" << filename << " : " << errno; + return false; + } + + std::string msg = sample.ToString(); + int32 size = msg.length() + sizeof(int32); + if (size > kMessageMaxLength) { + DLOG(ERROR) << "cannot write message: too long"; + return false; + } + + // The file containing the metrics samples will only be read by programs on + // the same device so we do not check endianness. + if (base::WriteFileDescriptor(file_descriptor.get(), + reinterpret_cast(&size), + sizeof(size)) != sizeof(size)) { + DPLOG(ERROR) << "error writing message length"; + return false; + } + + if (base::WriteFileDescriptor( + file_descriptor.get(), msg.c_str(), msg.size()) != + static_cast(msg.size())) { + DPLOG(ERROR) << "error writing message"; + return false; + } + + return true; +} + +} // namespace metrics diff --git a/metrics/serialization/serialization_utils.h b/metrics/serialization/serialization_utils.h new file mode 100644 index 000000000..5af61660f --- /dev/null +++ b/metrics/serialization/serialization_utils.h @@ -0,0 +1,48 @@ +// Copyright 2014 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef METRICS_SERIALIZATION_SERIALIZATION_UTILS_H_ +#define METRICS_SERIALIZATION_SERIALIZATION_UTILS_H_ + +#include + +#include "base/memory/scoped_ptr.h" +#include "base/memory/scoped_vector.h" + +namespace metrics { + +class MetricSample; + +// Metrics helpers to serialize and deserialize metrics collected by +// ChromeOS. +namespace SerializationUtils { + +// Deserializes a sample passed as a string and return a sample. +// The return value will either be a scoped_ptr to a Metric sample (if the +// deserialization was successful) or a NULL scoped_ptr. +scoped_ptr ParseSample(const std::string& sample); + +// Reads all samples from a file and truncate the file when done. +void ReadAndTruncateMetricsFromFile(const std::string& filename, + ScopedVector* metrics); + +// Serializes a sample and write it to filename. +// The format for the message is: +// message_size, serialized_message +// where +// * message_size is the total length of the message (message_size + +// serialized_message) on 4 bytes +// * serialized_message is the serialized version of sample (using ToString) +// +// NB: the file will never leave the device so message_size will be written +// with the architecture's endianness. +bool WriteMetricToFile(const MetricSample& sample, const std::string& filename); + +// Maximum length of a serialized message +static const int kMessageMaxLength = 1024; + +} // namespace SerializationUtils +} // namespace metrics + +#endif // METRICS_SERIALIZATION_SERIALIZATION_UTILS_H_ diff --git a/metrics/serialization/serialization_utils_unittest.cc b/metrics/serialization/serialization_utils_unittest.cc new file mode 100644 index 000000000..d47fbc847 --- /dev/null +++ b/metrics/serialization/serialization_utils_unittest.cc @@ -0,0 +1,169 @@ +// Copyright 2014 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "metrics/serialization/serialization_utils.h" + +#include +#include +#include +#include +#include + +#include "metrics/serialization/metric_sample.h" + +namespace metrics { +namespace { + +class SerializationUtilsTest : public testing::Test { + protected: + SerializationUtilsTest() { + bool success = temporary_dir.CreateUniqueTempDir(); + if (success) { + base::FilePath dir_path = temporary_dir.path(); + filename = dir_path.value() + "chromeossampletest"; + filepath = base::FilePath(filename); + } + } + + void SetUp() override { base::DeleteFile(filepath, false); } + + void TestSerialization(MetricSample* sample) { + std::string serialized(sample->ToString()); + ASSERT_EQ('\0', serialized[serialized.length() - 1]); + scoped_ptr deserialized = + SerializationUtils::ParseSample(serialized); + ASSERT_TRUE(deserialized); + EXPECT_TRUE(sample->IsEqual(*deserialized.get())); + } + + std::string filename; + base::ScopedTempDir temporary_dir; + base::FilePath filepath; +}; + +TEST_F(SerializationUtilsTest, CrashSerializeTest) { + TestSerialization(MetricSample::CrashSample("test").get()); +} + +TEST_F(SerializationUtilsTest, HistogramSerializeTest) { + TestSerialization( + MetricSample::HistogramSample("myhist", 13, 1, 100, 10).get()); +} + +TEST_F(SerializationUtilsTest, LinearSerializeTest) { + TestSerialization( + MetricSample::LinearHistogramSample("linearhist", 12, 30).get()); +} + +TEST_F(SerializationUtilsTest, SparseSerializeTest) { + TestSerialization(MetricSample::SparseHistogramSample("mysparse", 30).get()); +} + +TEST_F(SerializationUtilsTest, UserActionSerializeTest) { + TestSerialization(MetricSample::UserActionSample("myaction").get()); +} + +TEST_F(SerializationUtilsTest, IllegalNameAreFilteredTest) { + scoped_ptr sample1 = + MetricSample::SparseHistogramSample("no space", 10); + scoped_ptr sample2 = MetricSample::LinearHistogramSample( + base::StringPrintf("here%cbhe", '\0'), 1, 3); + + EXPECT_FALSE(SerializationUtils::WriteMetricToFile(*sample1.get(), filename)); + EXPECT_FALSE(SerializationUtils::WriteMetricToFile(*sample2.get(), filename)); + int64 size = 0; + + ASSERT_TRUE(!PathExists(filepath) || base::GetFileSize(filepath, &size)); + + EXPECT_EQ(0, size); +} + +TEST_F(SerializationUtilsTest, BadInputIsCaughtTest) { + std::string input( + base::StringPrintf("sparsehistogram%cname foo%c", '\0', '\0')); + EXPECT_EQ(NULL, MetricSample::ParseSparseHistogram(input).get()); +} + +TEST_F(SerializationUtilsTest, MessageSeparatedByZero) { + scoped_ptr crash = MetricSample::CrashSample("mycrash"); + + SerializationUtils::WriteMetricToFile(*crash.get(), filename); + int64 size = 0; + ASSERT_TRUE(base::GetFileSize(filepath, &size)); + // 4 bytes for the size + // 5 bytes for crash + // 7 bytes for mycrash + // 2 bytes for the \0 + // -> total of 18 + EXPECT_EQ(size, 18); +} + +TEST_F(SerializationUtilsTest, MessagesTooLongAreDiscardedTest) { + // Creates a message that is bigger than the maximum allowed size. + // As we are adding extra character (crash, \0s, etc), if the name is + // kMessageMaxLength long, it will be too long. + std::string name(SerializationUtils::kMessageMaxLength, 'c'); + + scoped_ptr crash = MetricSample::CrashSample(name); + EXPECT_FALSE(SerializationUtils::WriteMetricToFile(*crash.get(), filename)); + int64 size = 0; + ASSERT_TRUE(base::GetFileSize(filepath, &size)); + EXPECT_EQ(0, size); +} + +TEST_F(SerializationUtilsTest, ReadLongMessageTest) { + base::File test_file(filepath, + base::File::FLAG_OPEN_ALWAYS | base::File::FLAG_APPEND); + std::string message(SerializationUtils::kMessageMaxLength + 1, 'c'); + + int32 message_size = message.length() + sizeof(int32); + test_file.WriteAtCurrentPos(reinterpret_cast(&message_size), + sizeof(message_size)); + test_file.WriteAtCurrentPos(message.c_str(), message.length()); + test_file.Close(); + + scoped_ptr crash = MetricSample::CrashSample("test"); + SerializationUtils::WriteMetricToFile(*crash.get(), filename); + + ScopedVector samples; + SerializationUtils::ReadAndTruncateMetricsFromFile(filename, &samples); + ASSERT_EQ(size_t(1), samples.size()); + ASSERT_TRUE(samples[0] != NULL); + EXPECT_TRUE(crash->IsEqual(*samples[0])); +} + +TEST_F(SerializationUtilsTest, WriteReadTest) { + scoped_ptr hist = + MetricSample::HistogramSample("myhist", 1, 2, 3, 4); + scoped_ptr crash = MetricSample::CrashSample("mycrash"); + scoped_ptr lhist = + MetricSample::LinearHistogramSample("linear", 1, 10); + scoped_ptr shist = + MetricSample::SparseHistogramSample("mysparse", 30); + scoped_ptr action = MetricSample::UserActionSample("myaction"); + + SerializationUtils::WriteMetricToFile(*hist.get(), filename); + SerializationUtils::WriteMetricToFile(*crash.get(), filename); + SerializationUtils::WriteMetricToFile(*lhist.get(), filename); + SerializationUtils::WriteMetricToFile(*shist.get(), filename); + SerializationUtils::WriteMetricToFile(*action.get(), filename); + ScopedVector vect; + SerializationUtils::ReadAndTruncateMetricsFromFile(filename, &vect); + ASSERT_EQ(vect.size(), size_t(5)); + for (int i = 0; i < 5; i++) { + ASSERT_TRUE(vect[0] != NULL); + } + EXPECT_TRUE(hist->IsEqual(*vect[0])); + EXPECT_TRUE(crash->IsEqual(*vect[1])); + EXPECT_TRUE(lhist->IsEqual(*vect[2])); + EXPECT_TRUE(shist->IsEqual(*vect[3])); + EXPECT_TRUE(action->IsEqual(*vect[4])); + + int64 size = 0; + ASSERT_TRUE(base::GetFileSize(filepath, &size)); + ASSERT_EQ(0, size); +} + +} // namespace +} // namespace metrics diff --git a/metrics/uploader/metrics_hashes.cc b/metrics/uploader/metrics_hashes.cc new file mode 100644 index 000000000..87405a377 --- /dev/null +++ b/metrics/uploader/metrics_hashes.cc @@ -0,0 +1,39 @@ +// Copyright 2014 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "metrics/uploader/metrics_hashes.h" + +#include "base/logging.h" +#include "base/md5.h" +#include "base/sys_byteorder.h" + +namespace metrics { + +namespace { + +// Converts the 8-byte prefix of an MD5 hash into a uint64 value. +inline uint64_t HashToUInt64(const std::string& hash) { + uint64_t value; + DCHECK_GE(hash.size(), sizeof(value)); + memcpy(&value, hash.data(), sizeof(value)); + return base::HostToNet64(value); +} + +} // namespace + +uint64_t HashMetricName(const std::string& name) { + // Create an MD5 hash of the given |name|, represented as a byte buffer + // encoded as an std::string. + base::MD5Context context; + base::MD5Init(&context); + base::MD5Update(&context, name); + + base::MD5Digest digest; + base::MD5Final(&digest, &context); + + std::string hash_str(reinterpret_cast(digest.a), arraysize(digest.a)); + return HashToUInt64(hash_str); +} + +} // namespace metrics diff --git a/metrics/uploader/metrics_hashes.h b/metrics/uploader/metrics_hashes.h new file mode 100644 index 000000000..8679077e1 --- /dev/null +++ b/metrics/uploader/metrics_hashes.h @@ -0,0 +1,18 @@ +// Copyright 2014 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef METRICS_UPLOADER_METRICS_HASHES_H_ +#define METRICS_UPLOADER_METRICS_HASHES_H_ + +#include + +namespace metrics { + +// Computes a uint64 hash of a given string based on its MD5 hash. Suitable for +// metric names. +uint64_t HashMetricName(const std::string& name); + +} // namespace metrics + +#endif // METRICS_UPLOADER_METRICS_HASHES_H_ diff --git a/metrics/uploader/metrics_hashes_unittest.cc b/metrics/uploader/metrics_hashes_unittest.cc new file mode 100644 index 000000000..f7e390f64 --- /dev/null +++ b/metrics/uploader/metrics_hashes_unittest.cc @@ -0,0 +1,32 @@ +// Copyright 2014 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "metrics/uploader/metrics_hashes.h" + +#include +#include +#include +#include + +namespace metrics { + +// Make sure our ID hashes are the same as what we see on the server side. +TEST(MetricsUtilTest, HashMetricName) { + static const struct { + std::string input; + std::string output; + } cases[] = { + {"Back", "0x0557fa923dcee4d0"}, + {"Forward", "0x67d2f6740a8eaebf"}, + {"NewTab", "0x290eb683f96572f1"}, + }; + + for (size_t i = 0; i < arraysize(cases); ++i) { + uint64_t hash = HashMetricName(cases[i].input); + std::string hash_hex = base::StringPrintf("0x%016" PRIx64, hash); + EXPECT_EQ(cases[i].output, hash_hex); + } +} + +} // namespace metrics diff --git a/metrics/uploader/metrics_log.h b/metrics/uploader/metrics_log.h index 37a82bdd4..579632578 100644 --- a/metrics/uploader/metrics_log.h +++ b/metrics/uploader/metrics_log.h @@ -9,7 +9,7 @@ #include -#include "components/metrics/metrics_log_base.h" +#include "metrics/uploader/metrics_log_base.h" // This file defines a set of user experience metrics data recorded by // the MetricsService. This is the unit of data that is sent to the server. diff --git a/metrics/uploader/metrics_log_base.cc b/metrics/uploader/metrics_log_base.cc new file mode 100644 index 000000000..43cf82e29 --- /dev/null +++ b/metrics/uploader/metrics_log_base.cc @@ -0,0 +1,142 @@ +// Copyright 2014 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "metrics/uploader/metrics_log_base.h" + +#include "base/metrics/histogram_base.h" +#include "base/metrics/histogram_samples.h" +#include "components/metrics/proto/histogram_event.pb.h" +#include "components/metrics/proto/system_profile.pb.h" +#include "components/metrics/proto/user_action_event.pb.h" +#include "metrics/uploader/metrics_hashes.h" + +using base::Histogram; +using base::HistogramBase; +using base::HistogramSamples; +using base::SampleCountIterator; +using base::Time; +using base::TimeDelta; +using metrics::HistogramEventProto; +using metrics::SystemProfileProto; +using metrics::UserActionEventProto; + +namespace metrics { +namespace { + +// Any id less than 16 bytes is considered to be a testing id. +bool IsTestingID(const std::string& id) { + return id.size() < 16; +} + +} // namespace + +MetricsLogBase::MetricsLogBase(const std::string& client_id, + int session_id, + LogType log_type, + const std::string& version_string) + : num_events_(0), + locked_(false), + log_type_(log_type) { + DCHECK_NE(NO_LOG, log_type); + if (IsTestingID(client_id)) + uma_proto_.set_client_id(0); + else + uma_proto_.set_client_id(Hash(client_id)); + + uma_proto_.set_session_id(session_id); + uma_proto_.mutable_system_profile()->set_build_timestamp(GetBuildTime()); + uma_proto_.mutable_system_profile()->set_app_version(version_string); +} + +MetricsLogBase::~MetricsLogBase() {} + +// static +uint64_t MetricsLogBase::Hash(const std::string& value) { + uint64_t hash = metrics::HashMetricName(value); + + // The following log is VERY helpful when folks add some named histogram into + // the code, but forgot to update the descriptive list of histograms. When + // that happens, all we get to see (server side) is a hash of the histogram + // name. We can then use this logging to find out what histogram name was + // being hashed to a given MD5 value by just running the version of Chromium + // in question with --enable-logging. + DVLOG(1) << "Metrics: Hash numeric [" << value << "]=[" << hash << "]"; + + return hash; +} + +// static +int64_t MetricsLogBase::GetBuildTime() { + static int64_t integral_build_time = 0; + if (!integral_build_time) { + Time time; + const char* kDateTime = __DATE__ " " __TIME__ " GMT"; + bool result = Time::FromString(kDateTime, &time); + DCHECK(result); + integral_build_time = static_cast(time.ToTimeT()); + } + return integral_build_time; +} + +// static +int64_t MetricsLogBase::GetCurrentTime() { + return (base::TimeTicks::Now() - base::TimeTicks()).InSeconds(); +} + +void MetricsLogBase::CloseLog() { + DCHECK(!locked_); + locked_ = true; +} + +void MetricsLogBase::GetEncodedLog(std::string* encoded_log) { + DCHECK(locked_); + uma_proto_.SerializeToString(encoded_log); +} + +void MetricsLogBase::RecordUserAction(const std::string& key) { + DCHECK(!locked_); + + UserActionEventProto* user_action = uma_proto_.add_user_action_event(); + user_action->set_name_hash(Hash(key)); + user_action->set_time(GetCurrentTime()); + + ++num_events_; +} + +void MetricsLogBase::RecordHistogramDelta(const std::string& histogram_name, + const HistogramSamples& snapshot) { + DCHECK(!locked_); + DCHECK_NE(0, snapshot.TotalCount()); + + // We will ignore the MAX_INT/infinite value in the last element of range[]. + + HistogramEventProto* histogram_proto = uma_proto_.add_histogram_event(); + histogram_proto->set_name_hash(Hash(histogram_name)); + histogram_proto->set_sum(snapshot.sum()); + + for (scoped_ptr it = snapshot.Iterator(); !it->Done(); + it->Next()) { + HistogramBase::Sample min; + HistogramBase::Sample max; + HistogramBase::Count count; + it->Get(&min, &max, &count); + HistogramEventProto::Bucket* bucket = histogram_proto->add_bucket(); + bucket->set_min(min); + bucket->set_max(max); + bucket->set_count(count); + } + + // Omit fields to save space (see rules in histogram_event.proto comments). + for (int i = 0; i < histogram_proto->bucket_size(); ++i) { + HistogramEventProto::Bucket* bucket = histogram_proto->mutable_bucket(i); + if (i + 1 < histogram_proto->bucket_size() && + bucket->max() == histogram_proto->bucket(i + 1).min()) { + bucket->clear_max(); + } else if (bucket->max() == bucket->min() + 1) { + bucket->clear_min(); + } + } +} + +} // namespace metrics diff --git a/metrics/uploader/metrics_log_base.h b/metrics/uploader/metrics_log_base.h new file mode 100644 index 000000000..5a54c30af --- /dev/null +++ b/metrics/uploader/metrics_log_base.h @@ -0,0 +1,110 @@ +// Copyright 2014 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// This file defines a set of user experience metrics data recorded by +// the MetricsService. This is the unit of data that is sent to the server. + +#ifndef METRICS_UPLOADER_METRICS_LOG_BASE_H_ +#define METRICS_UPLOADER_METRICS_LOG_BASE_H_ + +#include + +#include "base/macros.h" +#include "base/metrics/histogram.h" +#include "base/time/time.h" +#include "components/metrics/proto/chrome_user_metrics_extension.pb.h" + +namespace base { +class HistogramSamples; +} // namespace base + +namespace metrics { + +// This class provides base functionality for logging metrics data. +class MetricsLogBase { + public: + // TODO(asvitkine): Remove the NO_LOG value. + enum LogType { + INITIAL_STABILITY_LOG, // The initial log containing stability stats. + ONGOING_LOG, // Subsequent logs in a session. + NO_LOG, // Placeholder value for when there is no log. + }; + + // Creates a new metrics log of the specified type. + // client_id is the identifier for this profile on this installation + // session_id is an integer that's incremented on each application launch + MetricsLogBase(const std::string& client_id, + int session_id, + LogType log_type, + const std::string& version_string); + virtual ~MetricsLogBase(); + + // Computes the MD5 hash of the given string, and returns the first 8 bytes of + // the hash. + static uint64_t Hash(const std::string& value); + + // Get the GMT buildtime for the current binary, expressed in seconds since + // January 1, 1970 GMT. + // The value is used to identify when a new build is run, so that previous + // reliability stats, from other builds, can be abandoned. + static int64_t GetBuildTime(); + + // Convenience function to return the current time at a resolution in seconds. + // This wraps base::TimeTicks, and hence provides an abstract time that is + // always incrementing for use in measuring time durations. + static int64_t GetCurrentTime(); + + // Records a user-initiated action. + void RecordUserAction(const std::string& key); + + // Record any changes in a given histogram for transmission. + void RecordHistogramDelta(const std::string& histogram_name, + const base::HistogramSamples& snapshot); + + // Stop writing to this record and generate the encoded representation. + // None of the Record* methods can be called after this is called. + void CloseLog(); + + // Fills |encoded_log| with the serialized protobuf representation of the + // record. Must only be called after CloseLog() has been called. + void GetEncodedLog(std::string* encoded_log); + + int num_events() { return num_events_; } + + void set_hardware_class(const std::string& hardware_class) { + uma_proto_.mutable_system_profile()->mutable_hardware()->set_hardware_class( + hardware_class); + } + + LogType log_type() const { return log_type_; } + + protected: + bool locked() const { return locked_; } + + metrics::ChromeUserMetricsExtension* uma_proto() { return &uma_proto_; } + const metrics::ChromeUserMetricsExtension* uma_proto() const { + return &uma_proto_; + } + + // TODO(isherman): Remove this once the XML pipeline is outta here. + int num_events_; // the number of events recorded in this log + + private: + // locked_ is true when record has been packed up for sending, and should + // no longer be written to. It is only used for sanity checking and is + // not a real lock. + bool locked_; + + // The type of the log, i.e. initial or ongoing. + const LogType log_type_; + + // Stores the protocol buffer representation for this log. + metrics::ChromeUserMetricsExtension uma_proto_; + + DISALLOW_COPY_AND_ASSIGN(MetricsLogBase); +}; + +} // namespace metrics + +#endif // METRICS_UPLOADER_METRICS_LOG_BASE_H_ diff --git a/metrics/uploader/metrics_log_base_unittest.cc b/metrics/uploader/metrics_log_base_unittest.cc new file mode 100644 index 000000000..fe02fea06 --- /dev/null +++ b/metrics/uploader/metrics_log_base_unittest.cc @@ -0,0 +1,126 @@ +// Copyright 2014 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "metrics/uploader/metrics_log_base.h" + +#include + +#include +#include +#include +#include + +#include "components/metrics/proto/chrome_user_metrics_extension.pb.h" + +namespace metrics { + +namespace { + +class TestMetricsLogBase : public MetricsLogBase { + public: + TestMetricsLogBase() + : MetricsLogBase("client_id", 1, MetricsLogBase::ONGOING_LOG, "1.2.3.4") { + } + virtual ~TestMetricsLogBase() {} + + using MetricsLogBase::uma_proto; + + private: + DISALLOW_COPY_AND_ASSIGN(TestMetricsLogBase); +}; + +} // namespace + +TEST(MetricsLogBaseTest, LogType) { + MetricsLogBase log1("id", 0, MetricsLogBase::ONGOING_LOG, "1.2.3"); + EXPECT_EQ(MetricsLogBase::ONGOING_LOG, log1.log_type()); + + MetricsLogBase log2("id", 0, MetricsLogBase::INITIAL_STABILITY_LOG, "1.2.3"); + EXPECT_EQ(MetricsLogBase::INITIAL_STABILITY_LOG, log2.log_type()); +} + +TEST(MetricsLogBaseTest, EmptyRecord) { + MetricsLogBase log("totally bogus client ID", 137, + MetricsLogBase::ONGOING_LOG, "bogus version"); + log.set_hardware_class("sample-class"); + log.CloseLog(); + + std::string encoded; + log.GetEncodedLog(&encoded); + + // A couple of fields are hard to mock, so these will be copied over directly + // for the expected output. + metrics::ChromeUserMetricsExtension parsed; + ASSERT_TRUE(parsed.ParseFromString(encoded)); + + metrics::ChromeUserMetricsExtension expected; + expected.set_client_id(5217101509553811875); // Hashed bogus client ID + expected.set_session_id(137); + expected.mutable_system_profile()->set_build_timestamp( + parsed.system_profile().build_timestamp()); + expected.mutable_system_profile()->set_app_version("bogus version"); + expected.mutable_system_profile()->mutable_hardware()->set_hardware_class( + "sample-class"); + + EXPECT_EQ(expected.SerializeAsString(), encoded); +} + +TEST(MetricsLogBaseTest, HistogramBucketFields) { + // Create buckets: 1-5, 5-7, 7-8, 8-9, 9-10, 10-11, 11-12. + base::BucketRanges ranges(8); + ranges.set_range(0, 1); + ranges.set_range(1, 5); + ranges.set_range(2, 7); + ranges.set_range(3, 8); + ranges.set_range(4, 9); + ranges.set_range(5, 10); + ranges.set_range(6, 11); + ranges.set_range(7, 12); + + base::SampleVector samples(&ranges); + samples.Accumulate(3, 1); // Bucket 1-5. + samples.Accumulate(6, 1); // Bucket 5-7. + samples.Accumulate(8, 1); // Bucket 8-9. (7-8 skipped) + samples.Accumulate(10, 1); // Bucket 10-11. (9-10 skipped) + samples.Accumulate(11, 1); // Bucket 11-12. + + TestMetricsLogBase log; + log.RecordHistogramDelta("Test", samples); + + const metrics::ChromeUserMetricsExtension* uma_proto = log.uma_proto(); + const metrics::HistogramEventProto& histogram_proto = + uma_proto->histogram_event(uma_proto->histogram_event_size() - 1); + + // Buckets with samples: 1-5, 5-7, 8-9, 10-11, 11-12. + // Should become: 1-/, 5-7, /-9, 10-/, /-12. + ASSERT_EQ(5, histogram_proto.bucket_size()); + + // 1-5 becomes 1-/ (max is same as next min). + EXPECT_TRUE(histogram_proto.bucket(0).has_min()); + EXPECT_FALSE(histogram_proto.bucket(0).has_max()); + EXPECT_EQ(1, histogram_proto.bucket(0).min()); + + // 5-7 stays 5-7 (no optimization possible). + EXPECT_TRUE(histogram_proto.bucket(1).has_min()); + EXPECT_TRUE(histogram_proto.bucket(1).has_max()); + EXPECT_EQ(5, histogram_proto.bucket(1).min()); + EXPECT_EQ(7, histogram_proto.bucket(1).max()); + + // 8-9 becomes /-9 (min is same as max - 1). + EXPECT_FALSE(histogram_proto.bucket(2).has_min()); + EXPECT_TRUE(histogram_proto.bucket(2).has_max()); + EXPECT_EQ(9, histogram_proto.bucket(2).max()); + + // 10-11 becomes 10-/ (both optimizations apply, omit max is prioritized). + EXPECT_TRUE(histogram_proto.bucket(3).has_min()); + EXPECT_FALSE(histogram_proto.bucket(3).has_max()); + EXPECT_EQ(10, histogram_proto.bucket(3).min()); + + // 11-12 becomes /-12 (last record must keep max, min is same as max - 1). + EXPECT_FALSE(histogram_proto.bucket(4).has_min()); + EXPECT_TRUE(histogram_proto.bucket(4).has_max()); + EXPECT_EQ(12, histogram_proto.bucket(4).max()); +} + +} // namespace metrics diff --git a/metrics/uploader/system_profile_cache.cc b/metrics/uploader/system_profile_cache.cc index cbb80512c..be3d8ecaa 100644 --- a/metrics/uploader/system_profile_cache.cc +++ b/metrics/uploader/system_profile_cache.cc @@ -13,9 +13,9 @@ #include "base/logging.h" #include "base/strings/string_number_conversions.h" #include "base/sys_info.h" -#include "components/metrics/metrics_log_base.h" #include "components/metrics/proto/chrome_user_metrics_extension.pb.h" #include "metrics/persistent_integer.h" +#include "metrics/uploader/metrics_log_base.h" #include "vboot/crossystem.h" namespace { diff --git a/metrics/uploader/upload_service.cc b/metrics/uploader/upload_service.cc index 7e856f925..cdec91e41 100644 --- a/metrics/uploader/upload_service.cc +++ b/metrics/uploader/upload_service.cc @@ -15,9 +15,9 @@ #include #include #include -#include -#include +#include "metrics/serialization/metric_sample.h" +#include "metrics/serialization/serialization_utils.h" #include "metrics/uploader/metrics_log.h" #include "metrics/uploader/sender_http.h" #include "metrics/uploader/system_profile_cache.h" diff --git a/metrics/uploader/upload_service_test.cc b/metrics/uploader/upload_service_test.cc index c48747e81..ff96f85ac 100644 --- a/metrics/uploader/upload_service_test.cc +++ b/metrics/uploader/upload_service_test.cc @@ -9,15 +9,15 @@ #include "base/files/scoped_temp_dir.h" #include "base/logging.h" #include "base/sys_info.h" -#include "components/metrics/chromeos/metric_sample.h" #include "components/metrics/proto/chrome_user_metrics_extension.pb.h" #include "components/metrics/proto/histogram_event.pb.h" #include "components/metrics/proto/system_profile.pb.h" -#include "uploader/metrics_log.h" -#include "uploader/mock/mock_system_profile_setter.h" -#include "uploader/mock/sender_mock.h" -#include "uploader/system_profile_cache.h" -#include "uploader/upload_service.h" +#include "metrics/serialization/metric_sample.h" +#include "metrics/uploader/metrics_log.h" +#include "metrics/uploader/mock/mock_system_profile_setter.h" +#include "metrics/uploader/mock/sender_mock.h" +#include "metrics/uploader/system_profile_cache.h" +#include "metrics/uploader/upload_service.h" static const char kMetricsServer[] = "https://clients4.google.com/uma/v2"; static const char kMetricsFilePath[] = "/var/run/metrics/uma-events";