// Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // This file was automatically generated on 2015-09-07T10:52:33.316574Z // Do not edit it directly #ifndef NONIUS_SINGLE_INCLUDE_HPP #define NONIUS_SINGLE_INCLUDE_HPP // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Single header root // #included from: nonius.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Main header #ifndef NONIUS_HPP #define NONIUS_HPP // #included from: clock.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Clocks #define NONIUS_CLOCK_HPP #if defined(_MSC_VER) && (_MSC_VER) < 1900 #if !defined(NONIUS_USE_BOOST_CHRONO) #define NONIUS_USE_BOOST_CHRONO #endif // NONIUS_USE_BOOST_CHRONO #endif // MSVC is borken and had little to no testing done before shipping (Dev14/VS15 CTP fixes it) #if defined(NONIUS_USE_BOOST_CHRONO) #include #else #include #include #endif namespace nonius { #if defined(NONIUS_USE_BOOST_CHRONO) namespace chrono = boost::chrono; template using ratio = boost::ratio; #else namespace chrono = std::chrono; template using ratio = std::ratio; #endif using milli = ratio<1, 1000>; using micro = ratio<1, 1000000>; using nano = ratio<1, 1000000000>; template using Duration = typename Clock::duration; template using FloatDuration = chrono::duration; template using TimePoint = typename Clock::time_point; using default_clock = chrono::high_resolution_clock; template struct now { TimePoint operator()() const { return Clock::now(); } }; using fp_seconds = chrono::duration>; } // namespace nonius // #included from: benchmark.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Benchmark #define NONIUS_BENCHMARK_HPP // #included from: configuration.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Run configuration #define NONIUS_CONFIGURATION_HPP #include namespace nonius { struct configuration { public: int samples = 100; double confidence_interval = 0.95; int resamples = 100000; std::string title = "benchmarks"; std::string output_file; std::string reporter; bool list_benchmarks = false; bool list_reporters = false; bool no_analysis = false; bool verbose = false; bool summary = false; bool help = false; }; } // namespace nonius // #included from: environment.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Environment information #define NONIUS_ENVIRONMENT_HPP // #included from: outlier_classification.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Outlier information #define NONIUS_OUTLIERS_HPP namespace nonius { struct outlier_classification { int samples_seen = 0; int low_severe = 0; // more than 3 times IQR below Q1 int low_mild = 0; // 1.5 to 3 times IQR below Q1 int high_mild = 0; // 1.5 to 3 times IQR above Q3 int high_severe = 0; // more than 3 times IQR above Q3 int total() const { return low_severe + low_mild + high_mild + high_severe; } }; } // namespace nonius namespace nonius { template struct environment_estimate { Duration mean; outlier_classification outliers; template operator environment_estimate() const { return { mean, outliers }; } }; template struct environment { environment_estimate> clock_resolution; environment_estimate> clock_cost; //estimate function_cost; }; } // namespace nonius // #included from: execution_plan.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Execution plan #define NONIUS_EXECUTION_PLAN_HPP namespace nonius { template struct execution_plan { int iterations_per_sample; Duration estimated_duration; template operator execution_plan() const { return { iterations_per_sample, estimated_duration }; } }; } // namespace nonius // #included from: chronometer.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // User-facing chronometer #define NONIUS_CHRONOMETER_HPP // #included from: detail/complete_invoke.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Invoke with a special case for void #define NONIUS_DETAIL_COMPLETE_INVOKE_HPP #include #include namespace nonius { namespace detail { template struct complete_type { using type = T; }; template <> struct complete_type { struct type {}; }; template using CompleteType = typename complete_type::type; template struct complete_invoker { template static Result invoke(Fun&& fun, Args&&... args) { return std::forward(fun)(std::forward(args)...); } }; template <> struct complete_invoker { template static CompleteType invoke(Fun&& fun, Args&&... args) { std::forward(fun)(std::forward(args)...); return {}; } }; template using ResultOf = typename std::result_of::type; // invoke and not return void :( template CompleteType> complete_invoke(Fun&& fun, Args&&... args) { return complete_invoker>::invoke(std::forward(fun), std::forward(args)...); } template struct always_true : std::true_type {}; struct is_callable_tester { template always_true()(std::declval()...))> static test(int); template std::false_type static test(...); }; template struct is_callable; template struct is_callable : decltype(is_callable_tester::test(0)) {}; } // namespace detail } // namespace nonius namespace nonius { namespace detail { struct chronometer_concept { virtual void start() = 0; virtual void finish() = 0; virtual ~chronometer_concept() = default; }; template struct chronometer_model final : public chronometer_concept { void start() override { started = Clock::now(); } void finish() override { finished = Clock::now(); } TimePoint started; TimePoint finished; }; } // namespace detail struct chronometer { public: template void measure(Fun&& fun) { measure(std::forward(fun), detail::is_callable()); } int runs() const { return k; } chronometer(detail::chronometer_concept& meter, int k) : impl(&meter), k(k) {} private: template void measure(Fun&& fun, std::false_type) { measure([&fun](int) { fun(); }); } template void measure(Fun&& fun, std::true_type) { impl->start(); for(int i = 0; i < k; ++i) fun(i); impl->finish(); } detail::chronometer_concept* impl; int k; }; } // namespace nonius // #included from: detail/measure.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Measure #define NONIUS_DETAIL_MEASURE_HPP // #included from: detail/timing.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Timing #define NONIUS_DETAIL_TIMING_HPP #include #include namespace nonius { template struct timing { Duration elapsed; Result result; int iterations; }; template using TimingOf = timing, detail::CompleteType>>; } // namespace nonius #include namespace nonius { namespace detail { template TimingOf measure(Fun&& fun, Args&&... args) { auto start = Clock::now(); auto&& r = detail::complete_invoke(fun, std::forward(args)...); auto end = Clock::now(); auto delta = end - start; return { delta, std::forward(r), 1 }; } } // namespace detail } // namespace nonius // #included from: detail/benchmark_function.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Dumb std::function implementation for consistent call overhead #define NONIUS_DETAIL_BENCHMARK_FUNCTION_HPP #include #include #include namespace nonius { namespace detail { template using Decay = typename std::decay::type; template struct is_related : std::is_same, Decay> {}; struct benchmark_function { private: struct concept { virtual void call(chronometer meter) const = 0; virtual concept* clone() const = 0; virtual ~concept() = default; }; template struct model : public concept { model(Fun&& fun) : fun(std::move(fun)) {} model(Fun const& fun) : fun(fun) {} model* clone() const override { return new model(*this); } void call(chronometer meter) const override { call(meter, is_callable()); } void call(chronometer meter, std::true_type) const { fun(meter); } void call(chronometer meter, std::false_type) const { meter.measure(fun); } Fun fun; }; public: template ::value, int>::type = 0> benchmark_function(Fun&& fun) : f(new model::type>(std::forward(fun))) {} benchmark_function(benchmark_function&& that) : f(std::move(that.f)) {} benchmark_function(benchmark_function const& that) : f(that.f->clone()) {} benchmark_function& operator=(benchmark_function&& that) { f = std::move(that.f); return *this; } benchmark_function& operator=(benchmark_function const& that) { f.reset(that.f->clone()); return *this; } void operator()(chronometer meter) const { f->call(meter); } private: std::unique_ptr f; }; } // namespace detail } // namespace nonius // #included from: detail/repeat.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // repeat algorithm #define NONIUS_DETAIL_REPEAT_HPP #include #include namespace nonius { namespace detail { template struct repeater { void operator()(int k) const { for(int i = 0; i < k; ++i) { fun(); } } Fun fun; }; template repeater::type> repeat(Fun&& fun) { return { std::forward(fun) }; } } // namespace detail } // namespace nonius // #included from: detail/run_for_at_least.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Run a function for a minimum amount of time #define NONIUS_RUN_FOR_AT_LEAST_HPP // #included from: timeout_error.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Exception to be thrown when a process takes too long to run #define NONIUS_TIMEOUT_ERROR_HPP // #included from: detail/noexcept.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Cross-compiler noexcept support #define NONIUS_DETAIL_NOEXCEPT_HPP #ifdef _MSC_VER #define NONIUS_NOEXCEPT throw() #else #define NONIUS_NOEXCEPT noexcept #endif // _MSC_VER #include #include #include namespace nonius { struct timeout_error : virtual std::exception { public: timeout_error(int seed, int iters) { std::ostringstream ss; ss << "took too long to run; seed: " << seed << ", iters: " << iters; message = ss.str(); } char const* what() const NONIUS_NOEXCEPT override { return message.c_str(); } private: std::string message; }; } // namespace nonius #include namespace nonius { namespace detail { template TimingOf run_for_at_least(Duration how_long, int seed, Fun&& fun) { auto iters = seed; auto start = Clock::now(); while(true) { auto now = Clock::now(); if(now - start > how_long * 10) { throw timeout_error(seed, iters); } auto r = detail::measure(fun, iters); if(r.elapsed >= how_long) { return { r.elapsed, std::move(r.result), iters }; } iters *= 2; } } } // namespace detail } // namespace nonius // #included from: detail/unique_name.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Unique name generator macro #define NONIUS_DETAIL_UNIQUE_NAME_HPP #define NONIUS_DETAIL_UNIQUE_NAME_LINE_CAT(name, id) NONIUS_ ## name ## _ ## id #define NONIUS_DETAIL_UNIQUE_NAME_LINE(name, id) NONIUS_DETAIL_UNIQUE_NAME_LINE_CAT(name, id) #ifdef __COUNTER__ #define NONIUS_DETAIL_UNIQUE_NAME(name) NONIUS_DETAIL_UNIQUE_NAME_LINE(name, __COUNTER__) #else // __COUNTER__ #define NONIUS_DETAIL_UNIQUE_NAME(name) NONIUS_DETAIL_UNIQUE_NAME_LINE(name, __LINE__) #endif // __COUNTER__ #include #include #include #include #include namespace nonius { namespace detail { const auto warmup_iterations = 10000; const auto warmup_time = chrono::milliseconds(100); const auto minimum_ticks = 1000; } // namespace detail struct benchmark { benchmark(std::string name, detail::benchmark_function fun) : name(std::move(name)), fun(std::move(fun)) {} void operator()(chronometer meter) const { fun(meter); } template execution_plan> prepare(configuration cfg, environment> env) const { auto min_time = env.clock_resolution.mean * detail::minimum_ticks; auto run_time = std::min(min_time, decltype(min_time)(detail::warmup_time)); auto&& test = detail::run_for_at_least(chrono::duration_cast>(run_time), 1, [this](int k) { detail::chronometer_model model; (*this)(chronometer(model, k)); }); int new_iters = static_cast(std::ceil(min_time * test.iterations / test.elapsed)); return { new_iters, test.elapsed / test.iterations * new_iters * cfg.samples }; } template std::vector> run(configuration cfg, environment> env, execution_plan> plan) const { // warmup a bit detail::run_for_at_least(chrono::duration_cast>(detail::warmup_time), detail::warmup_iterations, detail::repeat(now{})); std::vector> times; times.reserve(cfg.samples); std::generate_n(std::back_inserter(times), cfg.samples, [this, env, plan]{ detail::chronometer_model model; (*this)(chronometer(model, plan.iterations_per_sample)); auto elapsed = model.finished - model.started; auto sample_time = elapsed - env.clock_cost.mean; if(sample_time < FloatDuration::zero()) sample_time = FloatDuration::zero(); return (sample_time / plan.iterations_per_sample); }); return times; } std::string name; detail::benchmark_function fun; }; using benchmark_registry = std::vector; inline benchmark_registry& global_benchmark_registry() { static benchmark_registry registry; return registry; } struct benchmark_registrar { template benchmark_registrar(benchmark_registry& registry, std::string name, Fun&& registrant) { registry.emplace_back(std::move(name), std::forward(registrant)); } }; } // namespace nonius #define NONIUS_BENCHMARK(name, ...) \ namespace { static ::nonius::benchmark_registrar NONIUS_DETAIL_UNIQUE_NAME(benchmark_registrar) (::nonius::global_benchmark_registry(), name, __VA_ARGS__); } // #included from: constructor.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Eric Tremblay // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Constructor and destructor helpers #define NONIUS_CONSTRUCTOR_HPP #include namespace nonius { namespace detail { template struct object_storage { typedef typename std::aligned_storage::value>::type TStorage; object_storage() : data() {} object_storage(const object_storage& other) { new(&data) T(other.stored_object()); } object_storage(object_storage&& other) { new(&data) T(std::move(other.stored_object())); } ~object_storage() { destruct_on_exit(); } template void construct(Args&&... args) { new (&data) T(std::forward(args)...); } template typename std::enable_if::type destruct() { stored_object().~T(); } private: // If this is a constructor benchmark, destruct the underlying object template void destruct_on_exit(typename std::enable_if::type* = 0) { destruct(); } // Otherwise, don't template void destruct_on_exit(typename std::enable_if::type* = 0) { } T& stored_object() { return *static_cast(static_cast(&data)); } TStorage data; }; } template using storage_for = detail::object_storage; template using destructable_object = detail::object_storage; } // #included from: go.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Runner entry point #define NONIUS_GO_HPP // #included from: reporter.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Reporter interface #define NONIUS_REPORTER_HPP // #included from: sample_analysis.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Benchmark results #define NONIUS_BENCHMARK_RESULTS_HPP // #included from: estimate.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Statistics estimates #define NONIUS_ESTIMATE_HPP namespace nonius { template struct estimate { Duration point; Duration lower_bound; Duration upper_bound; double confidence_interval; template operator estimate() const { return { point, lower_bound, upper_bound, confidence_interval }; } }; } // namespace nonius #include #include #include #include namespace nonius { template struct sample_analysis { std::vector samples; estimate mean; estimate standard_deviation; outlier_classification outliers; double outlier_variance; template operator sample_analysis() const { std::vector samples2; samples2.reserve(samples.size()); std::transform(samples.begin(), samples.end(), std::back_inserter(samples2), [](Duration d) { return Duration2(d); }); return { std::move(samples2), mean, standard_deviation, outliers, outlier_variance, }; } }; } // namespace nonius #include #include #include #include #include #include #include #include #include namespace nonius { struct bad_stream : virtual std::exception { char const* what() const NONIUS_NOEXCEPT override { return "failed to open file"; } }; struct reporter { public: virtual ~reporter() = default; void configure(configuration& cfg) { if(cfg.output_file.empty()) { os = [&]() -> std::ostream& { return std::cout; }; } else { auto ofs = std::make_shared(cfg.output_file); os = [ofs]() -> std::ostream& { return *ofs; }; } report_stream().exceptions(std::ios::failbit); if(!report_stream()) throw bad_stream(); do_configure(cfg); } void warmup_start() { do_warmup_start(); } void warmup_end(int iterations) { do_warmup_end(iterations); } void estimate_clock_resolution_start() { do_estimate_clock_resolution_start(); } void estimate_clock_resolution_complete(environment_estimate estimate) { do_estimate_clock_resolution_complete(estimate); } void estimate_clock_cost_start() { do_estimate_clock_cost_start(); } void estimate_clock_cost_complete(environment_estimate estimate) { do_estimate_clock_cost_complete(estimate); } void suite_start() { do_suite_start(); } void benchmark_start(std::string const& name) { do_benchmark_start(name); } void measurement_start(execution_plan plan) { do_measurement_start(plan); } void measurement_complete(std::vector const& samples) { do_measurement_complete(samples); } void analysis_start() { do_analysis_start(); } void analysis_complete(sample_analysis const& analysis) { do_analysis_complete(analysis); } void benchmark_failure(std::exception_ptr error) { do_benchmark_failure(error); } void benchmark_complete() { do_benchmark_complete(); } void suite_complete() { do_suite_complete(); } virtual std::string description() = 0; private: virtual void do_configure(configuration& /*cfg*/) {} virtual void do_warmup_start() {} virtual void do_warmup_end(int /*iterations*/) {} virtual void do_estimate_clock_resolution_start() {} virtual void do_estimate_clock_resolution_complete(environment_estimate /*estimate*/) {} virtual void do_estimate_clock_cost_start() {} virtual void do_estimate_clock_cost_complete(environment_estimate /*estimate*/) {} virtual void do_suite_start() {} virtual void do_benchmark_start(std::string const& /*name*/) {} virtual void do_measurement_start(execution_plan /*plan*/) {} virtual void do_measurement_complete(std::vector const& /*samples*/) {} virtual void do_analysis_start() {} // TODO make generic? virtual void do_analysis_complete(sample_analysis const& /*analysis*/) {} virtual void do_benchmark_failure(std::exception_ptr /*error*/) {} virtual void do_benchmark_complete() {} virtual void do_suite_complete() {} protected: std::ostream& progress_stream() { return std::cout; } std::ostream& error_stream() { return std::cerr; } std::ostream& report_stream() { return os(); } private: std::function os; }; using reporter_registry = std::unordered_map>; inline reporter_registry& global_reporter_registry() { static reporter_registry registry; return registry; } struct reporter_registrar { reporter_registrar(reporter_registry& registry, std::string name, reporter* registrant) { registry.emplace(std::move(name), std::unique_ptr(registrant)); } }; } // namespace nonius #define NONIUS_REPORTER(name, ...) \ namespace { static ::nonius::reporter_registrar NONIUS_DETAIL_UNIQUE_NAME(reporter_registrar) (::nonius::global_reporter_registry(), name, new __VA_ARGS__()); } \ static_assert(true, "") // #included from: reporters/standard_reporter.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Standard reporter #define NONIUS_REPORTERS_STANDARD_REPORTER_HPP // #included from: detail/pretty_print.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Pretty printing routines #define NONIUS_PRETTY_PRINT_HPP #include #include #include #include #include #include #include #include namespace nonius { namespace detail { inline double get_magnitude(fp_seconds secs) { if(secs.count() >= 2.e0) { return 1.e0; } else if(secs.count() >= 2.e-3) { return 1.e3; } else if(secs.count() >= 2.e-6) { return 1.e6; } else { return 1.e9; } } inline std::string units_for_magnitude(double magnitude) { if(magnitude <= 1.e0) return "s"; else if(magnitude <= 1.e3) return "ms"; else if(magnitude <= 1.e6) return "μs"; else return "ns"; } inline std::string pretty_duration(fp_seconds secs) { auto magnitude = get_magnitude(secs); auto units = units_for_magnitude(magnitude); #ifdef _MSC_VER if(units == "μs") units = "us"; #endif std::ostringstream ss; ss << std::setprecision(ss.precision()); ss << (secs.count() * magnitude) << ' ' << units; return ss.str(); } inline std::string percentage(double d) { std::ostringstream ss; ss << std::setprecision(3); if(d != 0 && d < 1e-5) { ss << std::fixed; ss << 0.0001 << "%"; } else { ss.unsetf(std::ios::floatfield); ss << (100. * d) << "%"; } return ss.str(); } inline std::string percentage_ratio(double part, double whole) { return percentage(part / whole); } } // namespace detail } // namespace nonius #include #include #include #include #include #include #include namespace nonius { struct standard_reporter : reporter { private: std::string description() override { return "the standard reporter"; } void do_configure(configuration& cfg) override { n_samples = cfg.samples; verbose = cfg.verbose; summary = cfg.summary; n_resamples = cfg.resamples; } void do_warmup_start() override { if(verbose) report_stream() << "warming up\n"; } void do_estimate_clock_resolution_start() override { if(verbose) report_stream() << "estimating clock resolution\n"; } void do_estimate_clock_resolution_complete(environment_estimate estimate) override { if(!summary) { if(!verbose) report_stream() << "clock resolution: "; print_environment_estimate(estimate, estimate.outliers.samples_seen + 2); } } void do_estimate_clock_cost_start() override { if(verbose) report_stream() << "estimating cost of a clock call\n"; } void do_estimate_clock_cost_complete(environment_estimate estimate) override { if(verbose) print_environment_estimate(estimate, estimate.outliers.samples_seen); } void do_benchmark_start(std::string const& name) override { report_stream() << '\n'; if(!summary) report_stream() << "benchmarking "; report_stream() << name << "\n"; current = name; } void do_measurement_start(execution_plan plan) override { report_stream() << std::setprecision(7); report_stream().unsetf(std::ios::floatfield); if(!summary) report_stream() << "collecting " << n_samples << " samples, " << plan.iterations_per_sample << " iterations each, in estimated " << detail::pretty_duration(plan.estimated_duration) << "\n"; } void do_analysis_start() override { if(verbose) report_stream() << "bootstrapping with " << n_resamples << " resamples\n"; } void do_benchmark_failure(std::exception_ptr) override { error_stream() << current << " failed to run successfully\n"; report_stream() << "benchmark aborted\n"; } void do_analysis_complete(sample_analysis const& analysis) override { print_statistic_estimate("mean", analysis.mean); print_statistic_estimate("std dev", analysis.standard_deviation); if(!summary) print_outliers(analysis.outliers); if(verbose) report_stream() << "variance introduced by outliers: " << detail::percentage(analysis.outlier_variance) << "\n"; const char* effect; if(analysis.outlier_variance < 0.01) { effect = "unaffected"; } else if(analysis.outlier_variance < 0.1) { effect = "slightly inflated"; } else if(analysis.outlier_variance < 0.5) { effect = "moderately inflated"; } else { effect = "severely inflated"; } report_stream() << "variance is " << effect << " by outliers\n"; } void print_environment_estimate(environment_estimate e, int iterations) { report_stream() << std::setprecision(7); report_stream().unsetf(std::ios::floatfield); report_stream() << "mean is " << detail::pretty_duration(e.mean) << " (" << iterations << " iterations)\n"; if(verbose) print_outliers(e.outliers); } void print_outlier_count(const char* description, int count, int total) { if(count > 0) report_stream() << " " << count << " (" << detail::percentage_ratio(count, total) << ") " << description << "\n"; } void print_outliers(outlier_classification o) { report_stream() << "found " << o.total() << " outliers among " << o.samples_seen << " samples (" << detail::percentage_ratio(o.total(), o.samples_seen) << ")\n"; if(verbose) { print_outlier_count("low severe", o.low_severe, o.samples_seen); print_outlier_count("low mild", o.low_mild, o.samples_seen); print_outlier_count("high mild", o.high_mild, o.samples_seen); print_outlier_count("high severe", o.high_severe, o.samples_seen); } } void print_statistic_estimate(const char* name, estimate estimate) { report_stream() << std::setprecision(7); report_stream().unsetf(std::ios::floatfield); report_stream() << name << ": " << detail::pretty_duration(estimate.point); if(!summary) { report_stream() << ", lb " << detail::pretty_duration(estimate.lower_bound) << ", ub " << detail::pretty_duration(estimate.upper_bound) << ", ci " << std::setprecision(3) << estimate.confidence_interval; } report_stream() << "\n"; } int n_samples = 0; int n_resamples = 0; bool verbose = false; bool summary = false; std::string current; }; NONIUS_REPORTER("", standard_reporter); NONIUS_REPORTER("standard", standard_reporter); } // namespace nonius // #included from: detail/estimate_clock.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Environment measurement #define NONIUS_DETAIL_ENVIRONMENT_HPP // #included from: detail/stats.h++ // Nonius - C++ benchmarking tool // // Written in 2014-2015 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Statistical analysis tools #define NONIUS_DETAIL_ANALYSIS_HPP #include #include #include #include #include #include #include #include #include #include #include #include #include namespace nonius { namespace detail { using sample = std::vector; template double weighted_average_quantile(int k, int q, Iterator first, Iterator last) { auto count = last - first; double idx = (count - 1) * k /static_cast(q); int j = static_cast(idx); double g = idx - j; std::nth_element(first, first+j, last); auto xj = first[j]; if(g == 0) return xj; auto xj1 = *std::min_element(first+(j+1), last); return xj + g * (xj1 - xj); } template outlier_classification classify_outliers(Iterator first, Iterator last) { std::vector copy(first, last); auto q1 = weighted_average_quantile(1, 4, copy.begin(), copy.end()); auto q3 = weighted_average_quantile(3, 4, copy.begin(), copy.end()); auto iqr = q3 - q1; auto los = q1 - (iqr * 3.); auto lom = q1 - (iqr * 1.5); auto him = q3 + (iqr * 1.5); auto his = q3 + (iqr * 3.); outlier_classification o; for(; first != last; ++first) { auto&& t = *first; if(t < los) ++o.low_severe; else if(t < lom) ++o.low_mild; else if(t > his) ++o.high_severe; else if(t > him) ++o.high_mild; ++o.samples_seen; } return o; } template double mean(Iterator first, Iterator last) { auto count = last - first; double sum = std::accumulate(first, last, 0.); return sum / count; } template double standard_deviation(Iterator first, Iterator last) { auto m = mean(first, last); double variance = std::accumulate(first, last, 0., [m](double a, double b) { double diff = b - m; return a + diff*diff; }) / (last - first); return std::sqrt(variance); } template sample resample(URng& rng, int resamples, Iterator first, Iterator last, Estimator& estimator) { auto n = last - first; std::uniform_int_distribution dist(0, n-1); sample out; out.reserve(resamples); std::generate_n(std::back_inserter(out), resamples, [n, first, &estimator, &dist, &rng] { std::vector resampled; resampled.reserve(n); std::generate_n(std::back_inserter(resampled), n, [first, &dist, &rng] { return first[dist(rng)]; }); return estimator(resampled.begin(), resampled.end()); }); std::sort(out.begin(), out.end()); return out; } template sample jackknife(Estimator&& estimator, Iterator first, Iterator last) { auto n = last - first; auto second = std::next(first); sample results; results.reserve(n); for(auto it = first; it != last; ++it) { std::iter_swap(it, first); results.push_back(estimator(second, last)); } return results; } template estimate bootstrap(double confidence_level, Iterator first, Iterator last, sample const& resample, Estimator&& estimator) { namespace bm = boost::math; auto n_samples = last - first; double point = estimator(first, last); // Degenerate case with a single sample if(n_samples == 1) return { point, point, point, confidence_level }; sample jack = jackknife(estimator, first, last); double jack_mean = mean(jack.begin(), jack.end()); double sum_squares, sum_cubes; std::tie(sum_squares, sum_cubes) = std::accumulate(jack.begin(), jack.end(), std::make_pair(0., 0.), [jack_mean](std::pair sqcb, double x) -> std::pair { auto d = jack_mean - x; auto d2 = d * d; auto d3 = d2 * d; return { sqcb.first + d2, sqcb.second + d3 }; }); double accel = sum_cubes / (6 * std::pow(sum_squares, 1.5)); int n = static_cast(resample.size()); double prob_n = std::count_if(resample.begin(), resample.end(), [point](double x) { return x < point; }) /(double) n; // degenerate case with uniform samples if(prob_n == 0) return { point, point, point, confidence_level }; double bias = bm::quantile(bm::normal{}, prob_n); double z1 = bm::quantile(bm::normal{}, (1. - confidence_level) / 2.); auto cumn = [n](double x) -> int { return std::lround(bm::cdf(bm::normal{}, x) * n); }; auto a = [bias, accel](double b) { return bias + b / (1. - accel * b); }; double b1 = bias + z1; double b2 = bias - z1; double a1 = a(b1); double a2 = a(b2); auto lo = std::max(cumn(a1), 0); auto hi = std::min(cumn(a2), n - 1); return { point, resample[lo], resample[hi], confidence_level }; } inline double outlier_variance(estimate mean, estimate stddev, int n) { double sb = stddev.point; double mn = mean.point / n; double mg_min = mn / 2.; double sg = std::min(mg_min / 4., sb / std::sqrt(n)); double sg2 = sg * sg; double sb2 = sb * sb; auto c_max = [n, mn, sb2, sg2](double x) -> double { double k = mn - x; double d = k * k; double nd = n * d; double k0 = -n * nd; double k1 = sb2 - n * sg2 + nd; double det = k1 * k1 - 4 * sg2 * k0; return (int)(-2. * k0 / (k1 + std::sqrt(det))); }; auto var_out = [n, sb2, sg2](double c) { double nc = n - c; return (nc / n) * (sb2 - nc * sg2); }; return std::min(var_out(1), var_out(std::min(c_max(0.), c_max(mg_min)))) / sb2; } struct bootstrap_analysis { estimate mean; estimate standard_deviation; double outlier_variance; }; template bootstrap_analysis analyse_samples(double confidence_level, int n_resamples, Iterator first, Iterator last) { static std::random_device entropy; auto n = static_cast(last - first); // seriously, one can't use integral types without hell in C++ auto mean = &detail::mean; auto stddev = &detail::standard_deviation; auto estimate = [=](double(*f)(Iterator, Iterator)) { auto seed = entropy(); return std::async(std::launch::async, [=]{ std::mt19937 rng(seed); auto resampled = resample(rng, n_resamples, first, last, f); return bootstrap(confidence_level, first, last, resampled, f); }); }; auto mean_future = estimate(mean); auto stddev_future = estimate(stddev); auto mean_estimate = mean_future.get(); auto stddev_estimate = stddev_future.get(); double outlier_variance = detail::outlier_variance(mean_estimate, stddev_estimate, n); return { mean_estimate, stddev_estimate, outlier_variance }; } } // namespace detail } // namespace nonius #include #include #include #include #include namespace nonius { namespace detail { template std::vector resolution(int k) { std::vector> times; times.reserve(k+1); std::generate_n(std::back_inserter(times), k+1, now{}); std::vector deltas; deltas.reserve(k); std::transform(std::next(times.begin()), times.end(), times.begin(), std::back_inserter(deltas), [](TimePoint a, TimePoint b) { return static_cast((a - b).count()); }); return deltas; } const auto warmup_seed = 10000; const auto clock_resolution_estimation_time = chrono::milliseconds(500); const auto clock_cost_estimation_time_limit = chrono::seconds(1); const auto clock_cost_estimation_tick_limit = 100000; const auto clock_cost_estimation_time = chrono::milliseconds(10); const auto clock_cost_estimation_iterations = 10000; template int warmup() { return run_for_at_least(chrono::duration_cast>(warmup_time), warmup_seed, &resolution) .iterations; } template environment_estimate> estimate_clock_resolution(int iterations) { auto r = run_for_at_least(chrono::duration_cast>(clock_resolution_estimation_time), iterations, &resolution) .result; return { FloatDuration(mean(r.begin(), r.end())), classify_outliers(r.begin(), r.end()), }; } template environment_estimate> estimate_clock_cost(FloatDuration resolution) { auto time_limit = std::min(resolution * clock_cost_estimation_tick_limit, FloatDuration(clock_cost_estimation_time_limit)); auto time_clock = [](int k) { return detail::measure([k]{ for(int i = 0; i < k; ++i) { volatile auto ignored = Clock::now(); (void)ignored; } }).elapsed; }; time_clock(1); int iters = clock_cost_estimation_iterations; auto&& r = run_for_at_least(chrono::duration_cast>(clock_cost_estimation_time), iters, time_clock); std::vector times; int nsamples = static_cast(std::ceil(time_limit / r.elapsed)); times.reserve(nsamples); std::generate_n(std::back_inserter(times), nsamples, [time_clock, &r]{ return (time_clock(r.iterations) / r.iterations).count(); }); return { FloatDuration(mean(times.begin(), times.end())), classify_outliers(times.begin(), times.end()), }; } } // namespace detail } // namespace nonius // #included from: detail/analyse.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Run and analyse one benchmark #define NONIUS_DETAIL_ANALYSE_HPP #include #include #include namespace nonius { namespace detail { template sample_analysis analyse(configuration cfg, environment, Iterator first, Iterator last) { std::vector samples; samples.reserve(last - first); std::transform(first, last, std::back_inserter(samples), [](Duration d) { return d.count(); }); auto analysis = nonius::detail::analyse_samples(cfg.confidence_interval, cfg.resamples, samples.begin(), samples.end()); auto outliers = nonius::detail::classify_outliers(samples.begin(), samples.end()); auto wrap_estimate = [](estimate e) { return estimate { Duration(e.point), Duration(e.lower_bound), Duration(e.upper_bound), e.confidence_interval, }; }; std::vector samples2; samples2.reserve(samples.size()); std::transform(samples.begin(), samples.end(), std::back_inserter(samples2), [](double d) { return Duration(d); }); return { std::move(samples2), wrap_estimate(analysis.mean), wrap_estimate(analysis.standard_deviation), outliers, analysis.outlier_variance, }; } } // namespace detail } // namespace nonius #include #include #include #include namespace nonius { namespace detail { template environment> measure_environment(reporter& rep) { rep.warmup_start(); auto iters = detail::warmup(); rep.warmup_end(iters); rep.estimate_clock_resolution_start(); auto resolution = detail::estimate_clock_resolution(iters); rep.estimate_clock_resolution_complete(resolution); rep.estimate_clock_cost_start(); auto cost = detail::estimate_clock_cost(resolution.mean); rep.estimate_clock_cost_complete(cost); return { resolution, cost }; } } // namespace detail struct benchmark_user_error : virtual std::exception { char const* what() const NONIUS_NOEXCEPT override { return "a benchmark failed to run successfully"; } }; template detail::CompleteType> user_code(reporter& rep, Fun&& fun) { try { return detail::complete_invoke(std::forward(fun)); } catch(...) { rep.benchmark_failure(std::current_exception()); throw benchmark_user_error(); } } template void go(configuration cfg, Iterator first, Iterator last, reporter& rep) { rep.configure(cfg); auto env = detail::measure_environment(rep); rep.suite_start(); for(; first != last; ++first) { try { rep.benchmark_start(first->name); auto plan = user_code(rep, [&first, &cfg, &env]{ return first->template prepare(cfg, env); }); rep.measurement_start(plan); auto samples = user_code(rep, [&first, &cfg, &env, &plan]{ return first->template run(cfg, env, plan); }); rep.measurement_complete(std::vector(samples.begin(), samples.end())); if(!cfg.no_analysis) { rep.analysis_start(); auto analysis = detail::analyse(cfg, env, samples.begin(), samples.end()); rep.analysis_complete(analysis); } rep.benchmark_complete(); } catch(benchmark_user_error const&) { continue; } } rep.suite_complete(); } struct duplicate_benchmarks : virtual std::exception { char const* what() const NONIUS_NOEXCEPT override { return "two or more benchmarks with the same name were registered"; } }; template void validate_benchmarks(Iterator first, Iterator last) { struct strings_lt_through_pointer { bool operator()(std::string* a, std::string* b) const { return *a < *b; }; }; std::set names; for(; first != last; ++first) { if(!names.insert(&first->name).second) throw duplicate_benchmarks(); } } template void go(configuration cfg, Iterator first, Iterator last, reporter&& rep) { go(cfg, first, last, rep); } struct no_such_reporter : virtual std::exception { char const* what() const NONIUS_NOEXCEPT override { return "reporter could not be found"; } }; template void go(configuration cfg, benchmark_registry& benchmarks = global_benchmark_registry(), reporter_registry& reporters = global_reporter_registry()) { auto it = reporters.find(cfg.reporter); if(it == reporters.end()) throw no_such_reporter(); validate_benchmarks(benchmarks.begin(), benchmarks.end()); go(cfg, benchmarks.begin(), benchmarks.end(), *it->second); } } // namespace nonius #ifndef NONIUS_DISABLE_EXTRA_REPORTERS #ifndef NONIUS_DISABLE_CSV_REPORTER // #included from: reporters/csv_reporter.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // CSV raw data reporter #define NONIUS_REPORTERS_CSV_REPORTER_HPP #include #include #include #include #include #include #include #include namespace nonius { struct csv_reporter : reporter { private: std::string description() override { return "outputs samples to a CSV file"; } void do_configure(configuration& cfg) override { cfg.no_analysis = true; n_samples = cfg.samples; verbose = cfg.verbose; } void do_warmup_start() override { if(verbose) progress_stream() << "warming up\n"; } void do_estimate_clock_resolution_start() override { if(verbose) progress_stream() << "estimating clock resolution\n"; } void do_estimate_clock_cost_start() override { if(verbose) progress_stream() << "estimating cost of a clock call\n"; } void do_benchmark_start(std::string const& name) override { if(verbose) progress_stream() << "\nbenchmarking " << name << "\n"; current = name; } void do_measurement_start(execution_plan plan) override { report_stream() << std::setprecision(7); report_stream().unsetf(std::ios::floatfield); if(verbose) progress_stream() << "collecting " << n_samples << " samples, " << plan.iterations_per_sample << " iterations each, in estimated " << detail::pretty_duration(plan.estimated_duration) << "\n"; } void do_measurement_complete(std::vector const& samples) override { data[current] = samples; } void do_benchmark_failure(std::exception_ptr) override { error_stream() << current << " failed to run successfully\n"; } void do_suite_complete() override { if(verbose) progress_stream() << "\ngenerating CSV report\n"; report_stream() << std::fixed; report_stream().precision(std::numeric_limits::digits10); bool first = true; for(auto&& kv : data) { if(!first) report_stream() << ","; report_stream() << "\"" << escape(kv.first) << "\""; // TODO escape first = false; } report_stream() << "\n"; for(int i = 0; i < n_samples; ++i) { first = true; for(auto&& kv : data) { if(!first) report_stream() << ","; report_stream() << kv.second[i].count(); first = false; } report_stream() << "\n"; } if(verbose) progress_stream() << "done\n"; } private: static std::string escape(std::string const& source) { auto first = source.begin(); auto last = source.end(); auto quotes = std::count(first, last, '"'); if(quotes == 0) return source; std::string escaped; escaped.reserve(source.size() + quotes); while(first != last) { auto next_quote = std::find(first, last, '"'); std::copy(first, next_quote, std::back_inserter(escaped)); first = next_quote; if(first != last) { ++first; escaped.push_back('"'); escaped.push_back('"'); } } return escaped; } int n_samples; bool verbose; std::string current; std::unordered_map> data; }; NONIUS_REPORTER("csv", csv_reporter); } // namespace nonius #endif // NONIUS_DISABLE_CSV_REPORTER #ifndef NONIUS_DISABLE_JUNIT_REPORTER // #included from: reporters/junit_reporter.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // JUnit reporter #define NONIUS_REPORTERS_JUNIT_REPORTER_HPP // #included from: detail/escape.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // General escaping routines #define NONIUS_DETAIL_ESCAPE_HPP #include #include #include #include namespace nonius { namespace detail { inline std::string escape(std::string const& source, std::unordered_map const& escapes) { std::string magic; magic.reserve(escapes.size()); std::transform(escapes.begin(), escapes.end(), std::back_inserter(magic), [](std::pair const& p) { return p.first; }); auto first = source.begin(); auto last = source.end(); auto n_magic = std::count_if(first, last, [&magic](char c) { return magic.find(c) != std::string::npos; }); std::string escaped; escaped.reserve(source.size() + n_magic*6); while(first != last) { auto next_magic = std::find_first_of(first, last, magic.begin(), magic.end()); std::copy(first, next_magic, std::back_inserter(escaped)); first = next_magic; if(first != last) { auto it = escapes.find(*first); if(it != escapes.end()) { escaped += it->second; } ++first; } } return escaped; } } // namespace detail } // namespace nonius #include #include #include #include #include #include #include #include #include namespace nonius { struct junit_reporter : reporter { private: std::string description() override { return "outputs results to a JUnit-compatible XML file"; } void do_configure(configuration& cfg) override { n_samples = cfg.samples; confidence_interval = cfg.confidence_interval; resamples = cfg.resamples; verbose = cfg.verbose; title = cfg.title; } struct result { sample_analysis analysis; std::exception_ptr failure; }; void do_warmup_start() override { if(verbose) progress_stream() << "warming up\n"; } void do_estimate_clock_resolution_start() override { if(verbose) progress_stream() << "estimating clock resolution\n"; } void do_estimate_clock_cost_start() override { if(verbose) progress_stream() << "estimating cost of a clock call\n"; } void do_benchmark_start(std::string const& name) override { if(verbose) progress_stream() << "\nbenchmarking " << name << "\n"; current = name; } void do_measurement_start(execution_plan plan) override { report_stream() << std::setprecision(7); report_stream().unsetf(std::ios::floatfield); if(verbose) progress_stream() << "collecting " << n_samples << " samples, " << plan.iterations_per_sample << " iterations each, in estimated " << detail::pretty_duration(plan.estimated_duration) << "\n"; } void do_analysis_start() override { if(verbose) report_stream() << "analysing samples\n"; } void do_analysis_complete(sample_analysis const& analysis) override { data[current] = { analysis, nullptr }; } void do_benchmark_failure(std::exception_ptr e) override { data[current] = { sample_analysis(), e }; error_stream() << current << " failed to run successfully\n"; } void do_suite_complete() override { if(verbose) progress_stream() << "\ngenerating JUnit report\n"; report_stream() << "\n"; report_stream() << " const& p) { return static_cast(p.second.failure); }); if(failures > 0) report_stream() << " errors=\"" << failures << "\""; report_stream() << ">\n"; report_stream() << " \n"; report_stream() << " \n"; report_stream() << " \n"; report_stream() << " \n"; report_stream() << " \n"; for(auto tc : data) { report_stream() << " \n"; try { std::rethrow_exception(tc.second.failure); } catch(std::exception const& e) { report_stream() << " \n"; } catch(...) { report_stream() << " \n"; } report_stream() << " \n"; } else { report_stream() << std::fixed; report_stream().precision(std::numeric_limits::digits10); report_stream() << " time=\"" << tc.second.analysis.mean.point.count() << "\" />\n"; } } report_stream() << "\n"; report_stream() << std::flush; if(verbose) progress_stream() << "done\n"; } static std::string escape(std::string const& source) { static const std::unordered_map escapes { { '\'', "'" }, { '"', """ }, { '<', "<" }, { '>', ">" }, { '&', "&" }, }; return detail::escape(source, escapes); } int n_samples; double confidence_interval; int resamples; bool verbose; std::string title; std::string current; std::unordered_map data; }; NONIUS_REPORTER("junit", junit_reporter); } // namespace nonius #endif // NONIUS_DISABLE_JUNIT_REPORTER #ifndef NONIUS_DISABLE_HTML_REPORTER // #included from: reporters/html_reporter.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // HTML single-chart reporter #define NONIUS_REPORTERS_HTML_ALL_REPORTER_HPP // #included from: detail/cpptempl.h // cpptempl // ================= // This is a template engine for C++. // // Syntax // ================= // Variables: {$variable_name} // Loops: {% for person in people %}Name: {$person.name}{% endfor %} // If: {% for person.name == "Bob" %}Full name: Robert{% endif %} // // Copyright // ================== // Copyright (c) Ryan Ginstrom // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to // deal in the Software without restriction, including without limitation the // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or // sell copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS // IN THE SOFTWARE. // // Modified by: Martinho Fernandes // // Usage // ======================= // std::string text = "{% if item %}{$item}{% endif %}\n" // "{% if thing %}{$thing}{% endif %}" ; // cpptempl::data_map data ; // data["item"] = cpptempl::make_data("aaa") ; // data["thing"] = cpptempl::make_data("bbb") ; // // std::string result = cpptempl::parse(text, data) ; // // Handy Functions // ======================== // make_data() : Feed it a string, data_map, or data_list to create a data entry. // Example: // data_map person ; // person["name"] = make_data("Bob") ; // person["occupation"] = make_data("Plumber") ; // data_map data ; // data["person"] = make_data(person) ; // std::string result = parse(templ_text, data) ; #ifndef CPPTEMPL_H #define CPPTEMPL_H #ifdef _MSC_VER #pragma warning( disable : 4996 ) // 'std::copy': Function call with parameters that may be unsafe - this call relies on the caller to check that the passed values are correct. To disable this warning, use -D_SCL_SECURE_NO_WARNINGS. See documentation on how to use Visual C++ 'Checked Iterators' #pragma warning( disable : 4512 ) // 'std::copy': Function call with parameters that may be unsafe - this call relies on the caller to check that the passed values are correct. To disable this warning, use -D_SCL_SECURE_NO_WARNINGS. See documentation on how to use Visual C++ 'Checked Iterators' #endif #include #include #include #include #include #include #include #include #include namespace cpptempl { // various typedefs // data classes class Data ; class DataValue ; class DataList ; class DataMap ; class data_ptr { public: data_ptr() {} template data_ptr(const T& data) { this->operator =(data); } data_ptr(DataValue* data); data_ptr(DataList* data); data_ptr(DataMap* data); data_ptr(const data_ptr& data) { ptr = data.ptr; } template void operator = (const T& data); void push_back(const data_ptr& data); virtual ~data_ptr() {} Data* operator ->() { return ptr.get(); } private: std::shared_ptr ptr; }; typedef std::vector data_list ; class data_map { public: data_ptr& operator [](const std::string& key); bool empty(); bool has(const std::string& key); private: std::unordered_map data; }; template<> inline void data_ptr::operator = (const data_ptr& data); template<> void data_ptr::operator = (const std::string& data); template<> void data_ptr::operator = (const std::string& data); template<> void data_ptr::operator = (const data_map& data); template void data_ptr::operator = (const T& data) { std::string data_str = boost::lexical_cast(data); this->operator =(data_str); } // token classes class Token ; typedef std::shared_ptr token_ptr ; typedef std::vector token_vector ; // Custom exception class for library errors class TemplateException : public std::exception { public: TemplateException(std::string reason) : m_reason(std::move(reason)){} char const* what() const NONIUS_NOEXCEPT override { return m_reason.c_str(); } private: std::string m_reason; }; // Data types used in templates class Data { public: virtual bool empty() = 0 ; virtual std::string getvalue(); virtual data_list& getlist(); virtual data_map& getmap() ; }; class DataValue : public Data { std::string m_value ; public: DataValue(std::string value) : m_value(value){} std::string getvalue(); bool empty(); }; class DataList : public Data { data_list m_items ; public: DataList(const data_list &items) : m_items(items){} data_list& getlist() ; bool empty(); }; class DataMap : public Data { data_map m_items ; public: DataMap(const data_map &items) : m_items(items){} data_map& getmap(); bool empty(); }; inline data_ptr::data_ptr(DataValue* data) : ptr(data) {} inline data_ptr::data_ptr(DataList* data) : ptr(data) {} inline data_ptr::data_ptr(DataMap* data) : ptr(data) {} // convenience functions for making data objects inline data_ptr make_data(std::string val) { return data_ptr(new DataValue(val)) ; } inline data_ptr make_data(data_list &val) { return data_ptr(new DataList(val)) ; } inline data_ptr make_data(data_map &val) { return data_ptr(new DataMap(val)) ; } // get a data value from a data map // e.g. foo.bar => data["foo"]["bar"] data_ptr parse_val(std::string key, data_map &data) ; typedef enum { TOKEN_TYPE_NONE, TOKEN_TYPE_TEXT, TOKEN_TYPE_VAR, TOKEN_TYPE_IF, TOKEN_TYPE_FOR, TOKEN_TYPE_ENDIF, TOKEN_TYPE_ENDFOR, } TokenType; // Template tokens // base class for all token types class Token { public: virtual TokenType gettype() = 0 ; virtual void gettext(std::ostream &stream, data_map &data) = 0 ; virtual void set_children(token_vector &children); virtual token_vector & get_children(); }; // normal text class TokenText : public Token { std::string m_text ; public: TokenText(std::string text) : m_text(text){} TokenType gettype(); void gettext(std::ostream &stream, data_map &data); }; // variable class TokenVar : public Token { std::string m_key ; public: TokenVar(std::string key) : m_key(key){} TokenType gettype(); void gettext(std::ostream &stream, data_map &data); }; // for block class TokenFor : public Token { public: std::string m_key ; std::string m_val ; token_vector m_children ; TokenFor(std::string expr); TokenType gettype(); void gettext(std::ostream &stream, data_map &data); void set_children(token_vector &children); token_vector &get_children(); }; // if block class TokenIf : public Token { public: std::string m_expr ; token_vector m_children ; TokenIf(std::string expr) : m_expr(expr){} TokenType gettype(); void gettext(std::ostream &stream, data_map &data); bool is_true(std::string expr, data_map &data); void set_children(token_vector &children); token_vector &get_children(); }; // end of block class TokenEnd : public Token // end of control block { std::string m_type ; public: TokenEnd(std::string text) : m_type(text){} TokenType gettype(); void gettext(std::ostream &stream, data_map &data); }; std::string gettext(token_ptr token, data_map &data) ; void parse_tree(token_vector &tokens, token_vector &tree, TokenType until=TOKEN_TYPE_NONE) ; token_vector & tokenize(std::string text, token_vector &tokens) ; // The big daddy. Pass in the template and data, // and get out a completed doc. void parse(std::ostream &stream, std::string templ_text, data_map &data) ; std::string parse(std::string templ_text, data_map &data); std::string parse(std::string templ_text, data_map &data); // *********** Implementation ************ ////////////////////////////////////////////////////////////////////////// // Data classes ////////////////////////////////////////////////////////////////////////// // data_map inline data_ptr& data_map::operator [](const std::string& key) { return data[key]; } inline bool data_map::empty() { return data.empty(); } inline bool data_map::has(const std::string& key) { return data.find(key) != data.end(); } // data_ptr template<> inline void data_ptr::operator = (const data_ptr& data) { ptr = data.ptr; } template<> inline void data_ptr::operator = (const std::string& data) { ptr.reset(new DataValue(data)); } template<> inline void data_ptr::operator = (const data_map& data) { ptr.reset(new DataMap(data)); } inline void data_ptr::push_back(const data_ptr& data) { if (!ptr) { ptr.reset(new DataList(data_list())); } data_list& list = ptr->getlist(); list.push_back(data); } // base data inline std::string Data::getvalue() { throw TemplateException("Data item is not a value") ; } inline data_list& Data::getlist() { throw TemplateException("Data item is not a list") ; } inline data_map& Data::getmap() { throw TemplateException("Data item is not a dictionary") ; } // data value inline std::string DataValue::getvalue() { return m_value ; } inline bool DataValue::empty() { return m_value.empty(); } // data list inline data_list& DataList::getlist() { return m_items ; } inline bool DataList::empty() { return m_items.empty(); } // data map inline data_map& DataMap:: getmap() { return m_items ; } inline bool DataMap::empty() { return m_items.empty(); } ////////////////////////////////////////////////////////////////////////// // parse_val ////////////////////////////////////////////////////////////////////////// inline data_ptr parse_val(std::string key, data_map &data) { // quoted string if (key[0] == '\"') { return make_data(boost::trim_copy_if(key, [](char c){ return c == '"'; })); } // check for dotted notation, i.e [foo.bar] size_t index = key.find(".") ; if (index == std::string::npos) { if (!data.has(key)) { return make_data("{$" + key + "}") ; } return data[key] ; } std::string sub_key = key.substr(0, index) ; if (!data.has(sub_key)) { return make_data("{$" + key + "}") ; } data_ptr item = data[sub_key] ; return parse_val(key.substr(index+1), item->getmap()) ; } ////////////////////////////////////////////////////////////////////////// // Token classes ////////////////////////////////////////////////////////////////////////// // defaults, overridden by subclasses with children inline void Token::set_children( token_vector & ) { throw TemplateException("This token type cannot have children") ; } inline token_vector & Token::get_children() { throw TemplateException("This token type cannot have children") ; } // TokenText inline TokenType TokenText::gettype() { return TOKEN_TYPE_TEXT ; } inline void TokenText::gettext( std::ostream &stream, data_map & ) { stream << m_text ; } // TokenVar inline TokenType TokenVar::gettype() { return TOKEN_TYPE_VAR ; } inline void TokenVar::gettext( std::ostream &stream, data_map &data ) { stream << parse_val(m_key, data)->getvalue() ; } // TokenFor inline TokenFor::TokenFor(std::string expr) { std::vector elements ; boost::split(elements, expr, boost::is_space()) ; if (elements.size() != 4u) { throw TemplateException("Invalid syntax in for statement") ; } m_val = elements[1] ; m_key = elements[3] ; } inline TokenType TokenFor::gettype() { return TOKEN_TYPE_FOR ; } inline void TokenFor::gettext( std::ostream &stream, data_map &data ) { data_ptr value = parse_val(m_key, data) ; data_list &items = value->getlist() ; for (size_t i = 0 ; i < items.size() ; ++i) { data_map loop ; loop["index"] = make_data(boost::lexical_cast(i+1)) ; loop["index0"] = make_data(boost::lexical_cast(i)) ; data["loop"] = make_data(loop); data[m_val] = items[i] ; for(size_t j = 0 ; j < m_children.size() ; ++j) { m_children[j]->gettext(stream, data) ; } } } inline void TokenFor::set_children( token_vector &children ) { m_children.assign(children.begin(), children.end()) ; } inline token_vector & TokenFor::get_children() { return m_children; } // TokenIf inline TokenType TokenIf::gettype() { return TOKEN_TYPE_IF ; } inline void TokenIf::gettext( std::ostream &stream, data_map &data ) { if (is_true(m_expr, data)) { for(size_t j = 0 ; j < m_children.size() ; ++j) { m_children[j]->gettext(stream, data) ; } } } inline bool TokenIf::is_true( std::string expr, data_map &data ) { std::vector elements ; boost::split(elements, expr, boost::is_space()) ; if (elements[1] == "not") { return parse_val(elements[2], data)->empty() ; } if (elements.size() == 2) { return ! parse_val(elements[1], data)->empty() ; } data_ptr lhs = parse_val(elements[1], data) ; data_ptr rhs = parse_val(elements[3], data) ; if (elements[2] == "==") { return lhs->getvalue() == rhs->getvalue() ; } return lhs->getvalue() != rhs->getvalue() ; } inline void TokenIf::set_children( token_vector &children ) { m_children.assign(children.begin(), children.end()) ; } inline token_vector & TokenIf::get_children() { return m_children; } // TokenEnd inline TokenType TokenEnd::gettype() { return m_type == "endfor" ? TOKEN_TYPE_ENDFOR : TOKEN_TYPE_ENDIF ; } inline void TokenEnd::gettext( std::ostream &, data_map &) { throw TemplateException("End-of-control statements have no associated text") ; } // gettext // generic helper for getting text from tokens. inline std::string gettext(token_ptr token, data_map &data) { std::ostringstream stream ; token->gettext(stream, data) ; return stream.str() ; } ////////////////////////////////////////////////////////////////////////// // parse_tree // recursively parses list of tokens into a tree ////////////////////////////////////////////////////////////////////////// inline void parse_tree(token_vector &tokens, token_vector &tree, TokenType until) { while(! tokens.empty()) { // 'pops' first item off list token_ptr token = tokens[0] ; tokens.erase(tokens.begin()) ; if (token->gettype() == TOKEN_TYPE_FOR) { token_vector children ; parse_tree(tokens, children, TOKEN_TYPE_ENDFOR) ; token->set_children(children) ; } else if (token->gettype() == TOKEN_TYPE_IF) { token_vector children ; parse_tree(tokens, children, TOKEN_TYPE_ENDIF) ; token->set_children(children) ; } else if (token->gettype() == until) { return ; } tree.push_back(token) ; } } ////////////////////////////////////////////////////////////////////////// // tokenize // parses a template into tokens (text, for, if, variable) ////////////////////////////////////////////////////////////////////////// inline token_vector & tokenize(std::string text, token_vector &tokens) { while(! text.empty()) { size_t pos = text.find("{") ; if (pos == std::string::npos) { if (! text.empty()) { tokens.push_back(token_ptr(new TokenText(text))) ; } return tokens ; } std::string pre_text = text.substr(0, pos) ; if (! pre_text.empty()) { tokens.push_back(token_ptr(new TokenText(pre_text))) ; } text = text.substr(pos+1) ; if (text.empty()) { tokens.push_back(token_ptr(new TokenText("{"))) ; return tokens ; } // variable if (text[0] == '$') { pos = text.find("}") ; if (pos != std::string::npos) { tokens.push_back(token_ptr (new TokenVar(text.substr(1, pos-1)))) ; text = text.substr(pos+1) ; } } // control statement else if (text[0] == '%') { pos = text.find("}") ; if (pos != std::string::npos) { std::string expression = boost::trim_copy(text.substr(1, pos-2)) ; text = text.substr(pos+1) ; if (boost::starts_with(expression, "for")) { tokens.push_back(token_ptr (new TokenFor(expression))) ; } else if (boost::starts_with(expression, "if")) { tokens.push_back(token_ptr (new TokenIf(expression))) ; } else { tokens.push_back(token_ptr (new TokenEnd(boost::trim_copy(expression)))) ; } } } else { tokens.push_back(token_ptr(new TokenText("{"))) ; } } return tokens ; } inline std::string parse(std::string templ_text, data_map &data) { std::ostringstream stream ; parse(stream, templ_text, data) ; return stream.str() ; } inline void parse(std::ostream &stream, std::string templ_text, data_map &data) { token_vector tokens ; tokenize(templ_text, tokens) ; token_vector tree ; parse_tree(tokens, tree) ; for (size_t i = 0 ; i < tree.size() ; ++i) { // Recursively calls gettext on each node in the tree. // gettext returns the appropriate text for that node. // for text, itself; // for variable, substitution; // for control statement, recursively gets kids tree[i]->gettext(stream, data) ; } } } #endif // CPPTEMPL_H #include #include #include #include #include #include #include #include #include namespace nonius { struct html_reporter : reporter { private: static std::string const& template_string() { static char const* template_parts[] = { // generated content broken into pieces because MSVC is in the 1990s. // #included from: detail/html_report_template.g.h++ "\n" "\n" " \n" " \n" " {$title} - nonius report\n" " \n" " \n" " \n" "\n" " \n" "\n" " \n" "\n" " \n" "
\n" " \n" "\n" "\n" }; static std::string const the_template = []() -> std::string { std::string s; for(auto part : template_parts) { s += part; } return s; }(); return the_template; } std::string description() override { return "outputs an HTML file with a single interactive chart of all benchmarks"; } void do_configure(configuration& cfg) override { cfg.no_analysis = true; title = cfg.title; n_samples = cfg.samples; verbose = cfg.verbose; } void do_warmup_start() override { if(verbose) progress_stream() << "warming up\n"; } void do_estimate_clock_resolution_start() override { if(verbose) progress_stream() << "estimating clock resolution\n"; } void do_estimate_clock_cost_start() override { if(verbose) progress_stream() << "estimating cost of a clock call\n"; } void do_benchmark_start(std::string const& name) override { if(verbose) progress_stream() << "\nbenchmarking " << name << "\n"; current = name; } void do_measurement_start(execution_plan plan) override { progress_stream() << std::setprecision(7); progress_stream().unsetf(std::ios::floatfield); if(verbose) progress_stream() << "collecting " << n_samples << " samples, " << plan.iterations_per_sample << " iterations each, in estimated " << detail::pretty_duration(plan.estimated_duration) << "\n"; } void do_measurement_complete(std::vector const& samples) override { data[current] = samples; } void do_benchmark_failure(std::exception_ptr) override { error_stream() << current << " failed to run successfully\n"; } void do_suite_complete() override { if(verbose) progress_stream() << "\ngenerating HTML report\n"; auto&& templ = template_string(); auto magnitude = ideal_magnitude(); cpptempl::data_map map; map["title"] = escape(title); map["units"] = detail::units_for_magnitude(magnitude); for(auto d : data) { cpptempl::data_map item; item["name"] = escape(d.first); for(auto e : d.second) { item["times"].push_back(truncate(e.count() * magnitude)); } map["benchmarks"].push_back(item); } cpptempl::parse(report_stream(), templ, map); report_stream() << std::flush; if(verbose) progress_stream() << "done\n"; } static double truncate(double x) { return int(x * 1000.) / 1000.; } double ideal_magnitude() const { std::vector mins; mins.reserve(data.size()); for(auto d : data) { mins.push_back(*std::min_element(d.second.begin(), d.second.end())); } auto min = *std::min_element(mins.begin(), mins.end()); return detail::get_magnitude(min); } static std::string escape(std::string const& source) { static const std::unordered_map escapes { { '\'', "'" }, { '"', """ }, { '<', "<" }, { '>', ">" }, { '&', "&" }, { '\\', "\\\\" }, }; return detail::escape(source, escapes); } int n_samples; bool verbose; std::string title; std::string current; std::unordered_map> data; }; NONIUS_REPORTER("html", html_reporter); } // namespace nonius #endif // NONIUS_DISABLE_HTML_REPORTER #endif // NONIUS_DISABLE_EXTRA_REPORTERS #endif // NONIUS_HPP #ifdef NONIUS_RUNNER // #included from: main.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Executable building kit #define NONIUS_MAIN_HPP // #included from: detail/argparse.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // Command-line argument parsing #define NONIUS_ARGPARSE_HPP // #included from: detail/mismatch.h++ // Nonius - C++ benchmarking tool // // Written in 2014 by Martinho Fernandes // // To the extent possible under law, the author(s) have dedicated all copyright and related // and neighboring rights to this software to the public domain worldwide. This software is // distributed without any warranty. // // You should have received a copy of the CC0 Public Domain Dedication along with this software. // If not, see // mismatch algorithm #define NONIUS_DETAIL_MISMATCH_HPP #include namespace nonius { namespace detail { template std::pair mismatch(InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, BinaryPredicate p) { while(first1 != last1 && first2 != last2 && p(*first1, *first2)) { ++first1, ++first2; } return std::make_pair(first1, first2); } } // namespace detail } // namespace nonius #include #include #include #include #include #include #include #include namespace nonius { namespace detail { struct option { bool matches_short(std::string const& s) const { return s == ("-" + short_form); } std::tuple long_separator(std::string const& s) const { auto l = "--" + long_form; auto its = detail::mismatch(s.begin(), s.end(), l.begin(), l.end(), [](char a, char b) { return a == b; }); return std::make_tuple(its.second == l.end(), its.first); } bool matches_long(std::string const& s) const { return std::get<0>(long_separator(s)); } bool matches_long(std::string const& s, std::string& argument) const { bool match; std::string::const_iterator it; std::tie(match, it) = long_separator(s); if(match && it != s.end()) { if(*it == '=') argument.assign(it+1, s.end()); else return false; } return match; } option(std::string long_form, std::string short_form, std::string description, std::string argument = std::string(), bool multiple = false) : long_form(std::move(long_form)), short_form(std::move(short_form)), description(std::move(description)), argument(std::move(argument)), multiple(multiple) {} std::string long_form; std::string short_form; std::string description; std::string argument; bool multiple; }; using option_set = std::vector