BenchmarkData.h

Go to the documentation of this file.

/*
* Copyright Onix Solutions Limited [OnixS]. All rights reserved.
*
* This software owned by Onix Solutions Limited [OnixS] and is protected by copyright law
* and international copyright treaties.
*
* Access to and use of the software is governed by the terms of the applicable OnixS Software
* Services Agreement (the Agreement) and Customer end user license agreements granting
* a non-assignable, non-transferable and non-exclusive license to use the software
* for it's own data processing purposes under the terms defined in the Agreement.
*
* Except as otherwise granted within the terms of the Agreement, copying or reproduction of any part
* of this source code or associated reference material to any other location for further reproduction
* or redistribution, and any amendments to this copyright notice, are expressly prohibited.
*
* Any reproduction or redistribution for sale or hiring of the Software not in accordance with
* the terms of the Agreement is a violation of copyright law.
*/
#pragma once
 
#include "PerformanceCounter.h"
 
#include <numeric>
#include <string>
#include <vector>
#include <iostream>
#include <iomanip>
#include <cmath>
#include <fstream>
#include <ios>
#include <cstddef>
 
#ifndef _WIN32
#include <stdlib.h>
#endif
 
struct BenchmarkData
{
    struct SendMarks
    {
        PerformanceCounter::Count sendStart;
        PerformanceCounter::Count sendFinish;
        PerformanceCounter::Count overallSendFinish; // includes the OS kernel ::send(..) time
 
        SendMarks() = default;
 
        PerformanceCounter::Span sendSpanNano() const
        {
            return PerformanceCounter::nsSpan(sendFinish, sendStart);
        }
 
        PerformanceCounter::Span overallSendSpanNano() const
        {
            return PerformanceCounter::nsSpan(overallSendFinish, sendStart);
        }
    };
 
    struct ReceiveMarks
    {
        PerformanceCounter::Count recvStart;
        PerformanceCounter::Count recvFinish;
 
        ReceiveMarks() = default;
 
        PerformanceCounter::Span recvSpanNano() const
        {
            return PerformanceCounter::nsSpan(recvFinish, recvStart);
        }
    };
 
    // Measurement overhead
    struct Overhead
    {
        PerformanceCounter::Span receive;
        PerformanceCounter::Span send;
        PerformanceCounter::Span overallSend;
        PerformanceCounter::Span oneWay;
 
        Overhead() = default;
    };
 
    static PerformanceCounter::Span oneWaySpanNano(const SendMarks & sm, const ReceiveMarks & rm)
    {
        return PerformanceCounter::nsSpan(rm.recvFinish, sm.sendStart);
    }
 
    typedef std::vector<PerformanceCounter::Span> Latencies;
 
    struct SquareVariance
    {
        explicit SquareVariance(double mean)
            : mean_(mean)
        {}
 
        double operator()(double sum, PerformanceCounter::Span value) const
        {
            return sum + std::pow(static_cast<double>(value) - mean_, 2u);
        }
 
        double mean_;
    };
 
    static double stdDev(const Latencies& data)
    {
        const std::size_t size = data.size();
        assert(size > 1);
 
        const double mean = std::accumulate(data.begin(), data.end(), static_cast<double>(0.0f)) / size;
 
        return std::sqrt(
            std::accumulate(data.begin(), data.end(), static_cast<double>(0.0f), SquareVariance(mean)) / (size - 1));
    }
 
    static void processAndReportLatencies(const std::string & name, Latencies & latencies)
    {
        sort(latencies.begin(), latencies.end());
 
        const std::size_t NameWidth = 23;
 
        std::clog << std::setw(NameWidth) << std::left << name;
 
        if(name.length() > NameWidth)
            std::clog << std::endl << std::setw(NameWidth) << ' ';
 
        std::clog
            << std::right
            << "min: " << std::setw(7) << *latencies.begin()
            << "   median: " << std::setw(7) << latencies.at(latencies.size() / 2)
            << "   99th percentile: " << std::setw(6)
            << latencies.at(static_cast<std::size_t>(std::ceil((latencies.size() * 99) / 100.0)) - 1)
            << "   stdDev: " << std::setw(10) << std::fixed << std::setprecision(2) << stdDev(latencies)
            << std::endl;
    }
 
    static void reportResults(
            const std::string & name, const SendMarks * sendMarksArray, const ReceiveMarks * receiveMarksArray,
            std::size_t count, const Overhead & overhead)
    {
        const std::string baseName = name + '_';
 
        std::ofstream receiveStream(std::string(baseName + "recv.csv").c_str(), std::ios_base::ate);
        std::ofstream sendStream(std::string(baseName + "send.csv").c_str(), std::ios_base::ate);
        std::ofstream overallSendStream(std::string(baseName + "overallsend.csv").c_str(), std::ios_base::ate);
        std::ofstream sendAndReceiveStream(std::string(baseName + "sendrecv.csv").c_str(), std::ios_base::ate);
 
#define MEASURE_NETWORK_LATENCY 1 // Includes the network-related latency and CME's or emulator processing time.
 
#if MEASURE_NETWORK_LATENCY
        std::ofstream oneWayStream(std::string(baseName + "oneway.csv").c_str(), std::ios_base::ate);
        Latencies oneWayLatencies;
#endif
 
        Latencies sendLatencies, overallSendLatencies, receiveLatencies, sendAndReceiveLatencies;
 
        for(std::size_t i = 0; i < count; ++i)
        {
            const SendMarks & sendMarks = sendMarksArray[i];
            const ReceiveMarks & receiveMarks = receiveMarksArray[i];
 
            assert(!PerformanceCounter::isUndefinedValue(receiveMarks.recvStart));
 
            receiveLatencies.push_back(receiveMarks.recvSpanNano() - overhead.receive);
            receiveStream << *receiveLatencies.rbegin() << std::endl;
 
            sendLatencies.push_back(sendMarks.sendSpanNano() - overhead.send);
            sendStream << *sendLatencies.rbegin() << std::endl;
 
            overallSendLatencies.push_back(sendMarks.overallSendSpanNano() - overhead.overallSend);
            overallSendStream << *overallSendLatencies.rbegin() << std::endl;
 
            const PerformanceCounter::Span sendAndReceiveLatency = *sendLatencies.rbegin() + *receiveLatencies.rbegin();
            sendAndReceiveLatencies.push_back(sendAndReceiveLatency);
            sendAndReceiveStream << *sendAndReceiveLatencies.rbegin() << std::endl;
 
#if MEASURE_NETWORK_LATENCY
            oneWayLatencies.push_back((oneWaySpanNano(sendMarks, receiveMarks) - overhead.oneWay) / 2);
            oneWayStream << *oneWayLatencies.rbegin() << std::endl;
#endif
        }
 
        std::clog << std::endl << name << " (nanoseconds): \n" << std::endl;
 
        processAndReportLatencies("Handler Receive", receiveLatencies);
        processAndReportLatencies("Handler Send", sendLatencies);
        processAndReportLatencies("Handler Send+Receive", sendAndReceiveLatencies);
        processAndReportLatencies(
            "OS + Handler Send (depends on the network card, kernel bypass software, OS load)",
            overallSendLatencies
        );
 
#if MEASURE_NETWORK_LATENCY
        processAndReportLatencies("One-Way (Round-Trip/2)", oneWayLatencies);
#endif
 
#define REPORT_OVERHEAD 1
 
#if REPORT_OVERHEAD
        std::clog
            << std::endl << "Measurement overhead:" << std::endl
            << "Send: " << overhead.send << std::endl
            << "Receive: " << overhead.receive << std::endl
            << "Send+Receive: " << overhead.send + overhead.receive << std::endl
            << "OS + Handler Send: " << overhead.overallSend << std::endl;
#endif
 
#if MEASURE_NETWORK_LATENCY
        std::clog << "One way:" << overhead.oneWay << std::endl;
#endif
    }
};