Benchmark Sample

This sample shows how to measure latency and use the warm-up feature.

Using OpenOnload® Network Stack

OpenOnload is part of Solarflare’s suite of network acceleration technologies.

To achieve better results, use the latency profile. For example:

./run-under-onload.sh 54 192.168.28.1 20054

TCP Loopback Acceleration

The TCP loopback acceleration is turned off by default. It is configured via the environment variables EF_TCP_CLIENT_LOOPBACK and EF_TCP_SERVER_LOOPBACK.

Activate

export EF_TCP_CLIENT_LOOPBACK=1
export EF_TCP_SERVER_LOOPBACK=1

Verify

echo $EF_TCP_CLIENT_LOOPBACK
echo $EF_TCP_SERVER_LOOPBACK

Source code

using Benchmark.Latency;
using OnixS.Cme.ILink3.Testing;
using OnixS.Cme.ILink3;
using OnixS.SimpleBinaryEncoding;
using System.Collections.Generic;
using System.Configuration;
using System.Diagnostics;
using System.IO;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using System.Threading;
using System;
using Microsoft.CodeAnalysis;
using System.Security.Cryptography;

namespace Benchmark
{
    internal static class Benchmark
    {
        static SessionTimeMarks[] marks;
        static bool active;
        static int sendCounter = 0;
        static int recvCounter = 0;

        static readonly ManualResetEventSlim ready = new(true);

        static void FillSettings(SessionSettings settings, bool useEmulator)
        {
            settings.LicenseStore = "../../../../../license|../../../../license|.";
            settings.TradingSystemVersion = "1.1.0";
            settings.TradingSystemName = "Trading System";
            settings.TradingSystemVendor = "OnixS";

            if (useEmulator)
            {
                settings.SessionId = "XXX";
                settings.FirmId = "001";
                settings.AccessKey = "dGVzdHRlc3R0ZXN0dA==";
                settings.SecretKey = "dGVzdHRlc3R0ZXN0dGVzdHRlc3R0";
                settings.KeepAliveInterval = 5000;
                settings.ReconnectAttempts = 0;
            }
            else
            {
                settings.SessionId = ConfigurationManager.AppSettings["SessionId"];
                settings.FirmId = ConfigurationManager.AppSettings["FirmId"];
                settings.AccessKey = ConfigurationManager.AppSettings["AccessKey"];
                settings.SecretKey = ConfigurationManager.AppSettings["SecretKey"];
            }
        }


        [STAThread]
        static void Main(string[] args)
        {
            Console.WriteLine("\n\tUsage: Benchmark [MarketSegmentId] [Host] [Port] (numberOfMessages) (sendPeriodUsec) (warmupPeriodUsec)\n");

            const int MainThreadAffinity = 1;
            const int ReceivingThreadAffinity = 2;
            const int SendingThreadAffinity = 3;
            const int EmulatorThreadAffinity = 4;

            bool useEmulator = false;

            int marketSegmentId = 99;
            string host = "127.0.0.1";
            int port = 49152;

#if DEBUG
            int numberOfMessages = 1000;
            Console.WriteLine("\n\n\nWARNING: Don't use the Debug build for benchmarking. The debug version can run 10-100 times slower!!!\n\n\n");
            return;
#else
            int numberOfMessages = 20000;
#endif

            var sendPeriodUsec = 100;
            var warmupPeriodUsec = 10;

            if (args.Length < 3)
            {
                useEmulator = true;
                Console.WriteLine("WARNING: Using Gateway Emulator!");
            }
            else
            {
                marketSegmentId = int.Parse(args[0]);
                host = args[1];
                port = int.Parse(args[2]);

                numberOfMessages = args.Length > 3 ? int.Parse(args[3]) : numberOfMessages;
                sendPeriodUsec = args.Length > 4 ? int.Parse(args[4]) : sendPeriodUsec;
                warmupPeriodUsec = args.Length > 5 ? int.Parse(args[5]) : warmupPeriodUsec;
            }

            Console.WriteLine($"\nSettings:\n\tnumberOfMessages={numberOfMessages}\n\tsendPeriodUsec={sendPeriodUsec}\n\twarmupPeriodUsec={warmupPeriodUsec}"
                + $"\n\tmainThreadAffinity={MainThreadAffinity}\n\treceivingThreadAffinity={ReceivingThreadAffinity}\n\tsendingThreadAffinity={SendingThreadAffinity}\n\temulatorThreadAffinity={EmulatorThreadAffinity}\n");

            try
            {
                Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.RealTime;
            }
            catch (Exception ex)
            {
                Console.WriteLine($"\nWARNING: unable to the set real-time process priority: {ex.Message}.\n To set the real-time priority run this benchmark ");
                Console.WriteLine(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "under administrator." : "with root access (e.g. using 'sudo').");
            }

            ThisThread.SetAffinity(new int[] { MainThreadAffinity });

            marks = new SessionTimeMarks[numberOfMessages];

            try
            {
                var settings = new SessionSettings();
                FillSettings(settings, useEmulator);

                Task emulatorTask = null;
                Gateway emulator = null;
                if (useEmulator)
                {
                    emulator = new Gateway(port, settings);
                    emulator.SendReceiveSpinTimeout = TimeSpan.FromMilliseconds(10);
                    emulatorTask = Task.Run(() =>
                    {
                        ThisThread.SetAffinity(new int[] { EmulatorThreadAffinity });

                        emulator.AcceptSession();

                        var report = emulator.Encoder().Wrap(TemplateId.ExecutionReportTradeOutright);

                        emulator.WaitUntilTerminate(TimeSpan.FromMinutes(3), (IMessage message) =>
                            {
                                if (message.TemplateID == TemplateId.NewOrderSingle)
                                {
                                    report.SetUnsignedLong(Tag.UUID, emulator.Uuid);
                                    report.SetUnsignedInteger(Tag.SeqNum, message.GetUnsignedInteger(Tag.SeqNum));

                                    emulator.Send(report);
                                }
                            });
                    });
                }

                using (Session session = new (settings, marketSegmentId, SessionStorageType.MemoryBasedStorage))
                {
                    session.Warning += (object sender, SessionWarningEventArgs e) => Console.WriteLine("WARNING: " + e.Description);
                    session.Error += (object sender, SessionErrorEventArgs e) => Console.WriteLine("ERROR: " + e.Description);

                    session.InboundApplicationMessage += Session_InboundApplicationMessage;
                    session.BytesReceived += Session_OnBytesReceived;
                    session.MessageSending += Session_OnSendingMessage;
                    session.ReceivingThreadAffinity = new[] { ReceivingThreadAffinity };
                    session.SendingThreadAffinity = new[] { SendingThreadAffinity };

                    session.ReceiveSpinningTimeout = 10000;
                    session.ReuseEventArguments = true;
                    session.ReuseInboundMessage = true;

                    session.Reset();
                    session.Connect(host, port);

                    var order = CreateOrder(session.CreateEncoder());

                    const int MaxNumberOfWarmupMessages = 20000;
                    int numberOfWarmupMessages = Math.Min(MaxNumberOfWarmupMessages, numberOfMessages);

                    Console.WriteLine("Warm-up to avoid JIT compilation issues and make the first calls fast..");
                    Measure(session, order, numberOfWarmupMessages, sendPeriodUsec, warmupPeriodUsec);

                    Console.WriteLine("Measure..");
                    Measure(session, order, numberOfMessages, sendPeriodUsec, warmupPeriodUsec);

                    session.Disconnect();
                }

                ReportResults("Latency", marks, numberOfMessages);

                Console.WriteLine("Done.");

                if (useEmulator && emulatorTask != null)
                {
                    if(emulatorTask.Wait(TimeSpan.FromMinutes(1)) && emulator != null)
                        emulator.Dispose();
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine("Exception: " + ex.ToString());
                Console.WriteLine("Press Enter to exit...");
                Console.ReadLine();
            }

            NLog.LogManager.Shutdown();
        }

        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
        private static void Measure(Session session, IMessage order, int numberOfMessages, int sendPeriodUsec, int warmupPeriodUsec)
        {
            ResetSessionTimeMarks();

            active = true;
            recvCounter = 0;

            const string ShortClientId = "ClientID";
            const string LongClientId = "ClientIDClientIDClientIDClientID";

            for (sendCounter = 0; sendCounter < numberOfMessages; ++sendCounter)
            {
                ready.Wait();
                ready.Reset();

                order.SetString(Tag.ClOrdID, sendCounter % 2 == 0 ? ShortClientId : LongClientId);

                marks[sendCounter].SendStart = TimestampHelper.Ticks;

                session.Send(order);

                marks[sendCounter].OverallSendFinish = TimestampHelper.Ticks;

                if (warmupPeriodUsec > 0)
                {
                    var start = TimestampHelper.Ticks;

                    do
                    {
                        SpinWait(warmupPeriodUsec);
                        session.WarmUp(order);
                    } while (TimestampHelper.ElapsedMicroseconds(start) < sendPeriodUsec);
                }
                else if (sendPeriodUsec > 0)
                {
                    SpinWait(sendPeriodUsec);
                }
            }

            ready.Wait();
            active = false;
        }

        private static void ResetSessionTimeMarks()
        {
            sendCounter = 0;
            recvCounter = 0;

            for (var i = 0; i < marks.Length; ++i)
            {
                marks[i].Reset();
            }
        }

        private static void ProcessAndReportDurations(string durationName, List<double> durations)
        {
            durations.Sort();

            Console.WriteLine(
                    "\t{0,-22} min: {1,-8:F} median: {2,-8:F} 99%: {3,-8:F}",
                    durationName,
                    durations[0],
                    durations[durations.Count / 2],
                    durations[Convert.ToInt32(Math.Ceiling(durations.Count * 99 / 100.0)) - 1]
                );
        }

        private static void ReportResults(string testName, SessionTimeMarks[] marksArray, int count)
        {
            var baseName = testName + '_';

            using StreamWriter receiveStream = new(baseName + "recv.csv", true)
                , sendStream = new(baseName + "send.csv", true)
                , overallSendStream = new(baseName + "overallsend.csv", true)
                , oneWayStream = new(baseName + "oneway.csv", true)
                , sendAndReceiveStream = new(baseName + "sendrecv.csv", true);

            var sendDurations = new List<double>();
            var overallSendDurations = new List<double>();
            var receiveDurations = new List<double>();
            var sendAndReceiveDurations = new List<double>();
            var oneWayDurations = new List<double>();

            const int MaxNumberStringLength = 26;
            var buff = new char[MaxNumberStringLength];

            ReadOnlySpan<char> Format(double value)
            {
                const string NumberFormat = "0.000";

                if (value.TryFormat(buff, out var length, NumberFormat))
                    return new ReadOnlySpan<char>(buff, 0, length);

                return value.ToString(NumberFormat);
            }

            for (var i = 0; i < count; ++i)
            {
                var m = marksArray[i];

                var receiveDuration = m.RecvSpan / TimestampHelper.TicksPerMicrosecond;
                receiveDurations.Add(receiveDuration);
                receiveStream.WriteLine(Format(receiveDuration));

                var sendDuration = m.SendSpan / TimestampHelper.TicksPerMicrosecond;
                sendDurations.Add(sendDuration);
                sendStream.WriteLine(Format(sendDuration));

                var overallSendDuration = m.OverallSendSpan / TimestampHelper.TicksPerMicrosecond;
                overallSendDurations.Add(overallSendDuration);
                overallSendStream.WriteLine(Format(overallSendDuration));

                var sendAndReceiveDuration = sendDuration + receiveDuration;
                sendAndReceiveDurations.Add(sendAndReceiveDuration);
                sendAndReceiveStream.WriteLine(Format(sendAndReceiveDuration));

                var oneWayDuration = m.OneWaySpan / TimestampHelper.TicksPerMicrosecond;
                oneWayDurations.Add(oneWayDuration);
                oneWayStream.WriteLine(Format(oneWayDuration));
            }

            Console.WriteLine("\n{0} (microseconds): ", testName);

            ProcessAndReportDurations("Receive", receiveDurations);
            ProcessAndReportDurations("Send", sendDurations);
            ProcessAndReportDurations("Send+Receive", sendAndReceiveDurations);
            ProcessAndReportDurations("Overall Send", overallSendDurations);
            ProcessAndReportDurations("One-Way (Round-Trip/2)", oneWayDurations);
        }

        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
        private static void Session_OnBytesReceived(ReadOnlySpan<byte> args)
        {
            if (!active)
                return;

            marks[recvCounter].RecvStart = TimestampHelper.Ticks;
        }

        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
        private static void Session_InboundApplicationMessage(object sender, InboundMessageEventArgs e)
        {
            if (!active)
                return;

            marks[recvCounter].RecvFinish = TimestampHelper.Ticks;

            ++recvCounter;

            ready.Set();
        }

        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
        private static void Session_OnSendingMessage(ReadOnlySpan<byte> args)
        {
            if (!active)
                return;

            marks[sendCounter].SendFinish = TimestampHelper.Ticks;
        }

        private static IMessage CreateOrder(IEncoder encoder)
        {
            const int DefaultSecurityId = 5424; // Channel 54 - "CME Equity Options"
            const decimal DefaultPrice = -46190;
            const ulong PartyDetailsListReqId = 1;

            IMessage message = encoder.Wrap(TemplateId.NewOrderSingle);

            message
                .SetUnsignedLong(Tag.PartyDetailsListReqID, PartyDetailsListReqId)
                .SetByte(Tag.Side, (byte)Side.Buy)
                .SetString(Tag.SenderID, "GFP")
                .SetString(Tag.ClOrdID, "OrderId")
                .SetUnsignedLong(Tag.OrderRequestID, 1u)
                .SetString(Tag.Location, "UK")
                .SetChar(Tag.OrdType, (char)OrderType.Limit)
                .SetByte(Tag.TimeInForce, (byte)TimeInForce.Day)
                .SetInteger(Tag.SecurityID, DefaultSecurityId)
                .SetUnsignedInteger(Tag.OrderQty, 1)
                .SetDecimal(Tag.Price, DefaultPrice)
                .SetByte(Tag.ManualOrderIndicator, (byte)ManualOrdInd.Automated)
                .SetChar(Tag.ExecutionMode, (char)ExecMode.Aggressive)
                .SetByte(Tag.ExecInst, (byte)ExecInst.AON);

            return message;
        }

        private static void SpinWait(int microseconds)
        {
            var start = TimestampHelper.Ticks;

            while (TimestampHelper.ElapsedMicroseconds(start) < microseconds)
                Thread.SpinWait(10);
        }
    }
}