From d1aff5d54b0cc106608344e22977ab9f75b62ad3 Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Tue, 20 Aug 2019 11:02:30 +0800 Subject: [PATCH 01/23] POI pybind11 --- lightmatchingengine/__init__.py | 0 setup.py | 106 +++++++- src/lightmatchingengine.cpp | 31 +++ src/lightmatchingengine.h | 246 ++++++++++++++++++ .../lightmatchingengine.pyx | 0 5 files changed, 377 insertions(+), 6 deletions(-) delete mode 100644 lightmatchingengine/__init__.py create mode 100644 src/lightmatchingengine.cpp create mode 100644 src/lightmatchingengine.h rename {lightmatchingengine => src}/lightmatchingengine.pyx (100%) diff --git a/lightmatchingengine/__init__.py b/lightmatchingengine/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/setup.py b/setup.py index 5c736cb..6a448a0 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,97 @@ from setuptools import setup, find_packages, Extension +from setuptools.command.build_ext import build_ext +import sys +import setuptools + +class get_pybind_include(object): + """Helper class to determine the pybind11 include path + The purpose of this class is to postpone importing pybind11 + until it is actually installed, so that the ``get_include()`` + method can be invoked. """ + + def __init__(self, user=False): + self.user = user + + def __str__(self): + import pybind11 + return pybind11.get_include(self.user) + + +ext_modules = [ + Extension( + 'lightmatchingengine', + ['src/lightmatchingengine.cpp'], + include_dirs=[ + # Path to pybind11 headers + get_pybind_include(), + get_pybind_include(user=True) + ], + language='c++' + ), +] + + +# As of Python 3.6, CCompiler has a `has_flag` method. +# cf http://bugs.python.org/issue26689 +def has_flag(compiler, flagname): + """Return a boolean indicating whether a flag name is supported on + the specified compiler. + """ + import tempfile + with tempfile.NamedTemporaryFile('w', suffix='.cpp') as f: + f.write('int main (int argc, char **argv) { return 0; }') + try: + compiler.compile([f.name], extra_postargs=[flagname]) + except setuptools.distutils.errors.CompileError: + return False + return True + + +def cpp_flag(compiler): + """Return the -std=c++[11/14/17] compiler flag. + The newer version is prefered over c++11 (when it is available). + """ + flags = ['-std=c++17', '-std=c++14', '-std=c++11'] + + for flag in flags: + if has_flag(compiler, flag): return flag + + raise RuntimeError('Unsupported compiler -- at least C++11 support ' + 'is needed!') + + +class BuildExt(build_ext): + """A custom build extension for adding compiler-specific options.""" + c_opts = { + 'msvc': ['/EHsc'], + 'unix': [], + } + l_opts = { + 'msvc': [], + 'unix': [], + } + + if sys.platform == 'darwin': + darwin_opts = ['-stdlib=libc++', '-mmacosx-version-min=10.7'] + c_opts['unix'] += darwin_opts + l_opts['unix'] += darwin_opts + + def build_extensions(self): + ct = self.compiler.compiler_type + opts = self.c_opts.get(ct, []) + link_opts = self.l_opts.get(ct, []) + if ct == 'unix': + opts.append('-DVERSION_INFO="%s"' % self.distribution.get_version()) + opts.append(cpp_flag(self.compiler)) + if has_flag(self.compiler, '-fvisibility=hidden'): + opts.append('-fvisibility=hidden') + elif ct == 'msvc': + opts.append('/DVERSION_INFO=\\"%s\\"' % self.distribution.get_version()) + for ext in self.extensions: + ext.extra_compile_args = opts + ext.extra_link_args = link_opts + build_ext.build_extensions(self) + setup( name="lightmatchingengine", @@ -12,12 +105,13 @@ packages=find_packages(exclude=('tests',)), - use_scm_version=True, - install_requires=[], - setup_requires=['setuptools_scm', 'cython'], - ext_modules=[Extension( - 'lightmatchingengine.lightmatchingengine', - ['lightmatchingengine/lightmatchingengine.pyx'])], + # use_scm_version=True, + version='0.1.0', + install_requires=['pybind11'], + # setup_requires=['setuptools_scm', 'cython'], + setup_requires=['setuptools_scm', 'pybind11'], + ext_modules=ext_modules, + cmdclass={'build_ext': BuildExt}, tests_require=[ 'pytest' ], diff --git a/src/lightmatchingengine.cpp b/src/lightmatchingengine.cpp new file mode 100644 index 0000000..e0700ee --- /dev/null +++ b/src/lightmatchingengine.cpp @@ -0,0 +1,31 @@ +#include +#include "lightmatchingengine.h" + +namespace py = pybind11; + +PYBIND11_MODULE(lightmatchingengine, m) { + // Enum Side + py::enum_(m, "Side") + .value("BUY", Side::BUY) + .value("SELL", Side::SELL) + .export_values(); + + // Struct Order + py::class_(m, "Order") + .def(py::init< + int, + string&, + double, + double, + Side>()); + + // Struct Trade + py::class_(m, "Trade") + .def(py::init< + int, + const string&, + double, + double, + Side, + int>()); +} diff --git a/src/lightmatchingengine.h b/src/lightmatchingengine.h new file mode 100644 index 0000000..1c0fd47 --- /dev/null +++ b/src/lightmatchingengine.h @@ -0,0 +1,246 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +#define MIN_NPRICE LLONG_MIN +#define MAX_NPRICE LLONG_MAX +#define MIN_QUANTITY 1e-9 +#define MIN_TICK_SIZE 1e-9 +#define EPILSON 5e-10 +#define NORMALIZE_PRICE( x ) static_cast( x / MIN_TICK_SIZE + EPILSON ) +#define DENORMALIZE_PRICE( x ) ( x * MIN_TICK_SIZE ) +#define nprice_t long long +#define price_t double +#define qty_t double +#define id_t long long + +enum Side { + BUY = 1, + SELL = 2 +}; + +struct Order { + id_t order_id; + const string& instmt; + price_t price; + qty_t qty; + qty_t cum_qty; + qty_t leaves_qty; + Side side; + + Order(id_t order_id, const string& instmt, price_t price, qty_t qty, Side side): + order_id(order_id), + instmt(instmt), + price(price), + qty(qty), + cum_qty(0), + leaves_qty(qty), + side(side) {} +}; + +struct Trade { + id_t order_id; + const string& instmt; + price_t trade_price; + qty_t trade_qty; + Side trade_side; + id_t trade_id; + + Trade(id_t order_id, const string& instmt, price_t trade_price, qty_t trade_qty, + Side trade_side, id_t trade_id): + order_id(order_id), + instmt(instmt), + trade_price(trade_price), + trade_qty(trade_qty), + trade_side(trade_side), + trade_id(trade_id) {} +}; + +struct OrderBook { + map> bids; + map> asks; + unordered_map order_id_map; +}; + +class LightMatchingEngine { + public: + unordered_map& order_books() { + return __order_books; + } + + int curr_order_id() { + return __curr_order_id; + } + + int curr_trade_id() { + return __curr_trade_id; + } + + tuple> add_order( + const string& instmt, price_t price, qty_t qty, Side side) + { + vector trades; + id_t order_id = (__curr_order_id += 1); + Order order = Order(order_id, instmt, price, qty, side); + nprice_t nprice = NORMALIZE_PRICE(price); + + // Find the order book + auto order_book_it = __order_books.find(instmt); + if (order_book_it == __order_books.end()) { + order_book_it = __order_books.emplace(instmt, OrderBook()).first; + } + + auto order_book = order_book_it->second; + + if (side == Side::BUY) { + nprice_t best_nprice = MAX_NPRICE; + if (order_book.asks.size() > 0) { + best_nprice = order_book.asks.begin()->first; + } + + while (nprice >= best_nprice && order.leaves_qty > MIN_QUANTITY) { + auto nbbo = order_book.asks.begin()->second; + auto original_leaves_qty = order.leaves_qty; + + // Matching the ask queue + while (nbbo.size() > 0) { + auto front_nbbo = nbbo.front(); + qty_t matching_qty = min(order.leaves_qty, nbbo[0].leaves_qty); + order.leaves_qty -= matching_qty; + front_nbbo.leaves_qty -= matching_qty; + + // Trades on the passive order + trades.push_back(Trade( + front_nbbo.order_id, + instmt, + best_nprice, + matching_qty, + front_nbbo.side, + ++__curr_trade_id)); + + // Remove the order if it is fully executed + if (front_nbbo.leaves_qty < MIN_QUANTITY) { + order_book.order_id_map.erase(front_nbbo.order_id); + nbbo.pop_front(); + } + } + + // Trades from the original order + trades.push_back(Trade( + order.order_id, + instmt, + best_nprice, + original_leaves_qty - order.leaves_qty, + order.side, + ++__curr_trade_id)); + + // Remove the ask queue if the size = 0 + if (nbbo.size() == 0) { + order_book.asks.erase(order_book.asks.begin()); + } + + // Update the ask best prices + if (order_book.asks.size() > 0) { + best_nprice = order_book.asks.begin()->first; + } else { + best_nprice = MAX_NPRICE; + } + } + + // After matching the order, place the leaving order to the end + // of the order book queue, and create the order id mapping + if (order.leaves_qty > MIN_QUANTITY) { + auto nbbo_it = order_book.bids.find(nprice); + if (nbbo_it == order_book.bids.end()){ + nbbo_it = order_book.bids.emplace(nprice, deque()).first; + } + + auto nbbo = nbbo_it->second; + nbbo.emplace_back(order); + order_book.order_id_map.emplace(order.order_id, order); + } + } else { + nprice_t best_nprice = MIN_NPRICE; + if (order_book.bids.size() > 0) { + best_nprice = order_book.bids.begin()->first; + } + + while (nprice <= best_nprice && order.leaves_qty > MIN_QUANTITY) { + auto nbbo = order_book.bids.begin()->second; + auto original_leaves_qty = order.leaves_qty; + + // Matching the ask queue + while (nbbo.size() > 0) { + auto front_nbbo = nbbo.front(); + qty_t matching_qty = min(order.leaves_qty, nbbo[0].leaves_qty); + order.leaves_qty -= matching_qty; + front_nbbo.leaves_qty -= matching_qty; + + // Trades on the passive order + trades.push_back(Trade( + front_nbbo.order_id, + instmt, + best_nprice, + matching_qty, + front_nbbo.side, + ++__curr_trade_id)); + + // Remove the order if it is fully executed + if (front_nbbo.leaves_qty < MIN_QUANTITY) { + order_book.order_id_map.erase(front_nbbo.order_id); + nbbo.pop_front(); + } + } + + // Trades from the original order + trades.push_back(Trade( + order.order_id, + instmt, + best_nprice, + original_leaves_qty - order.leaves_qty, + order.side, + ++__curr_trade_id)); + + // Remove the bid queue if the size = 0 + if (nbbo.size() == 0) { + order_book.bids.erase(order_book.bids.begin()); + } + + // Update the bid best prices + if (order_book.bids.size() > 0) { + best_nprice = order_book.bids.begin()->first; + } else { + best_nprice = MIN_NPRICE; + } + } + + // After matching the order, place the leaving order to the end + // of the order book queue, and create the order id mapping + if (order.leaves_qty > MIN_QUANTITY) { + auto nbbo_it = order_book.asks.find(nprice); + if (nbbo_it == order_book.asks.end()){ + nbbo_it = order_book.asks.emplace(nprice, deque()).first; + } + + auto nbbo = nbbo_it->second; + nbbo.emplace_back(order); + order_book.order_id_map.emplace(order.order_id, order); + } + } + + return make_tuple(order, trades); + } + + private: + unordered_map __order_books; + int __curr_order_id; + int __curr_trade_id; +}; diff --git a/lightmatchingengine/lightmatchingengine.pyx b/src/lightmatchingengine.pyx similarity index 100% rename from lightmatchingengine/lightmatchingengine.pyx rename to src/lightmatchingengine.pyx From 2c1b6e12f6592d2ecb4d4f8e4adce528af915e76 Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Wed, 21 Aug 2019 12:54:12 +0800 Subject: [PATCH 02/23] Pybind11 support backward compatibility --- src/lightmatchingengine.cpp | 5 + src/lightmatchingengine.h | 57 +++++++- src/lightmatchingengine.pyx | 257 ------------------------------------ 3 files changed, 60 insertions(+), 259 deletions(-) delete mode 100644 src/lightmatchingengine.pyx diff --git a/src/lightmatchingengine.cpp b/src/lightmatchingengine.cpp index e0700ee..e7d1c67 100644 --- a/src/lightmatchingengine.cpp +++ b/src/lightmatchingengine.cpp @@ -28,4 +28,9 @@ PYBIND11_MODULE(lightmatchingengine, m) { double, Side, int>()); + + // Class LightMatchingEngine + py::class_(m, "LightMatchingEngine") + .def(py::init()) + .def("add_order", &LightMatchingEngine::add_order); } diff --git a/src/lightmatchingengine.h b/src/lightmatchingengine.h index 1c0fd47..d34f700 100644 --- a/src/lightmatchingengine.h +++ b/src/lightmatchingengine.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include using namespace std; @@ -22,20 +24,26 @@ using namespace std; #define qty_t double #define id_t long long -enum Side { +enum class Side { BUY = 1, SELL = 2 }; struct Order { id_t order_id; - const string& instmt; + string instmt; price_t price; qty_t qty; qty_t cum_qty; qty_t leaves_qty; Side side; + /* Order() = default; */ + /* Order(const Order&) = default; */ + /* Order& operator=(Order&) = default; */ + /* Order(Order&&) = default; */ + /* Order& operator=(Order&&) = default; */ + Order(id_t order_id, const string& instmt, price_t price, qty_t qty, Side side): order_id(order_id), instmt(instmt), @@ -239,6 +247,51 @@ class LightMatchingEngine { return make_tuple(order, trades); } + Order& cancel_order(id_t order_id, const string& instmt) { + auto order_book_it = __order_books.find(instmt); + if (order_book_it == __order_books.end()) { + auto err_message = string("Order books do not have the instrument ") + instmt; + throw runtime_error(err_message); + } + + auto order_book = order_book_it->second; + auto order_it = order_book.order_id_map.find(order_id); + if (order_it == order_book.order_id_map.end()) { + ostringstream sstream; + sstream << "Cannot find order " << order_id << " from instrument " << instmt; + throw runtime_error(sstream.str()); + } + + auto order = order_it->second; + auto nprice = NORMALIZE_PRICE(order.price); + + if (order.side == Side::BUY) { + auto order_queue_it = order_book.bids.find(nprice); + assert(order_queue_it != order_book.bids.end()); + auto order_queue = order_queue_it->second; + auto found_order = find_if( + order_queue.begin(), order_queue.end(), [&order](auto o) { return o.order_id == order.order_id; }); + + // Remove the order from the matching engine + order_queue.erase(found_order); + } else { + auto order_queue_it = order_book.asks.find(nprice); + assert(order_queue_it != order_book.asks.end()); + auto order_queue = order_queue_it->second; + auto found_order = find_if( + order_queue.begin(), order_queue.end(), [&order](auto o) { return o.order_id == order.order_id; }); + + // Remove the order from the matching engine + order_queue.erase(found_order); + } + + // Finally set the leaves qty to 0 + order.leaves_qty = 0.0; + order_book.order_id_map.erase(order_it); + + return order; + } + private: unordered_map __order_books; int __curr_order_id; diff --git a/src/lightmatchingengine.pyx b/src/lightmatchingengine.pyx deleted file mode 100644 index 61307a5..0000000 --- a/src/lightmatchingengine.pyx +++ /dev/null @@ -1,257 +0,0 @@ -#!/usr/bin/python3 -cpdef enum Side: - BUY = 1 - SELL = 2 - - -cdef class Order: - cdef public int order_id - cdef public str instmt - cdef public double price - cdef public double qty - cdef public double cum_qty - cdef public double leaves_qty - cdef public Side side - - def __init__(self, order_id, instmt, price, qty, side): - """ - Constructor - """ - self.order_id = order_id - self.instmt = instmt - self.price = price - self.qty = qty - self.cum_qty = 0 - self.leaves_qty = qty - self.side = side - - -cdef class OrderBook: - cdef public dict bids - cdef public dict asks - cdef public dict order_id_map - - def __init__(self): - """ - Constructor - """ - self.bids = {} - self.asks = {} - self.order_id_map = {} - - -cdef class Trade: - cdef public int order_id - cdef public str instmt - cdef public double trade_price - cdef public double trade_qty - cdef public Side trade_side - cdef public int trade_id - - def __init__(self, order_id, instmt, trade_price, trade_qty, trade_side, trade_id): - """ - Constructor - """ - self.order_id = order_id - self.instmt = instmt - self.trade_price = trade_price - self.trade_qty = trade_qty - self.trade_side = trade_side - self.trade_id = trade_id - - -cdef class LightMatchingEngine: - cdef public dict order_books - cdef public int curr_order_id - cdef public int curr_trade_id - - def __init__(self): - """ - Constructor - """ - self.order_books = {} - self.curr_order_id = 0 - self.curr_trade_id = 0 - - cpdef add_order(self, str instmt, double price, double qty, Side side): - """ - Add an order - :param instmt Instrument name - :param price Price, defined as zero if market order - :param qty Order quantity - :param side 1 for BUY, 2 for SELL. Defaulted as BUY. - :return The order and the list of trades. - Empty list if there is no matching. - """ - cdef list trades = [] - cdef int order_id - cdef Order order - - assert side == Side.BUY or side == Side.SELL, \ - "Invalid side %s" % side - - # Locate the order book - order_book = self.order_books.setdefault(instmt, OrderBook()) - - # Initialization - self.curr_order_id += 1 - order_id = self.curr_order_id - order = Order(order_id, instmt, price, qty, side) - - if side == Side.BUY: - # Buy - best_price = min(order_book.asks.keys()) if len(order_book.asks) > 0 \ - else None - while best_price is not None and \ - (price == 0.0 or price >= best_price ) and \ - order.leaves_qty >= 1e-9: - best_price_qty = sum([ask.leaves_qty for ask in order_book.asks[best_price]]) - match_qty = min(best_price_qty, order.leaves_qty) - assert match_qty > 0, "Match quantity must be larger than zero" - - # Generate aggressive order trade first - self.curr_trade_id += 1 - order.cum_qty += match_qty - order.leaves_qty -= match_qty - trades.append(Trade(order_id, instmt, best_price, match_qty, \ - Side.BUY, self.curr_trade_id)) - - # Generate the passive executions - while match_qty >= 1e-9: - # The order hit - hit_order = order_book.asks[best_price][0] - # The order quantity hit - order_match_qty = min(match_qty, hit_order.leaves_qty) - self.curr_trade_id += 1 - trades.append(Trade(hit_order.order_id, instmt, best_price, \ - order_match_qty, \ - Side.SELL, self.curr_trade_id)) - hit_order.cum_qty += order_match_qty - hit_order.leaves_qty -= order_match_qty - match_qty -= order_match_qty - if hit_order.leaves_qty < 1e-9: - del order_book.asks[best_price][0] - - # If the price does not have orders, delete the particular price depth - if len(order_book.asks[best_price]) == 0: - del order_book.asks[best_price] - - # Update the best price - best_price = min(order_book.asks.keys()) if len(order_book.asks) > 0 \ - else None - - # Add the remaining order into the depth - if order.leaves_qty > 0.0: - depth = order_book.bids.setdefault(price, []) - depth.append(order) - order_book.order_id_map[order_id] = order - else: - #Sell - best_price = max(order_book.bids.keys()) if len(order_book.bids) > 0 \ - else None - while best_price is not None and \ - (price == 0.0 or price <= best_price) and \ - order.leaves_qty >= 1e-9: - best_price_qty = sum([bid.leaves_qty for bid in order_book.bids[best_price]]) - match_qty = min(best_price_qty, order.leaves_qty) - assert match_qty >= 1e-9, "Match quantity must be larger than zero" - - # Generate aggressive order trade first - self.curr_trade_id += 1 - order.cum_qty += match_qty - order.leaves_qty -= match_qty - trades.append(Trade(order_id, instmt, best_price, match_qty, \ - Side.SELL, self.curr_trade_id)) - - # Generate the passive executions - while match_qty >= 1e-9: - # The order hit - hit_order = order_book.bids[best_price][0] - # The order quantity hit - order_match_qty = min(match_qty, hit_order.leaves_qty) - self.curr_trade_id += 1 - trades.append(Trade(hit_order.order_id, instmt, best_price, \ - order_match_qty, \ - Side.BUY, self.curr_trade_id)) - hit_order.cum_qty += order_match_qty - hit_order.leaves_qty -= order_match_qty - match_qty -= order_match_qty - if hit_order.leaves_qty < 1e-9: - del order_book.bids[best_price][0] - - # If the price does not have orders, delete the particular price depth - if len(order_book.bids[best_price]) == 0: - del order_book.bids[best_price] - - # Update the best price - best_price = max(order_book.bids.keys()) if len(order_book.bids) > 0 \ - else None - - # Add the remaining order into the depth - if order.leaves_qty >= 1e-9: - depth = order_book.asks.setdefault(price, []) - depth.append(order) - order_book.order_id_map[order_id] = order - - return order, trades - - cpdef cancel_order(self, int order_id, str instmt): - """ - Cancel order - :param order_id Order ID - :param instmt Instrument - :return The order if the cancellation is successful - """ - cdef Order order - cdef double order_price - cdef Side side - cdef int index - - assert instmt in self.order_books.keys(), \ - "Instrument %s is not valid in the order book" % instmt - order_book = self.order_books[instmt] - - if order_id not in order_book.order_id_map.keys(): - # Invalid order id - return None - - order = order_book.order_id_map[order_id] - order_price = order.price - order_id = order.order_id - side = order.side - - if side == Side.BUY: - assert order_price in order_book.bids.keys(), \ - "Order price %.6f is not in the bid price depth" % order_price - price_level = order_book.bids[order_price] - else: - assert order_price in order_book.asks.keys(), \ - "Order price %.6f is not in the ask price depth" % order_price - price_level = order_book.asks[order_price] - - index = 0 - price_level_len = len(price_level) - while index < price_level_len: - if price_level[index].order_id == order_id: - del price_level[index] - break - index += 1 - - if index == price_level_len: - # Cannot find the order ID. Incorrect side - return None - - if side == Side.BUY and len(order_book.bids[order_price]) == 0: - # Delete empty particular price level - del order_book.bids[order_price] - elif side == Side.SELL and len(order_book.asks[order_price]) == 0: - # Delete empty particular price level - del order_book.asks[order_price] - - # Delete the order id from the map - del order_book.order_id_map[order_id] - - # Zero out leaves qty - order.leaves_qty = 0 - - return order From 27e5dd2899bfefc066a8b3df032cfe761f795025 Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Wed, 21 Aug 2019 13:00:07 +0800 Subject: [PATCH 03/23] Support travis check for pybind11 --- .travis.yml | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 52515c2..6ffeb38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,24 @@ language: python -python: - - 2.7 - - 3.4 - - 3.5 - - 3.6 - - 3.7 - - pypy +# python: +# - 2.7 +# - 3.4 +# - 3.5 +# - 3.6 +# - 3.7 +# - pypy +matrix: + include: + - os: linux + compiler: gcc + addons: &gcc49 + apt: + sources: ['ubuntu-toolchain-r-test'] + packages: ['g++-4.9', 'gcc-4.9'] + env: + - CXX='g++-4.9' + - CC='gcc-4.9' + python: 3.4 install: - pip install .[performance] From 1a5b6eb1b43e38905d0b7681c24955e4061ba0da Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Thu, 22 Aug 2019 13:24:38 +0800 Subject: [PATCH 04/23] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6ffeb38..d64e79f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ matrix: env: - CXX='g++-4.9' - CC='gcc-4.9' - python: 3.4 + python: 3.4 install: - pip install .[performance] From 53ffca03147ca1bbfa1447e63def1fd66bfe97a7 Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 07:56:54 +0800 Subject: [PATCH 05/23] Update .travis.yml --- .travis.yml | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index d64e79f..99bce7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,16 +9,26 @@ language: python # - pypy matrix: include: - - os: linux - compiler: gcc - addons: &gcc49 - apt: - sources: ['ubuntu-toolchain-r-test'] - packages: ['g++-4.9', 'gcc-4.9'] - env: - - CXX='g++-4.9' - - CC='gcc-4.9' - python: 3.4 + - os: linux + dist: trusty + name: Python 3.5, c++14, gcc 6, Debug build + # N.B. `ensurepip` could be installed transitively by `python3.5-venv`, but + # seems to have apt conflicts (at least for Trusty). Use Docker instead. + services: docker + env: DOCKER=debian:stretch PYTHON=3.5 CPP=14 GCC=6 DEBUG=1 + - os: linux + dist: xenial + env: PYTHON=3.6 CPP=17 GCC=7 + name: Python 3.6, c++17, gcc 7 + addons: + apt: + sources: + - deadsnakes + - ubuntu-toolchain-r-test + packages: + - g++-7 + - python3.6-dev + - python3.6-venv install: - pip install .[performance] From ee610331d32b4349c3f29bd3c59bc941a569c23b Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 08:13:36 +0800 Subject: [PATCH 06/23] Update .travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 99bce7d..9921106 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,7 @@ matrix: - python3.6-venv install: + - pip install 'pybind11>=2.3' - pip install .[performance] script: From a8814879efe759ff32b5f70565621585af1c3593 Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 08:52:12 +0800 Subject: [PATCH 07/23] Update .travis.yml --- .travis.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9921106..c51aea4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,10 +30,17 @@ matrix: - python3.6-dev - python3.6-venv -install: - - pip install 'pybind11>=2.3' - - pip install .[performance] +before_install: + - | + virtualenv env + source env/bin/active +install: + - | + python setup.py sdist + python -m pip install 'pybind11>=2.3' + python -m pip install --verbose dist/*.tar.gz + script: - make test - python tests/performance/performance_test.py --freq 20 --num-orders 500 From 33b196d2b7510c781bc49f257312548b2bc1edfd Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 08:54:03 +0800 Subject: [PATCH 08/23] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c51aea4..711b905 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ matrix: before_install: - | virtualenv env - source env/bin/active + source env/bin/activate install: - | From f8bcefb93e436cab16323ff25614d82e04bd63af Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 08:57:13 +0800 Subject: [PATCH 09/23] Update .travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 711b905..5d628e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,7 @@ install: python setup.py sdist python -m pip install 'pybind11>=2.3' python -m pip install --verbose dist/*.tar.gz + python -m pip install pytest script: - make test From 915df0ad9b7d2f2e02caaa547ec8024cb31b4083 Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 09:00:25 +0800 Subject: [PATCH 10/23] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6a448a0..94a9f6f 100644 --- a/setup.py +++ b/setup.py @@ -106,7 +106,7 @@ def build_extensions(self): packages=find_packages(exclude=('tests',)), # use_scm_version=True, - version='0.1.0', + version='2019.2', install_requires=['pybind11'], # setup_requires=['setuptools_scm', 'cython'], setup_requires=['setuptools_scm', 'pybind11'], From 4db87c3ab6349249e89ee483c62176a6db423a25 Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 09:14:59 +0800 Subject: [PATCH 11/23] Update .travis.yml --- .travis.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5d628e4..42bd715 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,13 +9,6 @@ language: python # - pypy matrix: include: - - os: linux - dist: trusty - name: Python 3.5, c++14, gcc 6, Debug build - # N.B. `ensurepip` could be installed transitively by `python3.5-venv`, but - # seems to have apt conflicts (at least for Trusty). Use Docker instead. - services: docker - env: DOCKER=debian:stretch PYTHON=3.5 CPP=14 GCC=6 DEBUG=1 - os: linux dist: xenial env: PYTHON=3.6 CPP=17 GCC=7 @@ -37,11 +30,9 @@ before_install: install: - | - python setup.py sdist python -m pip install 'pybind11>=2.3' - python -m pip install --verbose dist/*.tar.gz - python -m pip install pytest + python -m pip install .[performance] script: - - make test + - python -m pytest tests/unit - python tests/performance/performance_test.py --freq 20 --num-orders 500 From 05f4de255844cef4cfb8e4af5bb073a29d729eee Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 10:59:17 +0800 Subject: [PATCH 12/23] Update .travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 42bd715..b2b2d04 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ matrix: before_install: - | - virtualenv env + virtualenv -p python3.6 env source env/bin/activate install: @@ -34,5 +34,5 @@ install: python -m pip install .[performance] script: - - python -m pytest tests/unit + - pytest tests/unit - python tests/performance/performance_test.py --freq 20 --num-orders 500 From 174cbe82b2c8c065361b09d7375517ef4a22d93b Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 11:04:14 +0800 Subject: [PATCH 13/23] Update .travis.yml --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index b2b2d04..b25ba11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,10 @@ install: - | python -m pip install 'pybind11>=2.3' python -m pip install .[performance] + python -m pip install pytest script: + - pip freeze + - which python - pytest tests/unit - python tests/performance/performance_test.py --freq 20 --num-orders 500 From f13d689816ee670a9d836c1efd480cf50efd624d Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 11:11:45 +0800 Subject: [PATCH 14/23] Update .travis.yml --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index b25ba11..502f4b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,5 +37,7 @@ install: script: - pip freeze - which python + - python -c "import lightmatchingengine" + - python -c "import lightmatchingengine.lightmatchingengine" - pytest tests/unit - python tests/performance/performance_test.py --freq 20 --num-orders 500 From 56448b46c1b1fa05463346317236a04afc3dce36 Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 11:16:30 +0800 Subject: [PATCH 15/23] Update .travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 502f4b9..3310090 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,7 @@ install: script: - pip freeze - which python + - ls -lah /home/travis/build/gavincyi/LightMatchingEngine/env/lib/python3.6/site-packages/lightmatchingengine - python -c "import lightmatchingengine" - python -c "import lightmatchingengine.lightmatchingengine" - pytest tests/unit From 92f9b5aeabe4a518217f16a38a947df09ed3d5db Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 11:19:17 +0800 Subject: [PATCH 16/23] Update .travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3310090..92ffe8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,7 @@ install: script: - pip freeze - which python + - ls -lah /home/travis/build/gavincyi/LightMatchingEngine/env/lib/ - ls -lah /home/travis/build/gavincyi/LightMatchingEngine/env/lib/python3.6/site-packages/lightmatchingengine - python -c "import lightmatchingengine" - python -c "import lightmatchingengine.lightmatchingengine" From dfd1264765a73f3fd75f4030be051083c6a629c7 Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 11:22:06 +0800 Subject: [PATCH 17/23] Update .travis.yml --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 92ffe8f..16a89d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,8 @@ install: script: - pip freeze - which python - - ls -lah /home/travis/build/gavincyi/LightMatchingEngine/env/lib/ + - ls -lah /home/travis/build/gavincyi/LightMatchingEngine/env/lib/python3.6 + - ls -lah /home/travis/build/gavincyi/LightMatchingEngine/env/lib/python3.6/site-packages - ls -lah /home/travis/build/gavincyi/LightMatchingEngine/env/lib/python3.6/site-packages/lightmatchingengine - python -c "import lightmatchingengine" - python -c "import lightmatchingengine.lightmatchingengine" From e166e1a2076a16851f83fad023b6988871c59723 Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 11:28:23 +0800 Subject: [PATCH 18/23] Update .travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 16a89d3..b72ce43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ before_install: - | virtualenv -p python3.6 env source env/bin/activate + python -m pip install -U pip setuptools wheel install: - | From bfa8a9de36bfea6f2b852748696a4107e0e2a44e Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 11:40:20 +0800 Subject: [PATCH 19/23] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 94a9f6f..065751c 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def __str__(self): ext_modules = [ Extension( - 'lightmatchingengine', + 'lightmatchingengine.lightmatchingengine', ['src/lightmatchingengine.cpp'], include_dirs=[ # Path to pybind11 headers From 1b6f9568660e1acd4e3ab6548a05adb4d6971a6f Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 11:47:56 +0800 Subject: [PATCH 20/23] Include pybind stl --- src/lightmatchingengine.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lightmatchingengine.h b/src/lightmatchingengine.h index d34f700..9cb04f0 100644 --- a/src/lightmatchingengine.h +++ b/src/lightmatchingengine.h @@ -1,3 +1,5 @@ +#include +#include #include #include #include From 3cacc0f86e51f4d538114eb15280d939a5f079ed Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 12:00:55 +0800 Subject: [PATCH 21/23] Binding cancel order and the properties --- src/lightmatchingengine.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lightmatchingengine.cpp b/src/lightmatchingengine.cpp index e7d1c67..6aa8dd4 100644 --- a/src/lightmatchingengine.cpp +++ b/src/lightmatchingengine.cpp @@ -32,5 +32,9 @@ PYBIND11_MODULE(lightmatchingengine, m) { // Class LightMatchingEngine py::class_(m, "LightMatchingEngine") .def(py::init()) - .def("add_order", &LightMatchingEngine::add_order); + .def("add_order", &LightMatchingEngine::add_order) + .def("cancel_order", &LightMatchingEngine::cancel_order) + .def_property_readonly("order_books", &LightMatchingEngine::order_books) + .def_property_readonly("curr_order_id", &LightMatchingEngine::curr_order_id) + .def_property_readonly("curr_trade_id", &LightMatchingEngine::curr_trade_id); } From c5f8ec0a98241a10e6fe2d45e40c967dcee56578 Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 23 Aug 2019 13:50:11 +0800 Subject: [PATCH 22/23] Fixed some issues --- src/lightmatchingengine.cpp | 30 ++++++- src/lightmatchingengine.h | 42 ++++++--- tests/unit/test_basic_orders.py | 152 ++++++++++++++++---------------- 3 files changed, 134 insertions(+), 90 deletions(-) diff --git a/src/lightmatchingengine.cpp b/src/lightmatchingengine.cpp index 6aa8dd4..0e0feed 100644 --- a/src/lightmatchingengine.cpp +++ b/src/lightmatchingengine.cpp @@ -1,4 +1,5 @@ #include +#include #include "lightmatchingengine.h" namespace py = pybind11; @@ -12,6 +13,13 @@ PYBIND11_MODULE(lightmatchingengine, m) { // Struct Order py::class_(m, "Order") + .def_readwrite("order_id", &Order::order_id) + .def_readwrite("instmt", &Order::instmt) + .def_readwrite("price", &Order::price) + .def_readwrite("qty", &Order::qty) + .def_readwrite("cum_qty", &Order::cum_qty) + .def_readwrite("leaves_qty", &Order::leaves_qty) + .def_readwrite("side", &Order::side) .def(py::init< int, string&, @@ -21,6 +29,12 @@ PYBIND11_MODULE(lightmatchingengine, m) { // Struct Trade py::class_(m, "Trade") + .def_readwrite("order_id", &Trade::order_id) + .def_readwrite("instmt", &Trade::instmt) + .def_readwrite("trade_price", &Trade::trade_price) + .def_readwrite("trade_qty", &Trade::trade_qty) + .def_readwrite("trade_side", &Trade::trade_side) + .def_readwrite("trade_id", &Trade::trade_id) .def(py::init< int, const string&, @@ -29,12 +43,26 @@ PYBIND11_MODULE(lightmatchingengine, m) { Side, int>()); + // Late binding + // py::bind_vector>(m, "DequeOrder"); + py::bind_map>>(m, "MapDoubleVectorOrder"); + py::bind_map>(m, "UnorderedMapIntOrder"); + + // Class OrderBook + py::class_(m, "OrderBook") + .def(py::init()) + .def_readwrite("bids", &OrderBook::bids) + .def_readwrite("asks", &OrderBook::asks) + .def_readwrite("order_id_map", &OrderBook::order_id_map); + + // Class LightMatchingEngine + py::bind_map>(m, "UnorderedMapStringOrderBook"); py::class_(m, "LightMatchingEngine") .def(py::init()) .def("add_order", &LightMatchingEngine::add_order) .def("cancel_order", &LightMatchingEngine::cancel_order) - .def_property_readonly("order_books", &LightMatchingEngine::order_books) + .def_property_readonly("order_books", &LightMatchingEngine::order_books, py::return_value_policy::reference) .def_property_readonly("curr_order_id", &LightMatchingEngine::curr_order_id) .def_property_readonly("curr_trade_id", &LightMatchingEngine::curr_trade_id); } diff --git a/src/lightmatchingengine.h b/src/lightmatchingengine.h index 9cb04f0..0d159ec 100644 --- a/src/lightmatchingengine.h +++ b/src/lightmatchingengine.h @@ -4,15 +4,12 @@ #include #include #include -#include -#include -#include -#include #include #include #include using namespace std; +namespace py = pybind11; #define MIN_NPRICE LLONG_MIN #define MAX_NPRICE LLONG_MAX @@ -58,7 +55,7 @@ struct Order { struct Trade { id_t order_id; - const string& instmt; + string instmt; price_t trade_price; qty_t trade_qty; Side trade_side; @@ -74,12 +71,23 @@ struct Trade { trade_id(trade_id) {} }; +PYBIND11_MAKE_OPAQUE(map>); +PYBIND11_MAKE_OPAQUE(unordered_map); + struct OrderBook { map> bids; map> asks; unordered_map order_id_map; + + OrderBook() = default; + OrderBook(const OrderBook&) = delete; + OrderBook(OrderBook&&) = default; + OrderBook& operator=(OrderBook&&) = default; + OrderBook& operator=(const OrderBook&) = delete; }; +PYBIND11_MAKE_OPAQUE(unordered_map); + class LightMatchingEngine { public: unordered_map& order_books() { @@ -108,7 +116,7 @@ class LightMatchingEngine { order_book_it = __order_books.emplace(instmt, OrderBook()).first; } - auto order_book = order_book_it->second; + auto& order_book = order_book_it->second; if (side == Side::BUY) { nprice_t best_nprice = MAX_NPRICE; @@ -117,14 +125,16 @@ class LightMatchingEngine { } while (nprice >= best_nprice && order.leaves_qty > MIN_QUANTITY) { - auto nbbo = order_book.asks.begin()->second; + auto& nbbo = order_book.asks.begin()->second; auto original_leaves_qty = order.leaves_qty; // Matching the ask queue while (nbbo.size() > 0) { auto front_nbbo = nbbo.front(); qty_t matching_qty = min(order.leaves_qty, nbbo[0].leaves_qty); + order.cum_qty += matching_qty; order.leaves_qty -= matching_qty; + front_nbbo.cum_qty += matching_qty; front_nbbo.leaves_qty -= matching_qty; // Trades on the passive order @@ -173,7 +183,7 @@ class LightMatchingEngine { nbbo_it = order_book.bids.emplace(nprice, deque()).first; } - auto nbbo = nbbo_it->second; + auto& nbbo = nbbo_it->second; nbbo.emplace_back(order); order_book.order_id_map.emplace(order.order_id, order); } @@ -184,14 +194,16 @@ class LightMatchingEngine { } while (nprice <= best_nprice && order.leaves_qty > MIN_QUANTITY) { - auto nbbo = order_book.bids.begin()->second; + auto& nbbo = order_book.bids.begin()->second; auto original_leaves_qty = order.leaves_qty; // Matching the ask queue while (nbbo.size() > 0) { auto front_nbbo = nbbo.front(); qty_t matching_qty = min(order.leaves_qty, nbbo[0].leaves_qty); + order.cum_qty += matching_qty; order.leaves_qty -= matching_qty; + front_nbbo.cum_qty += matching_qty; front_nbbo.leaves_qty -= matching_qty; // Trades on the passive order @@ -237,10 +249,14 @@ class LightMatchingEngine { if (order.leaves_qty > MIN_QUANTITY) { auto nbbo_it = order_book.asks.find(nprice); if (nbbo_it == order_book.asks.end()){ + order.cum_qty += matching_qty; + order.leaves_qty -= matching_qty; + front_nbbo.cum_qty += matching_qty; + front_nbbo.leaves_qty -= matching_qty; nbbo_it = order_book.asks.emplace(nprice, deque()).first; } - auto nbbo = nbbo_it->second; + auto& nbbo = nbbo_it->second; nbbo.emplace_back(order); order_book.order_id_map.emplace(order.order_id, order); } @@ -256,7 +272,7 @@ class LightMatchingEngine { throw runtime_error(err_message); } - auto order_book = order_book_it->second; + auto& order_book = order_book_it->second; auto order_it = order_book.order_id_map.find(order_id); if (order_it == order_book.order_id_map.end()) { ostringstream sstream; @@ -264,13 +280,13 @@ class LightMatchingEngine { throw runtime_error(sstream.str()); } - auto order = order_it->second; + auto& order = order_it->second; auto nprice = NORMALIZE_PRICE(order.price); if (order.side == Side::BUY) { auto order_queue_it = order_book.bids.find(nprice); assert(order_queue_it != order_book.bids.end()); - auto order_queue = order_queue_it->second; + auto& order_queue = order_queue_it->second; auto found_order = find_if( order_queue.begin(), order_queue.end(), [&order](auto o) { return o.order_id == order.order_id; }); diff --git a/tests/unit/test_basic_orders.py b/tests/unit/test_basic_orders.py index 6a4e831..82c3573 100644 --- a/tests/unit/test_basic_orders.py +++ b/tests/unit/test_basic_orders.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python3 import lightmatchingengine.lightmatchingengine as lme import unittest @@ -6,7 +6,7 @@ class TestBasicOrders(unittest.TestCase): instmt = "TestingInstrument" price = 100.0 lot_size = 1.0 - + def check_order(self, order, order_id, instmt, price, qty, side, cum_qty, leaves_qty): """ Check the order information @@ -19,11 +19,11 @@ def check_order(self, order, order_id, instmt, price, qty, side, cum_qty, leaves self.assertEqual(side, order.side) self.assertEqual(cum_qty, order.cum_qty) self.assertEqual(leaves_qty, order.leaves_qty) - + def check_trade(self, trade, order_id, instmt, trade_price, trade_qty, trade_side, trade_id): """ Check the trade information - """ + """ self.assertTrue(trade is not None) self.assertEqual(order_id, trade.order_id) self.assertEqual(instmt, trade.instmt) @@ -31,15 +31,15 @@ def check_trade(self, trade, order_id, instmt, trade_price, trade_qty, trade_sid self.assertEqual(trade_qty, trade.trade_qty) self.assertEqual(trade_side, trade.trade_side) self.assertEqual(trade_id, trade.trade_id) - + def check_order_book(self, me, instmt, num_bids_level, num_asks_level): """ Check the order book depth """ - self.assertTrue(instmt in me.order_books.keys()) + self.assertTrue(instmt in me.order_books) self.assertEqual(num_bids_level, len(me.order_books[instmt].bids)) - self.assertEqual(num_asks_level, len(me.order_books[instmt].asks)) - + self.assertEqual(num_asks_level, len(me.order_books[instmt].asks)) + def check_deleted_order(self, order, del_order): """ Check if the deleted order is same as the original order @@ -48,10 +48,10 @@ def check_deleted_order(self, order, del_order): self.assertTrue(del_order is not None) self.assertEqual(order, del_order) self.assertEqual(0, del_order.leaves_qty) - + def test_cancel_order(self): me = lme.LightMatchingEngine() - + # Place a buy order order, trades = me.add_order(TestBasicOrders.instmt, \ TestBasicOrders.price, \ @@ -61,13 +61,13 @@ def test_cancel_order(self): self.check_order_book(me, TestBasicOrders.instmt, 1, 0) self.check_order(order, 1, TestBasicOrders.instmt, TestBasicOrders.price, \ TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) - + # Cancel a buy order del_order = me.cancel_order(order.order_id, TestBasicOrders.instmt) self.assertEqual(0, len(trades)) self.check_order_book(me, TestBasicOrders.instmt, 0, 0) self.check_deleted_order(order, del_order) - + # Place a sell order order, trades = me.add_order(TestBasicOrders.instmt, \ TestBasicOrders.price, \ @@ -76,16 +76,16 @@ def test_cancel_order(self): self.assertEqual(0, len(trades)) self.check_order_book(me, TestBasicOrders.instmt, 0, 1) self.check_order(order, 2, TestBasicOrders.instmt, TestBasicOrders.price, \ - TestBasicOrders.lot_size, lme.Side.SELL, 0, TestBasicOrders.lot_size) - + TestBasicOrders.lot_size, lme.Side.SELL, 0, TestBasicOrders.lot_size) + # Cancel a sell order del_order = me.cancel_order(order.order_id, TestBasicOrders.instmt) self.check_order_book(me, TestBasicOrders.instmt, 0, 0) self.check_deleted_order(order, del_order) - + def test_fill_order(self): me = lme.LightMatchingEngine() - + # Place a buy order buy_order, trades = me.add_order(TestBasicOrders.instmt, \ TestBasicOrders.price, \ @@ -95,28 +95,28 @@ def test_fill_order(self): self.check_order_book(me, TestBasicOrders.instmt, 1, 0) self.check_order(buy_order, 1, TestBasicOrders.instmt, TestBasicOrders.price, \ TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) - + # Place a sell order sell_order, trades = me.add_order(TestBasicOrders.instmt, \ TestBasicOrders.price, \ TestBasicOrders.lot_size, \ - lme.Side.SELL) + lme.Side.SELL) self.check_order_book(me, TestBasicOrders.instmt, 0, 0) - self.assertEqual(2, len(trades)) + self.assertEqual(2, len(trades)) self.check_order(buy_order, 1, TestBasicOrders.instmt, TestBasicOrders.price, \ TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, 0) self.check_order(sell_order, 2, TestBasicOrders.instmt, TestBasicOrders.price, \ TestBasicOrders.lot_size, lme.Side.SELL, TestBasicOrders.lot_size, 0) - + # Check trades self.check_trade(trades[0], sell_order.order_id, sell_order.instmt, \ - sell_order.price, sell_order.qty, sell_order.side, 1) + sell_order.price, sell_order.qty, sell_order.side, 1) self.check_trade(trades[1], buy_order.order_id, buy_order.instmt, \ buy_order.price, buy_order.qty, buy_order.side, 2) - + def test_fill_multiple_orders_same_level(self): me = lme.LightMatchingEngine() - + # Place buy orders for i in range(1, 11): buy_order, trades = me.add_order(TestBasicOrders.instmt, \ @@ -127,32 +127,32 @@ def test_fill_multiple_orders_same_level(self): self.check_order_book(me, TestBasicOrders.instmt, 1, 0) self.assertEqual(i, len(me.order_books[TestBasicOrders.instmt].bids[TestBasicOrders.price])) self.check_order(buy_order, i, TestBasicOrders.instmt, TestBasicOrders.price, \ - TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) - + TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) + # Place sell orders sell_order, trades = me.add_order(TestBasicOrders.instmt, \ TestBasicOrders.price, \ 10.0 * TestBasicOrders.lot_size, \ - lme.Side.SELL) + lme.Side.SELL) self.check_order_book(me, TestBasicOrders.instmt, 0, 0) - self.assertEqual(11, len(trades)) + self.assertEqual(11, len(trades)) self.check_order(buy_order, 10, TestBasicOrders.instmt, TestBasicOrders.price, \ - TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, 0) + TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, 0) self.check_order(sell_order, 11, TestBasicOrders.instmt, TestBasicOrders.price, \ - 10*TestBasicOrders.lot_size, lme.Side.SELL, 10*TestBasicOrders.lot_size, 0) - + 10*TestBasicOrders.lot_size, lme.Side.SELL, 10*TestBasicOrders.lot_size, 0) + # Check aggressive hit orders self.check_trade(trades[0], sell_order.order_id, sell_order.instmt, \ sell_order.price, sell_order.qty, sell_order.side, 1) - + # Check passive hit orders for i in range(1, 11): self.check_trade(trades[i], i, buy_order.instmt, \ - buy_order.price, buy_order.qty, buy_order.side, i+1) - + buy_order.price, buy_order.qty, buy_order.side, i+1) + def test_fill_multiple_orders_different_level(self): me = lme.LightMatchingEngine() - + # Place buy orders for i in range(1, 11): buy_order, trades = me.add_order(TestBasicOrders.instmt, \ @@ -163,20 +163,20 @@ def test_fill_multiple_orders_different_level(self): self.check_order_book(me, TestBasicOrders.instmt, i, 0) self.assertEqual(1, len(me.order_books[TestBasicOrders.instmt].bids[TestBasicOrders.price+i])) self.check_order(buy_order, i, TestBasicOrders.instmt, TestBasicOrders.price+i, \ - TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) - + TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) + # Place sell orders sell_order, trades = me.add_order(TestBasicOrders.instmt, \ TestBasicOrders.price, \ 10.0 * TestBasicOrders.lot_size, \ - lme.Side.SELL) + lme.Side.SELL) self.check_order_book(me, TestBasicOrders.instmt, 0, 0) - self.assertEqual(20, len(trades)) + self.assertEqual(20, len(trades)) self.check_order(buy_order, 10, TestBasicOrders.instmt, TestBasicOrders.price+10, \ - TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, 0) + TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, 0) self.check_order(sell_order, 11, TestBasicOrders.instmt, TestBasicOrders.price, \ - 10*TestBasicOrders.lot_size, lme.Side.SELL, 10*TestBasicOrders.lot_size, 0) - + 10*TestBasicOrders.lot_size, lme.Side.SELL, 10*TestBasicOrders.lot_size, 0) + for i in range(0, 10): match_price = sell_order.price+10-i self.check_trade(trades[2*i], sell_order.order_id, sell_order.instmt, \ @@ -186,7 +186,7 @@ def test_fill_multiple_orders_different_level(self): def test_cancel_partial_fill_orders(self): me = lme.LightMatchingEngine() - + # Place a buy order buy1_order, trades = me.add_order(TestBasicOrders.instmt, \ TestBasicOrders.price + 0.1, \ @@ -196,7 +196,7 @@ def test_cancel_partial_fill_orders(self): self.check_order_book(me, TestBasicOrders.instmt, 1, 0) self.check_order(buy1_order, 1, TestBasicOrders.instmt, TestBasicOrders.price + 0.1, \ TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) - + # Place a buy order buy2_order, trades = me.add_order(TestBasicOrders.instmt, \ TestBasicOrders.price, \ @@ -205,40 +205,40 @@ def test_cancel_partial_fill_orders(self): self.assertEqual(0, len(trades)) self.check_order_book(me, TestBasicOrders.instmt, 2, 0) self.check_order(buy2_order, 2, TestBasicOrders.instmt, TestBasicOrders.price, \ - 2 * TestBasicOrders.lot_size, lme.Side.BUY, 0, 2 * TestBasicOrders.lot_size) - + 2 * TestBasicOrders.lot_size, lme.Side.BUY, 0, 2 * TestBasicOrders.lot_size) + # Place a sell order sell_order, trades = me.add_order(TestBasicOrders.instmt, \ TestBasicOrders.price, \ 2 * TestBasicOrders.lot_size, \ - lme.Side.SELL) + lme.Side.SELL) self.check_order_book(me, TestBasicOrders.instmt, 1, 0) - self.assertEqual(4, len(trades)) + self.assertEqual(4, len(trades)) self.check_order(buy1_order, 1, TestBasicOrders.instmt, TestBasicOrders.price + 0.1, \ TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, 0) self.check_order(buy2_order, 2, TestBasicOrders.instmt, TestBasicOrders.price, \ - 2*TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, TestBasicOrders.lot_size) + 2*TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, TestBasicOrders.lot_size) self.check_order(sell_order, 3, TestBasicOrders.instmt, TestBasicOrders.price, \ 2*TestBasicOrders.lot_size, lme.Side.SELL, 2*TestBasicOrders.lot_size, 0) - + # Check trades self.check_trade(trades[0], sell_order.order_id, sell_order.instmt, \ - TestBasicOrders.price + 0.1, TestBasicOrders.lot_size, sell_order.side, 1) + TestBasicOrders.price + 0.1, TestBasicOrders.lot_size, sell_order.side, 1) self.check_trade(trades[1], buy1_order.order_id, buy1_order.instmt, \ - TestBasicOrders.price + 0.1, TestBasicOrders.lot_size, buy1_order.side, 2) + TestBasicOrders.price + 0.1, TestBasicOrders.lot_size, buy1_order.side, 2) self.check_trade(trades[2], sell_order.order_id, sell_order.instmt, \ - TestBasicOrders.price, TestBasicOrders.lot_size, sell_order.side, 3) + TestBasicOrders.price, TestBasicOrders.lot_size, sell_order.side, 3) self.check_trade(trades[3], buy2_order.order_id, buy1_order.instmt, \ - TestBasicOrders.price, TestBasicOrders.lot_size, buy2_order.side, 4) - + TestBasicOrders.price, TestBasicOrders.lot_size, buy2_order.side, 4) + # Cancel the second order del_order = me.cancel_order(buy2_order.order_id, TestBasicOrders.instmt) self.check_order_book(me, TestBasicOrders.instmt, 0, 0) self.check_deleted_order(buy2_order, del_order) - + def test_fill_multiple_orders_same_level_market_order(self): me = lme.LightMatchingEngine() - + # Place buy orders for i in range(1, 11): buy_order, trades = me.add_order(TestBasicOrders.instmt, \ @@ -249,32 +249,32 @@ def test_fill_multiple_orders_same_level_market_order(self): self.check_order_book(me, TestBasicOrders.instmt, 1, 0) self.assertEqual(i, len(me.order_books[TestBasicOrders.instmt].bids[TestBasicOrders.price])) self.check_order(buy_order, i, TestBasicOrders.instmt, TestBasicOrders.price, \ - TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) - + TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) + # Place sell orders sell_order, trades = me.add_order(TestBasicOrders.instmt, \ 0, \ 10.0 * TestBasicOrders.lot_size, \ - lme.Side.SELL) + lme.Side.SELL) self.check_order_book(me, TestBasicOrders.instmt, 0, 0) - self.assertEqual(11, len(trades)) + self.assertEqual(11, len(trades)) self.check_order(buy_order, 10, TestBasicOrders.instmt, TestBasicOrders.price, \ - TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, 0) + TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, 0) self.check_order(sell_order, 11, TestBasicOrders.instmt, 0, \ - 10*TestBasicOrders.lot_size, lme.Side.SELL, 10*TestBasicOrders.lot_size, 0) - + 10*TestBasicOrders.lot_size, lme.Side.SELL, 10*TestBasicOrders.lot_size, 0) + # Check aggressive hit orders - Trade price is same as the passive hit limit price self.check_trade(trades[0], sell_order.order_id, sell_order.instmt, \ buy_order.price, sell_order.qty, sell_order.side, 1) - + # Check passive hit orders for i in range(1, 11): self.check_trade(trades[i], i, buy_order.instmt, \ - buy_order.price, buy_order.qty, buy_order.side, i+1) - + buy_order.price, buy_order.qty, buy_order.side, i+1) + def test_fill_multiple_orders_different_level_market_order(self): me = lme.LightMatchingEngine() - + # Place buy orders for i in range(1, 11): buy_order, trades = me.add_order(TestBasicOrders.instmt, \ @@ -285,26 +285,26 @@ def test_fill_multiple_orders_different_level_market_order(self): self.check_order_book(me, TestBasicOrders.instmt, i, 0) self.assertEqual(1, len(me.order_books[TestBasicOrders.instmt].bids[TestBasicOrders.price+i])) self.check_order(buy_order, i, TestBasicOrders.instmt, TestBasicOrders.price+i, \ - TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) - + TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) + # Place sell orders sell_order, trades = me.add_order(TestBasicOrders.instmt, \ 0, \ 10.0 * TestBasicOrders.lot_size, \ - lme.Side.SELL) + lme.Side.SELL) self.check_order_book(me, TestBasicOrders.instmt, 0, 0) - self.assertEqual(20, len(trades)) + self.assertEqual(20, len(trades)) self.check_order(buy_order, 10, TestBasicOrders.instmt, TestBasicOrders.price+10, \ - TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, 0) + TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, 0) self.check_order(sell_order, 11, TestBasicOrders.instmt, 0, \ - 10*TestBasicOrders.lot_size, lme.Side.SELL, 10*TestBasicOrders.lot_size, 0) - + 10*TestBasicOrders.lot_size, lme.Side.SELL, 10*TestBasicOrders.lot_size, 0) + for i in range(0, 10): match_price = TestBasicOrders.price+10-i self.check_trade(trades[2*i], sell_order.order_id, sell_order.instmt, \ match_price, TestBasicOrders.lot_size, sell_order.side, 2*i+1) self.check_trade(trades[2*i+1], 10-i, buy_order.instmt, \ match_price, TestBasicOrders.lot_size, buy_order.side, 2*i+2) - + if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 5678a562e488c000db203c306616db7e17a18c2e Mon Sep 17 00:00:00 2001 From: Gavin Chan Date: Fri, 30 Aug 2019 15:48:24 +0800 Subject: [PATCH 23/23] Fix all the bugs --- src/lightmatchingengine.cpp | 18 ++- src/lightmatchingengine.h | 171 ++++++++++++++++++-------- tests/performance/performance_test.py | 22 +++- tests/unit/test_basic_orders.py | 74 +++++------ 4 files changed, 188 insertions(+), 97 deletions(-) diff --git a/src/lightmatchingengine.cpp b/src/lightmatchingengine.cpp index 0e0feed..0db255d 100644 --- a/src/lightmatchingengine.cpp +++ b/src/lightmatchingengine.cpp @@ -25,7 +25,9 @@ PYBIND11_MODULE(lightmatchingengine, m) { string&, double, double, - Side>()); + Side>()) + .def("__str__", &Order::to_string) + .def("__repr__", &Order::to_string); // Struct Trade py::class_(m, "Trade") @@ -41,12 +43,14 @@ PYBIND11_MODULE(lightmatchingengine, m) { double, double, Side, - int>()); + int>()) + .def("__str__", &Trade::to_string) + .def("__repr__", &Trade::to_string); // Late binding // py::bind_vector>(m, "DequeOrder"); - py::bind_map>>(m, "MapDoubleVectorOrder"); - py::bind_map>(m, "UnorderedMapIntOrder"); + py::bind_map>>(m, "MapDoubleVectorOrder"); + py::bind_map>(m, "UnorderedMapIntOrder"); // Class OrderBook py::class_(m, "OrderBook") @@ -60,8 +64,10 @@ PYBIND11_MODULE(lightmatchingengine, m) { py::bind_map>(m, "UnorderedMapStringOrderBook"); py::class_(m, "LightMatchingEngine") .def(py::init()) - .def("add_order", &LightMatchingEngine::add_order) - .def("cancel_order", &LightMatchingEngine::cancel_order) + .def("add_order", &LightMatchingEngine::add_order, py::return_value_policy::reference_internal) + .def("cancel_order", &LightMatchingEngine::cancel_order, py::return_value_policy::reference_internal) + .def("get_bid_queue", &LightMatchingEngine::get_bid_queue, py::return_value_policy::reference) + .def("get_ask_queue", &LightMatchingEngine::get_ask_queue, py::return_value_policy::reference) .def_property_readonly("order_books", &LightMatchingEngine::order_books, py::return_value_policy::reference) .def_property_readonly("curr_order_id", &LightMatchingEngine::curr_order_id) .def_property_readonly("curr_trade_id", &LightMatchingEngine::curr_trade_id); diff --git a/src/lightmatchingengine.h b/src/lightmatchingengine.h index 0d159ec..882ea4c 100644 --- a/src/lightmatchingengine.h +++ b/src/lightmatchingengine.h @@ -16,12 +16,12 @@ namespace py = pybind11; #define MIN_QUANTITY 1e-9 #define MIN_TICK_SIZE 1e-9 #define EPILSON 5e-10 -#define NORMALIZE_PRICE( x ) static_cast( x / MIN_TICK_SIZE + EPILSON ) -#define DENORMALIZE_PRICE( x ) ( x * MIN_TICK_SIZE ) #define nprice_t long long #define price_t double #define qty_t double #define id_t long long +#define NORMALIZE_PRICE( x ) static_cast( x / MIN_TICK_SIZE + EPILSON ) +#define DENORMALIZE_PRICE( x ) ( static_cast(x) * MIN_TICK_SIZE ) enum class Side { BUY = 1, @@ -37,12 +37,6 @@ struct Order { qty_t leaves_qty; Side side; - /* Order() = default; */ - /* Order(const Order&) = default; */ - /* Order& operator=(Order&) = default; */ - /* Order(Order&&) = default; */ - /* Order& operator=(Order&&) = default; */ - Order(id_t order_id, const string& instmt, price_t price, qty_t qty, Side side): order_id(order_id), instmt(instmt), @@ -51,6 +45,18 @@ struct Order { cum_qty(0), leaves_qty(qty), side(side) {} + + string to_string() { + ostringstream sstream; + sstream << "Order Id: " << order_id << ", " + << "instmt: " << instmt << ", " + << "price: " << price << ", " + << "qty: " << qty << ", " + << "cum_qty: " << cum_qty << ", " + << "leaves_qty: " << leaves_qty << ", " + << "side: " << (side == Side::BUY ? "Buy" : "Sell"); + return sstream.str(); + } }; struct Trade { @@ -69,20 +75,31 @@ struct Trade { trade_qty(trade_qty), trade_side(trade_side), trade_id(trade_id) {} + + string to_string() { + ostringstream sstream; + sstream << "Trade Id: " << trade_id << ", " + << "order Id: " << order_id << ", " + << "instmt: " << instmt << ", " + << "trade_price: " << trade_price << ", " + << "trade_qty: " << trade_qty << ", " + << "trade_side: " << (trade_side == Side::BUY ? "Buy" : "Sell"); + return sstream.str(); + } }; -PYBIND11_MAKE_OPAQUE(map>); -PYBIND11_MAKE_OPAQUE(unordered_map); +PYBIND11_MAKE_OPAQUE(map>); +PYBIND11_MAKE_OPAQUE(unordered_map); struct OrderBook { - map> bids; - map> asks; - unordered_map order_id_map; + map> bids; + map> asks; + unordered_map order_id_map; OrderBook() = default; - OrderBook(const OrderBook&) = delete; + /* OrderBook(const OrderBook&) = delete; */ OrderBook(OrderBook&&) = default; - OrderBook& operator=(OrderBook&&) = default; + /* OrderBook& operator=(OrderBook&&) = default; */ OrderBook& operator=(const OrderBook&) = delete; }; @@ -102,12 +119,41 @@ class LightMatchingEngine { return __curr_trade_id; } - tuple> add_order( + deque* get_bid_queue(const string& instmt, price_t price) { + auto& order_book = *__get_order_book(instmt); + auto nprice = NORMALIZE_PRICE(price); + auto order_queue_it = order_book.bids.find(nprice); + if (order_queue_it == order_book.bids.end()) { + ostringstream sstream; + sstream << "Order price " << price << " cannot be found " + << "in the " << instmt << " bid order book"; + throw runtime_error(sstream.str()); + } + + return &(order_queue_it->second); + } + + deque* get_ask_queue(const string& instmt, price_t price) { + auto& order_book = *__get_order_book(instmt); + auto nprice = NORMALIZE_PRICE(price); + auto order_queue_it = order_book.asks.find(nprice); + if (order_queue_it == order_book.asks.end()) { + ostringstream sstream; + sstream << "Order price " << price << " cannot be found " + << "in the " << instmt << " bid order book"; + throw runtime_error(sstream.str()); + } + + return &(order_queue_it->second); + } + + tuple> add_order( const string& instmt, price_t price, qty_t qty, Side side) { vector trades; id_t order_id = (__curr_order_id += 1); - Order order = Order(order_id, instmt, price, qty, side); + Order* order_ptr = new Order(order_id, instmt, price, qty, side); + auto& order = *order_ptr; nprice_t nprice = NORMALIZE_PRICE(price); // Find the order book @@ -129,9 +175,10 @@ class LightMatchingEngine { auto original_leaves_qty = order.leaves_qty; // Matching the ask queue - while (nbbo.size() > 0) { - auto front_nbbo = nbbo.front(); - qty_t matching_qty = min(order.leaves_qty, nbbo[0].leaves_qty); + while (nbbo.size() > 0 && order.leaves_qty > MIN_QUANTITY) { + auto& front_nbbo = *(nbbo.front()); + qty_t matching_qty = min(order.leaves_qty, front_nbbo.leaves_qty); + assert(matching_qty >= MIN_QUANTITY); order.cum_qty += matching_qty; order.leaves_qty -= matching_qty; front_nbbo.cum_qty += matching_qty; @@ -141,7 +188,7 @@ class LightMatchingEngine { trades.push_back(Trade( front_nbbo.order_id, instmt, - best_nprice, + DENORMALIZE_PRICE(best_nprice), matching_qty, front_nbbo.side, ++__curr_trade_id)); @@ -157,7 +204,7 @@ class LightMatchingEngine { trades.push_back(Trade( order.order_id, instmt, - best_nprice, + DENORMALIZE_PRICE(best_nprice), original_leaves_qty - order.leaves_qty, order.side, ++__curr_trade_id)); @@ -180,27 +227,28 @@ class LightMatchingEngine { if (order.leaves_qty > MIN_QUANTITY) { auto nbbo_it = order_book.bids.find(nprice); if (nbbo_it == order_book.bids.end()){ - nbbo_it = order_book.bids.emplace(nprice, deque()).first; + nbbo_it = order_book.bids.emplace(nprice, deque()).first; } auto& nbbo = nbbo_it->second; - nbbo.emplace_back(order); - order_book.order_id_map.emplace(order.order_id, order); + nbbo.emplace_back(&order); + order_book.order_id_map.emplace(order.order_id, &order); } } else { nprice_t best_nprice = MIN_NPRICE; if (order_book.bids.size() > 0) { - best_nprice = order_book.bids.begin()->first; + best_nprice = order_book.bids.rbegin()->first; } while (nprice <= best_nprice && order.leaves_qty > MIN_QUANTITY) { - auto& nbbo = order_book.bids.begin()->second; + auto& nbbo = order_book.bids.rbegin()->second; auto original_leaves_qty = order.leaves_qty; // Matching the ask queue - while (nbbo.size() > 0) { - auto front_nbbo = nbbo.front(); - qty_t matching_qty = min(order.leaves_qty, nbbo[0].leaves_qty); + while (nbbo.size() > 0 && order.leaves_qty > MIN_QUANTITY) { + auto& front_nbbo = *(nbbo.front()); + qty_t matching_qty = min(order.leaves_qty, front_nbbo.leaves_qty); + assert(matching_qty >= MIN_QUANTITY); order.cum_qty += matching_qty; order.leaves_qty -= matching_qty; front_nbbo.cum_qty += matching_qty; @@ -210,7 +258,7 @@ class LightMatchingEngine { trades.push_back(Trade( front_nbbo.order_id, instmt, - best_nprice, + DENORMALIZE_PRICE(best_nprice), matching_qty, front_nbbo.side, ++__curr_trade_id)); @@ -226,19 +274,19 @@ class LightMatchingEngine { trades.push_back(Trade( order.order_id, instmt, - best_nprice, + DENORMALIZE_PRICE(best_nprice), original_leaves_qty - order.leaves_qty, order.side, ++__curr_trade_id)); // Remove the bid queue if the size = 0 if (nbbo.size() == 0) { - order_book.bids.erase(order_book.bids.begin()); + order_book.bids.erase(next(order_book.bids.rbegin()).base()); } // Update the bid best prices if (order_book.bids.size() > 0) { - best_nprice = order_book.bids.begin()->first; + best_nprice = order_book.bids.rbegin()->first; } else { best_nprice = MIN_NPRICE; } @@ -249,27 +297,24 @@ class LightMatchingEngine { if (order.leaves_qty > MIN_QUANTITY) { auto nbbo_it = order_book.asks.find(nprice); if (nbbo_it == order_book.asks.end()){ - order.cum_qty += matching_qty; - order.leaves_qty -= matching_qty; - front_nbbo.cum_qty += matching_qty; - front_nbbo.leaves_qty -= matching_qty; - nbbo_it = order_book.asks.emplace(nprice, deque()).first; + nbbo_it = order_book.asks.emplace(nprice, deque()).first; } auto& nbbo = nbbo_it->second; - nbbo.emplace_back(order); - order_book.order_id_map.emplace(order.order_id, order); + nbbo.emplace_back(&order); + order_book.order_id_map.emplace(order.order_id, &order); } } - return make_tuple(order, trades); + return make_tuple(&order, trades); } - Order& cancel_order(id_t order_id, const string& instmt) { + Order* cancel_order(id_t order_id, const string& instmt) { auto order_book_it = __order_books.find(instmt); if (order_book_it == __order_books.end()) { - auto err_message = string("Order books do not have the instrument ") + instmt; - throw runtime_error(err_message); + ostringstream sstream; + sstream << "Order books do not have the instrument " << instmt; + throw runtime_error(sstream.str()); } auto& order_book = order_book_it->second; @@ -280,7 +325,7 @@ class LightMatchingEngine { throw runtime_error(sstream.str()); } - auto& order = order_it->second; + auto& order = *order_it->second; auto nprice = NORMALIZE_PRICE(order.price); if (order.side == Side::BUY) { @@ -288,30 +333,56 @@ class LightMatchingEngine { assert(order_queue_it != order_book.bids.end()); auto& order_queue = order_queue_it->second; auto found_order = find_if( - order_queue.begin(), order_queue.end(), [&order](auto o) { return o.order_id == order.order_id; }); + order_queue.begin(), order_queue.end(), + [&order](auto o) { return o->order_id == order.order_id; }); + + assert(found_order != order_queue.end()); // Remove the order from the matching engine order_queue.erase(found_order); + + // Remove the nbbo if no queue on it + if (order_queue.size() == 0) { + order_book.bids.erase(nprice); + } } else { auto order_queue_it = order_book.asks.find(nprice); assert(order_queue_it != order_book.asks.end()); - auto order_queue = order_queue_it->second; + auto& order_queue = order_queue_it->second; auto found_order = find_if( - order_queue.begin(), order_queue.end(), [&order](auto o) { return o.order_id == order.order_id; }); + order_queue.begin(), order_queue.end(), + [&order](auto o) { return o->order_id == order.order_id; }); + + assert(found_order != order_queue.end()); // Remove the order from the matching engine order_queue.erase(found_order); + + // Remove the nbbo if no queue on it + if (order_queue.size() == 0) { + order_book.asks.erase(nprice); + } } // Finally set the leaves qty to 0 order.leaves_qty = 0.0; order_book.order_id_map.erase(order_it); - return order; + return ℴ } private: unordered_map __order_books; int __curr_order_id; int __curr_trade_id; + + OrderBook* __get_order_book(const string& instmt) { + auto order_book_it = __order_books.find(instmt); + if (order_book_it == __order_books.end()) { + auto err_message = string("Order books do not have the instrument ") + instmt; + throw runtime_error(err_message); + } + + return &(order_book_it->second); + } }; diff --git a/tests/performance/performance_test.py b/tests/performance/performance_test.py index 8fe5a50..0a18e66 100644 --- a/tests/performance/performance_test.py +++ b/tests/performance/performance_test.py @@ -15,6 +15,7 @@ --tick-size= Tick size. [Default: 0.1] --gamma-quantity= Gamma value in the gamma distribution for the order quantity. [Default: 2] + --debug Debug mode. """ from docopt import docopt import logging @@ -67,16 +68,21 @@ def run(args): if uniform(0, 1) <= add_order_prob or len(orders) == 0: price = np.random.standard_normal() * std_price + mean_price price = int(price / tick_size) * tick_size - quantity = np.random.gamma(gamma_quantity) + 1 + quantity = int(np.random.gamma(gamma_quantity)) + 1 side = Side.BUY if uniform(0, 1) <= 0.5 else Side.SELL + LOGGER.debug('Adding order at side %s, price %s ' + 'and quantity %s', + side, price, quantity) + # Add the order with Timer() as timer: order, trades = engine.add_order(symbol, price, quantity, side) LOGGER.debug('Order %s is added at side %s, price %s ' - 'and quantity %s', - order.order_id, order.side, order.price, order.qty) + 'and quantity %s, with %s trades', + order.order_id, order.side, order.price, order.qty, + len(trades)) # Save the order if there is any quantity left if order.leaves_qty > 0.0: @@ -84,6 +90,8 @@ def run(args): # Remove the trades for trade in trades: + LOGGER.debug('Trade of order id %s is detected with trade ' + 'qty %s', trade.order_id, trade.trade_qty) if (trade.order_id != order.order_id and orders[trade.order_id].leaves_qty < 1e-9): del orders[trade.order_id] @@ -100,10 +108,13 @@ def run(args): order_id = list(orders.keys())[index] + LOGGER.debug('Deleting order %s', order_id) + with Timer() as timer: engine.cancel_order(order_id, order.instmt) LOGGER.debug('Order %s is deleted', order_id) + del orders[order_id] # Save the statistics @@ -153,7 +164,10 @@ def describe_statistics(add_statistics, cancel_statistics): if __name__ == '__main__': args = docopt(__doc__, version='1.0.0') - logging.basicConfig(level=logging.INFO) + if args['--debug']: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) LOGGER.info('Running the performance benchmark') add_statistics, cancel_statistics = run(args) diff --git a/tests/unit/test_basic_orders.py b/tests/unit/test_basic_orders.py index 82c3573..32a6a0b 100644 --- a/tests/unit/test_basic_orders.py +++ b/tests/unit/test_basic_orders.py @@ -109,10 +109,10 @@ def test_fill_order(self): TestBasicOrders.lot_size, lme.Side.SELL, TestBasicOrders.lot_size, 0) # Check trades - self.check_trade(trades[0], sell_order.order_id, sell_order.instmt, \ - sell_order.price, sell_order.qty, sell_order.side, 1) - self.check_trade(trades[1], buy_order.order_id, buy_order.instmt, \ - buy_order.price, buy_order.qty, buy_order.side, 2) + self.check_trade(trades[0], buy_order.order_id, buy_order.instmt, \ + buy_order.price, buy_order.qty, buy_order.side, 1) + self.check_trade(trades[1], sell_order.order_id, sell_order.instmt, \ + sell_order.price, sell_order.qty, sell_order.side, 2) def test_fill_multiple_orders_same_level(self): me = lme.LightMatchingEngine() @@ -125,7 +125,7 @@ def test_fill_multiple_orders_same_level(self): lme.Side.BUY) self.assertEqual(0, len(trades)) self.check_order_book(me, TestBasicOrders.instmt, 1, 0) - self.assertEqual(i, len(me.order_books[TestBasicOrders.instmt].bids[TestBasicOrders.price])) + self.assertEqual(i, len(me.get_bid_queue(TestBasicOrders.instmt, TestBasicOrders.price))) self.check_order(buy_order, i, TestBasicOrders.instmt, TestBasicOrders.price, \ TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) @@ -141,14 +141,14 @@ def test_fill_multiple_orders_same_level(self): self.check_order(sell_order, 11, TestBasicOrders.instmt, TestBasicOrders.price, \ 10*TestBasicOrders.lot_size, lme.Side.SELL, 10*TestBasicOrders.lot_size, 0) - # Check aggressive hit orders - self.check_trade(trades[0], sell_order.order_id, sell_order.instmt, \ - sell_order.price, sell_order.qty, sell_order.side, 1) - # Check passive hit orders for i in range(1, 11): - self.check_trade(trades[i], i, buy_order.instmt, \ - buy_order.price, buy_order.qty, buy_order.side, i+1) + self.check_trade(trades[i-1], i, buy_order.instmt, \ + buy_order.price, buy_order.qty, buy_order.side, i) + + # Check aggressive hit orders + self.check_trade(trades[10], sell_order.order_id, sell_order.instmt, \ + sell_order.price, sell_order.qty, sell_order.side, 11) def test_fill_multiple_orders_different_level(self): me = lme.LightMatchingEngine() @@ -161,7 +161,7 @@ def test_fill_multiple_orders_different_level(self): lme.Side.BUY) self.assertEqual(0, len(trades)) self.check_order_book(me, TestBasicOrders.instmt, i, 0) - self.assertEqual(1, len(me.order_books[TestBasicOrders.instmt].bids[TestBasicOrders.price+i])) + self.assertEqual(1, len(me.get_bid_queue(TestBasicOrders.instmt, TestBasicOrders.price+i))) self.check_order(buy_order, i, TestBasicOrders.instmt, TestBasicOrders.price+i, \ TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) @@ -179,22 +179,22 @@ def test_fill_multiple_orders_different_level(self): for i in range(0, 10): match_price = sell_order.price+10-i - self.check_trade(trades[2*i], sell_order.order_id, sell_order.instmt, \ - match_price, TestBasicOrders.lot_size, sell_order.side, 2*i+1) - self.check_trade(trades[2*i+1], 10-i, buy_order.instmt, \ - match_price, TestBasicOrders.lot_size, buy_order.side, 2*i+2) + self.check_trade(trades[2*i], 10-i, buy_order.instmt, \ + match_price, TestBasicOrders.lot_size, buy_order.side, 2*i+1) + self.check_trade(trades[2*i+1], sell_order.order_id, sell_order.instmt, \ + match_price, TestBasicOrders.lot_size, sell_order.side, 2*i+2) def test_cancel_partial_fill_orders(self): me = lme.LightMatchingEngine() # Place a buy order buy1_order, trades = me.add_order(TestBasicOrders.instmt, \ - TestBasicOrders.price + 0.1, \ + TestBasicOrders.price + 1, \ TestBasicOrders.lot_size, \ lme.Side.BUY) self.assertEqual(0, len(trades)) self.check_order_book(me, TestBasicOrders.instmt, 1, 0) - self.check_order(buy1_order, 1, TestBasicOrders.instmt, TestBasicOrders.price + 0.1, \ + self.check_order(buy1_order, 1, TestBasicOrders.instmt, TestBasicOrders.price + 1, \ TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) # Place a buy order @@ -214,7 +214,7 @@ def test_cancel_partial_fill_orders(self): lme.Side.SELL) self.check_order_book(me, TestBasicOrders.instmt, 1, 0) self.assertEqual(4, len(trades)) - self.check_order(buy1_order, 1, TestBasicOrders.instmt, TestBasicOrders.price + 0.1, \ + self.check_order(buy1_order, 1, TestBasicOrders.instmt, TestBasicOrders.price + 1, \ TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, 0) self.check_order(buy2_order, 2, TestBasicOrders.instmt, TestBasicOrders.price, \ 2*TestBasicOrders.lot_size, lme.Side.BUY, TestBasicOrders.lot_size, TestBasicOrders.lot_size) @@ -222,14 +222,14 @@ def test_cancel_partial_fill_orders(self): 2*TestBasicOrders.lot_size, lme.Side.SELL, 2*TestBasicOrders.lot_size, 0) # Check trades - self.check_trade(trades[0], sell_order.order_id, sell_order.instmt, \ - TestBasicOrders.price + 0.1, TestBasicOrders.lot_size, sell_order.side, 1) - self.check_trade(trades[1], buy1_order.order_id, buy1_order.instmt, \ - TestBasicOrders.price + 0.1, TestBasicOrders.lot_size, buy1_order.side, 2) - self.check_trade(trades[2], sell_order.order_id, sell_order.instmt, \ - TestBasicOrders.price, TestBasicOrders.lot_size, sell_order.side, 3) - self.check_trade(trades[3], buy2_order.order_id, buy1_order.instmt, \ - TestBasicOrders.price, TestBasicOrders.lot_size, buy2_order.side, 4) + self.check_trade(trades[0], buy1_order.order_id, buy1_order.instmt, \ + TestBasicOrders.price + 1, TestBasicOrders.lot_size, buy1_order.side, 1) + self.check_trade(trades[1], sell_order.order_id, sell_order.instmt, \ + TestBasicOrders.price + 1, TestBasicOrders.lot_size, sell_order.side, 2) + self.check_trade(trades[2], buy2_order.order_id, buy1_order.instmt, \ + TestBasicOrders.price, TestBasicOrders.lot_size, buy2_order.side, 3) + self.check_trade(trades[3], sell_order.order_id, sell_order.instmt, \ + TestBasicOrders.price, TestBasicOrders.lot_size, sell_order.side, 4) # Cancel the second order del_order = me.cancel_order(buy2_order.order_id, TestBasicOrders.instmt) @@ -247,7 +247,7 @@ def test_fill_multiple_orders_same_level_market_order(self): lme.Side.BUY) self.assertEqual(0, len(trades)) self.check_order_book(me, TestBasicOrders.instmt, 1, 0) - self.assertEqual(i, len(me.order_books[TestBasicOrders.instmt].bids[TestBasicOrders.price])) + self.assertEqual(i, len(me.get_bid_queue(TestBasicOrders.instmt, TestBasicOrders.price))) self.check_order(buy_order, i, TestBasicOrders.instmt, TestBasicOrders.price, \ TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) @@ -264,13 +264,13 @@ def test_fill_multiple_orders_same_level_market_order(self): 10*TestBasicOrders.lot_size, lme.Side.SELL, 10*TestBasicOrders.lot_size, 0) # Check aggressive hit orders - Trade price is same as the passive hit limit price - self.check_trade(trades[0], sell_order.order_id, sell_order.instmt, \ - buy_order.price, sell_order.qty, sell_order.side, 1) + self.check_trade(trades[10], sell_order.order_id, sell_order.instmt, \ + buy_order.price, sell_order.qty, sell_order.side, 11) # Check passive hit orders for i in range(1, 11): - self.check_trade(trades[i], i, buy_order.instmt, \ - buy_order.price, buy_order.qty, buy_order.side, i+1) + self.check_trade(trades[i-1], i, buy_order.instmt, \ + buy_order.price, buy_order.qty, buy_order.side, i) def test_fill_multiple_orders_different_level_market_order(self): me = lme.LightMatchingEngine() @@ -283,7 +283,7 @@ def test_fill_multiple_orders_different_level_market_order(self): lme.Side.BUY) self.assertEqual(0, len(trades)) self.check_order_book(me, TestBasicOrders.instmt, i, 0) - self.assertEqual(1, len(me.order_books[TestBasicOrders.instmt].bids[TestBasicOrders.price+i])) + self.assertEqual(1, len(me.get_bid_queue(TestBasicOrders.instmt, TestBasicOrders.price+i))) self.check_order(buy_order, i, TestBasicOrders.instmt, TestBasicOrders.price+i, \ TestBasicOrders.lot_size, lme.Side.BUY, 0, TestBasicOrders.lot_size) @@ -301,10 +301,10 @@ def test_fill_multiple_orders_different_level_market_order(self): for i in range(0, 10): match_price = TestBasicOrders.price+10-i - self.check_trade(trades[2*i], sell_order.order_id, sell_order.instmt, \ - match_price, TestBasicOrders.lot_size, sell_order.side, 2*i+1) - self.check_trade(trades[2*i+1], 10-i, buy_order.instmt, \ - match_price, TestBasicOrders.lot_size, buy_order.side, 2*i+2) + self.check_trade(trades[2*i+1], sell_order.order_id, sell_order.instmt, \ + match_price, TestBasicOrders.lot_size, sell_order.side, 2*i+2) + self.check_trade(trades[2*i], 10-i, buy_order.instmt, \ + match_price, TestBasicOrders.lot_size, buy_order.side, 2*i+1) if __name__ == '__main__': unittest.main()