Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement high-precision 128-bit value types #2072

Merged
merged 99 commits into from
Jan 15, 2025
Merged
Show file tree
Hide file tree
Changes from 86 commits
Commits
Show all changes
99 commits
Select commit Hold shift + click to select a range
7c6ee6f
Feature flag i128 precision for raw price
twitu Nov 24, 2024
dd9da30
Refactor raw price type checks
twitu Nov 30, 2024
ac755bf
Fix clippy lints
twitu Nov 30, 2024
9a2b7e0
Fix improper ctype calling convention lint
twitu Nov 30, 2024
370a939
Use fixed size binary encoding for high precision
twitu Nov 30, 2024
dda2d73
Fix decode batch using macro
twitu Dec 2, 2024
42d8993
Fix depth tests
twitu Dec 3, 2024
d91e90c
Refactor get_raw_price function
twitu Dec 3, 2024
a6bb854
Clippy fixes
twitu Dec 3, 2024
7c4e922
Clippy fixes
twitu Dec 3, 2024
bb47645
Fix tests
twitu Dec 3, 2024
8734184
Fix clippy
twitu Dec 3, 2024
e3e1290
Turn off databento for high precision
twitu Dec 3, 2024
95c81c1
Merge branch 'develop' into high-precision
twitu Dec 3, 2024
f727e3e
Merge branch 'develop' into high-precision
twitu Dec 8, 2024
d710167
Merge branch 'develop' into high-precision
twitu Dec 21, 2024
3595210
Run high-precision experiments
twitu Dec 22, 2024
de24723
Make fixed sized binary the default encoding for price values
twitu Dec 22, 2024
f0fb7f6
Refactor tests and imports
twitu Dec 22, 2024
c94ec35
Fix databento high precision price usage
twitu Dec 22, 2024
ad9b252
Nit
twitu Dec 22, 2024
6be0128
Merge branch 'develop' into high-precision
twitu Dec 27, 2024
1a1d09c
High precision i128 backing Quantity
twitu Dec 30, 2024
7838ac7
Fix databento raw value decoding
twitu Dec 31, 2024
199c089
Fix scalar usage in serialization
twitu Dec 31, 2024
a567a2f
Fix serialization tests
twitu Dec 31, 2024
d18470f
High precision money
twitu Jan 1, 2025
0b25231
Fix serializer tests
twitu Jan 1, 2025
934275a
Fix for databento tests
twitu Jan 1, 2025
e331526
TODO: overwritten constants
twitu Jan 2, 2025
4034bf0
High precision issue test cases
twitu Jan 2, 2025
feae416
Port test data to new schema
twitu Jan 2, 2025
9c969d2
Fix orderbook levels tests
twitu Jan 2, 2025
72cbeb5
Compile feature flagged high precision types (#2164)
twitu Jan 4, 2025
ce092cf
Merge branch 'develop' into high-precision
twitu Jan 4, 2025
9720fd4
Merge branch 'develop' into high-precision
cjdsellers Jan 5, 2025
62f69e1
Fix compiler warnings
cjdsellers Jan 5, 2025
08e60c6
Fix some tests and corral high-precision issue
cjdsellers Jan 5, 2025
0e12e43
Fix rounding issues
cjdsellers Jan 5, 2025
9543a84
Fix money max and min
cjdsellers Jan 5, 2025
b7db9ea
Merge branch 'develop' into high-precision
cjdsellers Jan 5, 2025
43c6b3e
Merge branch 'develop' into high-precision
cjdsellers Jan 5, 2025
3d4856e
Cleanup imports
cjdsellers Jan 5, 2025
430799b
Pause CI for now
cjdsellers Jan 5, 2025
bff5187
Pause CI for now
cjdsellers Jan 5, 2025
79d88bb
Merge branch 'develop' into high-precision
cjdsellers Jan 7, 2025
956cf4c
Merge branch 'develop' into high-precision
cjdsellers Jan 7, 2025
4b04875
Fix from_raw conversions and some tests
cjdsellers Jan 8, 2025
2fc8d34
Merge branch 'develop' into high-precision
cjdsellers Jan 9, 2025
6abffa6
Fix object subpackage tests
cjdsellers Jan 9, 2025
a31186e
Merge branch 'develop' into high-precision
cjdsellers Jan 10, 2025
80192cb
Fix more tests for 64-bit precision
cjdsellers Jan 10, 2025
dafccf5
Fix clippy lints
cjdsellers Jan 10, 2025
5e60b08
Standardize high-precision feature config ordering
cjdsellers Jan 10, 2025
f1a0c1a
Fix and standardize from_raw
cjdsellers Jan 10, 2025
f7c6847
Fix quote_tick_new C alignment issue
cjdsellers Jan 10, 2025
029fb15
Merge branch 'develop' into high-precision
cjdsellers Jan 11, 2025
c27bdff
Add price and quantity raw create functions
cjdsellers Jan 11, 2025
724e51b
Fix bar for high-precision
cjdsellers Jan 11, 2025
298e340
Fix currency tests
cjdsellers Jan 11, 2025
95c8f79
Fix order book and remaining data types
cjdsellers Jan 11, 2025
bf5da52
Alias integer types
cjdsellers Jan 11, 2025
9fa2357
Standardize raw value aliasing
cjdsellers Jan 11, 2025
02325de
Standardize raw int method naming
cjdsellers Jan 11, 2025
ab83464
Fix legacy wranglers and data
cjdsellers Jan 11, 2025
1b48422
Merge branch 'develop' into high-precision
cjdsellers Jan 11, 2025
72d6ff9
Remove high precision issue ignores
cjdsellers Jan 11, 2025
44a911a
Fix and add fixed precision function tests
cjdsellers Jan 12, 2025
c3f93b8
Fix price constants and market fills
cjdsellers Jan 12, 2025
b1f65cb
Fix wrangler default volume calc
cjdsellers Jan 12, 2025
b5c60a2
Fix adapter scaling and tests
cjdsellers Jan 12, 2025
fa917e2
Fix wranglers v2 and refine constants
cjdsellers Jan 12, 2025
2a0033d
Fix transformer tests
cjdsellers Jan 12, 2025
9ee2926
Remove trivial_copy feature
cjdsellers Jan 12, 2025
f68b935
Improve build and constants
cjdsellers Jan 13, 2025
282fe35
Add benchmark for arithmetic operators
twitu Jan 13, 2025
58bff52
WIP: Compiling HIGH_PRECISION=false
twitu Jan 13, 2025
7b4c959
Fix build and import warnings
cjdsellers Jan 13, 2025
b302a0c
Fix import warnings and tests
cjdsellers Jan 13, 2025
5affb1c
Fix tests for low precision
cjdsellers Jan 13, 2025
70c6ac4
Fix export rename
cjdsellers Jan 13, 2025
5a959e4
Remove redundant helper functions
cjdsellers Jan 13, 2025
32750d0
Fix build for high_precision
cjdsellers Jan 14, 2025
9bc68a6
Fix Rust tests for low precision
cjdsellers Jan 14, 2025
48d6519
Fix persistence tests for both 64 and 128 bit
twitu Jan 14, 2025
d6c020c
Refactor test data path helper
twitu Jan 14, 2025
7351326
Merge branch 'develop' into high-precision
cjdsellers Jan 14, 2025
178042b
Refine databento decoder and type conversions
cjdsellers Jan 14, 2025
5879c20
Refine build, flags and CI
cjdsellers Jan 15, 2025
34b220c
Fix build for windows
cjdsellers Jan 15, 2025
5f0db07
Refine build
cjdsellers Jan 15, 2025
0f5fdac
Refine build and CI
cjdsellers Jan 15, 2025
82b2f2d
Improve low-precision job name
cjdsellers Jan 15, 2025
ad9b3b4
Fix clippy which needs low-precision as default
cjdsellers Jan 15, 2025
1dabc1b
Refine CI pre-commit job
cjdsellers Jan 15, 2025
6bc2f4d
Refine high-precision feature flag naming
cjdsellers Jan 15, 2025
a29071c
Cleanup CI coverage workflow
cjdsellers Jan 15, 2025
1ed6fe8
Merge branch 'develop' into high-precision
cjdsellers Jan 15, 2025
045db4d
Cleanups
cjdsellers Jan 15, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ name: build
on:
push:
branches: [master, nightly, develop]
pull_request:
branches: [develop]
# pull_request: # TODO: Pause CI for now
# branches: [develop]

jobs:
pre-commit:
Expand Down
12 changes: 11 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,17 @@ cargo-test:
echo "cargo-nextest is not installed. You can install it using 'cargo install cargo-nextest'"; \
exit 1; \
fi
RUST_BACKTRACE=1 && (cd nautilus_core && cargo nextest run --workspace)
RUST_BACKTRACE=1 && (cd nautilus_core && cargo nextest run --workspace --features "python,ffi,high_precision")


.PHONY: cargo-test-low-precision
cargo-test-low-precision:
@if ! cargo nextest --version >/dev/null 2>&1; then \
echo "cargo-nextest is not installed. You can install it using 'cargo install cargo-nextest'"; \
exit 1; \
fi
RUST_BACKTRACE=1 && (cd nautilus_core && cargo nextest run --workspace --features "python,ffi")


.PHONY: cargo-test-coverage
cargo-test-coverage:
Expand Down
18 changes: 14 additions & 4 deletions build.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,13 @@
# If ANNOTATION mode is enabled, generate an annotated HTML version of the input source files
ANNOTATION_MODE = bool(os.getenv("ANNOTATION_MODE", ""))
# If PARALLEL build is enabled, uses all CPUs for compile stage of build
PARALLEL_BUILD = os.getenv("PARALLEL_BUILD", "true") == "true"
PARALLEL_BUILD = os.getenv("PARALLEL_BUILD", "true").lower() == "true"
# If COPY_TO_SOURCE is enabled, copy built *.so files back into the source tree
COPY_TO_SOURCE = os.getenv("COPY_TO_SOURCE", "true") == "true"
COPY_TO_SOURCE = os.getenv("COPY_TO_SOURCE", "true").lower() == "true"
# If PyO3 only then don't build C extensions to reduce compilation time
PYO3_ONLY = os.getenv("PYO3_ONLY", "") != ""
PYO3_ONLY = os.getenv("PYO3_ONLY", "").lower() != ""
# If high_precision mode is enabled (value types use 128-bit integers, rather than 64-bit)
HIGH_PRECISION = os.getenv("HIGH_PRECISION", "true").lower() == "true"

if PROFILE_MODE:
# For subsequent debugging, the C source needs to be in the same tree as
Expand Down Expand Up @@ -99,11 +101,17 @@ def _build_rust_libs() -> None:

build_options = " --release" if BUILD_MODE == "release" else ""

if HIGH_PRECISION:
features = ["--all-features"]
else:
# Enable features needed for main build, but not high_precision
features = ["--features", "ffi,python,extension-module"]

cmd_args = [
"cargo",
"build",
*build_options.split(),
"--all-features",
*features,
]

if RUST_TOOLCHAIN == "nightly":
Expand Down Expand Up @@ -143,6 +151,7 @@ def _build_rust_libs() -> None:
"profile": PROFILE_MODE, # If we're debugging or profiling
"linetrace": PROFILE_MODE, # If we're debugging or profiling
"warn.maybe_uninitialized": True,
"warn.deprecated.IF": False,
}


Expand Down Expand Up @@ -371,6 +380,7 @@ def build() -> None:
print(f"PARALLEL_BUILD={PARALLEL_BUILD}")
print(f"COPY_TO_SOURCE={COPY_TO_SOURCE}")
print(f"PYO3_ONLY={PYO3_ONLY}")
print(f"HIGH_PRECISION={HIGH_PRECISION}")
print(f"CC={os.environ['CC']}") if "CC" in os.environ else None
print(f"CXX={os.environ['CXX']}") if "CXX" in os.environ else None
print(f"LDSHARED={os.environ['LDSHARED']}") if "LDSHARED" in os.environ else None
Expand Down
Loading