diff --git a/Cargo.lock b/Cargo.lock index 579988f..a15f7c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,7 +34,7 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "betfair_data" -version = "0.2.0" +version = "0.2.1" dependencies = [ "bzip2-rs", "chrono", @@ -46,7 +46,6 @@ dependencies = [ "pyo3", "pyo3-log", "rayon", - "rayon-seq-iter", "self_cell", "serde", "serde_json", @@ -538,14 +537,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "rayon-seq-iter" -version = "0.1.0-SNAPSHOT" -source = "git+https://github.com/nwtgck/rayon-seq-iter#d3582ce853e3b205a880cb6862569934c9984feb" -dependencies = [ - "rayon", -] - [[package]] name = "redox_syscall" version = "0.2.10" diff --git a/Cargo.toml b/Cargo.toml index 23c7c92..d84f82b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "betfair_data" -version = "0.2.0" +version = "0.2.1" edition = "2021" [lib] @@ -23,7 +23,7 @@ chrono = "0.4.19" simdutf8 = { version = "0.1", features = ["std", "aarch64_neon"] } rayon = "1.5" flate2 = "1.0" -rayon-seq-iter = { git = "https://github.com/nwtgck/rayon-seq-iter" } +# rayon-seq-iter = { git = "https://github.com/nwtgck/rayon-seq-iter" } bzip2-rs = { git = "https://github.com/paolobarbolini/bzip2-rs", features = ["rayon", "nightly"]} ouroboros = "0.14.2" diff --git a/README.md b/README.md index 90d5858..daf1a23 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ Betfair Data is a very fast Betfair historical data file parsing library for python. It currently supports tar archives containing BZ2 compressed NLJSON files (the standard format provided by [Betfair's historic data portal](https://historicdata.betfair.com/#/home)). -The library is written in Rust and uses advanced performance enhancing techniques, like in place json deserialization and decompressing Bz2 encoded data on worker threads and is ideal for parsing large quantities of historic data that could otherwise take hours or days to parse. +The library is written in Rust and uses advanced performance enhancing techniques, like in place json deserialization and decompressing Bz2/Gzip encoded data on worker threads and is ideal for parsing large quantities of historic data that could otherwise take hours or days to parse. + +This library is a work in progress and is still subject to breaking changes. ## Installation @@ -38,80 +40,76 @@ for market in betfair_data.TarBz2(paths).mutable(): print(f"Markets {market_count} Updates {update_count}") ``` -## Types -IDE's should automatically detect the types and provide checking and auto complete. See the [pyi stub file](betfair_data.pyi) for a comprehensive view of the types and method available. - -
- -## Benchmarks - -| Betfair Data (this) | [Betfairlightweight](https://github.com/liampauling/betfair/) | -| ---------------------|---------------------| -| 3m 37sec | 1hour 1min 45sec | -| ~101 markets/sec | ~6 markets/sec | -| ~768,000 updates/sec | ~45,500 updates/sec | - -Benchmarks were run against 3 months of Australian racing markets comprising roughly 22,000 markets. Benchmarks were run on a M1 Macbook Pro with 32GB ram. -These results should only be used as a rough comparison, different machines, different sports and even different months can effect the performance and overall markets/updates per second. +## Loading Files -No disrespect is intended towards betfairlightweight, which remains an amazing library and a top choice for working with the Betfair API. Every effort was made to have its benchmark below run as fast as possible, and any improvements are welcome. - -
+You can read in self recorded stream files. Make sure to set cumulative_runner_tv to False for self recorded files to make sure you get the correct runner and market volumes. +```python +import betfair_data +import glob -Betfair_Data benchmark show in the example above. -
Betfairlightweight Benchmark +paths = glob.glob("data/*.gz") +files = betfair_data.Files(paths, cumulative_runner_tv=False) +``` +Or you can read official Betfair Tar archives with bz2 encoded market files. ```python -from typing import Sequence - -import unittest.mock -import tarfile -import bz2 -import betfairlightweight +import betfair_data +import glob -trading = betfairlightweight.APIClient("username", "password", "appkey") -listener = betfairlightweight.StreamListener( - max_latency=None, lightweight=True, update_clk=False, output_queue=None, cumulative_runner_tv=True, calculate_market_tv=True -) +paths = glob.glob("data/*.tar") +files = betfair_data.TarBz2(paths, cumulative_runner_tv=True) +``` -paths = [ - "data/2021_10_OctRacingAUPro.tar", - "data/2021_11_NovRacingAUPro.tar", - "data/2021_12_DecRacingAUPro.tar" -] +Or load the file through any other means and pass the bytes and name into the object constructors. -def load_tar(file_paths: Sequence[str]): - for file_path in file_paths: - with tarfile.TarFile(file_path) as archive: - for file in archive: - yield bz2.open(archive.extractfile(file)) - return None +```python +# generator to read in files +def load_files(paths: str): + for path in glob.glob(paths, recursive=True): + with open(path, "rb") as file: + yield (path, file.read()) + +# iterate over the files and convert into bflw iterator +for name, bs in load_files("markets/*.json"): + for market_books in bflw.BflwIter(name, bs): + for market_book in market_books: + # do stuff + pass +``` -market_count = 0 -update_count = 0 +## Object Types -for file_obj in load_tar(paths): - with unittest.mock.patch("builtins.open", lambda f, _: f): - stream = trading.streaming.create_historical_generator_stream( - file_path=file_obj, - listener=listener, - ) - gen = stream.get_generator() +You can use differnt styles of objects, with pros or depending on your needs - market_count += 1 - for market_books in gen(): - for market_book in market_books: - update_count += 1 +Mutable objects, generally the fastest, but can be hard to use. If you find yourself calling market.copy a lot, you may find immutable faster +``` python +# where files is loaded from a TarBz2 or Files source like above +mut_iter = files.mutable() +for market in mut_iter: # different markets per file + while market.update(): # update the market in place + pass +``` - print(f"Markets {market_count} Updates {update_count}", end='\r') -print(f"Markets {market_count} Updates {update_count}") +Immutable objects, slightly slower but can be easier to use. Equilivent of calling market.copy() on every update but faster, as only objects that change make new copies. ```NOT YET FINISHED``` +``` python +immut_iter = files.immutable() +for market_iter in immut_iter: # different files + for market in market_iter: # each update of a market/file + pass +``` +Betfairlightweight compatible version, drop in replacement for bflw objects. +```python +bflw_iter = files.bflw() +for file in bflw_iter: # different files + for market_books in file: # different books per update + for market in market_books: # each update of a market + pass ``` -
-
-
+## Types +IDE's should automatically detect the types and provide checking and auto complete. See the [pyi stub file](betfair_data.pyi) for a comprehensive view of the types and method available. ## Logging diff --git a/betfair_data/betfair_data.abi3.so b/betfair_data/betfair_data.abi3.so index 73fcd7c..8978735 100755 Binary files a/betfair_data/betfair_data.abi3.so and b/betfair_data/betfair_data.abi3.so differ diff --git a/pyproject.toml b/pyproject.toml index 76146cf..2f2fc46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "maturin" [project] name = "betfair_data" requires-python = ">=3.6" -version = "0.2.0" +version = "0.2.1" description = "Fast Python Betfair historic data file parser" authors = [ {name = "Robert Tarbath", email = "rtarbath@gmail.com"} ] license = {file = "LICENSE"} diff --git a/src/bflw/mod.rs b/src/bflw/mod.rs index 84275da..b9aeacc 100644 --- a/src/bflw/mod.rs +++ b/src/bflw/mod.rs @@ -5,7 +5,6 @@ pub mod market_definition; pub mod market_definition_runner; pub mod runner_book; mod float_str; -mod runner_book_ex; mod runner_book_sp; diff --git a/src/bflw/runner_book.rs b/src/bflw/runner_book.rs index 032100a..ca6e48f 100644 --- a/src/bflw/runner_book.rs +++ b/src/bflw/runner_book.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Deserializer}; use serde_json::value::RawValue; use super::market_definition_runner::MarketDefRunnerUpdate; -use super::runner_book_ex::{RunnerBookEX, RunnerBookEXUpdate}; +use crate::immutable::runner_book_ex::{RunnerBookEX, RunnerBookEXUpdate}; use super::runner_book_sp::{RunnerBookSP, RunnerBookSPUpdate}; use crate::bflw::float_str::FloatStr; use crate::bflw::RoundToCents; diff --git a/src/bflw/runner_book_ex.rs b/src/bflw/runner_book_ex.rs deleted file mode 100644 index 534f6e4..0000000 --- a/src/bflw/runner_book_ex.rs +++ /dev/null @@ -1,56 +0,0 @@ -use std::sync::Arc; - -use pyo3::prelude::*; - -use crate::immutable::container::SyncObj; -use crate::price_size::PriceSize; - -#[derive(Clone, Default)] -#[pyclass] -pub struct RunnerBookEX { - pub available_to_back: SyncObj>>, - pub available_to_lay: SyncObj>>, - pub traded_volume: SyncObj>>, -} - -pub struct RunnerBookEXUpdate { - pub available_to_back: Option>, - pub available_to_lay: Option>, - pub traded_volume: Option>, -} - -impl RunnerBookEX { - pub fn update(&self, update: RunnerBookEXUpdate, py: Python) -> Py { - Py::new( - py, - Self { - available_to_back: update - .available_to_back - .map_or_else(|| self.available_to_back.clone(), |ps| SyncObj::new(Arc::new(ps))), - available_to_lay: update - .available_to_lay - .map_or_else(|| self.available_to_lay.clone(), |ps| SyncObj::new(Arc::new(ps))), - traded_volume: update - .traded_volume - .map_or_else(|| self.traded_volume.clone(), |ps| SyncObj::new(Arc::new(ps))), - }, - ) - .unwrap() - } -} - -#[pymethods] -impl RunnerBookEX { - #[getter(available_to_back)] - fn get_available_to_back(&self, py: Python) -> PyObject { - self.available_to_back.to_object(py) - } - #[getter(available_to_lay)] - fn get_available_to_lay(&self, py: Python) -> PyObject { - self.available_to_lay.to_object(py) - } - #[getter(traded_volume)] - fn get_traded_volume(&self, py: Python) -> PyObject { - self.traded_volume.to_object(py) - } -} diff --git a/src/immutable/datetime.rs b/src/immutable/datetime.rs index 4f63ce6..1015701 100644 --- a/src/immutable/datetime.rs +++ b/src/immutable/datetime.rs @@ -80,7 +80,7 @@ impl PartialEq for &str { } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, Default)] pub struct DateTime(u64); impl DateTime { diff --git a/src/immutable/definition.rs b/src/immutable/definition.rs new file mode 100644 index 0000000..6bb4be1 --- /dev/null +++ b/src/immutable/definition.rs @@ -0,0 +1,903 @@ +// use core::fmt; +// use pyo3::prelude::*; +// use serde::de::{DeserializeSeed, MapAccess, Visitor}; +// use serde::{de, Deserialize, Deserializer}; +// use std::borrow::Cow; +// use std::sync::Arc; + +// use super::runner_book_ex::RunnerBookEX; +// use super::runner_book_sp::{RunnerBookSP, RunnerBookSPUpdate}; +// use crate::enums::{MarketBettingType, MarketStatus, SelectionStatus}; +// use crate::ids::{EventID, EventTypeID, SelectionID}; +// use crate::immutable::container::SyncObj; +// use crate::immutable::datetime::DateTimeString; +// use crate::immutable::market::PyMarket; +// use crate::immutable::runner::PyRunner; +// use crate::market_source::SourceConfig; +// use crate::price_size::F64OrStr; +// use crate::strings::FixedSizeString; + +// #[derive(Debug, Default)] +// pub struct MarketDefinitionUpdate<'a> { +// bet_delay: Option, +// betting_type: Option, +// bsp_market: Option, +// bsp_reconciled: Option, +// complete: Option, +// cross_matching: Option, +// discount_allowed: Option, +// event_id: Option, +// event_type_id: Option, +// in_play: Option, +// market_base_rate: Option, +// market_time: Option<&'a str>, +// market_type: Option<&'a str>, +// number_of_active_runners: Option, +// number_of_winners: Option, +// open_date: Option<&'a str>, +// persistence_enabled: Option, +// regulators: Option>, +// runners_voidable: Option, +// settled_time: Option<&'a str>, +// status: Option, +// suspend_time: Option<&'a str>, +// timezone: Option<&'a str>, +// turn_in_play_enabled: Option, +// venue: Option<&'a str>, +// version: Option, +// country_code: Option<&'a str>, +// market_name: Option>, +// event_name: Option>, +// } + +// impl<'a, 'b> MarketDefinitionUpdate<'a> { +// fn update(self, next: &'b mut Option) { +// match next { +// Some(market) => { +// market.bet_delay = self.bet_delay.unwrap_or(market.bet_delay); +// market.betting_type = self.betting_type.unwrap_or(market.betting_type); +// market.bsp_market = self.bsp_market.unwrap_or(market.bsp_market); +// market.bsp_reconciled = self.bsp_reconciled.unwrap_or(market.bsp_reconciled); +// market.complete = self.complete.unwrap_or(market.complete); +// market.cross_matching = self.cross_matching.unwrap_or(market.cross_matching); +// market.discount_allowed = self.discount_allowed.unwrap_or(market.discount_allowed); +// market.event_id = self.event_id.unwrap_or(market.event_id); +// market.event_type_id = self.event_type_id.unwrap_or(market.event_type_id); +// market.in_play = self.in_play.unwrap_or(market.in_play); +// market.market_base_rate = self.market_base_rate.unwrap_or(market.market_base_rate); +// market.number_of_winners = +// self.number_of_winners.unwrap_or(market.number_of_winners); +// market.persistence_enabled = self +// .persistence_enabled +// .unwrap_or(market.persistence_enabled); +// market.runners_voidable = self.runners_voidable.unwrap_or(market.runners_voidable); +// market.version = self.version.unwrap_or(market.version); +// market.status = self.status.unwrap_or(market.status); +// market.turn_in_play_enabled = self +// .turn_in_play_enabled +// .unwrap_or(market.turn_in_play_enabled); +// market.number_of_active_runners = self +// .number_of_active_runners +// .unwrap_or(market.number_of_active_runners); +// market.market_time = self +// .market_time +// .map(|s| SyncObj::new(DateTimeString::new(s).unwrap())) +// .unwrap_or_else(|| market.market_time.clone()); +// market.market_type = self +// .market_type +// .map(|s| SyncObj::new(Arc::new(String::from(s)))) +// .unwrap_or_else(|| market.market_type.clone()); +// market.regulators = self +// .regulators +// .map(|v| SyncObj::new(Arc::new(v.iter().map(|s| s.to_string()).collect()))) +// .unwrap_or_else(|| market.regulators.clone()); +// market.timezone = self +// .timezone +// .map(|s| SyncObj::new(Arc::new(String::from(s)))) +// .unwrap_or_else(|| market.timezone.clone()); +// market.venue = self +// .venue +// .map(|s| Some(SyncObj::new(Arc::new(String::from(s))))) +// .unwrap_or_else(|| market.venue.clone()); +// market.country_code = self +// .country_code +// .map(|s| SyncObj::new(FixedSizeString::try_from(s).unwrap())) // todo +// .unwrap_or_else(|| market.country_code.clone()); +// market.open_date = self +// .open_date +// .map(|s| SyncObj::new(DateTimeString::new(s).unwrap())) +// .unwrap_or_else(|| market.open_date.clone()); +// market.settled_time = self +// .settled_time +// .map(|s| SyncObj::new(DateTimeString::new(s).unwrap())) +// .or_else(|| market.settled_time.clone()); +// market.suspend_time = self +// .suspend_time +// .map(|s| SyncObj::new(DateTimeString::new(s).unwrap())) +// .or_else(|| market.suspend_time.clone()); +// market.market_name = self +// .market_name +// .map(|s| SyncObj::new(Arc::new(s.into_owned()))) +// .or_else(|| market.market_name.clone()); +// market.event_name = self +// .event_name +// .map(|s| SyncObj::new(Arc::new(s.into_owned()))) +// .or_else(|| market.event_name.clone()); +// } +// None => { +// *next = Some(PyMarket { +// market_id: Default::default(), +// file: Default::default(), +// clk: Default::default(), +// publish_time: Default::default(), +// each_way_divisor: Default::default(), +// total_matched: Default::default(), +// runners: Default::default(), + +// bet_delay: self.bet_delay.unwrap_or_default(), +// betting_type: self.betting_type.unwrap_or_default(), +// regulators: self +// .regulators +// .map(|v| SyncObj::new(Arc::new(v.iter().map(|s| s.to_string()).collect()))) +// .unwrap_or_default(), +// bsp_reconciled: self.bsp_reconciled.unwrap_or_default(), +// bsp_market: self.bsp_market.unwrap_or_default(), +// complete: self.complete.unwrap_or_default(), +// cross_matching: self.cross_matching.unwrap_or_default(), +// discount_allowed: self.discount_allowed.unwrap_or_default(), +// event_id: self.event_id.unwrap_or_default(), +// event_type_id: self.event_type_id.unwrap_or_default(), +// in_play: self.in_play.unwrap_or_default(), +// market_base_rate: self.market_base_rate.unwrap_or_default(), +// number_of_winners: self.number_of_winners.unwrap_or_default(), +// persistence_enabled: self.persistence_enabled.unwrap_or_default(), +// runners_voidable: self.runners_voidable.unwrap_or_default(), +// version: self.version.unwrap_or_default(), +// status: self.status.unwrap_or_default(), +// turn_in_play_enabled: self.turn_in_play_enabled.unwrap_or_default(), +// number_of_active_runners: self.number_of_active_runners.unwrap_or_default(), +// market_time: self +// .market_time +// .map(|s| SyncObj::new(DateTimeString::new(s).unwrap())) +// .unwrap(), +// market_type: self +// .market_type +// .map(|s| SyncObj::new(Arc::new(String::from(s)))) +// .unwrap(), +// timezone: self +// .timezone +// .map(|s| SyncObj::new(Arc::new(String::from(s)))) +// .unwrap(), +// venue: self.venue.map(|s| SyncObj::new(Arc::new(String::from(s)))), +// country_code: self +// .country_code +// .map(|s| SyncObj::new(FixedSizeString::try_from(s).unwrap())) +// .unwrap(), // todo +// open_date: self +// .open_date +// .map(|s| SyncObj::new(DateTimeString::new(s).unwrap())) +// .unwrap(), +// settled_time: self +// .settled_time +// .map(|s| SyncObj::new(DateTimeString::new(s).unwrap())), +// suspend_time: self +// .suspend_time +// .map(|s| SyncObj::new(DateTimeString::new(s).unwrap())), +// market_name: self +// .market_name +// .map(|s| SyncObj::new(Arc::new(s.into_owned()))), +// event_name: self +// .event_name +// .map(|s| SyncObj::new(Arc::new(s.into_owned()))), +// }); +// } +// } +// } +// } + +// pub struct MarketDefinitionDeser<'a, 'py> { +// pub market: Option>, +// pub next: &'a mut Option, +// pub next_runners: &'a mut Option>>, +// pub py: Python<'py>, +// pub config: SourceConfig, +// } +// impl<'de, 'a, 'py> DeserializeSeed<'de> for MarketDefinitionDeser<'a, 'py> { +// type Value = (); + +// fn deserialize(self, deserializer: D) -> Result +// where +// D: Deserializer<'de>, +// { +// #[derive(Debug, Deserialize)] +// #[serde(field_identifier, rename_all = "camelCase")] +// enum Field { +// BetDelay, +// BettingType, +// BspMarket, +// BspReconciled, +// Complete, +// CountryCode, +// CrossMatching, +// DiscountAllowed, +// EachWayDivisor, +// EventId, +// EventName, +// EventTypeId, +// InPlay, +// KeyLineDefiniton, +// LineMaxUnit, +// LineMinUnit, +// LineInterval, +// MarketBaseRate, +// MarketTime, +// MarketType, +// Name, +// NumberOfActiveRunners, +// NumberOfWinners, +// OpenDate, +// PersistenceEnabled, +// PriceLadderDefinition, +// RaceType, +// Regulators, +// Runners, +// RunnersVoidable, +// SettledTime, +// Status, +// SuspendTime, +// Timezone, +// TurnInPlayEnabled, +// Venue, +// Version, +// } + +// struct MarketDefinitionVisitor<'a, 'py> { +// market: Option>, +// next: &'a mut Option, +// next_runners: &'a mut Option>>, +// py: Python<'py>, +// config: SourceConfig, +// } +// impl<'de, 'a, 'py> Visitor<'de> for MarketDefinitionVisitor<'a, 'py> { +// type Value = (); + +// fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { +// formatter.write_str("") +// } + +// fn visit_map(self, mut map: V) -> Result +// where +// V: MapAccess<'de>, +// { +// let mut upt: MarketDefinitionUpdate = MarketDefinitionUpdate::default(); +// let mut changed = false; + +// while let Some(key) = map.next_key()? { +// match key { +// Field::Runners => { +// let rs: Option<&[Py]> = +// self.market.as_ref().map(|m| (**m.runners).as_ref()); +// map.next_value_seed(RunnerDefSeq { +// runners: rs, +// next: self.next_runners, +// py: self.py, +// config: self.config, +// })?; +// } +// Field::BspMarket => { +// let bsp_market = map.next_value()?; +// if self.market.is_some_with(|def| def.bsp_market != bsp_market) +// || self.market.is_none() +// { +// upt.bsp_market = Some(bsp_market); +// changed = true; +// } +// } +// Field::Name => { +// let market_name = map.next_value::>()?; +// if self.market.is_some_with(|def| { +// !def.market_name.contains(&market_name.as_ref()) +// }) || self.market.is_none() +// { +// upt.market_name = Some(market_name); +// changed = true; +// } +// } +// Field::TurnInPlayEnabled => { +// let turn_in_play_enabled = map.next_value()?; +// if self.market.is_some_with(|def| { +// def.turn_in_play_enabled != turn_in_play_enabled +// }) || self.market.is_none() +// { +// upt.turn_in_play_enabled = Some(turn_in_play_enabled); +// changed = true; +// } +// } +// Field::InPlay => { +// let in_play = map.next_value()?; +// if self.market.is_some_with(|def| def.in_play != in_play) +// || self.market.is_none() +// { +// upt.in_play = Some(in_play); +// changed = true; +// } +// } +// Field::PersistenceEnabled => { +// let persistence_enabled = map.next_value()?; +// if self +// .market +// .is_some_with(|def| def.persistence_enabled != persistence_enabled) +// || self.market.is_none() +// { +// upt.persistence_enabled = Some(persistence_enabled); +// changed = true; +// } +// } +// Field::BspReconciled => { +// let bsp_reconciled = map.next_value()?; +// if self +// .market +// .is_some_with(|def| def.bsp_reconciled != bsp_reconciled) +// || self.market.is_none() +// { +// upt.bsp_reconciled = Some(bsp_reconciled); +// changed = true; +// } +// } +// Field::Complete => { +// let complete = map.next_value()?; +// if self.market.is_some_with(|def| def.complete != complete) +// || self.market.is_none() +// { +// upt.complete = Some(complete); +// changed = true; +// } +// } +// Field::CrossMatching => { +// let cross_matching = map.next_value()?; +// if self +// .market +// .is_some_with(|def| def.cross_matching != cross_matching) +// || self.market.is_none() +// { +// upt.cross_matching = Some(cross_matching); +// changed = true; +// } +// } +// Field::RunnersVoidable => { +// let runners_voidable = map.next_value()?; +// if self +// .market +// .is_some_with(|def| def.runners_voidable != runners_voidable) +// || self.market.is_none() +// { +// upt.runners_voidable = Some(runners_voidable); +// changed = true; +// } +// } +// Field::DiscountAllowed => { +// let discount_allowed = map.next_value()?; +// if self +// .market +// .is_some_with(|def| def.discount_allowed != discount_allowed) +// || self.market.is_none() +// { +// upt.discount_allowed = Some(discount_allowed); +// changed = true; +// } +// } +// Field::Timezone => { +// let timezone = map.next_value::<&str>()?; +// if self +// .market +// .is_some_with(|def| def.timezone.as_str() != timezone) +// || self.market.is_none() +// { +// upt.timezone = Some(timezone); +// changed = true; +// } +// } + +// Field::EventName => { +// let event_name = map.next_value::>()?; +// if self +// .market +// .is_some_with(|def| !def.event_name.contains(&event_name.as_ref())) +// || self.market.is_none() +// { +// upt.event_name = Some(event_name); +// changed = true; +// } +// } +// Field::CountryCode => { +// let country_code = map.next_value::<&str>()?; +// if self +// .market +// .is_some_with(|def| !def.country_code.contains(&country_code)) +// || self.market.is_none() +// { +// upt.country_code = Some(country_code); +// changed = true; +// } +// } +// Field::Venue => { +// let venue = map.next_value::<&str>()?; +// if self.market.is_some_with(|def| !def.venue.contains(&venue)) +// || self.market.is_none() +// { +// upt.venue = Some(venue); +// changed = true; +// } +// } +// Field::Status => { +// let status = map.next_value()?; +// if self.market.is_some_with(|def| def.status != status) +// || self.market.is_none() +// { +// upt.status = Some(status); +// changed = true; +// } +// } +// Field::MarketBaseRate => { +// let market_base_rate = map.next_value::()?; +// if self +// .market +// .is_some_with(|def| def.market_base_rate != market_base_rate) +// || self.market.is_none() +// { +// upt.market_base_rate = Some(market_base_rate); +// changed = true; +// } +// } +// Field::NumberOfWinners => { +// let number_of_winners = map.next_value::()? as u8; +// if self +// .market +// .is_some_with(|def| def.number_of_winners != number_of_winners) +// || self.market.is_none() +// { +// upt.number_of_winners = Some(number_of_winners); +// changed = true; +// } +// } +// Field::NumberOfActiveRunners => { +// let number_of_active_runners = map.next_value()?; +// if self.market.is_some_with(|def| { +// def.number_of_active_runners != number_of_active_runners +// }) || self.market.is_none() +// { +// upt.number_of_active_runners = Some(number_of_active_runners); +// changed = true; +// } +// } +// Field::BetDelay => { +// let bet_delay = map.next_value()?; +// if self.market.is_some_with(|def| def.bet_delay != bet_delay) +// || self.market.is_none() +// { +// upt.bet_delay = Some(bet_delay); +// changed = true; +// } +// } +// Field::EventId => { +// let event_id = map +// .next_value::<&str>()? +// .parse() +// .map_err(de::Error::custom)?; +// if self.market.is_some_with(|def| def.event_id != event_id) +// || self.market.is_none() +// { +// upt.event_id = Some(event_id); +// changed = true; +// } +// } +// Field::EventTypeId => { +// let event_type_id = map +// .next_value::<&str>()? +// .parse() +// .map_err(de::Error::custom)?; +// if self +// .market +// .is_some_with(|def| def.event_type_id != event_type_id) +// || self.market.is_none() +// { +// upt.event_type_id = Some(event_type_id); +// changed = true; +// } +// } +// Field::Version => { +// let version = map.next_value()?; +// if self.market.is_some_with(|def| def.version != version) +// || self.market.is_none() +// { +// upt.version = Some(version); +// changed = true; +// } +// } +// Field::MarketType => { +// let market_type = map.next_value::<&str>()?; +// if self +// .market +// .is_some_with(|def| def.market_type.as_str() != market_type) +// || self.market.is_none() +// { +// upt.market_type = Some(market_type); +// changed = true; +// } +// } +// Field::BettingType => { +// let betting_type = map.next_value()?; +// if self +// .market +// .is_some_with(|def| def.betting_type != betting_type) +// { +// upt.betting_type = Some(betting_type); +// changed = true; +// } +// } + +// Field::MarketTime => { +// let market_time = map.next_value()?; +// if self +// .market +// .is_some_with(|def| def.market_time.as_str() != market_time) +// || self.market.is_none() +// { +// upt.market_time = Some(market_time); +// changed = true; +// } +// } +// Field::SuspendTime => { +// let suspend_time = map.next_value::<&str>()?; +// if !self +// .market +// .is_some_with(|def| def.suspend_time.contains(&suspend_time)) +// || self.market.is_none() +// { +// upt.suspend_time = Some(suspend_time); +// changed = true; +// } +// } +// Field::SettledTime => { +// let settled_time = map.next_value::<&str>()?; +// if !self +// .market +// .is_some_with(|def| def.settled_time.contains(&settled_time)) +// || self.market.is_none() +// { +// upt.settled_time = Some(settled_time); +// changed = true; +// } +// } +// Field::OpenDate => { +// let open_date = map.next_value()?; +// if self +// .market +// .is_some_with(|def| def.open_date.as_str() != open_date) +// || self.market.is_none() +// { +// upt.open_date = Some(open_date); +// changed = true; +// } +// } + +// Field::Regulators => { +// let v = map.next_value::>()?; + +// if self.market.is_some_with(|def| { +// (def.regulators.is_empty() && !v.is_empty()) +// || !def.regulators.iter().eq(v.iter()) +// }) || self.market.is_none() +// { +// upt.regulators = Some(v); +// changed = true; +// } +// } + +// Field::EachWayDivisor => { +// map.next_value::()?; +// } +// Field::RaceType => { +// map.next_value::()?; +// } +// Field::KeyLineDefiniton => { +// map.next_value::()?; +// } +// Field::PriceLadderDefinition => { +// map.next_value::()?; +// } +// Field::LineMaxUnit => { +// map.next_value::()?; +// } +// Field::LineMinUnit => { +// map.next_value::()?; +// } +// Field::LineInterval => { +// map.next_value::()?; +// } +// } +// } + +// if changed { +// upt.update(self.next); +// } + +// Ok(()) +// } +// } + +// const FIELDS: &[&str] = &[ +// "keyLineDefiniton", +// "priceLadderDefinition", +// "raceType", +// "lineMaxUnit", +// "lineMinUnit", +// "lineInterval", +// "bspMarket", +// "turnInPlayEnabled", +// "persistenceEnabled", +// "marketBaseRate", +// "eventId", +// "eventTypeId", +// "numberOfWinners", +// "bettingType", +// "marketType", +// "marketTime", +// "suspendTime", +// "bspReconciled", +// "complete", +// "inPlay", +// "crossMatching", +// "runnersVoidable", +// "numberOfActiveRunners", +// "betDelay", +// "status", +// "runners", +// "regulators", +// "countryCode", +// "discountAllowed", +// "timezone", +// "openDate", +// "version", +// "name", +// "eventName", +// "venue", +// "settledTime", +// "eachWayDivisor", +// ]; + +// deserializer.deserialize_struct( +// "MarketDefinition", +// FIELDS, +// MarketDefinitionVisitor { +// market: self.market, +// next: self.next, +// next_runners: self.next_runners, +// py: self.py, +// config: self.config, +// }, +// ) +// } +// } + +// pub struct RunnerDefSeq<'a, 'py> { +// pub runners: Option<&'a [Py]>, +// pub next: &'a mut Option>>, +// pub py: Python<'py>, +// pub config: SourceConfig, +// } +// impl<'de, 'a, 'py> DeserializeSeed<'de> for RunnerDefSeq<'a, 'py> { +// type Value = (); + +// fn deserialize(self, deserializer: D) -> Result +// where +// D: Deserializer<'de>, +// { +// struct RunnerSeqVisitor<'a, 'py> { +// runners: Option<&'a [Py]>, +// next: &'a mut Option>>, +// py: Python<'py>, +// #[allow(dead_code)] +// config: SourceConfig, +// } +// impl<'de, 'a, 'py> Visitor<'de> for RunnerSeqVisitor<'a, 'py> { +// type Value = (); + +// fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { +// formatter.write_str("") +// } + +// fn visit_seq(self, mut seq: A) -> Result +// where +// A: serde::de::SeqAccess<'de>, +// { +// while let Some(change) = seq.next_element::()? { +// let index = self.runners.and_then(|rs| { +// rs.iter() +// .map(|r| r.borrow_mut(self.py)) +// .position(|r| r.selection_id == change.id) +// }); + +// match (self.runners, index) { +// (Some(from), Some(index)) => { +// let runner = unsafe { from.get_unchecked(index).borrow(self.py) }; + +// if change.diff(&runner, self.py) { +// match self.next.as_mut() { +// // TODO handle previous initilization on self +// Some(defs) => { +// defs.push( +// Py::new(self.py, change.update(&runner, self.py)) +// .unwrap(), +// ); +// } +// None => { +// *self.next = { +// let mut next = Vec::with_capacity(from.len() + 1); +// next.push( +// Py::new(self.py, change.update(&runner, self.py)) +// .unwrap(), +// ); +// Some(next) +// }; +// } +// }; +// } +// } +// (Some(from), None) => { +// let runner = Py::new(self.py, change.create(self.py)).unwrap(); + +// match self.next.as_mut() { +// Some(next) => next.push(runner), +// None => { +// let mut d = +// Vec::with_capacity(std::cmp::max(from.len() + 1, 10)); +// d.push(runner); +// *self.next = Some(d); +// } +// }; +// } +// (None, None) => { +// let runner = Py::new(self.py, change.create(self.py)).unwrap(); + +// match self.next.as_mut() { +// Some(next) => next.push(runner), +// None => { +// let mut d = Vec::with_capacity(10); +// d.push(runner); +// *self.next = Some(d); +// } +// }; +// } +// _ => unreachable!(), +// } +// } + +// Ok(()) +// } +// } + +// deserializer.deserialize_seq(RunnerSeqVisitor { +// runners: self.runners, +// next: self.next, +// py: self.py, +// config: self.config, +// }) +// } +// } + +// #[derive(Deserialize)] +// #[serde(rename_all = "camelCase")] +// struct RunnerDefUpdate<'a> { +// id: SelectionID, +// adjustment_factor: Option, +// status: SelectionStatus, +// sort_priority: u16, +// name: Option<&'a str>, +// bsp: Option, +// removal_date: Option<&'a str>, +// hc: Option, +// } + +// impl<'a> RunnerDefUpdate<'a> { +// fn create(&self, py: Python) -> PyRunner { +// let sp = RunnerBookSP { +// actual_sp: self.bsp.map(|f| *f), +// ..Default::default() +// }; + +// PyRunner { +// selection_id: self.id, +// status: self.status, +// adjustment_factor: self.adjustment_factor, +// handicap: self.hc.map(|f| *f), +// sort_priority: self.sort_priority, +// name: self.name.map(|s| SyncObj::new(Arc::new(String::from(s)))), +// removal_date: self +// .removal_date +// .map(|s| SyncObj::new(DateTimeString::new(s).unwrap())), +// sp: Py::new(py, sp).unwrap(), +// ex: Py::new(py, RunnerBookEX::default()).unwrap(), +// total_matched: 0.0, +// last_price_traded: None, +// } +// } + +// fn diff(&self, runner: &PyRunner, py: Python) -> bool { +// runner.status != self.status +// || runner.adjustment_factor != self.adjustment_factor +// || runner.sort_priority != self.sort_priority +// || runner.sp.borrow(py).actual_sp != self.bsp.map(|f| *f) +// || runner.handicap != self.hc.map(|f| *f) +// || ((runner.name.is_none() && self.name.is_some()) +// || runner +// .name +// .is_some_with(|s| !self.name.contains(&s.as_str()))) +// || ((runner.removal_date.is_none() && self.removal_date.is_some()) +// || runner +// .removal_date +// .is_some_with(|s| !self.removal_date.contains(&s.as_str()))) +// } + +// fn update(&self, runner: &PyRunner, py: Python) -> PyRunner { +// let (ex, sp) = if self.status == SelectionStatus::Removed +// || self.status == SelectionStatus::RemovedVacant +// { +// ( +// Py::new(py, RunnerBookEX::default()).unwrap(), +// runner.sp.clone_ref(py), +// ) +// } else if self.bsp.is_some() { +// // need to update sp obj with bsp value +// let sp = runner.sp.borrow(py); +// if sp.actual_sp != self.bsp.map(|f| *f) { +// let upt = RunnerBookSPUpdate { +// actual_sp: self.bsp.map(|f| *f), +// ..Default::default() +// }; +// (runner.ex.clone_ref(py), sp.update(upt, py)) +// } else { +// (runner.ex.clone_ref(py), runner.sp.clone_ref(py)) +// } +// } else { +// (runner.ex.clone_ref(py), runner.sp.clone_ref(py)) +// }; + +// PyRunner { +// selection_id: runner.selection_id, +// status: self.status, +// adjustment_factor: self.adjustment_factor.or(runner.adjustment_factor), +// handicap: self.hc.map(|f| *f).or(runner.handicap), +// sort_priority: if runner.sort_priority != self.sort_priority { +// self.sort_priority +// } else { +// runner.sort_priority +// }, +// name: self +// .name +// .and_then(|n| { +// if runner.name.contains(&n) { +// runner.name.clone() +// } else { +// Some(SyncObj::new(Arc::new(String::from(n)))) +// } +// }) +// .or_else(|| runner.name.clone()), + +// removal_date: self +// .removal_date +// .and_then(|n| { +// if runner.removal_date.contains(&n) { +// runner.removal_date.clone() +// } else { +// Some(SyncObj::new(DateTimeString::new(n).unwrap())) +// } +// }) +// .or_else(|| runner.removal_date.clone()), +// total_matched: runner.total_matched, +// last_price_traded: runner.last_price_traded, +// sp, +// ex, +// } +// } +// } diff --git a/src/immutable/market.rs b/src/immutable/market.rs index 2713532..4a182a0 100644 --- a/src/immutable/market.rs +++ b/src/immutable/market.rs @@ -1,141 +1,123 @@ -// use log::warn; -// use pyo3::{exceptions, prelude::*}; -// use serde::de::{Error, IgnoredAny}; -// use serde::{ -// de::{self, DeserializeSeed, MapAccess, Visitor}, -// Deserialize, Deserializer, -// }; -// use serde_json::value::RawValue; +// use core::fmt; // use std::borrow::Cow; -// use std::fmt; // use std::path::PathBuf; // use std::sync::Arc; +// use pyo3::{prelude::*}; +// use serde::{Deserializer, Deserialize}; +// use serde::de::{DeserializeSeed, Visitor, MapAccess, IgnoredAny, Error}; +// use serde_json::value::RawValue; -// use crate::deser::DeserializerWithData; // use crate::enums::{MarketBettingType, MarketStatus}; -// use crate::errors::DeserErr; // use crate::ids::{EventID, EventTypeID, MarketID}; -// use crate::market_source::SourceItem; -// use crate::mutable::runner::{PyRunner, PyRunnerChangeSeq, PyRunnerDefSeq}; -// use crate::strings::{StringSetExtNeq, FixedSizeString}; - -// use super::config::Config; +// use crate::immutable::datetime::DateTime; +// use crate::immutable::definition::MarketDefinitionDeser; +// use crate::immutable::runner::RunnerChangeSeq; +// use crate::market_source::SourceConfig; +// use crate::strings::{FixedSizeString}; +// use super::definition::MarketDefinitionUpdate; +// use super::runner::PyRunner; // use super::container::SyncObj; // use super::datetime::DateTimeString; // #[pyclass(name = "MarketImage")] -// pub struct PyMarketBase { +// pub struct PyMarket { // #[pyo3(get)] -// file: SyncObj, +// pub file: SyncObj, // #[pyo3(get)] -// bet_delay: u16, +// pub bet_delay: u16, // #[pyo3(get)] -// bsp_market: bool, +// pub bsp_market: bool, // #[pyo3(get)] -// bsp_reconciled: bool, +// pub bsp_reconciled: bool, // #[pyo3(get)] -// clk: SyncObj>, +// pub clk: SyncObj>, // #[pyo3(get)] -// complete: bool, +// pub complete: bool, // #[pyo3(get)] -// cross_matching: bool, +// pub cross_matching: bool, // #[pyo3(get)] -// discount_allowed: bool, +// pub discount_allowed: bool, // #[pyo3(get)] -// each_way_divisor: Option, +// pub each_way_divisor: Option, // #[pyo3(get)] -// event_id: EventID, +// pub event_id: EventID, // #[pyo3(get)] -// event_name: Option>>, +// pub event_name: Option>>, // #[pyo3(get)] -// event_type_id: EventTypeID, +// pub event_type_id: EventTypeID, // #[pyo3(get)] -// in_play: bool, +// pub in_play: bool, // #[pyo3(get)] -// market_base_rate: u8, +// pub market_base_rate: f32, // #[pyo3(get)] -// market_type: SyncObj, +// pub market_type: SyncObj>, // #[pyo3(get)] -// market_name: Option>>, +// pub market_name: Option>>, // #[pyo3(get)] -// number_of_active_runners: u16, +// pub number_of_active_runners: u16, // #[pyo3(get)] -// number_of_winners: u8, +// pub number_of_winners: u8, // #[pyo3(get)] -// persistence_enabled: bool, +// pub persistence_enabled: bool, // #[pyo3(get)] -// publish_time: u64, +// pub publish_time: DateTime, // #[pyo3(get)] -// runners_voidable: bool, +// pub runners_voidable: bool, // #[pyo3(get)] -// timezone: SyncObj, +// pub timezone: SyncObj>, // #[pyo3(get)] -// total_matched: f64, +// pub total_matched: f64, // #[pyo3(get)] -// turn_in_play_enabled: bool, +// pub turn_in_play_enabled: bool, // #[pyo3(get)] -// venue: Option>>, +// pub venue: Option>>, // #[pyo3(get)] -// version: u64, +// pub version: u64, // #[pyo3(get)] -// runners: SyncObj>>>, +// pub runners: SyncObj>>>, // #[pyo3(get)] -// status: MarketStatus, +// pub status: MarketStatus, // #[pyo3(get)] -// betting_type: MarketBettingType, +// pub betting_type: MarketBettingType, // #[pyo3(get)] -// market_time: SyncObj, +// pub market_time: SyncObj, // #[pyo3(get)] -// open_date: SyncObj, +// pub open_date: SyncObj, // #[pyo3(get)] -// suspend_time: Option>, +// pub suspend_time: Option>, // #[pyo3(get)] -// settled_time: Option>, +// pub settled_time: Option>, // #[pyo3(get)] -// market_id: SyncObj, +// pub market_id: SyncObj, // #[pyo3(get)] -// country_code: SyncObj>, -// } - -// impl PyMarketBase { - - - - +// pub country_code: SyncObj>, +// #[pyo3(get)] +// pub regulators: SyncObj>>, +// } -// fn update(mut self_: PyRefMut, py: Python) -> PyResult { -// let config = self_.config; -// let mut deser = self_.deser.take().expect("Market without deser"); -// let base = self_.as_mut(); +// #[derive(Debug, Default)] +// struct MarketBookUpdate<'a> { +// market_id: &'a str, +// definition: Option<&'a MarketDefinitionUpdate<'a>>, +// runners: Option>>, +// total_volume: Option, +// } -// let r = Self::drive_deserialize(&mut deser, base, config, py) -// .map(|_| true) -// .unwrap_or_else(|err| { -// if !err.is_eof() { -// warn!(target: "betfair_data", "file: {} err: (JSON Parse Error) {}", base.file.to_string_lossy(), err); -// } -// false -// }); -// self_.deser = Some(deser); +// // we could detect multiple market ids and only do the 1 specified, and potentially repeat the deserializer +// // multiple times 1 for each market -// Ok(r) -// } +// // or just intersperse the markets throughout the iter - probably better -// fn copy(self_: PyRef, py: Python) -> PyResult> { -// Py::new(py, self_.as_ref().clone(py)) -// } +// pub struct PyMarketsDeser<'a, 'py> { +// pub markets: &'a [Py], +// pub py: Python<'py>, +// pub config: SourceConfig, // } - -// struct PyMarketToken<'a, 'py> { -// market: &'a mut PyMarketBase, -// config: Config, -// py: Python<'py>, -// } -// impl<'de, 'a, 'py> DeserializeSeed<'de> for PyMarketToken<'a, 'py> { -// type Value = (); +// impl<'de, 'a, 'py> DeserializeSeed<'de> for PyMarketsDeser<'a, 'py> { +// type Value = Vec>; // fn deserialize(self, deserializer: D) -> Result // where @@ -150,76 +132,90 @@ // Mc, // } -// struct PyMarketOuterVisitor<'a, 'py> { -// market: &'a mut PyMarketBase, -// config: Config, +// struct PyMarketsDeserVisitor<'a, 'py> { +// markets: &'a [Py], // py: Python<'py>, +// config: SourceConfig, // } -// impl<'de, 'a, 'py> Visitor<'de> for PyMarketOuterVisitor<'a, 'py> { -// type Value = (); +// impl<'de, 'a, 'py> Visitor<'de> for PyMarketsDeserVisitor<'a, 'py> { +// type Value = Vec>; // fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { // formatter.write_str("") // } -// fn visit_map(mut self, mut map: V) -> Result +// fn visit_map(self, mut map: V) -> Result // where // V: MapAccess<'de>, // { +// let mut pt: Option = None; +// let mut clk: Option> = None; +// let mut books: Vec> = Vec::new(); + // while let Some(key) = map.next_key()? { // match key { -// Field::Op => { -// map.next_value::()?; +// Field::Pt => { +// pt = Some(DateTime::new(map.next_value::()?)); +// } +// Field::Mc => { +// books = map.next_value_seed(MarketMcSeq { +// markets: self.markets, +// py: self.py, +// config: self.config, +// })?; // } -// Field::Pt => self.market.publish_time = map.next_value()?, -// Field::Mc => map.next_value_seed(PyMarketMcSeq { -// market: self.market, -// config: self.config, -// py: self.py, -// })?, // Field::Clk => { -// self.market.clk.set_if_ne(map.next_value::<&str>()?); +// clk = Some(map.next_value::>()?); +// } +// Field::Op => { +// map.next_value::()?; // } // } // } -// Ok(()) +// if let Some(pt) = pt { +// books +// .iter_mut() +// .for_each(|mb| mb.borrow_mut(self.py).publish_time = pt); +// } + +// Ok(books) // } // } // const FIELDS: &[&str] = &["op", "pt", "clk", "mc"]; // deserializer.deserialize_struct( -// "MarketBook", +// "PyMarket", // FIELDS, -// PyMarketOuterVisitor { -// market: self.market, -// config: self.config, +// PyMarketsDeserVisitor { +// markets: self.markets, // py: self.py, +// config: self.config, // }, // ) // } // } // // Used for serializing in place over the marketChange `mc` array -// struct PyMarketMcSeq<'a, 'py> { -// market: &'a mut PyMarketBase, -// config: Config, +// struct MarketMcSeq<'a, 'py> { +// markets: &'a [Py], // py: Python<'py>, +// config: SourceConfig, // } -// impl<'de, 'a, 'py> DeserializeSeed<'de> for PyMarketMcSeq<'a, 'py> { -// type Value = (); +// impl<'de, 'a, 'py> DeserializeSeed<'de> for MarketMcSeq<'a, 'py> { +// type Value = Vec>; // fn deserialize(self, deserializer: D) -> Result // where // D: Deserializer<'de>, // { -// struct PyMarketMcSeqVisitor<'a, 'py> { -// market: &'a mut PyMarketBase, -// config: Config, +// struct MarketMcSeqVisitor<'a, 'py> { +// markets: &'a [Py], // py: Python<'py>, +// config: SourceConfig, // } -// impl<'de, 'a, 'py> Visitor<'de> for PyMarketMcSeqVisitor<'a, 'py> { -// type Value = (); +// impl<'de, 'a, 'py> Visitor<'de> for MarketMcSeqVisitor<'a, 'py> { +// type Value = Vec>; // fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { // formatter.write_str("") @@ -230,45 +226,64 @@ // A: serde::de::SeqAccess<'de>, // { // #[derive(Deserialize)] -// struct IDandImg { +// struct IdImg { +// id: MarketID, // img: Option, // } +// let mut next_books: Vec> = Vec::new(); + // while let Some(raw) = seq.next_element::<&RawValue>()? { -// let mut deserializer = serde_json::Deserializer::from_str(raw.get()); -// let idimg: IDandImg = serde_json::from_str(raw.get()).map_err(Error::custom)?; +// let mut deser = serde_json::Deserializer::from_str(raw.get()); +// let mid: IdImg = +// serde_json::from_str(raw.get()).map_err(Error::custom)?; -// PyMarketMc { -// market: self.market, -// config: self.config, +// let mb = next_books +// .iter() +// // search already created markets +// .find(|m| (*m).borrow(self.py).market_id.as_str() == mid.id) +// // search markets passed in originally +// .or_else(|| { +// self.markets.iter().find(|m| { +// (*m).borrow(self.py).market_id.as_str() == mid.id +// }) +// }) +// .map(|o| o.borrow(self.py)); + +// let next_mb = MarketMc { +// id: mid.id, +// image: mid.img.contains(&true), +// market: mb, // py: self.py, -// img: idimg.img.unwrap_or(false), +// config: self.config, // } -// .deserialize(&mut deserializer) +// .deserialize(&mut deser) // .map_err(Error::custom)?; + +// next_books.push(Py::new(self.py, next_mb).unwrap()); // } -// Ok(()) +// Ok(next_books) // } // } -// deserializer.deserialize_seq(PyMarketMcSeqVisitor { -// market: self.market, -// config: self.config, +// deserializer.deserialize_seq(MarketMcSeqVisitor { +// markets: self.markets, // py: self.py, +// config: self.config, // }) // } // } -// // Used for serializing in place over the marketChange `mc` objects -// struct PyMarketMc<'a, 'py> { -// market: &'a mut PyMarketBase, -// config: Config, +// struct MarketMc<'py> { +// id: MarketID, +// market: Option>, // py: Python<'py>, -// img: bool, +// image: bool, +// config: SourceConfig, // } -// impl<'de, 'a, 'py> DeserializeSeed<'de> for PyMarketMc<'a, 'py> { -// type Value = (); +// impl<'de, 'py> DeserializeSeed<'de> for MarketMc<'py> { +// type Value = PyMarket; // fn deserialize(self, deserializer: D) -> Result // where @@ -289,54 +304,70 @@ // StreamId, // } -// struct PyMarketMcVisitor<'a, 'py> { -// market: &'a mut PyMarketBase, -// config: Config, -// img: bool, +// struct MarketMcVisitor<'py> { +// market: Option>, // py: Python<'py>, +// config: SourceConfig, // } -// impl<'de, 'a, 'py> Visitor<'de> for PyMarketMcVisitor<'a, 'py> { -// type Value = (); +// impl<'de, 'py> Visitor<'de> for MarketMcVisitor<'py> { +// type Value = PyMarket; // fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { // formatter.write_str("") // } -// fn visit_map(mut self, mut map: V) -> Result +// fn visit_map(self, mut map: V) -> Result // where // V: MapAccess<'de>, // { +// let mut upt = MarketBookUpdate::default(); + +// let mut next: &mut Option = &mut None; +// let mut next_runners: &mut Option>> = &mut None; + + // while let Some(key) = map.next_key()? { // match key { // Field::Id => { -// // Do not currently support files that have multiple markets in them. -// // There are event level files that betfair provide, which have every -// // market for an event -// // with filtering: if a market has already been initted and then -// // changes then we must have a multi market file. -// let is_init = !self.market.market_id.is_empty(); -// if self.market.market_id.set_if_ne(map.next_value::<&str>()?) && is_init -// { -// return Err(Error::custom( -// "multiple markets per file is not supported", -// )); +// let s = map.next_value::<&str>()?; +// upt.market_id = s; +// } +// Field::MarketDefinition => { +// map.next_value_seed(MarketDefinitionDeser { +// market: self.market, +// next, +// next_runners, +// py: self.py, +// config: self.config, +// })?; +// } +// Field::Rc => { +// let runners: Option<&[Py]> = +// self.market.as_ref().map(|m| (**m.runners).as_ref()); + +// upt.runners = Some(map.next_value_seed(RunnerChangeSeq { +// runners, +// next: next_runners, +// py: self.py, +// config: self.config, +// })?); + +// // if cumulative_runner_tv is on, then tv shouldnt be sent at a market level and will have +// // to be derived from the sum of runner tv's. This happens when using the data provided +// // from betfair historical data service, not saved from the actual stream +// if self.config.cumulative_runner_tv { +// upt.total_volume = upt +// .runners +// .as_ref() +// .map(|rs| { +// rs.iter().map(|r| r.borrow(self.py).total_matched).sum() +// }) +// .map(|f: f64| f.round_cent()); // } // } -// Field::MarketDefinition => map.next_value_seed(PyMarketDefinition { -// market: self.market, -// config: self.config, -// img: self.img, -// py: self.py, -// })?, -// Field::Rc => map.next_value_seed(PyRunnerChangeSeq { -// runners: &mut self.market.runners, -// img: self.img, -// config: self.config, -// py: self.py, -// })?, // Field::Tv => { // if !self.config.cumulative_runner_tv { -// self.market.total_matched += map.next_value::()?; +// upt.total_volume = Some(map.next_value::()?.round_cent()); // } else { // map.next_value::()?; // } @@ -345,7 +376,6 @@ // map.next_value::()?; // } // Field::Img => { -// // TODO I need to handle this and clear the market // map.next_value::()?; // } // _ => { @@ -354,19 +384,16 @@ // } // } -// // if cumulative_runner_tv is on, then tv shouldnt be sent at a market level and will have -// // to be derived from the sum of runner tv's. This happens when using the data provided -// // from betfair historical data service, not saved from the actual stream -// if self.config.cumulative_runner_tv { -// self.market.total_matched = self -// .market -// .runners -// .iter() -// .map(|r| r.borrow(self.py).total_matched) -// .sum(); -// } +// let mb = match (self.market, &upt.definition) { +// (Some(mb), Some(_)) => mb.update_from_change(upt, self.py), +// (Some(mb), None) => mb.update_from_change(upt, self.py), +// (None, Some(_)) => PyMarket::new(upt, self.py), +// (None, None) => { +// return Err(Error::custom("missing definition on initial market update")) +// } +// }; -// Ok(()) +// Ok(mb) // } // } @@ -374,315 +401,12 @@ // deserializer.deserialize_struct( // "MarketChange", // FIELDS, -// PyMarketMcVisitor { +// MarketMcVisitor { // market: self.market, -// config: self.config, -// img: self.img, // py: self.py, -// }, -// ) -// } -// } - -// // Used for serializing in place over the mc marketDefinition object -// struct PyMarketDefinition<'a, 'py> { -// market: &'a mut PyMarketBase, -// config: Config, -// img: bool, -// py: Python<'py>, -// } -// impl<'de, 'a, 'py> DeserializeSeed<'de> for PyMarketDefinition<'a, 'py> { -// type Value = (); - -// fn deserialize(self, deserializer: D) -> Result -// where -// D: Deserializer<'de>, -// { -// #[derive(Debug, Deserialize)] -// #[serde(field_identifier, rename_all = "camelCase")] -// enum Field { -// BetDelay, -// BettingType, -// BspMarket, -// BspReconciled, -// Complete, -// CountryCode, -// CrossMatching, -// DiscountAllowed, -// EachWayDivisor, -// EventId, -// EventName, -// EventTypeId, -// InPlay, -// KeyLineDefiniton, -// LineMaxUnit, -// LineMinUnit, -// LineInterval, -// MarketBaseRate, -// MarketTime, -// MarketType, -// Name, -// NumberOfActiveRunners, -// NumberOfWinners, -// OpenDate, -// PersistenceEnabled, -// PriceLadderDefinition, -// RaceType, -// Regulators, -// Runners, -// RunnersVoidable, -// SettledTime, -// Status, -// SuspendTime, -// Timezone, -// TurnInPlayEnabled, -// Venue, -// Version, -// } - -// struct PyMarketDefinitionVisitor<'a, 'py> { -// market: &'a mut PyMarketBase, -// config: Config, -// img: bool, -// py: Python<'py>, -// } -// impl<'de, 'a, 'py> Visitor<'de> for PyMarketDefinitionVisitor<'a, 'py> { -// type Value = (); - -// fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { -// formatter.write_str("") -// } - -// fn visit_map(mut self, mut map: V) -> Result -// where -// V: MapAccess<'de>, -// { -// while let Some(key) = map.next_key()? { -// match key { -// Field::BspMarket => self.market.bsp_market = map.next_value()?, -// Field::TurnInPlayEnabled => { -// self.market.turn_in_play_enabled = map.next_value()? -// } -// Field::InPlay => self.market.in_play = map.next_value()?, -// Field::PersistenceEnabled => { -// self.market.persistence_enabled = map.next_value()? -// } -// Field::BspReconciled => self.market.bsp_reconciled = map.next_value()?, -// Field::Complete => self.market.complete = map.next_value()?, -// Field::CrossMatching => self.market.cross_matching = map.next_value()?, -// Field::RunnersVoidable => { -// self.market.runners_voidable = map.next_value()? -// } -// Field::DiscountAllowed => { -// self.market.discount_allowed = map.next_value()? -// } -// Field::Timezone => { -// self.market.timezone.set_if_ne(map.next_value::<&str>()?); -// } -// Field::Name => { -// self.market -// .market_name -// .set_if_ne(map.next_value::>()?); -// } -// Field::EventName => { -// self.market -// .event_name -// .set_if_ne(map.next_value::>()?); -// } -// Field::CountryCode => { -// self.market -// .country_code -// .set_if_ne(map.next_value::<&str>()?); -// } -// Field::Venue => { -// self.market.venue.set_if_ne(map.next_value::>()?); -// } -// Field::Status => self.market.status = map.next_value()?, -// Field::MarketBaseRate => { -// self.market.market_base_rate = map.next_value::()? as u8 -// } // TODO: why is this needed -// Field::NumberOfWinners => { -// self.market.number_of_winners = map.next_value::()? as u8 -// } // TODO: why is this needed -// Field::NumberOfActiveRunners => { -// self.market.number_of_active_runners = map.next_value()? -// } -// Field::BetDelay => self.market.bet_delay = map.next_value()?, -// Field::EventId => { -// self.market.event_id = map -// .next_value::<&str>()? -// .parse() -// .map_err(de::Error::custom)?; -// } -// Field::EventTypeId => { -// self.market.event_type_id = map -// .next_value::<&str>()? -// .parse() -// .map_err(de::Error::custom)?; -// } -// Field::Version => self.market.version = map.next_value()?, -// Field::Runners => map.next_value_seed(PyRunnerDefSeq { -// runners: &mut self.market.runners, -// config: self.config, -// img: self.img, -// py: self.py, -// })?, -// Field::MarketType => { -// self.market.market_type.set_if_ne(map.next_value::<&str>()?); -// } -// Field::BettingType => self.market.betting_type = map.next_value()?, -// Field::EachWayDivisor => { -// self.market.each_way_divisor = Some(map.next_value::()?) -// } -// Field::MarketTime => { -// let s = map.next_value()?; -// if self.market.market_time_str.set_if_ne(s) { -// let ts = chrono::DateTime::parse_from_rfc3339(s) -// .map_err(de::Error::custom)? -// .timestamp_millis(); - -// self.market.market_time = ts; -// } -// } -// Field::SuspendTime => { -// let s = map.next_value()?; -// if self.market.suspend_time_str.set_if_ne(s) { -// let ts = chrono::DateTime::parse_from_rfc3339(s) -// .map_err(de::Error::custom)? -// .timestamp_millis(); -// // let d = PyDateTime::from_timestamp(self.1, ts as f64, None).unwrap(); -// // self.market.suspend_time = Some(d.into_py(self.1)); -// self.market.suspend_time = Some(ts); -// } -// } -// Field::SettledTime => { -// let s = map.next_value()?; -// if self.market.settled_time_str.set_if_ne(s) { -// let ts = chrono::DateTime::parse_from_rfc3339(s) -// .map_err(de::Error::custom)? -// .timestamp_millis(); -// // let d = PyDateTime::from_timestamp(self.1, ts as f64, None).unwrap(); -// // self.market.settled_time = Some(d.into_py(self.1)); -// self.market.settled_time = Some(ts); -// } -// } -// Field::OpenDate => { -// let s = map.next_value()?; -// if self.market.open_date_str.set_if_ne(s) { -// let ts = chrono::DateTime::parse_from_rfc3339(s) -// .map_err(de::Error::custom)? -// .timestamp_millis(); -// // let d = PyDateTime::from_timestamp(self.1, ts as f64, None).unwrap(); -// // self.market.open_date = Some(d.into_py(self.1)); -// self.market.open_date = ts; -// } -// } -// Field::Regulators => { -// map.next_value::()?; -// } - -// // after searching over 200k markets, I cant find these values in any data sets :/ -// Field::RaceType => { -// map.next_value::()?; -// // panic!("{} {}", self.0.source, self.0.file); -// } -// Field::KeyLineDefiniton => { -// map.next_value::()?; -// // panic!("{} {}", self.0.source, self.0.file); -// } -// Field::PriceLadderDefinition => { -// map.next_value::()?; -// // panic!("{} {}", self.0.source, self.0.file); -// } -// Field::LineMaxUnit => { -// map.next_value::()?; -// // panic!("{} {}", self.0.source, self.0.file); -// } -// Field::LineMinUnit => { -// map.next_value::()?; -// // panic!("{} {}", self.0.source, self.0.file); -// } -// Field::LineInterval => { -// map.next_value::()?; -// // panic!("{} {}", self.0.source, self.0.file); -// } -// } -// } -// Ok(()) -// } -// } - -// const FIELDS: &[&str] = &[ -// "keyLineDefiniton", -// "priceLadderDefinition", -// "raceType", -// "lineMaxUnit", -// "lineMinUnit", -// "lineInterval", -// "bspMarket", -// "turnInPlayEnabled", -// "persistenceEnabled", -// "marketBaseRate", -// "eventId", -// "eventTypeId", -// "numberOfWinners", -// "bettingType", -// "marketType", -// "marketTime", -// "suspendTime", -// "bspReconciled", -// "complete", -// "inPlay", -// "crossMatching", -// "runnersVoidable", -// "numberOfActiveRunners", -// "betDelay", -// "status", -// "runners", -// "regulators", -// "countryCode", -// "discountAllowed", -// "timezone", -// "openDate", -// "version", -// "name", -// "eventName", -// "venue", -// "settledTime", -// "eachWayDivisor", -// ]; -// deserializer.deserialize_struct( -// "MarketDefinition", -// FIELDS, -// PyMarketDefinitionVisitor { -// market: self.market, // config: self.config, -// img: self.img, -// py: self.py, // }, // ) // } // } -// #[cfg(test)] -// mod tests { - -// // test disabled awaiting merge which fixes cargo test -// // https://github.com/PyO3/pyo3/pull/2135 -// /* -// use super::*; - -// #[test] -// fn test_multiple_markets() { -// let mut m = PyMarketBase::new("".to_owned(), "".to_owned()); -// let py = unsafe { Python::assume_gil_acquired() }; - -// let config = Config{cumulative_runner_tv: true, stable_runner_index: false}; - -// let mut deser = serde_json::Deserializer::from_str(r#"{"id": "1.123456789"}{"id":"1.987654321"}"#); - -// PyMarketMc(&mut m, py, config).deserialize(&mut deser).expect("1st market_id deser ok"); -// PyMarketMc(&mut m, py, config).deserialize(&mut deser).expect_err("2nd market_id deser error"); -// } -// */ -// } diff --git a/src/immutable/mod.rs b/src/immutable/mod.rs index 10b2e16..76cf2e6 100644 --- a/src/immutable/mod.rs +++ b/src/immutable/mod.rs @@ -5,4 +5,6 @@ pub mod runner_book_sp; pub mod market; pub mod runner; pub mod config; -pub mod datetime; \ No newline at end of file +pub mod datetime; + +mod definition; \ No newline at end of file diff --git a/src/immutable/price_size.rs b/src/immutable/price_size.rs index d40b8f3..51d67f8 100644 --- a/src/immutable/price_size.rs +++ b/src/immutable/price_size.rs @@ -397,8 +397,8 @@ mod tests { ]; let ps10 = ImmutablePriceSizeLayLadder(&ps9) - .deserialize(&mut deser) - .expect("failed to deserialize"); + .deserialize(&mut deser) + .expect("failed to deserialize"); let ans10 = vec![ PriceSize::new(4.0, 5.0), PriceSize::new(3.1, 3.0), @@ -434,6 +434,5 @@ mod tests { assert_eq!(ps8, ans8); assert_eq!(ps9, ans9); assert_eq!(ps10, ans10); - } } diff --git a/src/immutable/runner.rs b/src/immutable/runner.rs index fd31edd..1f39dd2 100644 --- a/src/immutable/runner.rs +++ b/src/immutable/runner.rs @@ -1,17 +1,20 @@ +// use core::fmt; // use pyo3::prelude::*; -// use serde::de::{DeserializeSeed, Error, IgnoredAny, MapAccess, Visitor}; +// use pyo3::types::PyList; +// use serde::de::{DeserializeSeed, Error, Visitor}; // use serde::{Deserialize, Deserializer}; // use serde_json::value::RawValue; -// use std::{borrow::Cow, fmt}; +// use std::borrow::Borrow; +// use std::sync::Arc; +// use super::container::{PyRep, SyncObj}; +// use super::datetime::DateTimeString; // use crate::enums::SelectionStatus; // use crate::ids::SelectionID; -// use crate::price_size::{F64OrStr, PriceSize}; -// use super::datetime::DateTimeString; -// use super::price_size::{ImmutablePriceSizeBackLadder, ImmutablePriceSizeLayLadder}; -// use super::config::Config; - -// use crate::strings::StringSetExtNeq; +// use crate::market_source::SourceConfig; +// // use super::definition::RunnerDefUpdate; +// use super::runner_book_ex::RunnerBookEX; +// use super::runner_book_sp::RunnerBookSP; // #[pyclass(name = "Runner")] // pub struct PyRunner { @@ -20,7 +23,7 @@ // #[pyo3(get)] // pub selection_id: SelectionID, // #[pyo3(get)] -// pub name: String, +// pub name: Option>>, // #[pyo3(get)] // pub last_price_traded: Option, // #[pyo3(get)] @@ -30,275 +33,42 @@ // #[pyo3(get)] // pub handicap: Option, // #[pyo3(get)] -// pub ex: Py, +// pub ex: Py, // #[pyo3(get)] -// pub sp: Py, +// pub sp: Py, // #[pyo3(get)] // pub sort_priority: u16, // #[pyo3(get)] -// pub removal_date: Option, -// // removal_date: Option>, -// pub removal_date_str: Option, -// } - -// impl PyRunner { -// fn new(py: Python) -> Self { -// let ex: PyRunnerBookEX = Default::default(); -// let sp: PyRunnerBookSP = Default::default(); - -// PyRunner { -// selection_id: Default::default(), -// status: Default::default(), -// name: Default::default(), -// last_price_traded: Default::default(), -// total_matched: Default::default(), -// adjustment_factor: Default::default(), -// handicap: Default::default(), -// sort_priority: Default::default(), -// removal_date_str: Default::default(), -// removal_date: Default::default(), -// ex: Py::new(py, ex).unwrap(), -// sp: Py::new(py, sp).unwrap(), -// } -// } - -// pub fn clone(&self, py: Python) -> Self { -// let ex: PyRunnerBookEX = self.ex.borrow(py).clone(); -// let sp: PyRunnerBookSP = self.sp.borrow(py).clone(); - -// Self { -// selection_id: self.selection_id, -// status: self.status, -// name: self.name.clone(), -// last_price_traded: self.last_price_traded, -// total_matched: self.total_matched, -// adjustment_factor: self.adjustment_factor, -// handicap: self.handicap, -// sort_priority: self.sort_priority, -// removal_date_str: self.removal_date_str.clone(), -// removal_date: self.removal_date, -// ex: Py::new(py, ex).unwrap(), -// sp: Py::new(py, sp).unwrap(), -// } -// } +// pub removal_date: Option>, // } -// pub struct PyRunnerDefSeq<'a, 'py> { -// pub runners: &'a mut Vec>, -// pub config: Config, -// pub img: bool, -// pub py: Python<'py>, -// } -// impl<'de, 'a, 'py> DeserializeSeed<'de> for PyRunnerDefSeq<'a, 'py> { -// type Value = (); - -// fn deserialize(self, deserializer: D) -> Result -// where -// D: Deserializer<'de>, -// { -// struct RunnerSeqVisitor<'a, 'py> { -// runners: &'a mut Vec>, -// config: Config, -// img: bool, -// py: Python<'py>, -// } -// impl<'de, 'a, 'py> Visitor<'de> for RunnerSeqVisitor<'a, 'py> { -// type Value = (); - -// fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { -// formatter.write_str("") -// } - -// fn visit_seq(self, mut seq: A) -> Result -// where -// A: serde::de::SeqAccess<'de>, -// { -// // grow an empty vec -// if self.runners.capacity() == 0 { -// match seq.size_hint() { -// Some(s) => self.runners.reserve_exact(s + 2), -// None => self.runners.reserve_exact(12), -// } -// } - -// // WARNING -// // So this grossness, is due to us needing to get the selection_id to find the -// // runner to deserialize in place into. The id is not the first element of the -// // object, so we'd need to defer parsing the other properties, then come back -// // to them once we know the correct selection to use as the seed. -// // What we do here is parse the json twice, once pulling out only the id, then -// // again as normal -// #[derive(Deserialize)] -// struct RunnerWithID { -// id: SelectionID, -// } - -// while let Some(raw) = seq.next_element::<&RawValue>()? { -// let mut deser = serde_json::Deserializer::from_str(raw.get()); -// let rid: RunnerWithID = -// serde_json::from_str(raw.get()).map_err(Error::custom)?; - -// let index = self -// .runners -// .iter() -// .map(|r| r.borrow_mut(self.py)) -// .position(|r| r.selection_id == rid.id); -// match index { -// Some(index) => { -// let mut runner = -// unsafe { self.runners.get_unchecked(index).borrow_mut(self.py) }; -// PyRunnerDefinitonDeser { -// runner: &mut runner, -// config: self.config, -// img: self.img, -// py: self.py, -// } -// .deserialize(&mut deser) -// .map_err(Error::custom)? -// } -// None => { -// let mut runner = PyRunner::new(self.py); -// PyRunnerDefinitonDeser { -// runner: &mut runner, -// config: self.config, -// img: self.img, -// py: self.py, -// } -// .deserialize(&mut deser) -// .map_err(Error::custom)?; - -// self.runners.push(Py::new(self.py, runner).unwrap()); -// } -// } -// } - -// // this config flag will reorder the runners into the order specified in the sort priority -// // as seen in the data files -// if !self.config.stable_runner_index { -// self.runners -// .sort_by_key(|r| r.borrow(self.py).sort_priority); -// } - -// Ok(()) -// } -// } - -// deserializer.deserialize_seq(RunnerSeqVisitor { -// runners: self.runners, -// config: self.config, -// img: self.img, -// py: self.py, -// }) +// // pub struct RunnerDefUpdate<'a> { +// // id: SelectionID, +// // adjustment_factor: Option, +// // status: SelectionStatus, +// // sort_priority: u16, +// // name: Option<&'a str>, +// // bsp: Option, +// // removal_date: Option<&'a str>, +// // hc: Option, +// // } + +// impl PyRunner {} + +// impl PyRep for Vec> { +// fn py_rep(&self, py: Python) -> PyObject { +// PyList::new(py, self.iter().map(|ps| ps.into_py(py))).into_py(py) // } // } -// struct PyRunnerDefinitonDeser<'a, 'py> { -// pub runner: &'a mut PyRunner, -// pub config: Config, -// pub img: bool, +// pub struct RunnerChangeSeq<'a, 'py> { +// pub runners: Option<&'a [Py]>, +// pub next: &'a mut Option>>, // pub py: Python<'py>, -// } -// impl<'de, 'a, 'py> DeserializeSeed<'de> for PyRunnerDefinitonDeser<'a, 'py> { -// type Value = (); - -// fn deserialize(self, deserializer: D) -> Result -// where -// D: Deserializer<'de>, -// { -// #[derive(Debug, Deserialize)] -// #[serde(field_identifier, rename_all = "camelCase")] -// enum Field { -// AdjustmentFactor, -// Status, -// SortPriority, -// Id, -// Name, -// Bsp, -// RemovalDate, -// Hc, -// } - -// struct RunnerDefVisitor<'a, 'py> { -// runner: &'a mut PyRunner, -// _config: Config, -// _img: bool, -// py: Python<'py>, -// } -// impl<'de, 'a, 'py> Visitor<'de> for RunnerDefVisitor<'a, 'py> { -// type Value = (); - -// fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { -// formatter.write_str("") -// } - -// fn visit_map(mut self, mut map: V) -> Result -// where -// V: MapAccess<'de>, -// { -// while let Some(key) = map.next_key()? { -// match key { -// Field::Id => self.runner.selection_id = map.next_value()?, -// Field::AdjustmentFactor => { -// self.runner.adjustment_factor = -// Some(map.next_value::()?.into()) -// } -// Field::Status => self.runner.status = map.next_value()?, -// Field::SortPriority => self.runner.sort_priority = map.next_value()?, -// Field::Hc => { -// self.runner.handicap = Some(map.next_value::()?.into()) -// } -// Field::Name => { -// self.runner.name.set_if_ne(map.next_value::>()?); -// } -// Field::RemovalDate => { -// let s = map.next_value()?; -// if self.runner.removal_date_str.set_if_ne(s) { -// let ts = chrono::DateTime::parse_from_rfc3339(s) -// .map_err(Error::custom)? -// .timestamp_millis(); -// self.runner.removal_date = Some(ts); -// } -// } -// Field::Bsp => { -// let mut sp = self.runner.sp.borrow_mut(self.py); -// sp.actual_sp = Some(map.next_value::()?.into()); -// } -// } -// } - -// Ok(()) -// } -// } - -// const FIELDS: &[&str] = &[ -// "adjustmentFactor", -// "status", -// "sortPriority", -// "id", -// "name", -// "bsp", -// "removalDate", -// ]; -// deserializer.deserialize_struct( -// "RunnerDef", -// FIELDS, -// RunnerDefVisitor { -// runner: self.runner, -// _config: self.config, -// _img: self.img, -// py: self.py, -// }, -// ) -// } +// pub config: SourceConfig, // } -// pub struct PyRunnerChangeSeq<'a, 'py> { -// pub runners: &'a mut Vec>, -// pub config: Config, -// pub img: bool, -// pub py: Python<'py>, -// } -// impl<'de, 'a, 'py> DeserializeSeed<'de> for PyRunnerChangeSeq<'a, 'py> { +// impl<'de, 'a, 'py> DeserializeSeed<'de> for RunnerChangeSeq<'a, 'py> { // type Value = (); // fn deserialize(self, deserializer: D) -> Result @@ -306,10 +76,10 @@ // D: Deserializer<'de>, // { // struct RunnerSeqVisitor<'a, 'py> { -// runners: &'a mut Vec>, -// config: Config, -// img: bool, +// runners: Option<&'a Vec>>, +// next: &'a mut Option>>, // py: Python<'py>, +// config: SourceConfig, // } // impl<'de, 'a, 'py> Visitor<'de> for RunnerSeqVisitor<'a, 'py> { // type Value = (); @@ -322,13 +92,10 @@ // where // A: serde::de::SeqAccess<'de>, // { -// // grow an empty vec -// if self.runners.capacity() == 0 { -// match seq.size_hint() { -// Some(s) => self.runners.reserve_exact(s + 2), -// None => self.runners.reserve_exact(12), -// } -// } +// let mut v = self +// .runners +// .map(|v| v.iter().map(|r| r.clone_ref(self.py)).collect::>()) +// .unwrap_or_else(|| Vec::with_capacity(10)); // #[derive(Deserialize)] // struct RunnerWithID { @@ -340,41 +107,48 @@ // let rid: RunnerWithID = // serde_json::from_str(raw.get()).map_err(Error::custom)?; -// let index = self -// .runners +// let runner = v // .iter() -// .map(|r| r.borrow_mut(self.py)) -// .position(|r| r.selection_id == rid.id); -// match index { -// Some(index) => { -// let mut runner = -// unsafe { self.runners.get_unchecked(index).borrow_mut(self.py) }; -// PyRunnerChangeDeser { -// runner: &mut runner, -// img: self.img, -// config: self.config, -// py: self.py, -// } -// .deserialize(&mut deser) -// .map_err(Error::custom)?; +// .map(|r| r.borrow(self.py)) +// .find(|r| r.selection_id == rid.id); + +// let next_runner = self.next.and_then(|rs| { +// rs.iter_mut() +// .map(|r| r.borrow_mut(self.py)) +// .find(|r| r.selection_id == rid.id) +// }); + +// match (runner, next_runner) { +// (Some(index), Some(next_runner)) => { + +// RunnerBookChangeDeser { +// runner: &runner, +// next +// py: self.py, +// config: self.config, +// } +// .deserialize(&mut deser) +// .map_err(Error::custom)? +// }; + +// v[index] = Py::new(self.py, runner).unwrap(); // } // None => { -// let mut runner = PyRunner::new(self.py); -// PyRunnerChangeDeser { -// runner: &mut runner, -// config: self.config, -// img: self.img, +// let runner = RunnerBook::new(rid.id, self.py); +// let runner = RunnerBookChangeDeser { +// runner: &runner, // py: self.py, +// config: self.config, // } // .deserialize(&mut deser) // .map_err(Error::custom)?; -// self.runners.push(Py::new(self.py, runner).unwrap()); +// v.push(Py::new(self.py, runner).unwrap()); // } // } // } -// Ok(()) +// Ok(v) // } // } @@ -382,19 +156,17 @@ // runners: self.runners, // py: self.py, // config: self.config, -// img: self.img, // }) // } // } -// struct PyRunnerChangeDeser<'a, 'py> { -// pub runner: &'a mut PyRunner, -// pub config: Config, -// pub img: bool, -// pub py: Python<'py>, +// struct RunnerBookChangeDeser<'a, 'py> { +// runner: &'a RunnerBook, +// py: Python<'py>, +// config: SourceConfig, // } -// impl<'de, 'a, 'py> DeserializeSeed<'de> for PyRunnerChangeDeser<'a, 'py> { -// type Value = (); +// impl<'de, 'a, 'py> DeserializeSeed<'de> for RunnerBookChangeDeser<'a, 'py> { +// type Value = RunnerBook; // fn deserialize(self, deserializer: D) -> Result // where @@ -417,87 +189,86 @@ // } // struct RunnerChangeVisitor<'a, 'py> { -// runner: &'a mut PyRunner, -// config: Config, -// img: bool, +// runner: &'a RunnerBook, // py: Python<'py>, +// config: SourceConfig, // } // impl<'de, 'a, 'py> Visitor<'de> for RunnerChangeVisitor<'a, 'py> { -// type Value = (); +// type Value = RunnerBook; // fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { // formatter.write_str("") // } -// fn visit_map(mut self, mut map: V) -> Result +// fn visit_map(self, mut map: V) -> Result // where // V: MapAccess<'de>, // { +// let mut atb: Option> = None; +// let mut atl: Option> = None; +// let mut trd: Option> = None; + +// let mut spb: Option> = None; +// let mut spl: Option> = None; +// let mut spn: Option = None; +// let mut spf: Option = None; + +// let mut tv: Option = None; +// let mut ltp: Option = None; +// let mut hc: Option = None; + // while let Some(key) = map.next_key()? { // match key { -// Field::Id => self.runner.selection_id = map.next_value()?, +// Field::Id => { +// let id = map.next_value::()?; +// debug_assert!(id == self.runner.selection_id); +// } // Field::Atb => { -// let mut ex = self.runner.ex.borrow_mut(self.py); -// let atb = &mut ex.available_to_back; -// if self.img { -// atb.clear(); -// } -// map.next_value_seed(PriceSizeLayLadder(atb))?; +// let ex = self.runner.ex.borrow(self.py); +// atb = Some(map.next_value_seed(ImmutablePriceSizeLayLadder( +// &ex.available_to_back, +// ))?); // } // Field::Atl => { -// let mut ex = self.runner.ex.borrow_mut(self.py); -// let atl = &mut ex.available_to_lay; -// if self.img { -// atl.clear(); -// } -// map.next_value_seed(PriceSizeBackLadder(atl))?; +// let ex = self.runner.ex.borrow(self.py); +// atl = Some(map.next_value_seed(ImmutablePriceSizeBackLadder( +// &ex.available_to_lay, +// ))?); // } // Field::Trd => { -// let mut ex = self.runner.ex.borrow_mut(self.py); -// let trd = &mut ex.traded_volume; -// if self.img { -// trd.clear(); -// } - -// map.next_value_seed(PriceSizeBackLadder(trd))?; +// let ex = self.runner.ex.borrow(self.py); +// let l = map +// .next_value_seed(ImmutablePriceSizeBackLadder(&ex.traded_volume))?; // if self.config.cumulative_runner_tv { -// self.runner.total_matched = -// ex.traded_volume.iter().map(|ps| ps.size).sum(); +// tv = Some(l.iter().map(|ps| ps.size).sum::().round_cent()); // } + +// trd = Some(l); // } // Field::Spb => { -// let mut sp = self.runner.sp.borrow_mut(self.py); -// let spb = &mut sp.lay_liability_taken; -// if self.img { -// spb.clear(); -// } - -// map.next_value_seed(PriceSizeLayLadder(spb))?; +// let sp = self.runner.sp.borrow(self.py); +// spl = Some(map.next_value_seed(ImmutablePriceSizeLayLadder( +// &sp.lay_liability_taken, +// ))?); // } // Field::Spl => { -// let mut sp = self.runner.sp.borrow_mut(self.py); -// let spl = &mut sp.back_stake_taken; -// if self.img { -// spl.clear(); -// } - -// map.next_value_seed(PriceSizeBackLadder(spl))?; +// let sp = self.runner.sp.borrow(self.py); +// spb = Some(map.next_value_seed(ImmutablePriceSizeBackLadder( +// &sp.back_stake_taken, +// ))?); // } // Field::Spn => { -// let mut sp = self.runner.sp.borrow_mut(self.py); -// sp.near_price = Some(map.next_value::()?.into()); +// spn = Some(map.next_value::()?); // } // Field::Spf => { -// let mut sp = self.runner.sp.borrow_mut(self.py); -// sp.far_price = Some(map.next_value::()?.into()); +// spf = Some(map.next_value::()?); // } // Field::Ltp => { -// self.runner.last_price_traded = -// Some(map.next_value::()?.into()) +// ltp = Some(map.next_value::()?); // } // Field::Hc => { -// self.runner.handicap = Some(map.next_value::()?.into()) +// hc = Some(map.next_value::()?); // } // // The betfair historic data files differ from the stream here, they send tv deltas // // that need to be accumulated, whereas the stream sends the value itself. @@ -505,13 +276,49 @@ // if self.config.cumulative_runner_tv { // map.next_value::()?; // } else { -// self.runner.total_matched = map.next_value::()?.into(); +// let v: f64 = map.next_value::()?.into(); +// let v = v.round_cent(); +// tv = Some(v); // } // } // }; // } -// Ok(()) +// let ex = if atb.is_some() || atl.is_some() || trd.is_some() { +// let upt = RunnerBookEXUpdate { +// available_to_back: atb, +// available_to_lay: atl, +// traded_volume: trd, +// }; + +// Some(self.runner.ex.borrow(self.py).update(upt, self.py)) +// } else { +// None +// }; + +// let sp = if spl.is_some() || spb.is_some() || spn.is_some() || spf.is_some() { +// let upt = RunnerBookSPUpdate { +// actual_sp: None, +// far_price: spf, +// near_price: spn, +// back_stake_taken: spb, +// lay_liability_taken: spl, +// }; + +// Some(self.runner.sp.borrow(self.py).update(upt, self.py)) +// } else { +// None +// }; + +// let update = RunnerChangeUpdate { +// handicap: hc, +// last_price_traded: ltp, +// total_matched: tv, +// ex, +// sp, +// }; + +// Ok(self.runner.update_from_change(update, self.py)) // } // } @@ -523,9 +330,8 @@ // FIELDS, // RunnerChangeVisitor { // runner: self.runner, -// config: self.config, -// img: self.img, // py: self.py, +// config: self.config, // }, // ) // } diff --git a/src/immutable/runner_book_ex.rs b/src/immutable/runner_book_ex.rs index 2db8615..9f27179 100644 --- a/src/immutable/runner_book_ex.rs +++ b/src/immutable/runner_book_ex.rs @@ -1,14 +1,15 @@ +use std::sync::Arc; use pyo3::prelude::*; -use super::container::SyncObj; +use crate::immutable::container::SyncObj; use crate::price_size::PriceSize; #[derive(Clone, Default)] #[pyclass] pub struct RunnerBookEX { - pub available_to_back: SyncObj>, - pub available_to_lay: SyncObj>, - pub traded_volume: SyncObj>, + pub available_to_back: SyncObj>>, + pub available_to_lay: SyncObj>>, + pub traded_volume: SyncObj>>, } pub struct RunnerBookEXUpdate { @@ -24,13 +25,13 @@ impl RunnerBookEX { Self { available_to_back: update .available_to_back - .map_or_else(|| self.available_to_back.clone(), SyncObj::new), + .map_or_else(|| self.available_to_back.clone(), |ps| SyncObj::new(Arc::new(ps))), available_to_lay: update .available_to_lay - .map_or_else(|| self.available_to_lay.clone(), SyncObj::new), + .map_or_else(|| self.available_to_lay.clone(), |ps| SyncObj::new(Arc::new(ps))), traded_volume: update .traded_volume - .map_or_else(|| self.traded_volume.clone(), SyncObj::new), + .map_or_else(|| self.traded_volume.clone(), |ps| SyncObj::new(Arc::new(ps))), }, ) .unwrap() diff --git a/src/immutable/runner_book_sp.rs b/src/immutable/runner_book_sp.rs index b3cd3fa..2d060d8 100644 --- a/src/immutable/runner_book_sp.rs +++ b/src/immutable/runner_book_sp.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use pyo3::prelude::*; use super::container::SyncObj; @@ -12,9 +14,8 @@ pub struct RunnerBookSP { pub far_price: Option, #[pyo3(get)] pub near_price: Option, - - pub back_stake_taken: SyncObj>, - pub lay_liability_taken: SyncObj>, + pub back_stake_taken: SyncObj>>, + pub lay_liability_taken: SyncObj>>, } #[derive(Default)] @@ -36,10 +37,10 @@ impl RunnerBookSP { near_price: update.near_price.or(self.near_price), back_stake_taken: update .back_stake_taken - .map_or_else(|| self.back_stake_taken.clone(), SyncObj::new), + .map_or_else(|| self.back_stake_taken.clone(), |ps| SyncObj::new(Arc::new(ps))), lay_liability_taken: update .lay_liability_taken - .map_or_else(|| self.lay_liability_taken.clone(), SyncObj::new), + .map_or_else(|| self.lay_liability_taken.clone(), |ps| SyncObj::new(Arc::new(ps))), }, ) .unwrap() diff --git a/src/lib.rs b/src/lib.rs index a36d0c1..aef9558 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,7 +8,6 @@ option_result_contains, is_some_with, min_specialization, - )] mod bflw; @@ -57,7 +56,6 @@ fn betfair_data(py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; - let bflw = PyModule::new(py, "bflw")?; bflw.add_class::()?; bflw.add_class::()?; diff --git a/src/price_size.rs b/src/price_size.rs index 73f8f36..8e4147c 100644 --- a/src/price_size.rs +++ b/src/price_size.rs @@ -71,6 +71,15 @@ impl From for f64 { } } +impl std::ops::Deref for F64OrStr { + type Target = f64; + + #[inline] + fn deref(&self) -> &f64 { + &self.0 + } +} + impl<'de> Deserialize<'de> for F64OrStr { fn deserialize(deserializer: D) -> Result where