From fbe5695edaf0be2f67ef097f705171478e84d02b Mon Sep 17 00:00:00 2001 From: Linlang Date: Tue, 5 Mar 2024 17:01:13 +0800 Subject: [PATCH 01/26] download orderbook data --- examples/orderbook_data/README.md | 3 +-- qlib/data/dataset/utils.py | 8 ++++++-- qlib/tests/data.py | 3 +++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/examples/orderbook_data/README.md b/examples/orderbook_data/README.md index 059ee27056..623d84a7ce 100644 --- a/examples/orderbook_data/README.md +++ b/examples/orderbook_data/README.md @@ -27,8 +27,7 @@ pip install arctic # NOTE: pip may fail to resolve the right package dependency 2. Please follow following steps to download example data ```bash cd examples/orderbook_data/ -wget http://fintech.msra.cn/stock_data/downloads/highfreq_orderboook_example_data.tar.bz2 -tar xf highfreq_orderboook_example_data.tar.bz2 +python scripts/get_data.py qlib_data --target_dir "~/.qlib/orderbook_data" --name orderbook_data ``` 3. Please import the example data to your mongo db diff --git a/qlib/data/dataset/utils.py b/qlib/data/dataset/utils.py index f19dfe08fa..76f3ed4048 100644 --- a/qlib/data/dataset/utils.py +++ b/qlib/data/dataset/utils.py @@ -71,11 +71,15 @@ def fetch_df_by_index( if fetch_orig: for slc in idx_slc: if slc != slice(None, None): - return df.loc[pd.IndexSlice[idx_slc],] # noqa: E231 + return df.loc[ + pd.IndexSlice[idx_slc], + ] # noqa: E231 else: # pylint: disable=W0120 return df else: - return df.loc[pd.IndexSlice[idx_slc],] # noqa: E231 + return df.loc[ + pd.IndexSlice[idx_slc], + ] # noqa: E231 def fetch_df_by_col(df: pd.DataFrame, col_set: Union[str, List[str]]) -> pd.DataFrame: diff --git a/qlib/tests/data.py b/qlib/tests/data.py index f6bd780905..43c75c83e3 100644 --- a/qlib/tests/data.py +++ b/qlib/tests/data.py @@ -183,6 +183,9 @@ def qlib_data( def _get_file_name_with_version(qlib_version, dataset_version): dataset_version = "v2" if dataset_version is None else dataset_version file_name_with_version = f"{dataset_version}/{name}_{region.lower()}_{interval.lower()}_{qlib_version}.zip" + file_name_with_version = ( + "highfreq_orderbook_example_data.zip" if name == "orderbook_data" else file_name_with_version + ) return file_name_with_version file_name = _get_file_name_with_version(qlib_version, dataset_version=version) From 6ea921bd844f7ac60edac622c861916426d4a8f7 Mon Sep 17 00:00:00 2001 From: Linlang Date: Tue, 5 Mar 2024 17:14:36 +0800 Subject: [PATCH 02/26] fix CI error --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 508fd8c3a4..adafefd614 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,8 @@ def get_version(rel_path: str) -> str: # To ensure stable operation of the experiment manager, we have limited the version of mlflow, # and we need to verify whether version 2.0 of mlflow can serve qlib properly. "mlflow>=1.12.1, <=1.30.0", + # mlflow 1.30.0 requires packaging<22, so we limit the packaging version, otherwise the CI will fail. + "packaging<22", "tqdm", "loguru", "lightgbm>=3.3.0", From 8cf7bb3aaf2783a56f13e656b189cb6baf874e0a Mon Sep 17 00:00:00 2001 From: Linlang Date: Tue, 5 Mar 2024 17:24:03 +0800 Subject: [PATCH 03/26] fix CI error --- examples/benchmarks/TRA/src/model.py | 3 --- qlib/backtest/__init__.py | 16 +++++++++------- qlib/backtest/report.py | 8 +++++--- qlib/contrib/eva/alpha.py | 1 + qlib/contrib/model/pytorch_tra.py | 3 --- qlib/contrib/strategy/signal_strategy.py | 1 - qlib/data/dataset/utils.py | 8 ++------ qlib/model/ens/ensemble.py | 2 -- qlib/model/riskmodel/shrink.py | 4 +--- qlib/workflow/online/strategy.py | 1 - scripts/dump_bin.py | 4 +--- scripts/dump_pit.py | 8 +++++--- 12 files changed, 24 insertions(+), 35 deletions(-) diff --git a/examples/benchmarks/TRA/src/model.py b/examples/benchmarks/TRA/src/model.py index affb115a10..ebafd6a521 100644 --- a/examples/benchmarks/TRA/src/model.py +++ b/examples/benchmarks/TRA/src/model.py @@ -324,7 +324,6 @@ def predict(self, dataset, segment="test"): class LSTM(nn.Module): - """LSTM Model Args: @@ -414,7 +413,6 @@ def forward(self, x): class Transformer(nn.Module): - """Transformer Model Args: @@ -475,7 +473,6 @@ def forward(self, x): class TRA(nn.Module): - """Temporal Routing Adaptor (TRA) TRA takes historical prediction errors & latent representation as inputs, diff --git a/qlib/backtest/__init__.py b/qlib/backtest/__init__.py index d784aed57e..9daba91153 100644 --- a/qlib/backtest/__init__.py +++ b/qlib/backtest/__init__.py @@ -162,13 +162,15 @@ def create_account_instance( init_cash=init_cash, position_dict=position_dict, pos_type=pos_type, - benchmark_config={} - if benchmark is None - else { - "benchmark": benchmark, - "start_time": start_time, - "end_time": end_time, - }, + benchmark_config=( + {} + if benchmark is None + else { + "benchmark": benchmark, + "start_time": start_time, + "end_time": end_time, + } + ), ) diff --git a/qlib/backtest/report.py b/qlib/backtest/report.py index 8e7440ba9e..e7c6041efd 100644 --- a/qlib/backtest/report.py +++ b/qlib/backtest/report.py @@ -622,9 +622,11 @@ def cal_trade_indicators( print( "[Indicator({}) {}]: FFR: {}, PA: {}, POS: {}".format( freq, - trade_start_time - if isinstance(trade_start_time, str) - else trade_start_time.strftime("%Y-%m-%d %H:%M:%S"), + ( + trade_start_time + if isinstance(trade_start_time, str) + else trade_start_time.strftime("%Y-%m-%d %H:%M:%S") + ), fulfill_rate, price_advantage, positive_rate, diff --git a/qlib/contrib/eva/alpha.py b/qlib/contrib/eva/alpha.py index 95ec9b91e9..86d366d205 100644 --- a/qlib/contrib/eva/alpha.py +++ b/qlib/contrib/eva/alpha.py @@ -3,6 +3,7 @@ The interface should be redesigned carefully in the future. """ + import pandas as pd from typing import Tuple from qlib import get_module_logger diff --git a/qlib/contrib/model/pytorch_tra.py b/qlib/contrib/model/pytorch_tra.py index 964febf11c..bc9a6aa977 100644 --- a/qlib/contrib/model/pytorch_tra.py +++ b/qlib/contrib/model/pytorch_tra.py @@ -511,7 +511,6 @@ def predict(self, dataset, segment="test"): class RNN(nn.Module): - """RNN Model Args: @@ -601,7 +600,6 @@ def forward(self, x): class Transformer(nn.Module): - """Transformer Model Args: @@ -649,7 +647,6 @@ def forward(self, x): class TRA(nn.Module): - """Temporal Routing Adaptor (TRA) TRA takes historical prediction errors & latent representation as inputs, diff --git a/qlib/contrib/strategy/signal_strategy.py b/qlib/contrib/strategy/signal_strategy.py index 9ba960eebd..bad19ddfdc 100644 --- a/qlib/contrib/strategy/signal_strategy.py +++ b/qlib/contrib/strategy/signal_strategy.py @@ -373,7 +373,6 @@ def generate_trade_decision(self, execute_result=None): class EnhancedIndexingStrategy(WeightStrategyBase): - """Enhanced Indexing Strategy Enhanced indexing combines the arts of active management and passive management, diff --git a/qlib/data/dataset/utils.py b/qlib/data/dataset/utils.py index 76f3ed4048..f19dfe08fa 100644 --- a/qlib/data/dataset/utils.py +++ b/qlib/data/dataset/utils.py @@ -71,15 +71,11 @@ def fetch_df_by_index( if fetch_orig: for slc in idx_slc: if slc != slice(None, None): - return df.loc[ - pd.IndexSlice[idx_slc], - ] # noqa: E231 + return df.loc[pd.IndexSlice[idx_slc],] # noqa: E231 else: # pylint: disable=W0120 return df else: - return df.loc[ - pd.IndexSlice[idx_slc], - ] # noqa: E231 + return df.loc[pd.IndexSlice[idx_slc],] # noqa: E231 def fetch_df_by_col(df: pd.DataFrame, col_set: Union[str, List[str]]) -> pd.DataFrame: diff --git a/qlib/model/ens/ensemble.py b/qlib/model/ens/ensemble.py index ede1f8e3ad..1ebb16f18b 100644 --- a/qlib/model/ens/ensemble.py +++ b/qlib/model/ens/ensemble.py @@ -30,7 +30,6 @@ def __call__(self, ensemble_dict: dict, *args, **kwargs): class SingleKeyEnsemble(Ensemble): - """ Extract the object if there is only one key and value in the dict. Make the result more readable. {Only key: Only value} -> Only value @@ -64,7 +63,6 @@ def __call__(self, ensemble_dict: Union[dict, object], recursion: bool = True) - class RollingEnsemble(Ensemble): - """Merge a dict of rolling dataframe like `prediction` or `IC` into an ensemble. NOTE: The values of dict must be pd.DataFrame, and have the index "datetime". diff --git a/qlib/model/riskmodel/shrink.py b/qlib/model/riskmodel/shrink.py index b2594f707d..c3c0e48ef8 100644 --- a/qlib/model/riskmodel/shrink.py +++ b/qlib/model/riskmodel/shrink.py @@ -247,9 +247,7 @@ def _get_shrink_param_lw_single_factor(self, X: np.ndarray, S: np.ndarray, F: np v1 = y.T.dot(z) / t - cov_mkt[:, None] * S roff1 = np.sum(v1 * cov_mkt[:, None].T) / var_mkt - np.sum(np.diag(v1) * cov_mkt) / var_mkt v3 = z.T.dot(z) / t - var_mkt * S - roff3 = ( - np.sum(v3 * np.outer(cov_mkt, cov_mkt)) / var_mkt**2 - np.sum(np.diag(v3) * cov_mkt**2) / var_mkt**2 - ) + roff3 = np.sum(v3 * np.outer(cov_mkt, cov_mkt)) / var_mkt**2 - np.sum(np.diag(v3) * cov_mkt**2) / var_mkt**2 roff = 2 * roff1 - roff3 rho = rdiag + roff diff --git a/qlib/workflow/online/strategy.py b/qlib/workflow/online/strategy.py index f2988d843f..d545e4bc9a 100644 --- a/qlib/workflow/online/strategy.py +++ b/qlib/workflow/online/strategy.py @@ -90,7 +90,6 @@ def get_collector(self) -> Collector: class RollingStrategy(OnlineStrategy): - """ This example strategy always uses the latest rolling model sas online models. """ diff --git a/scripts/dump_bin.py b/scripts/dump_bin.py index 92abc8beec..a65b1f58ee 100644 --- a/scripts/dump_bin.py +++ b/scripts/dump_bin.py @@ -146,9 +146,7 @@ def get_dump_fields(self, df_columns: Iterable[str]) -> Iterable[str]: return ( self._include_fields if self._include_fields - else set(df_columns) - set(self._exclude_fields) - if self._exclude_fields - else df_columns + else set(df_columns) - set(self._exclude_fields) if self._exclude_fields else df_columns ) @staticmethod diff --git a/scripts/dump_pit.py b/scripts/dump_pit.py index 34d304ed78..1ca9cfc942 100644 --- a/scripts/dump_pit.py +++ b/scripts/dump_pit.py @@ -132,9 +132,11 @@ def get_dump_fields(self, df: Iterable[str]) -> Iterable[str]: return ( set(self._include_fields) if self._include_fields - else set(df[self.field_column_name]) - set(self._exclude_fields) - if self._exclude_fields - else set(df[self.field_column_name]) + else ( + set(df[self.field_column_name]) - set(self._exclude_fields) + if self._exclude_fields + else set(df[self.field_column_name]) + ) ) def get_filenames(self, symbol, field, interval): From 9f1808378d6c16b7cc31d2a6c43214cef18a482e Mon Sep 17 00:00:00 2001 From: Linlang Date: Tue, 5 Mar 2024 22:25:22 +0800 Subject: [PATCH 04/26] test fix CI error --- .github/workflows/test_qlib_from_source_slow.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test_qlib_from_source_slow.yml b/.github/workflows/test_qlib_from_source_slow.yml index caab6f444e..d36d217adf 100644 --- a/.github/workflows/test_qlib_from_source_slow.yml +++ b/.github/workflows/test_qlib_from_source_slow.yml @@ -37,6 +37,12 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Set up Python ${{ matrix.python-version }} + if: (matrix.python-version != '3.8') + uses: actions/setup-python@v4 + with: + python-version: "3.8.17" + - name: Set up Python tools run: | python -m pip install --upgrade pip From e71ee221f9bc1d21eaa169b707df32f42b1dc4a7 Mon Sep 17 00:00:00 2001 From: Linlang Date: Tue, 5 Mar 2024 23:00:56 +0800 Subject: [PATCH 05/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 8 +++++++- .github/workflows/test_qlib_from_source_slow.yml | 6 ------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index 9205a13641..123cfe1fb2 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -22,6 +22,12 @@ jobs: - name: Test qlib from source uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + if: (matrix.python-version == '3.8') + uses: actions/setup-python@v4 + with: + python-version: "3.8.17" + # Since version 3.7 of python for MacOS is installed in CI, version 3.7.17, this version causes "_bz not found error". # So we make the version number of python 3.7 for MacOS more specific. # refs: https://github.com/actions/setup-python/issues/682 @@ -32,7 +38,7 @@ jobs: python-version: "3.7.16" - name: Set up Python ${{ matrix.python-version }} - if: (matrix.os != 'macos-latest' || matrix.python-version != '3.7') && (matrix.os != 'macos-11' || matrix.python-version != '3.7') + if: (matrix.os != 'macos-latest' && matrix.python-version == '3.7') uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test_qlib_from_source_slow.yml b/.github/workflows/test_qlib_from_source_slow.yml index d36d217adf..caab6f444e 100644 --- a/.github/workflows/test_qlib_from_source_slow.yml +++ b/.github/workflows/test_qlib_from_source_slow.yml @@ -37,12 +37,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Set up Python ${{ matrix.python-version }} - if: (matrix.python-version != '3.8') - uses: actions/setup-python@v4 - with: - python-version: "3.8.17" - - name: Set up Python tools run: | python -m pip install --upgrade pip From 3f41787deac94c53de94bbcc236ce4d4af6e01de Mon Sep 17 00:00:00 2001 From: Linlang Date: Tue, 5 Mar 2024 23:03:49 +0800 Subject: [PATCH 06/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index 123cfe1fb2..301935c60e 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -26,7 +26,7 @@ jobs: if: (matrix.python-version == '3.8') uses: actions/setup-python@v4 with: - python-version: "3.8.17" + python-version: "3.8.16" # Since version 3.7 of python for MacOS is installed in CI, version 3.7.17, this version causes "_bz not found error". # So we make the version number of python 3.7 for MacOS more specific. From 5af52c2472f85e28cf9adc8a606f217b12227716 Mon Sep 17 00:00:00 2001 From: Linlang Date: Tue, 5 Mar 2024 23:09:09 +0800 Subject: [PATCH 07/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index 301935c60e..0889f6c7a6 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Python ${{ matrix.python-version }} if: (matrix.python-version == '3.8') - uses: actions/setup-python@v4 + uses: actions/setup-python@v3 with: python-version: "3.8.16" From 66fad0a92ece108cd6bdd9556ebd0a05f6c5ca82 Mon Sep 17 00:00:00 2001 From: Linlang Date: Tue, 5 Mar 2024 23:10:28 +0800 Subject: [PATCH 08/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index 0889f6c7a6..a13b177e91 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -26,7 +26,7 @@ jobs: if: (matrix.python-version == '3.8') uses: actions/setup-python@v3 with: - python-version: "3.8.16" + python-version: "3.8.17" # Since version 3.7 of python for MacOS is installed in CI, version 3.7.17, this version causes "_bz not found error". # So we make the version number of python 3.7 for MacOS more specific. From 77b6fcb92a44c350f4f4369df23e0ab114b4080c Mon Sep 17 00:00:00 2001 From: Linlang Date: Tue, 5 Mar 2024 23:13:23 +0800 Subject: [PATCH 09/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index a13b177e91..394d53d5c1 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Python ${{ matrix.python-version }} if: (matrix.python-version == '3.8') - uses: actions/setup-python@v3 + uses: actions/setup-python@v2 with: python-version: "3.8.17" From 835ef12c20c3166526102d3908fe6b595fcb905d Mon Sep 17 00:00:00 2001 From: Linlang Date: Tue, 5 Mar 2024 23:21:22 +0800 Subject: [PATCH 10/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index 394d53d5c1..cc17fddb82 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -24,9 +24,9 @@ jobs: - name: Set up Python ${{ matrix.python-version }} if: (matrix.python-version == '3.8') - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: "3.8.17" + python-version: "3.8.10" # Since version 3.7 of python for MacOS is installed in CI, version 3.7.17, this version causes "_bz not found error". # So we make the version number of python 3.7 for MacOS more specific. From f095792231f70d8dbccf70d61cb36ec2209fcb13 Mon Sep 17 00:00:00 2001 From: Linlang Date: Tue, 5 Mar 2024 23:29:20 +0800 Subject: [PATCH 11/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index cc17fddb82..adef6d99f3 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -23,11 +23,17 @@ jobs: uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - if: (matrix.python-version == '3.8') + if: (matrix.python-version == '3.8' && matrix.os == 'windows-latest') uses: actions/setup-python@v4 with: python-version: "3.8.10" + - name: Set up Python ${{ matrix.python-version }} + if: (matrix.python-version == '3.8' && matrix.os != 'windows-latest') + uses: actions/setup-python@v4 + with: + python-version: "3.8.17" + # Since version 3.7 of python for MacOS is installed in CI, version 3.7.17, this version causes "_bz not found error". # So we make the version number of python 3.7 for MacOS more specific. # refs: https://github.com/actions/setup-python/issues/682 From 1f390feafd22e2af39455f47e7f57b2506936720 Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 10:14:28 +0800 Subject: [PATCH 12/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index adef6d99f3..856136b05d 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -22,18 +22,6 @@ jobs: - name: Test qlib from source uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - if: (matrix.python-version == '3.8' && matrix.os == 'windows-latest') - uses: actions/setup-python@v4 - with: - python-version: "3.8.10" - - - name: Set up Python ${{ matrix.python-version }} - if: (matrix.python-version == '3.8' && matrix.os != 'windows-latest') - uses: actions/setup-python@v4 - with: - python-version: "3.8.17" - # Since version 3.7 of python for MacOS is installed in CI, version 3.7.17, this version causes "_bz not found error". # So we make the version number of python 3.7 for MacOS more specific. # refs: https://github.com/actions/setup-python/issues/682 @@ -44,7 +32,7 @@ jobs: python-version: "3.7.16" - name: Set up Python ${{ matrix.python-version }} - if: (matrix.os != 'macos-latest' && matrix.python-version == '3.7') + if: (matrix.os != 'macos-latest' || matrix.python-version != '3.7') && (matrix.os != 'macos-11' || matrix.python-version != '3.7') uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -72,6 +60,12 @@ jobs: run: | python -m pip install --upgrade cython python -m pip install -e .[dev] + python -m pip install Mako==1.3.0 + python -m pip install gitpython==3.1.40 + python -m pip install importlib-resources==6.1.1 + python -m pip install prometheus-client==0.19.0 + python -m pip install python-dateutil==2.8.2 + python -m pip install sqlalchemy==1.4.51 - name: Lint with Black # Python 3.7 will use a black with low level. So we use python with higher version for black check From f568c2f126d0547e8c0b7e866410f8ed30e9ab92 Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 10:19:54 +0800 Subject: [PATCH 13/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index 856136b05d..ea1c4b1a78 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -60,12 +60,12 @@ jobs: run: | python -m pip install --upgrade cython python -m pip install -e .[dev] - python -m pip install Mako==1.3.0 - python -m pip install gitpython==3.1.40 - python -m pip install importlib-resources==6.1.1 - python -m pip install prometheus-client==0.19.0 - python -m pip install python-dateutil==2.8.2 - python -m pip install sqlalchemy==1.4.51 + python -m pip install "Mako<=1.3.0" + python -m pip install "gitpython<=3.1.40" + python -m pip install "importlib-resources<=6.1.1" + python -m pip install "prometheus-client<=0.19.0" + python -m pip install "python-dateutil<=2.8.2" + python -m pip install "sqlalchemy<=1.4.51" - name: Lint with Black # Python 3.7 will use a black with low level. So we use python with higher version for black check From 25a6ff08123cabb7efe00d7f6efaa560898a2229 Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 10:42:26 +0800 Subject: [PATCH 14/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index ea1c4b1a78..2d07dfdc10 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -66,6 +66,14 @@ jobs: python -m pip install "prometheus-client<=0.19.0" python -m pip install "python-dateutil<=2.8.2" python -m pip install "sqlalchemy<=1.4.51" + python -m pip uninstall certifi + python -m pip uninstall charset-normalizer + python -m pip uninstall requests + python -m pip uninstall urllib3 + python -m pip uninstall certifi + python -m pip uninstall charset-normalizer + python -m pip uninstall requests + python -m pip uninstall urllib3 - name: Lint with Black # Python 3.7 will use a black with low level. So we use python with higher version for black check From 240cdff0f3c9bc63dc1a71c58aa917baa99be5e1 Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 10:47:21 +0800 Subject: [PATCH 15/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index 2d07dfdc10..8c1a0a6802 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -66,14 +66,10 @@ jobs: python -m pip install "prometheus-client<=0.19.0" python -m pip install "python-dateutil<=2.8.2" python -m pip install "sqlalchemy<=1.4.51" - python -m pip uninstall certifi - python -m pip uninstall charset-normalizer - python -m pip uninstall requests - python -m pip uninstall urllib3 - python -m pip uninstall certifi - python -m pip uninstall charset-normalizer - python -m pip uninstall requests - python -m pip uninstall urllib3 + python -m pip uninstall -y certifi + python -m pip uninstall -y charset-normalizer + python -m pip uninstall -y requests + python -m pip uninstall -y urllib3 - name: Lint with Black # Python 3.7 will use a black with low level. So we use python with higher version for black check From 1dc5e7308d2082552381c592da9cf308675477c4 Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 10:51:55 +0800 Subject: [PATCH 16/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index 8c1a0a6802..00fe11b106 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -68,7 +68,6 @@ jobs: python -m pip install "sqlalchemy<=1.4.51" python -m pip uninstall -y certifi python -m pip uninstall -y charset-normalizer - python -m pip uninstall -y requests python -m pip uninstall -y urllib3 - name: Lint with Black From cb0712a953629861d208c9577680ff01fe7495e9 Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 10:55:07 +0800 Subject: [PATCH 17/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index 00fe11b106..e06f4237ad 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -68,7 +68,6 @@ jobs: python -m pip install "sqlalchemy<=1.4.51" python -m pip uninstall -y certifi python -m pip uninstall -y charset-normalizer - python -m pip uninstall -y urllib3 - name: Lint with Black # Python 3.7 will use a black with low level. So we use python with higher version for black check From cac66e9c9dba3565e523db916cc2ce5b877bd70f Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 11:17:41 +0800 Subject: [PATCH 18/26] test fix CI error --- .github/workflows/test_qlib_from_source.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml index e06f4237ad..9205a13641 100644 --- a/.github/workflows/test_qlib_from_source.yml +++ b/.github/workflows/test_qlib_from_source.yml @@ -60,14 +60,6 @@ jobs: run: | python -m pip install --upgrade cython python -m pip install -e .[dev] - python -m pip install "Mako<=1.3.0" - python -m pip install "gitpython<=3.1.40" - python -m pip install "importlib-resources<=6.1.1" - python -m pip install "prometheus-client<=0.19.0" - python -m pip install "python-dateutil<=2.8.2" - python -m pip install "sqlalchemy<=1.4.51" - python -m pip uninstall -y certifi - python -m pip uninstall -y charset-normalizer - name: Lint with Black # Python 3.7 will use a black with low level. So we use python with higher version for black check From f206f0a6da33667d9b4c53eeabfeae2447d1e4fe Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 12:32:13 +0800 Subject: [PATCH 19/26] test fix CI error --- tests/test_workflow.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 129abc0fbb..2e60a0d959 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -10,6 +10,8 @@ class WorkflowTest(TestAutoData): TMP_PATH = Path("./.mlruns_tmp/") + TMP_PATH.mkdir() + print("创建的文件夹路径:", TMP_PATH.resolve()) def tearDown(self) -> None: if self.TMP_PATH.exists(): From 4023277874812ce2c5ce12ff9780a2a1b855a1b0 Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 12:53:52 +0800 Subject: [PATCH 20/26] test fix CI error --- tests/test_workflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 2e60a0d959..04bfb2ce2a 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -10,8 +10,6 @@ class WorkflowTest(TestAutoData): TMP_PATH = Path("./.mlruns_tmp/") - TMP_PATH.mkdir() - print("创建的文件夹路径:", TMP_PATH.resolve()) def tearDown(self) -> None: if self.TMP_PATH.exists(): @@ -19,6 +17,8 @@ def tearDown(self) -> None: def test_get_local_dir(self): """ """ + self.TMP_PATH.mkdir() + print(self.TMP_PATH.resolve()) with R.start(uri=str(self.TMP_PATH)): pass From e979590083d85f25dd21d65f7a428df529b05f03 Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 13:20:42 +0800 Subject: [PATCH 21/26] test fix CI error --- tests/test_workflow.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 04bfb2ce2a..b71c664ae9 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -17,7 +17,8 @@ def tearDown(self) -> None: def test_get_local_dir(self): """ """ - self.TMP_PATH.mkdir() + folder_path = Path("./.mlruns_tmp/.trash") + folder_path.mkdir(parents=True, exist_ok=True) print(self.TMP_PATH.resolve()) with R.start(uri=str(self.TMP_PATH)): pass From 0c14952136a3b1aef4b046aacbee191916df2e42 Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 13:57:37 +0800 Subject: [PATCH 22/26] test fix CI error --- tests/test_workflow.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_workflow.py b/tests/test_workflow.py index b71c664ae9..56a8e80ce3 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -9,7 +9,7 @@ class WorkflowTest(TestAutoData): - TMP_PATH = Path("./.mlruns_tmp/") + TMP_PATH = Path("./.mlruns_tmp/.trash") def tearDown(self) -> None: if self.TMP_PATH.exists(): @@ -17,9 +17,8 @@ def tearDown(self) -> None: def test_get_local_dir(self): """ """ - folder_path = Path("./.mlruns_tmp/.trash") - folder_path.mkdir(parents=True, exist_ok=True) - print(self.TMP_PATH.resolve()) + self.TMP_PATH.mkdir(parents=True, exist_ok=True) + with R.start(uri=str(self.TMP_PATH)): pass From fd0863b0bbb1e97a8efddd41c17554de372251ea Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 15:17:04 +0800 Subject: [PATCH 23/26] optimize get_data code --- examples/orderbook_data/README.md | 2 +- qlib/tests/data.py | 41 +++++++++++++++++++++++++++++++ tests/test_workflow.py | 2 ++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/examples/orderbook_data/README.md b/examples/orderbook_data/README.md index 623d84a7ce..93e6498296 100644 --- a/examples/orderbook_data/README.md +++ b/examples/orderbook_data/README.md @@ -27,7 +27,7 @@ pip install arctic # NOTE: pip may fail to resolve the right package dependency 2. Please follow following steps to download example data ```bash cd examples/orderbook_data/ -python scripts/get_data.py qlib_data --target_dir "~/.qlib/orderbook_data" --name orderbook_data +python get_data.py other_data --target_dir ~/.qlib/other_data/orderbook_data --name highfreq_orderbook_example_data.zip ``` 3. Please import the example data to your mongo db diff --git a/qlib/tests/data.py b/qlib/tests/data.py index 43c75c83e3..bbd0565229 100644 --- a/qlib/tests/data.py +++ b/qlib/tests/data.py @@ -130,6 +130,47 @@ def _delete_qlib_data(file_dir: Path): logger.warning(f"delete: {_p}") shutil.rmtree(_p) + def other_data( + self, + target_dir="~/.qlib/other_data", + name=None, + version=None, + delete_old=True, + ): + """Specifies a file name to download a file from a remote location. + + Parameters + ---------- + target_dir: str + data save directory + name: str + filename, by default None + version: str + data version, value from [v1, ...], by default None(use script to specify version) + delete_old: bool + delete an existing directory, by default True + + Examples + --------- + # get orderbook data + python get_data.py other_data --target_dir ~/.qlib/other_data/orderbook_data --name highfreq_orderbook_example_data.zip + When this command is run, the data will be downloaded from this link: https://qlibpublic.blob.core.windows.net/data/default/stock_data/highfreq_orderbook_example_data.zip?{token} + ------- + + """ + if name == None: + logger.warning("Specify the name of the file to be downloaded.") + return + + file_name = f"{version}/{name}" + file_name = name if version == None else file_name + + if not self.check_dataset(file_name): + logger.warning("The file you specified does not exist in the remote repository.") + return + + self.download_data(file_name.lower(), target_dir, delete_old) + def qlib_data( self, name="qlib_data", diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 56a8e80ce3..cf17b3d18a 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -9,6 +9,8 @@ class WorkflowTest(TestAutoData): + # Creating the directory manually doesn't work with mlflow, + # so we add a subfolder named .trash when we create the directory. TMP_PATH = Path("./.mlruns_tmp/.trash") def tearDown(self) -> None: From e85580600ce460be7387965748019c43c0a2a795 Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 15:17:42 +0800 Subject: [PATCH 24/26] optimize get_data code --- qlib/tests/data.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/qlib/tests/data.py b/qlib/tests/data.py index bbd0565229..c7de506cb5 100644 --- a/qlib/tests/data.py +++ b/qlib/tests/data.py @@ -224,9 +224,6 @@ def qlib_data( def _get_file_name_with_version(qlib_version, dataset_version): dataset_version = "v2" if dataset_version is None else dataset_version file_name_with_version = f"{dataset_version}/{name}_{region.lower()}_{interval.lower()}_{qlib_version}.zip" - file_name_with_version = ( - "highfreq_orderbook_example_data.zip" if name == "orderbook_data" else file_name_with_version - ) return file_name_with_version file_name = _get_file_name_with_version(qlib_version, dataset_version=version) From 467b352553228aeb2c934773490986f1ad1df777 Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 15:46:26 +0800 Subject: [PATCH 25/26] optimize get_data code --- examples/orderbook_data/README.md | 2 +- qlib/tests/data.py | 41 ------------------------------- 2 files changed, 1 insertion(+), 42 deletions(-) diff --git a/examples/orderbook_data/README.md b/examples/orderbook_data/README.md index 93e6498296..d5c83e22ee 100644 --- a/examples/orderbook_data/README.md +++ b/examples/orderbook_data/README.md @@ -27,7 +27,7 @@ pip install arctic # NOTE: pip may fail to resolve the right package dependency 2. Please follow following steps to download example data ```bash cd examples/orderbook_data/ -python get_data.py other_data --target_dir ~/.qlib/other_data/orderbook_data --name highfreq_orderbook_example_data.zip +python get_data.py download_data --target_dir ~/.qlib/orderbook_data --file_name highfreq_orderbook_example_data.zip ``` 3. Please import the example data to your mongo db diff --git a/qlib/tests/data.py b/qlib/tests/data.py index c7de506cb5..f6bd780905 100644 --- a/qlib/tests/data.py +++ b/qlib/tests/data.py @@ -130,47 +130,6 @@ def _delete_qlib_data(file_dir: Path): logger.warning(f"delete: {_p}") shutil.rmtree(_p) - def other_data( - self, - target_dir="~/.qlib/other_data", - name=None, - version=None, - delete_old=True, - ): - """Specifies a file name to download a file from a remote location. - - Parameters - ---------- - target_dir: str - data save directory - name: str - filename, by default None - version: str - data version, value from [v1, ...], by default None(use script to specify version) - delete_old: bool - delete an existing directory, by default True - - Examples - --------- - # get orderbook data - python get_data.py other_data --target_dir ~/.qlib/other_data/orderbook_data --name highfreq_orderbook_example_data.zip - When this command is run, the data will be downloaded from this link: https://qlibpublic.blob.core.windows.net/data/default/stock_data/highfreq_orderbook_example_data.zip?{token} - ------- - - """ - if name == None: - logger.warning("Specify the name of the file to be downloaded.") - return - - file_name = f"{version}/{name}" - file_name = name if version == None else file_name - - if not self.check_dataset(file_name): - logger.warning("The file you specified does not exist in the remote repository.") - return - - self.download_data(file_name.lower(), target_dir, delete_old) - def qlib_data( self, name="qlib_data", From 304f586f454ea8eb396b1fb17432d24617010316 Mon Sep 17 00:00:00 2001 From: Linlang Date: Wed, 6 Mar 2024 19:22:30 +0800 Subject: [PATCH 26/26] optimize README --- examples/orderbook_data/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/orderbook_data/README.md b/examples/orderbook_data/README.md index d5c83e22ee..890e11f41e 100644 --- a/examples/orderbook_data/README.md +++ b/examples/orderbook_data/README.md @@ -27,12 +27,11 @@ pip install arctic # NOTE: pip may fail to resolve the right package dependency 2. Please follow following steps to download example data ```bash cd examples/orderbook_data/ -python get_data.py download_data --target_dir ~/.qlib/orderbook_data --file_name highfreq_orderbook_example_data.zip +python ../../scripts/get_data.py download_data --target_dir . --file_name highfreq_orderbook_example_data.zip ``` 3. Please import the example data to your mongo db ```bash -cd examples/orderbook_data/ python create_dataset.py initialize_library # Initialization Libraries python create_dataset.py import_data # Initialization Libraries ``` @@ -41,7 +40,6 @@ python create_dataset.py import_data # Initialization Libraries After importing these data, you run `example.py` to create some high-frequency features. ```bash -cd examples/orderbook_data/ pytest -s --disable-warnings example.py # If you want run all examples pytest -s --disable-warnings example.py::TestClass::test_exp_10 # If you want to run specific example ```