From eab2c96f9c52cc5f4f8aa83f634fbbc72d394a6a Mon Sep 17 00:00:00 2001 From: ZingLix Date: Mon, 8 Jan 2024 11:50:24 +0800 Subject: [PATCH 01/11] readme: add feedback badge --- README.md | 3 ++- README.pypi.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 27961a08..c5af381b 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![Release Notes](https://img.shields.io/github/release/baidubce/bce-qianfan-sdk)](https://github.com/baidubce/bce-qianfan-sdk/releases) [![PyPI version](https://badge.fury.io/py/qianfan.svg)](https://pypi.org/project/qianfan/) [![Documentation Status](https://readthedocs.org/projects/qianfan/badge/?version=stable)](https://qianfan.readthedocs.io/en/stable/README.html) +[![Feedback](https://img.shields.io/badge/feedback-here-green)](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) ## 简介 @@ -171,7 +172,7 @@ trainer.run() > Check [**API References**](https://qianfan.readthedocs.io/en/stable/qianfan.html) for more details. -### 联系我们 +## 联系我们 如使用过程中遇到什么问题，或对SDK功能有建议，可通过如下方式联系我们 - [GitHub issues](https://github.com/baidubce/bce-qianfan-sdk/issues) - [百度智能云工单](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) （百度专家即时服务） diff --git a/README.pypi.md b/README.pypi.md index deca4e00..58220393 100644 --- a/README.pypi.md +++ b/README.pypi.md @@ -4,6 +4,7 @@ [![Release Notes](https://img.shields.io/github/release/baidubce/bce-qianfan-sdk)](https://github.com/baidubce/bce-qianfan-sdk/releases) [![PyPI version](https://badge.fury.io/py/qianfan.svg)](https://pypi.org/project/qianfan/) [![Documentation Status](https://readthedocs.org/projects/qianfan/badge/?version=stable)](https://qianfan.readthedocs.io/en/stable/README.html) +[![Feedback](https://img.shields.io/badge/feedback-here-green)](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) [Documentation](https://qianfan.readthedocs.io/en/stable/README.html) | [GitHub](https://github.com/baidubce/bce-qianfan-sdk) | [Cookbook](https://github.com/baidubce/bce-qianfan-sdk/tree/main/cookbook) From 052d82300bc0a3431ce1dd4b6997bfab635f92fb Mon Sep 17 00:00:00 2001 From: ZingLix Date: Mon, 8 Jan 2024 11:58:03 +0800 Subject: [PATCH 02/11] update badge --- README.md | 3 ++- README.pypi.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c5af381b..0c3e6fd6 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ [![Release Notes](https://img.shields.io/github/release/baidubce/bce-qianfan-sdk)](https://github.com/baidubce/bce-qianfan-sdk/releases) [![PyPI version](https://badge.fury.io/py/qianfan.svg)](https://pypi.org/project/qianfan/) [![Documentation Status](https://readthedocs.org/projects/qianfan/badge/?version=stable)](https://qianfan.readthedocs.io/en/stable/README.html) -[![Feedback](https://img.shields.io/badge/feedback-here-green)](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) +[![Feedback Issue](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-GitHub_Issue-green)](https://github.com/baidubce/bce-qianfan-sdk/issues) +[![Feedback Ticket](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-%E7%99%BE%E5%BA%A6%E6%99%BA%E8%83%BD%E4%BA%91%E5%B7%A5%E5%8D%95-green)](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) ## 简介 diff --git a/README.pypi.md b/README.pypi.md index 58220393..ca222d41 100644 --- a/README.pypi.md +++ b/README.pypi.md @@ -4,7 +4,8 @@ [![Release Notes](https://img.shields.io/github/release/baidubce/bce-qianfan-sdk)](https://github.com/baidubce/bce-qianfan-sdk/releases) [![PyPI version](https://badge.fury.io/py/qianfan.svg)](https://pypi.org/project/qianfan/) [![Documentation Status](https://readthedocs.org/projects/qianfan/badge/?version=stable)](https://qianfan.readthedocs.io/en/stable/README.html) -[![Feedback](https://img.shields.io/badge/feedback-here-green)](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) +[![Feedback Issue](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-GitHub_Issue-green)](https://github.com/baidubce/bce-qianfan-sdk/issues) +[![Feedback Ticket](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-%E7%99%BE%E5%BA%A6%E6%99%BA%E8%83%BD%E4%BA%91%E5%B7%A5%E5%8D%95-green)](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) [Documentation](https://qianfan.readthedocs.io/en/stable/README.html) | [GitHub](https://github.com/baidubce/bce-qianfan-sdk) | [Cookbook](https://github.com/baidubce/bce-qianfan-sdk/tree/main/cookbook) From e4257fd42873ece1806a99bc70f3898c92b4ef0b Mon Sep 17 00:00:00 2001 From: ZingLix Date: Mon, 8 Jan 2024 11:59:21 +0800 Subject: [PATCH 03/11] update badge --- README.md | 4 ++-- README.pypi.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0c3e6fd6..009de298 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ [![Release Notes](https://img.shields.io/github/release/baidubce/bce-qianfan-sdk)](https://github.com/baidubce/bce-qianfan-sdk/releases) [![PyPI version](https://badge.fury.io/py/qianfan.svg)](https://pypi.org/project/qianfan/) [![Documentation Status](https://readthedocs.org/projects/qianfan/badge/?version=stable)](https://qianfan.readthedocs.io/en/stable/README.html) -[![Feedback Issue](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-GitHub_Issue-green)](https://github.com/baidubce/bce-qianfan-sdk/issues) -[![Feedback Ticket](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-%E7%99%BE%E5%BA%A6%E6%99%BA%E8%83%BD%E4%BA%91%E5%B7%A5%E5%8D%95-green)](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) +[![Feedback Issue](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-GitHub_Issue-brightgreen)](https://github.com/baidubce/bce-qianfan-sdk/issues) +[![Feedback Ticket](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-%E7%99%BE%E5%BA%A6%E6%99%BA%E8%83%BD%E4%BA%91%E5%B7%A5%E5%8D%95-brightgreen)](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) ## 简介 diff --git a/README.pypi.md b/README.pypi.md index ca222d41..1d04d892 100644 --- a/README.pypi.md +++ b/README.pypi.md @@ -4,8 +4,8 @@ [![Release Notes](https://img.shields.io/github/release/baidubce/bce-qianfan-sdk)](https://github.com/baidubce/bce-qianfan-sdk/releases) [![PyPI version](https://badge.fury.io/py/qianfan.svg)](https://pypi.org/project/qianfan/) [![Documentation Status](https://readthedocs.org/projects/qianfan/badge/?version=stable)](https://qianfan.readthedocs.io/en/stable/README.html) -[![Feedback Issue](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-GitHub_Issue-green)](https://github.com/baidubce/bce-qianfan-sdk/issues) -[![Feedback Ticket](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-%E7%99%BE%E5%BA%A6%E6%99%BA%E8%83%BD%E4%BA%91%E5%B7%A5%E5%8D%95-green)](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) +[![Feedback Issue](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-GitHub_Issue-brightgreen)](https://github.com/baidubce/bce-qianfan-sdk/issues) +[![Feedback Ticket](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-%E7%99%BE%E5%BA%A6%E6%99%BA%E8%83%BD%E4%BA%91%E5%B7%A5%E5%8D%95-brightgreen)](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) [Documentation](https://qianfan.readthedocs.io/en/stable/README.html) | [GitHub](https://github.com/baidubce/bce-qianfan-sdk) | [Cookbook](https://github.com/baidubce/bce-qianfan-sdk/tree/main/cookbook) From 2f08f768cf67afec3534cb87423ad424faf20f42 Mon Sep 17 00:00:00 2001 From: ZingLix Date: Mon, 8 Jan 2024 12:02:21 +0800 Subject: [PATCH 04/11] update pypi readme --- README.pypi.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.pypi.md b/README.pypi.md index 1d04d892..136cdca3 100644 --- a/README.pypi.md +++ b/README.pypi.md @@ -110,10 +110,15 @@ print(resp["result"]) - Tokenizer [[Doc](https://qianfan.readthedocs.io/en/stable/docs/utils.html)][[GitHub](https://github.com/baidubce/bce-qianfan-sdk/blob/main/docs/utils.md)] - 接口流控 [[Doc](https://qianfan.readthedocs.io/en/stable/docs/configurable.html)][[GitHub](https://github.com/baidubce/bce-qianfan-sdk/blob/main/docs/configurable.md)] -详细信息请参考相应的文档。如果有任何问题，欢迎前往 [GitHub](https://github.com/baidubce/bce-qianfan-sdk) 提交 issue。 - > 还可以通过 [**API References**](https://qianfan.readthedocs.io/en/stable/qianfan.html) 查看每个接口的详细说明。 +## 联系我们 + +如使用过程中遇到什么问题，或对SDK功能有建议，可通过如下方式联系我们 + +- [GitHub issues](https://github.com/baidubce/bce-qianfan-sdk/issues) +- [百度智能云工单](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) （百度专家即时服务） + ## License Apache-2.0 From 8c672d249909b4b8aea7fb8e4345ff1f9952724f Mon Sep 17 00:00:00 2001 From: Dobiichi-Origami <56953648+Dobiichi-Origami@users.noreply.github.com> Date: Mon, 8 Jan 2024 15:50:59 +0800 Subject: [PATCH 05/11] switch pydantic-settings to dynaconf --- pyproject.toml | 4 +- src/qianfan/config.py | 222 ++++++++++++++++++++++++------------------ 2 files changed, 131 insertions(+), 95 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e2c546cc..b2bdbfa0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,8 +22,7 @@ aiolimiter = ">=1.1.0" importlib-metadata = { version = ">=1.4.0", python = "<=3.7" } bce-python-sdk = ">=0.8.79" typing-extensions = { version = ">=4.0.0", python = "<=3.10" } -pydantic = ">=2" -pydantic-settings = ">=2.0.3" +pydantic = "*" langchain = { version = ">=0.0.321", python = ">=3.8.1", optional = true } numpy = [ { version = "<1.22.0", python = ">=3.7 <3.8" }, @@ -37,6 +36,7 @@ python-dateutil = "^2.8.2" rich = ">=13.0.0" typer = ">=0.9.0" tenacity = "^8.2.3" +dynaconf = "<=3.1.11" [tool.poetry.scripts] qianfan = "qianfan.common.client.main:main" diff --git a/src/qianfan/config.py b/src/qianfan/config.py index 09e4505f..2370bda7 100644 --- a/src/qianfan/config.py +++ b/src/qianfan/config.py @@ -11,109 +11,145 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os from typing import Optional -from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict +from dynaconf import Dynaconf, Validator from typing_extensions import deprecated -from qianfan.consts import DefaultValue, Env +from qianfan.consts import DefaultValue +_GLOBAL_CONFIG: Optional[Dynaconf] = None -class GlobalConfig(BaseSettings): - """ - The global config of whole qianfan sdk - """ - model_config = SettingsConfigDict(env_prefix="QIANFAN_", case_sensitive=True) - - AK: Optional[str] = Field(default=None) - SK: Optional[str] = Field(default=None) - ACCESS_KEY: Optional[str] = Field(default=None) - SECRET_KEY: Optional[str] = Field(default=None) - ACCESS_TOKEN: Optional[str] = Field(default=None) - BASE_URL: str = Field(default=DefaultValue.BaseURL) - AUTH_TIMEOUT: float = Field(default=DefaultValue.AuthTimeout) - DISABLE_EB_SDK: bool = Field(default=DefaultValue.DisableErnieBotSDK) - EB_SDK_INSTALLED: bool = Field(default=False) - IAM_SIGN_EXPIRATION_SEC: int = Field(default=DefaultValue.IAMSignExpirationSeconds) - CONSOLE_API_BASE_URL: str = Field(default=DefaultValue.ConsoleAPIBaseURL) - ACCESS_TOKEN_REFRESH_MIN_INTERVAL: float = Field( - default=DefaultValue.AccessTokenRefreshMinInterval - ) - QPS_LIMIT: float = Field(default=DefaultValue.QpsLimit) - APPID: Optional[int] = Field(default=None) - - # for private - ENABLE_PRIVATE: bool = Field(default=DefaultValue.EnablePrivate) - # todo 补充 ENABLE_AUTH 的默认值和使用方法 - ENABLE_AUTH: Optional[bool] = Field(default=None) - ACCESS_CODE: Optional[str] = Field(default=None) - IMPORT_STATUS_POLLING_INTERVAL: float = Field( - default=DefaultValue.ImportStatusPollingInterval - ) - EXPORT_STATUS_POLLING_INTERVAL: float = Field( - default=DefaultValue.ExportStatusPollingInterval - ) - RELEASE_STATUS_POLLING_INTERVAL: float = Field( - default=DefaultValue.ReleaseStatusPollingInterval - ) - EXPORT_FILE_SIZE_LIMIT: int = Field(default=DefaultValue.ExportFileSizeLimit) - ETL_STATUS_POLLING_INTERVAL: float = Field( - default=DefaultValue.ETLStatusPollingInterval - ) - GET_ENTITY_CONTENT_FAILED_RETRY_TIMES: int = Field( - default=DefaultValue.GetEntityContentFailedRetryTimes - ) - TRAIN_STATUS_POLLING_INTERVAL: float = Field( - default=DefaultValue.TrainStatusPollingInterval - ) - TRAINER_STATUS_POLLING_BACKOFF_FACTOR: float = Field( - default=DefaultValue.TrainerStatusPollingBackoffFactor - ) - TRAINER_STATUS_POLLING_RETRY_TIMES: float = Field( - default=DefaultValue.TrainerStatusPollingRetryTimes - ) - MODEL_PUBLISH_STATUS_POLLING_INTERVAL: float = Field( - default=DefaultValue.ModelPublishStatusPollingInterval - ) - BATCH_RUN_STATUS_POLLING_INTERVAL: float = Field( - default=DefaultValue.BatchRunStatusPollingInterval - ) - DEPLOY_STATUS_POLLING_INTERVAL: float = Field( - default=DefaultValue.DeployStatusPollingInterval - ) - DEFAULT_FINE_TUNE_TRAIN_TYPE: str = Field( - default=DefaultValue.DefaultFinetuneTrainType - ) - LLM_API_RETRY_COUNT: int = Field(default=DefaultValue.RetryCount) - LLM_API_RETRY_TIMEOUT: int = Field(default=DefaultValue.RetryTimeout) - LLM_API_RETRY_BACKOFF_FACTOR: float = Field(default=DefaultValue.RetryBackoffFactor) - LLM_API_RETRY_JITTER: float = Field(default=DefaultValue.RetryJitter) - LLM_API_RETRY_ERR_CODES: set = Field(default=DefaultValue.RetryErrCodes) - CONSOLE_API_RETRY_COUNT: int = Field(default=DefaultValue.ConsoleRetryCount) - CONSOLE_API_RETRY_TIMEOUT: int = Field(default=DefaultValue.ConsoleRetryTimeout) - CONSOLE_API_RETRY_JITTER: float = Field(default=DefaultValue.ConsoleRetryJitter) - CONSOLE_API_RETRY_ERR_CODES: set = Field(default=DefaultValue.ConsoleRetryErrCodes) - CONSOLE_API_RETRY_BACKOFF_FACTOR: int = Field( - default=DefaultValue.ConsoleRetryBackoffFactor - ) - EVALUATION_ONLINE_POLLING_INTERVAL: float = Field( - default=DefaultValue.EvaluationOnlinePollingInterval - ) - BOS_HOST_REGION: str = Field(default=DefaultValue.BosHostRegion) - - -_GLOBAL_CONFIG: Optional[GlobalConfig] = None - - -def get_config() -> GlobalConfig: +def get_config() -> Dynaconf: global _GLOBAL_CONFIG if not _GLOBAL_CONFIG: try: - _GLOBAL_CONFIG = GlobalConfig( # type: ignore - _env_file=os.getenv(Env.DotEnvConfigFile, DefaultValue.DotEnvConfigFile) + _none_default_value_key_list = [ + "AK", + "SK", + "ACCESS_KEY", + "SECRET_KEY", + "ACCESS_TOKEN", + "APPID", + "ENABLE_AUTH", + "ACCESS_CODE", + ] + + _GLOBAL_CONFIG = Dynaconf( + validators=[ + Validator(*_none_default_value_key_list, default=None), + Validator("BASE_URL", default=DefaultValue.BaseURL), + Validator("AUTH_TIMEOUT", default=DefaultValue.AuthTimeout), + Validator( + "DISABLE_EB_SDK", default=DefaultValue.DisableErnieBotSDK + ), + Validator("EB_SDK_INSTALLED", default=False), + Validator( + "IAM_SIGN_EXPIRATION_SEC", + default=DefaultValue.IAMSignExpirationSeconds, + ), + Validator( + "CONSOLE_API_BASE_URL", default=DefaultValue.ConsoleAPIBaseURL + ), + Validator( + "ACCESS_TOKEN_REFRESH_MIN_INTERVAL", + default=DefaultValue.AccessTokenRefreshMinInterval, + ), + Validator("QPS_LIMIT", default=DefaultValue.QpsLimit), + Validator("ENABLE_PRIVATE", default=DefaultValue.EnablePrivate), + Validator( + "IMPORT_STATUS_POLLING_INTERVAL", + default=DefaultValue.ImportStatusPollingInterval, + ), + Validator( + "EXPORT_STATUS_POLLING_INTERVAL", + default=DefaultValue.ExportStatusPollingInterval, + ), + Validator( + "RELEASE_STATUS_POLLING_INTERVAL", + default=DefaultValue.ReleaseStatusPollingInterval, + ), + Validator( + "EXPORT_FILE_SIZE_LIMIT", + default=DefaultValue.ExportFileSizeLimit, + ), + Validator( + "ETL_STATUS_POLLING_INTERVAL", + default=DefaultValue.ETLStatusPollingInterval, + ), + Validator( + "GET_ENTITY_CONTENT_FAILED_RETRY_TIMES", + default=DefaultValue.GetEntityContentFailedRetryTimes, + ), + Validator( + "TRAIN_STATUS_POLLING_INTERVAL", + default=DefaultValue.TrainStatusPollingInterval, + ), + Validator( + "TRAINER_STATUS_POLLING_BACKOFF_FACTOR", + default=DefaultValue.TrainerStatusPollingBackoffFactor, + ), + Validator( + "TRAINER_STATUS_POLLING_RETRY_TIMES", + default=DefaultValue.TrainerStatusPollingRetryTimes, + ), + Validator( + "MODEL_PUBLISH_STATUS_POLLING_INTERVAL", + default=DefaultValue.ModelPublishStatusPollingInterval, + ), + Validator( + "BATCH_RUN_STATUS_POLLING_INTERVAL", + default=DefaultValue.BatchRunStatusPollingInterval, + ), + Validator( + "DEPLOY_STATUS_POLLING_INTERVAL", + default=DefaultValue.DeployStatusPollingInterval, + ), + Validator( + "DEFAULT_FINE_TUNE_TRAIN_TYPE", + default=DefaultValue.DefaultFinetuneTrainType, + ), + Validator("LLM_API_RETRY_COUNT", default=DefaultValue.RetryCount), + Validator( + "LLM_API_RETRY_TIMEOUT", default=DefaultValue.RetryTimeout + ), + Validator( + "LLM_API_RETRY_BACKOFF_FACTOR", + default=DefaultValue.RetryBackoffFactor, + ), + Validator("LLM_API_RETRY_JITTER", default=DefaultValue.RetryJitter), + Validator( + "LLM_API_RETRY_ERR_CODES", default=DefaultValue.RetryErrCodes + ), + Validator( + "CONSOLE_API_RETRY_COUNT", + default=DefaultValue.ConsoleRetryCount, + ), + Validator( + "CONSOLE_API_RETRY_TIMEOUT", + default=DefaultValue.ConsoleRetryTimeout, + ), + Validator( + "CONSOLE_API_RETRY_JITTER", + default=DefaultValue.ConsoleRetryJitter, + ), + Validator( + "CONSOLE_API_RETRY_ERR_CODES", + default=DefaultValue.ConsoleRetryErrCodes, + ), + Validator( + "CONSOLE_API_RETRY_BACKOFF_FACTOR", + default=DefaultValue.ConsoleRetryBackoffFactor, + ), + Validator( + "EVALUATION_ONLINE_POLLING_INTERVAL", + default=DefaultValue.EvaluationOnlinePollingInterval, + ), + Validator("BOS_HOST_REGION", default=DefaultValue.BosHostRegion), + ], + load_dotenv=True, + envvar_prefix="QIANFAN", ) except Exception as e: # todo 解决引入 Logger 带来的循环引用问题 From 1597436d6e91c7efae3cc96513db7fa352fbfa92 Mon Sep 17 00:00:00 2001 From: ZingLix Date: Mon, 8 Jan 2024 16:03:22 +0800 Subject: [PATCH 06/11] fix ci --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index e2c546cc..38c0e4f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ sphinx-rtd-theme = ">=1.2.0" mypy = ">=1.4.0" myst-parser = ">=0.19.2" pytest-mock = "3.11.1" +types-protobuf = "4.24.0.4" [tool.poetry.extras] langchain = ["langchain"] From bfc5b832fe9d1d7ddf03d82de34e5cb278419f42 Mon Sep 17 00:00:00 2001 From: Dobiichi-Origami <56953648+Dobiichi-Origami@users.noreply.github.com> Date: Mon, 8 Jan 2024 16:45:44 +0800 Subject: [PATCH 07/11] pydantic compatibility --- src/qianfan/common/tool/base_tool.py | 9 +++---- src/qianfan/dataset/data_operator.py | 2 +- src/qianfan/dataset/data_source.py | 26 +++++++++++--------- src/qianfan/dataset/dataset.py | 2 +- src/qianfan/dataset/table.py | 2 +- src/qianfan/evaluation/evaluation_manager.py | 4 +-- src/qianfan/evaluation/evaluator.py | 15 +++++------ src/qianfan/model/configs.py | 2 +- src/qianfan/pydantic.py | 22 +++++++++++++++++ src/qianfan/tests/dataset/dataset_test.py | 2 +- src/qianfan/tests/tool_test.py | 4 +-- src/qianfan/trainer/configs.py | 2 +- 12 files changed, 58 insertions(+), 34 deletions(-) create mode 100644 src/qianfan/pydantic.py diff --git a/src/qianfan/common/tool/base_tool.py b/src/qianfan/common/tool/base_tool.py index 71ffc118..85d4e674 100644 --- a/src/qianfan/common/tool/base_tool.py +++ b/src/qianfan/common/tool/base_tool.py @@ -18,20 +18,19 @@ import string from typing import Any, Dict, List, Optional, Type -from pydantic import BaseModel -from pydantic.v1 import ( +from qianfan.pydantic import ( BaseModel as PydanticV1BaseModel, ) -from pydantic.v1 import ( +from qianfan.pydantic import ( Field as PydanticV1Field, ) -from pydantic.v1 import ( +from qianfan.pydantic import ( create_model as create_pydantic_v1_model, ) from qianfan.utils.utils import assert_package_installed -class ToolParameter(BaseModel): +class ToolParameter(PydanticV1BaseModel): """ Tool parameters, used to define the inputs when calling a tool and to describe the parameters needed when calling the tool to the model. diff --git a/src/qianfan/dataset/data_operator.py b/src/qianfan/dataset/data_operator.py index 5ff37624..2242648d 100644 --- a/src/qianfan/dataset/data_operator.py +++ b/src/qianfan/dataset/data_operator.py @@ -17,7 +17,7 @@ """ -from pydantic import BaseModel, Field +from qianfan.pydantic import BaseModel, Field class QianfanOperator(BaseModel): diff --git a/src/qianfan/dataset/data_source.py b/src/qianfan/dataset/data_source.py index a49e90b0..e3ff1167 100644 --- a/src/qianfan/dataset/data_source.py +++ b/src/qianfan/dataset/data_source.py @@ -27,7 +27,7 @@ import dateutil.parser import requests -from pydantic import BaseModel, Field, model_validator +from qianfan.pydantic import BaseModel, Field, root_validator from qianfan.config import get_config from qianfan.dataset.consts import QianfanDatasetLocalCacheDir @@ -261,24 +261,26 @@ def set_format_type(self, format_type: FormatType) -> None: """ self.file_format = format_type - @model_validator(mode="after") - def _format_check(self) -> "FileDataSource": - if self.file_format: - return self + @root_validator + def _format_check(cls, values: Dict[str, Any]) -> Dict[str, Any]: + if values["file_format"]: + return values + + path = values["path"] try: - index = self.path.rfind(".") + index = path.rfind(".") # 读文件夹或查询不到的情况下默认使用纯文本格式 - if os.path.isdir(self.path) or index == -1: + if os.path.isdir(path) or index == -1: log_warn(f"use default format type {FormatType.Text}") - self.file_format = FormatType.Text - return self - suffix = self.path[index + 1 :] + values["file_format"] = FormatType.Text + return values + suffix = path[index + 1 :] for t in FormatType: if t.value == suffix: - self.file_format = t + values["file_format"] = t log_info(f"use format type {t}") - return self + return values raise ValueError(f"cannot match proper format type for {suffix}") except Exception as e: log_error(str(e)) diff --git a/src/qianfan/dataset/dataset.py b/src/qianfan/dataset/dataset.py index 392e9d4c..15efd9c0 100644 --- a/src/qianfan/dataset/dataset.py +++ b/src/qianfan/dataset/dataset.py @@ -690,7 +690,7 @@ def start_online_data_process_task(self, operators: List[QianfanOperator]) -> in "desensitization": [], } for operator in operators: - attr_dict = operator.model_dump() + attr_dict = operator.dict() attr_dict.pop("operator_name") attr_dict.pop("operator_type") diff --git a/src/qianfan/dataset/table.py b/src/qianfan/dataset/table.py index f6bf6dc1..9afb4daf 100644 --- a/src/qianfan/dataset/table.py +++ b/src/qianfan/dataset/table.py @@ -19,7 +19,7 @@ import pyarrow import pyarrow.compute as pc from pyarrow import Table as PyarrowTable -from pydantic import BaseModel +from qianfan.pydantic import BaseModel from typing_extensions import Self from qianfan.dataset.consts import ( diff --git a/src/qianfan/evaluation/evaluation_manager.py b/src/qianfan/evaluation/evaluation_manager.py index c71cd576..9d7f95a8 100644 --- a/src/qianfan/evaluation/evaluation_manager.py +++ b/src/qianfan/evaluation/evaluation_manager.py @@ -22,7 +22,7 @@ from concurrent.futures import ALL_COMPLETED, Future, ThreadPoolExecutor, wait from typing import Any, Dict, List, Optional, Set, Union -from pydantic import BaseModel, Field, model_validator +from qianfan.pydantic import BaseModel, Field, root_validator from qianfan import get_config from qianfan.dataset import Dataset, QianfanDataSource @@ -55,7 +55,7 @@ class EvaluationManager(BaseModel): local_evaluators: Optional[List[LocalEvaluator]] = Field(default=None) qianfan_evaluators: Optional[List[QianfanEvaluator]] = Field(default=None) - @model_validator(mode="before") + @root_validator @classmethod def _check_evaluators(cls, input_dict: Any) -> Any: """校验传入的参数""" diff --git a/src/qianfan/evaluation/evaluator.py b/src/qianfan/evaluation/evaluator.py index 7c4c961d..eb6256b5 100644 --- a/src/qianfan/evaluation/evaluator.py +++ b/src/qianfan/evaluation/evaluator.py @@ -20,7 +20,7 @@ from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional, Union -from pydantic import BaseModel, Field, model_validator +from qianfan.pydantic import BaseModel, Field, root_validator from qianfan.evaluation.consts import ( QianfanRefereeEvaluatorDefaultMaxScore, @@ -85,7 +85,7 @@ class QianfanManualEvaluator(QianfanEvaluator): default=[ManualEvaluatorDimension(dimension="满意度")] ) - @model_validator(mode="before") + @root_validator @classmethod def dimension_validation(cls, input_dict: Any) -> Any: assert isinstance(input_dict, dict) @@ -120,16 +120,17 @@ class Config: open_compass_evaluator: BaseEvaluator - @model_validator(mode="after") - def _check_open_compass_evaluator(self) -> "OpenCompassLocalEvaluator": - signature = inspect.signature(self.open_compass_evaluator.score) + @root_validator + def _check_open_compass_evaluator(cls, values: Dict[str, Any]) -> Dict[str, Any]: + open_compass_evaluator = values["open_compass_evaluator"] + signature = inspect.signature(open_compass_evaluator.score) params = list(signature.parameters.keys()) params.sort() if params != ["predictions", "references"]: raise ValueError( - f"unsupported opencompass evaluator {self.open_compass_evaluator}" + f"unsupported opencompass evaluator {type(open_compass_evaluator)}" ) - return self + return values def evaluate( self, input: Union[str, List[Dict[str, Any]]], reference: str, output: str diff --git a/src/qianfan/model/configs.py b/src/qianfan/model/configs.py index 052ba52d..8340fc00 100644 --- a/src/qianfan/model/configs.py +++ b/src/qianfan/model/configs.py @@ -1,6 +1,6 @@ from typing import Any -from pydantic import BaseModel +from qianfan.pydantic import BaseModel from qianfan.model.consts import ServiceType from qianfan.resources.console import consts as console_consts diff --git a/src/qianfan/pydantic.py b/src/qianfan/pydantic.py new file mode 100644 index 00000000..f145d6ae --- /dev/null +++ b/src/qianfan/pydantic.py @@ -0,0 +1,22 @@ +# Copyright (c) 2023 Baidu, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Pydantic compatible layer +""" + +try: + from pydantic.v1 import * # noqa: F403 +except ImportError: + from pydantic import * # noqa: F403 \ No newline at end of file diff --git a/src/qianfan/tests/dataset/dataset_test.py b/src/qianfan/tests/dataset/dataset_test.py index eb927ae4..0dea6c2e 100644 --- a/src/qianfan/tests/dataset/dataset_test.py +++ b/src/qianfan/tests/dataset/dataset_test.py @@ -19,7 +19,7 @@ from unittest.mock import patch import pytest -from pydantic import BaseModel +from qianfan.pydantic import BaseModel from qianfan.dataset.consts import QianfanDataGroupColumnName from qianfan.dataset.data_operator import FilterCheckNumberWords diff --git a/src/qianfan/tests/tool_test.py b/src/qianfan/tests/tool_test.py index 36ac4c56..55e97c19 100644 --- a/src/qianfan/tests/tool_test.py +++ b/src/qianfan/tests/tool_test.py @@ -108,7 +108,7 @@ def test_tool_from_langchain_tool(): return from langchain.tools.base import BaseTool as LangchainBaseTool - from pydantic.v1 import BaseModel, Field + from qianfan.pydantic import BaseModel, Field class CalculatorToolSchema(BaseModel): a: int = Field(description="a description") @@ -145,7 +145,7 @@ def test_tool_from_langchain_func_tool(): return from langchain.tools.base import Tool as LangchainTool - from pydantic.v1 import BaseModel, Field + from qianfan.pydantic import BaseModel, Field def hello(a: str, b: str) -> str: return f"hello {a} {b}" diff --git a/src/qianfan/trainer/configs.py b/src/qianfan/trainer/configs.py index 7e1519c9..3e1a7656 100644 --- a/src/qianfan/trainer/configs.py +++ b/src/qianfan/trainer/configs.py @@ -13,7 +13,7 @@ # limitations under the License. from typing import Any, Dict, List, Optional, Tuple, Union -from pydantic import BaseModel +from qianfan.pydantic import BaseModel from qianfan.trainer.consts import PeftType From 5a5f74eeb49e4b33db3efbb5b47172cec4c7db47 Mon Sep 17 00:00:00 2001 From: Dobiichi-Origami <56953648+Dobiichi-Origami@users.noreply.github.com> Date: Mon, 8 Jan 2024 17:00:26 +0800 Subject: [PATCH 08/11] format --- pyproject.toml | 2 +- src/qianfan/dataset/data_source.py | 2 +- src/qianfan/dataset/table.py | 2 +- src/qianfan/evaluation/evaluation_manager.py | 3 +-- src/qianfan/evaluation/evaluator.py | 7 ++++--- .../extensions/langchain/agents/baidu_qianfan_endpoint.py | 2 +- src/qianfan/model/configs.py | 3 +-- src/qianfan/{pydantic.py => pydantic/__init__.py} | 4 ++-- src/qianfan/tests/dataset/dataset_test.py | 2 +- src/qianfan/tests/tool_test.py | 2 ++ src/qianfan/trainer/configs.py | 1 - 11 files changed, 15 insertions(+), 15 deletions(-) rename src/qianfan/{pydantic.py => pydantic/__init__.py} (88%) diff --git a/pyproject.toml b/pyproject.toml index b2bdbfa0..7f8588ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ preview = true [tool.mypy] ignore_missing_imports = "True" disallow_untyped_defs = "True" -exclude = ["qianfan/tests"] +exclude = ["qianfan/tests", "qianfan/pydantic"] [build-system] diff --git a/src/qianfan/dataset/data_source.py b/src/qianfan/dataset/data_source.py index e3ff1167..1e063774 100644 --- a/src/qianfan/dataset/data_source.py +++ b/src/qianfan/dataset/data_source.py @@ -27,11 +27,11 @@ import dateutil.parser import requests -from qianfan.pydantic import BaseModel, Field, root_validator from qianfan.config import get_config from qianfan.dataset.consts import QianfanDatasetLocalCacheDir from qianfan.errors import FileSizeOverflow, QianfanRequestError +from qianfan.pydantic import BaseModel, Field, root_validator from qianfan.resources.console.consts import ( DataExportDestinationType, DataExportStatus, diff --git a/src/qianfan/dataset/table.py b/src/qianfan/dataset/table.py index 9afb4daf..f91ee696 100644 --- a/src/qianfan/dataset/table.py +++ b/src/qianfan/dataset/table.py @@ -19,7 +19,6 @@ import pyarrow import pyarrow.compute as pc from pyarrow import Table as PyarrowTable -from qianfan.pydantic import BaseModel from typing_extensions import Self from qianfan.dataset.consts import ( @@ -32,6 +31,7 @@ Processable, ) from qianfan.dataset.table_utils import _construct_table_from_nest_sequence +from qianfan.pydantic import BaseModel from qianfan.utils import log_debug, log_error, log_info, log_warn diff --git a/src/qianfan/evaluation/evaluation_manager.py b/src/qianfan/evaluation/evaluation_manager.py index 9d7f95a8..c04893c7 100644 --- a/src/qianfan/evaluation/evaluation_manager.py +++ b/src/qianfan/evaluation/evaluation_manager.py @@ -22,8 +22,6 @@ from concurrent.futures import ALL_COMPLETED, Future, ThreadPoolExecutor, wait from typing import Any, Dict, List, Optional, Set, Union -from qianfan.pydantic import BaseModel, Field, root_validator - from qianfan import get_config from qianfan.dataset import Dataset, QianfanDataSource from qianfan.dataset.consts import ( @@ -43,6 +41,7 @@ QianfanRuleEvaluator, ) from qianfan.model import Model, Service +from qianfan.pydantic import BaseModel, Field, root_validator from qianfan.resources import Model as ResourceModel from qianfan.resources.console.consts import EvaluationTaskStatus from qianfan.utils import log_debug, log_error, log_info, log_warn diff --git a/src/qianfan/evaluation/evaluator.py b/src/qianfan/evaluation/evaluator.py index eb6256b5..a0c43af5 100644 --- a/src/qianfan/evaluation/evaluator.py +++ b/src/qianfan/evaluation/evaluator.py @@ -20,13 +20,12 @@ from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional, Union -from qianfan.pydantic import BaseModel, Field, root_validator - from qianfan.evaluation.consts import ( QianfanRefereeEvaluatorDefaultMaxScore, QianfanRefereeEvaluatorDefaultMetrics, QianfanRefereeEvaluatorDefaultSteps, ) +from qianfan.pydantic import BaseModel, Field, root_validator from qianfan.utils import log_error, log_warn @@ -121,7 +120,9 @@ class Config: open_compass_evaluator: BaseEvaluator @root_validator - def _check_open_compass_evaluator(cls, values: Dict[str, Any]) -> Dict[str, Any]: + def _check_open_compass_evaluator( + cls, values: Dict[str, Any] + ) -> Dict[str, Any]: open_compass_evaluator = values["open_compass_evaluator"] signature = inspect.signature(open_compass_evaluator.score) params = list(signature.parameters.keys()) diff --git a/src/qianfan/extensions/langchain/agents/baidu_qianfan_endpoint.py b/src/qianfan/extensions/langchain/agents/baidu_qianfan_endpoint.py index a8822e3b..340fcf0f 100644 --- a/src/qianfan/extensions/langchain/agents/baidu_qianfan_endpoint.py +++ b/src/qianfan/extensions/langchain/agents/baidu_qianfan_endpoint.py @@ -92,7 +92,7 @@ def _agent_input_keys() -> List[str]: def _agent_validate_logical_core(values: dict) -> dict: """check if llm is valid""" - if not isinstance(values["llm"], QianfanChatEndpoint): + if not isinstance(values["llm"], QianfanChatEndpoint): # type: ignore raise ValueError("Only supported with QianfanChatEndpoint models.") if not (values["llm"].model == "ERNIE-Bot" or values["llm"].model == "ERNIE-Bot-4"): raise ValueError( diff --git a/src/qianfan/model/configs.py b/src/qianfan/model/configs.py index 8340fc00..8ecf152f 100644 --- a/src/qianfan/model/configs.py +++ b/src/qianfan/model/configs.py @@ -1,8 +1,7 @@ from typing import Any -from qianfan.pydantic import BaseModel - from qianfan.model.consts import ServiceType +from qianfan.pydantic import BaseModel from qianfan.resources.console import consts as console_consts diff --git a/src/qianfan/pydantic.py b/src/qianfan/pydantic/__init__.py similarity index 88% rename from src/qianfan/pydantic.py rename to src/qianfan/pydantic/__init__.py index f145d6ae..69762618 100644 --- a/src/qianfan/pydantic.py +++ b/src/qianfan/pydantic/__init__.py @@ -17,6 +17,6 @@ """ try: - from pydantic.v1 import * # noqa: F403 + from pydantic.v1 import * # noqa except ImportError: - from pydantic import * # noqa: F403 \ No newline at end of file + from pydantic import * # noqa diff --git a/src/qianfan/tests/dataset/dataset_test.py b/src/qianfan/tests/dataset/dataset_test.py index 0dea6c2e..69c5e242 100644 --- a/src/qianfan/tests/dataset/dataset_test.py +++ b/src/qianfan/tests/dataset/dataset_test.py @@ -19,7 +19,6 @@ from unittest.mock import patch import pytest -from qianfan.pydantic import BaseModel from qianfan.dataset.consts import QianfanDataGroupColumnName from qianfan.dataset.data_operator import FilterCheckNumberWords @@ -30,6 +29,7 @@ QianfanNonSortedConversation, QianfanSortedConversation, ) +from qianfan.pydantic import BaseModel from qianfan.resources.console.consts import DataTemplateType diff --git a/src/qianfan/tests/tool_test.py b/src/qianfan/tests/tool_test.py index 55e97c19..ca0a7539 100644 --- a/src/qianfan/tests/tool_test.py +++ b/src/qianfan/tests/tool_test.py @@ -108,6 +108,7 @@ def test_tool_from_langchain_tool(): return from langchain.tools.base import BaseTool as LangchainBaseTool + from qianfan.pydantic import BaseModel, Field class CalculatorToolSchema(BaseModel): @@ -145,6 +146,7 @@ def test_tool_from_langchain_func_tool(): return from langchain.tools.base import Tool as LangchainTool + from qianfan.pydantic import BaseModel, Field def hello(a: str, b: str) -> str: diff --git a/src/qianfan/trainer/configs.py b/src/qianfan/trainer/configs.py index 3e1a7656..12be7819 100644 --- a/src/qianfan/trainer/configs.py +++ b/src/qianfan/trainer/configs.py @@ -14,7 +14,6 @@ from typing import Any, Dict, List, Optional, Tuple, Union from qianfan.pydantic import BaseModel - from qianfan.trainer.consts import PeftType From fe90b41bf8eddd29a8980993d5c5c5f0867363f0 Mon Sep 17 00:00:00 2001 From: Dobiichi-Origami <56953648+Dobiichi-Origami@users.noreply.github.com> Date: Mon, 8 Jan 2024 17:21:02 +0800 Subject: [PATCH 09/11] rollback --- pyproject.toml | 2 +- src/qianfan/config.py | 223 +++++++++++++------------------ src/qianfan/tests/config_test.py | 11 ++ 3 files changed, 106 insertions(+), 130 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7f8588ae..ade30313 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ importlib-metadata = { version = ">=1.4.0", python = "<=3.7" } bce-python-sdk = ">=0.8.79" typing-extensions = { version = ">=4.0.0", python = "<=3.10" } pydantic = "*" +python-dotenv = "<=0.21.1" langchain = { version = ">=0.0.321", python = ">=3.8.1", optional = true } numpy = [ { version = "<1.22.0", python = ">=3.7 <3.8" }, @@ -36,7 +37,6 @@ python-dateutil = "^2.8.2" rich = ">=13.0.0" typer = ">=0.9.0" tenacity = "^8.2.3" -dynaconf = "<=3.1.11" [tool.poetry.scripts] qianfan = "qianfan.common.client.main:main" diff --git a/src/qianfan/config.py b/src/qianfan/config.py index 2370bda7..54b6b3f7 100644 --- a/src/qianfan/config.py +++ b/src/qianfan/config.py @@ -11,145 +11,110 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os from typing import Optional -from dynaconf import Dynaconf, Validator from typing_extensions import deprecated -from qianfan.consts import DefaultValue +from qianfan.consts import DefaultValue, Env +from qianfan.pydantic import BaseSettings, Field -_GLOBAL_CONFIG: Optional[Dynaconf] = None +class GlobalConfig(BaseSettings): + """ + The global config of whole qianfan sdk + """ -def get_config() -> Dynaconf: + class Config: + env_file_encoding = 'utf-8' + env_prefix = "QIANFAN_" + case_sensitive = True + + AK: Optional[str] = Field(default=None) + SK: Optional[str] = Field(default=None) + ACCESS_KEY: Optional[str] = Field(default=None) + SECRET_KEY: Optional[str] = Field(default=None) + ACCESS_TOKEN: Optional[str] = Field(default=None) + BASE_URL: str = Field(default=DefaultValue.BaseURL) + AUTH_TIMEOUT: float = Field(default=DefaultValue.AuthTimeout) + DISABLE_EB_SDK: bool = Field(default=DefaultValue.DisableErnieBotSDK) + EB_SDK_INSTALLED: bool = Field(default=False) + IAM_SIGN_EXPIRATION_SEC: int = Field(default=DefaultValue.IAMSignExpirationSeconds) + CONSOLE_API_BASE_URL: str = Field(default=DefaultValue.ConsoleAPIBaseURL) + ACCESS_TOKEN_REFRESH_MIN_INTERVAL: float = Field( + default=DefaultValue.AccessTokenRefreshMinInterval + ) + QPS_LIMIT: float = Field(default=DefaultValue.QpsLimit) + APPID: Optional[int] = Field(default=None) + + # for private + ENABLE_PRIVATE: bool = Field(default=DefaultValue.EnablePrivate) + ENABLE_AUTH: Optional[bool] = Field(default=None) + ACCESS_CODE: Optional[str] = Field(default=None) + IMPORT_STATUS_POLLING_INTERVAL: float = Field( + default=DefaultValue.ImportStatusPollingInterval + ) + EXPORT_STATUS_POLLING_INTERVAL: float = Field( + default=DefaultValue.ExportStatusPollingInterval + ) + RELEASE_STATUS_POLLING_INTERVAL: float = Field( + default=DefaultValue.ReleaseStatusPollingInterval + ) + EXPORT_FILE_SIZE_LIMIT: int = Field(default=DefaultValue.ExportFileSizeLimit) + ETL_STATUS_POLLING_INTERVAL: float = Field( + default=DefaultValue.ETLStatusPollingInterval + ) + GET_ENTITY_CONTENT_FAILED_RETRY_TIMES: int = Field( + default=DefaultValue.GetEntityContentFailedRetryTimes + ) + TRAIN_STATUS_POLLING_INTERVAL: float = Field( + default=DefaultValue.TrainStatusPollingInterval + ) + TRAINER_STATUS_POLLING_BACKOFF_FACTOR: float = Field( + default=DefaultValue.TrainerStatusPollingBackoffFactor + ) + TRAINER_STATUS_POLLING_RETRY_TIMES: float = Field( + default=DefaultValue.TrainerStatusPollingRetryTimes + ) + MODEL_PUBLISH_STATUS_POLLING_INTERVAL: float = Field( + default=DefaultValue.ModelPublishStatusPollingInterval + ) + BATCH_RUN_STATUS_POLLING_INTERVAL: float = Field( + default=DefaultValue.BatchRunStatusPollingInterval + ) + DEPLOY_STATUS_POLLING_INTERVAL: float = Field( + default=DefaultValue.DeployStatusPollingInterval + ) + DEFAULT_FINE_TUNE_TRAIN_TYPE: str = Field( + default=DefaultValue.DefaultFinetuneTrainType + ) + LLM_API_RETRY_COUNT: int = Field(default=DefaultValue.RetryCount) + LLM_API_RETRY_TIMEOUT: int = Field(default=DefaultValue.RetryTimeout) + LLM_API_RETRY_BACKOFF_FACTOR: float = Field(default=DefaultValue.RetryBackoffFactor) + LLM_API_RETRY_JITTER: float = Field(default=DefaultValue.RetryJitter) + LLM_API_RETRY_ERR_CODES: set = Field(default=DefaultValue.RetryErrCodes) + CONSOLE_API_RETRY_COUNT: int = Field(default=DefaultValue.ConsoleRetryCount) + CONSOLE_API_RETRY_TIMEOUT: int = Field(default=DefaultValue.ConsoleRetryTimeout) + CONSOLE_API_RETRY_JITTER: float = Field(default=DefaultValue.ConsoleRetryJitter) + CONSOLE_API_RETRY_ERR_CODES: set = Field(default=DefaultValue.ConsoleRetryErrCodes) + CONSOLE_API_RETRY_BACKOFF_FACTOR: int = Field( + default=DefaultValue.ConsoleRetryBackoffFactor + ) + EVALUATION_ONLINE_POLLING_INTERVAL: float = Field( + default=DefaultValue.EvaluationOnlinePollingInterval + ) + BOS_HOST_REGION: str = Field(default=DefaultValue.BosHostRegion) + + +_GLOBAL_CONFIG: Optional[GlobalConfig] = None + + +def get_config() -> GlobalConfig: global _GLOBAL_CONFIG if not _GLOBAL_CONFIG: try: - _none_default_value_key_list = [ - "AK", - "SK", - "ACCESS_KEY", - "SECRET_KEY", - "ACCESS_TOKEN", - "APPID", - "ENABLE_AUTH", - "ACCESS_CODE", - ] - - _GLOBAL_CONFIG = Dynaconf( - validators=[ - Validator(*_none_default_value_key_list, default=None), - Validator("BASE_URL", default=DefaultValue.BaseURL), - Validator("AUTH_TIMEOUT", default=DefaultValue.AuthTimeout), - Validator( - "DISABLE_EB_SDK", default=DefaultValue.DisableErnieBotSDK - ), - Validator("EB_SDK_INSTALLED", default=False), - Validator( - "IAM_SIGN_EXPIRATION_SEC", - default=DefaultValue.IAMSignExpirationSeconds, - ), - Validator( - "CONSOLE_API_BASE_URL", default=DefaultValue.ConsoleAPIBaseURL - ), - Validator( - "ACCESS_TOKEN_REFRESH_MIN_INTERVAL", - default=DefaultValue.AccessTokenRefreshMinInterval, - ), - Validator("QPS_LIMIT", default=DefaultValue.QpsLimit), - Validator("ENABLE_PRIVATE", default=DefaultValue.EnablePrivate), - Validator( - "IMPORT_STATUS_POLLING_INTERVAL", - default=DefaultValue.ImportStatusPollingInterval, - ), - Validator( - "EXPORT_STATUS_POLLING_INTERVAL", - default=DefaultValue.ExportStatusPollingInterval, - ), - Validator( - "RELEASE_STATUS_POLLING_INTERVAL", - default=DefaultValue.ReleaseStatusPollingInterval, - ), - Validator( - "EXPORT_FILE_SIZE_LIMIT", - default=DefaultValue.ExportFileSizeLimit, - ), - Validator( - "ETL_STATUS_POLLING_INTERVAL", - default=DefaultValue.ETLStatusPollingInterval, - ), - Validator( - "GET_ENTITY_CONTENT_FAILED_RETRY_TIMES", - default=DefaultValue.GetEntityContentFailedRetryTimes, - ), - Validator( - "TRAIN_STATUS_POLLING_INTERVAL", - default=DefaultValue.TrainStatusPollingInterval, - ), - Validator( - "TRAINER_STATUS_POLLING_BACKOFF_FACTOR", - default=DefaultValue.TrainerStatusPollingBackoffFactor, - ), - Validator( - "TRAINER_STATUS_POLLING_RETRY_TIMES", - default=DefaultValue.TrainerStatusPollingRetryTimes, - ), - Validator( - "MODEL_PUBLISH_STATUS_POLLING_INTERVAL", - default=DefaultValue.ModelPublishStatusPollingInterval, - ), - Validator( - "BATCH_RUN_STATUS_POLLING_INTERVAL", - default=DefaultValue.BatchRunStatusPollingInterval, - ), - Validator( - "DEPLOY_STATUS_POLLING_INTERVAL", - default=DefaultValue.DeployStatusPollingInterval, - ), - Validator( - "DEFAULT_FINE_TUNE_TRAIN_TYPE", - default=DefaultValue.DefaultFinetuneTrainType, - ), - Validator("LLM_API_RETRY_COUNT", default=DefaultValue.RetryCount), - Validator( - "LLM_API_RETRY_TIMEOUT", default=DefaultValue.RetryTimeout - ), - Validator( - "LLM_API_RETRY_BACKOFF_FACTOR", - default=DefaultValue.RetryBackoffFactor, - ), - Validator("LLM_API_RETRY_JITTER", default=DefaultValue.RetryJitter), - Validator( - "LLM_API_RETRY_ERR_CODES", default=DefaultValue.RetryErrCodes - ), - Validator( - "CONSOLE_API_RETRY_COUNT", - default=DefaultValue.ConsoleRetryCount, - ), - Validator( - "CONSOLE_API_RETRY_TIMEOUT", - default=DefaultValue.ConsoleRetryTimeout, - ), - Validator( - "CONSOLE_API_RETRY_JITTER", - default=DefaultValue.ConsoleRetryJitter, - ), - Validator( - "CONSOLE_API_RETRY_ERR_CODES", - default=DefaultValue.ConsoleRetryErrCodes, - ), - Validator( - "CONSOLE_API_RETRY_BACKOFF_FACTOR", - default=DefaultValue.ConsoleRetryBackoffFactor, - ), - Validator( - "EVALUATION_ONLINE_POLLING_INTERVAL", - default=DefaultValue.EvaluationOnlinePollingInterval, - ), - Validator("BOS_HOST_REGION", default=DefaultValue.BosHostRegion), - ], - load_dotenv=True, - envvar_prefix="QIANFAN", + _GLOBAL_CONFIG = GlobalConfig( # type: ignore + _env_file=os.getenv(Env.DotEnvConfigFile, DefaultValue.DotEnvConfigFile) ) except Exception as e: # todo 解决引入 Logger 带来的循环引用问题 diff --git a/src/qianfan/tests/config_test.py b/src/qianfan/tests/config_test.py index 4a38951e..8b271054 100644 --- a/src/qianfan/tests/config_test.py +++ b/src/qianfan/tests/config_test.py @@ -15,6 +15,8 @@ """ Unit test for config """ +import os + from qianfan import get_config from qianfan.consts import DefaultValue @@ -24,3 +26,12 @@ def test_rewrite_config_through_code(): assert get_config().AUTH_TIMEOUT == DefaultValue.AuthTimeout config_center.AUTH_TIMEOUT = 114514 assert get_config().AUTH_TIMEOUT == 114514 + + +def test_read_from_dot_env(): + try: + with open(".env", "w") as f: + f.write('QIANFAN_ACCESS_TOKEN="test_token"') + assert get_config().ACCESS_TOKEN == "test_token" + finally: + os.remove(".env") From d79acc3328b5f08efcadef385d1ba4f12cc4320b Mon Sep 17 00:00:00 2001 From: Dobiichi-Origami <56953648+Dobiichi-Origami@users.noreply.github.com> Date: Mon, 8 Jan 2024 17:21:30 +0800 Subject: [PATCH 10/11] format --- src/qianfan/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qianfan/config.py b/src/qianfan/config.py index 54b6b3f7..46e146e0 100644 --- a/src/qianfan/config.py +++ b/src/qianfan/config.py @@ -26,7 +26,7 @@ class GlobalConfig(BaseSettings): """ class Config: - env_file_encoding = 'utf-8' + env_file_encoding = "utf-8" env_prefix = "QIANFAN_" case_sensitive = True From 401f57e9ebb90903dac2f22b44ced8ccbc220e4a Mon Sep 17 00:00:00 2001 From: ZingLix Date: Mon, 8 Jan 2024 17:35:43 +0800 Subject: [PATCH 11/11] fix lint --- .../extensions/langchain/agents/baidu_qianfan_endpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qianfan/extensions/langchain/agents/baidu_qianfan_endpoint.py b/src/qianfan/extensions/langchain/agents/baidu_qianfan_endpoint.py index a8822e3b..340fcf0f 100644 --- a/src/qianfan/extensions/langchain/agents/baidu_qianfan_endpoint.py +++ b/src/qianfan/extensions/langchain/agents/baidu_qianfan_endpoint.py @@ -92,7 +92,7 @@ def _agent_input_keys() -> List[str]: def _agent_validate_logical_core(values: dict) -> dict: """check if llm is valid""" - if not isinstance(values["llm"], QianfanChatEndpoint): + if not isinstance(values["llm"], QianfanChatEndpoint): # type: ignore raise ValueError("Only supported with QianfanChatEndpoint models.") if not (values["llm"].model == "ERNIE-Bot" or values["llm"].model == "ERNIE-Bot-4"): raise ValueError(