diff --git a/README.md b/README.md index 27961a08..009de298 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ [![Release Notes](https://img.shields.io/github/release/baidubce/bce-qianfan-sdk)](https://github.com/baidubce/bce-qianfan-sdk/releases) [![PyPI version](https://badge.fury.io/py/qianfan.svg)](https://pypi.org/project/qianfan/) [![Documentation Status](https://readthedocs.org/projects/qianfan/badge/?version=stable)](https://qianfan.readthedocs.io/en/stable/README.html) +[![Feedback Issue](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-GitHub_Issue-brightgreen)](https://github.com/baidubce/bce-qianfan-sdk/issues) +[![Feedback Ticket](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-%E7%99%BE%E5%BA%A6%E6%99%BA%E8%83%BD%E4%BA%91%E5%B7%A5%E5%8D%95-brightgreen)](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) ## 简介 @@ -171,7 +173,7 @@ trainer.run() > Check [**API References**](https://qianfan.readthedocs.io/en/stable/qianfan.html) for more details. -### 联系我们 +## 联系我们 如使用过程中遇到什么问题，或对SDK功能有建议，可通过如下方式联系我们 - [GitHub issues](https://github.com/baidubce/bce-qianfan-sdk/issues) - [百度智能云工单](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) （百度专家即时服务） diff --git a/README.pypi.md b/README.pypi.md index deca4e00..136cdca3 100644 --- a/README.pypi.md +++ b/README.pypi.md @@ -4,6 +4,8 @@ [![Release Notes](https://img.shields.io/github/release/baidubce/bce-qianfan-sdk)](https://github.com/baidubce/bce-qianfan-sdk/releases) [![PyPI version](https://badge.fury.io/py/qianfan.svg)](https://pypi.org/project/qianfan/) [![Documentation Status](https://readthedocs.org/projects/qianfan/badge/?version=stable)](https://qianfan.readthedocs.io/en/stable/README.html) +[![Feedback Issue](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-GitHub_Issue-brightgreen)](https://github.com/baidubce/bce-qianfan-sdk/issues) +[![Feedback Ticket](https://img.shields.io/badge/%E8%81%94%E7%B3%BB%E6%88%91%E4%BB%AC-%E7%99%BE%E5%BA%A6%E6%99%BA%E8%83%BD%E4%BA%91%E5%B7%A5%E5%8D%95-brightgreen)](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) [Documentation](https://qianfan.readthedocs.io/en/stable/README.html) | [GitHub](https://github.com/baidubce/bce-qianfan-sdk) | [Cookbook](https://github.com/baidubce/bce-qianfan-sdk/tree/main/cookbook) @@ -108,10 +110,15 @@ print(resp["result"]) - Tokenizer [[Doc](https://qianfan.readthedocs.io/en/stable/docs/utils.html)][[GitHub](https://github.com/baidubce/bce-qianfan-sdk/blob/main/docs/utils.md)] - 接口流控 [[Doc](https://qianfan.readthedocs.io/en/stable/docs/configurable.html)][[GitHub](https://github.com/baidubce/bce-qianfan-sdk/blob/main/docs/configurable.md)] -详细信息请参考相应的文档。如果有任何问题，欢迎前往 [GitHub](https://github.com/baidubce/bce-qianfan-sdk) 提交 issue。 - > 还可以通过 [**API References**](https://qianfan.readthedocs.io/en/stable/qianfan.html) 查看每个接口的详细说明。 +## 联系我们 + +如使用过程中遇到什么问题，或对SDK功能有建议，可通过如下方式联系我们 + +- [GitHub issues](https://github.com/baidubce/bce-qianfan-sdk/issues) +- [百度智能云工单](https://console.bce.baidu.com/ticket/#/ticket/create?productId=279) （百度专家即时服务） + ## License Apache-2.0 diff --git a/pyproject.toml b/pyproject.toml index e2c546cc..d019b4af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,8 +22,8 @@ aiolimiter = ">=1.1.0" importlib-metadata = { version = ">=1.4.0", python = "<=3.7" } bce-python-sdk = ">=0.8.79" typing-extensions = { version = ">=4.0.0", python = "<=3.10" } -pydantic = ">=2" -pydantic-settings = ">=2.0.3" +pydantic = "*" +python-dotenv = "<=0.21.1" langchain = { version = ">=0.0.321", python = ">=3.8.1", optional = true } numpy = [ { version = "<1.22.0", python = ">=3.7 <3.8" }, @@ -53,6 +53,7 @@ sphinx-rtd-theme = ">=1.2.0" mypy = ">=1.4.0" myst-parser = ">=0.19.2" pytest-mock = "3.11.1" +types-protobuf = "4.24.0.4" [tool.poetry.extras] langchain = ["langchain"] @@ -72,7 +73,7 @@ preview = true [tool.mypy] ignore_missing_imports = "True" disallow_untyped_defs = "True" -exclude = ["qianfan/tests"] +exclude = ["qianfan/tests", "qianfan/pydantic"] [build-system] diff --git a/src/qianfan/common/tool/base_tool.py b/src/qianfan/common/tool/base_tool.py index 71ffc118..85d4e674 100644 --- a/src/qianfan/common/tool/base_tool.py +++ b/src/qianfan/common/tool/base_tool.py @@ -18,20 +18,19 @@ import string from typing import Any, Dict, List, Optional, Type -from pydantic import BaseModel -from pydantic.v1 import ( +from qianfan.pydantic import ( BaseModel as PydanticV1BaseModel, ) -from pydantic.v1 import ( +from qianfan.pydantic import ( Field as PydanticV1Field, ) -from pydantic.v1 import ( +from qianfan.pydantic import ( create_model as create_pydantic_v1_model, ) from qianfan.utils.utils import assert_package_installed -class ToolParameter(BaseModel): +class ToolParameter(PydanticV1BaseModel): """ Tool parameters, used to define the inputs when calling a tool and to describe the parameters needed when calling the tool to the model. diff --git a/src/qianfan/config.py b/src/qianfan/config.py index 09e4505f..46e146e0 100644 --- a/src/qianfan/config.py +++ b/src/qianfan/config.py @@ -14,11 +14,10 @@ import os from typing import Optional -from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict from typing_extensions import deprecated from qianfan.consts import DefaultValue, Env +from qianfan.pydantic import BaseSettings, Field class GlobalConfig(BaseSettings): @@ -26,7 +25,10 @@ class GlobalConfig(BaseSettings): The global config of whole qianfan sdk """ - model_config = SettingsConfigDict(env_prefix="QIANFAN_", case_sensitive=True) + class Config: + env_file_encoding = "utf-8" + env_prefix = "QIANFAN_" + case_sensitive = True AK: Optional[str] = Field(default=None) SK: Optional[str] = Field(default=None) @@ -47,7 +49,6 @@ class GlobalConfig(BaseSettings): # for private ENABLE_PRIVATE: bool = Field(default=DefaultValue.EnablePrivate) - # todo 补充 ENABLE_AUTH 的默认值和使用方法 ENABLE_AUTH: Optional[bool] = Field(default=None) ACCESS_CODE: Optional[str] = Field(default=None) IMPORT_STATUS_POLLING_INTERVAL: float = Field( diff --git a/src/qianfan/dataset/data_operator.py b/src/qianfan/dataset/data_operator.py index 5ff37624..2242648d 100644 --- a/src/qianfan/dataset/data_operator.py +++ b/src/qianfan/dataset/data_operator.py @@ -17,7 +17,7 @@ """ -from pydantic import BaseModel, Field +from qianfan.pydantic import BaseModel, Field class QianfanOperator(BaseModel): diff --git a/src/qianfan/dataset/data_source.py b/src/qianfan/dataset/data_source.py index a49e90b0..1e063774 100644 --- a/src/qianfan/dataset/data_source.py +++ b/src/qianfan/dataset/data_source.py @@ -27,11 +27,11 @@ import dateutil.parser import requests -from pydantic import BaseModel, Field, model_validator from qianfan.config import get_config from qianfan.dataset.consts import QianfanDatasetLocalCacheDir from qianfan.errors import FileSizeOverflow, QianfanRequestError +from qianfan.pydantic import BaseModel, Field, root_validator from qianfan.resources.console.consts import ( DataExportDestinationType, DataExportStatus, @@ -261,24 +261,26 @@ def set_format_type(self, format_type: FormatType) -> None: """ self.file_format = format_type - @model_validator(mode="after") - def _format_check(self) -> "FileDataSource": - if self.file_format: - return self + @root_validator + def _format_check(cls, values: Dict[str, Any]) -> Dict[str, Any]: + if values["file_format"]: + return values + + path = values["path"] try: - index = self.path.rfind(".") + index = path.rfind(".") # 读文件夹或查询不到的情况下默认使用纯文本格式 - if os.path.isdir(self.path) or index == -1: + if os.path.isdir(path) or index == -1: log_warn(f"use default format type {FormatType.Text}") - self.file_format = FormatType.Text - return self - suffix = self.path[index + 1 :] + values["file_format"] = FormatType.Text + return values + suffix = path[index + 1 :] for t in FormatType: if t.value == suffix: - self.file_format = t + values["file_format"] = t log_info(f"use format type {t}") - return self + return values raise ValueError(f"cannot match proper format type for {suffix}") except Exception as e: log_error(str(e)) diff --git a/src/qianfan/dataset/dataset.py b/src/qianfan/dataset/dataset.py index 392e9d4c..15efd9c0 100644 --- a/src/qianfan/dataset/dataset.py +++ b/src/qianfan/dataset/dataset.py @@ -690,7 +690,7 @@ def start_online_data_process_task(self, operators: List[QianfanOperator]) -> in "desensitization": [], } for operator in operators: - attr_dict = operator.model_dump() + attr_dict = operator.dict() attr_dict.pop("operator_name") attr_dict.pop("operator_type") diff --git a/src/qianfan/dataset/table.py b/src/qianfan/dataset/table.py index f6bf6dc1..f91ee696 100644 --- a/src/qianfan/dataset/table.py +++ b/src/qianfan/dataset/table.py @@ -19,7 +19,6 @@ import pyarrow import pyarrow.compute as pc from pyarrow import Table as PyarrowTable -from pydantic import BaseModel from typing_extensions import Self from qianfan.dataset.consts import ( @@ -32,6 +31,7 @@ Processable, ) from qianfan.dataset.table_utils import _construct_table_from_nest_sequence +from qianfan.pydantic import BaseModel from qianfan.utils import log_debug, log_error, log_info, log_warn diff --git a/src/qianfan/evaluation/evaluation_manager.py b/src/qianfan/evaluation/evaluation_manager.py index c71cd576..c04893c7 100644 --- a/src/qianfan/evaluation/evaluation_manager.py +++ b/src/qianfan/evaluation/evaluation_manager.py @@ -22,8 +22,6 @@ from concurrent.futures import ALL_COMPLETED, Future, ThreadPoolExecutor, wait from typing import Any, Dict, List, Optional, Set, Union -from pydantic import BaseModel, Field, model_validator - from qianfan import get_config from qianfan.dataset import Dataset, QianfanDataSource from qianfan.dataset.consts import ( @@ -43,6 +41,7 @@ QianfanRuleEvaluator, ) from qianfan.model import Model, Service +from qianfan.pydantic import BaseModel, Field, root_validator from qianfan.resources import Model as ResourceModel from qianfan.resources.console.consts import EvaluationTaskStatus from qianfan.utils import log_debug, log_error, log_info, log_warn @@ -55,7 +54,7 @@ class EvaluationManager(BaseModel): local_evaluators: Optional[List[LocalEvaluator]] = Field(default=None) qianfan_evaluators: Optional[List[QianfanEvaluator]] = Field(default=None) - @model_validator(mode="before") + @root_validator @classmethod def _check_evaluators(cls, input_dict: Any) -> Any: """校验传入的参数""" diff --git a/src/qianfan/evaluation/evaluator.py b/src/qianfan/evaluation/evaluator.py index 7c4c961d..a0c43af5 100644 --- a/src/qianfan/evaluation/evaluator.py +++ b/src/qianfan/evaluation/evaluator.py @@ -20,13 +20,12 @@ from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional, Union -from pydantic import BaseModel, Field, model_validator - from qianfan.evaluation.consts import ( QianfanRefereeEvaluatorDefaultMaxScore, QianfanRefereeEvaluatorDefaultMetrics, QianfanRefereeEvaluatorDefaultSteps, ) +from qianfan.pydantic import BaseModel, Field, root_validator from qianfan.utils import log_error, log_warn @@ -85,7 +84,7 @@ class QianfanManualEvaluator(QianfanEvaluator): default=[ManualEvaluatorDimension(dimension="满意度")] ) - @model_validator(mode="before") + @root_validator @classmethod def dimension_validation(cls, input_dict: Any) -> Any: assert isinstance(input_dict, dict) @@ -120,16 +119,19 @@ class Config: open_compass_evaluator: BaseEvaluator - @model_validator(mode="after") - def _check_open_compass_evaluator(self) -> "OpenCompassLocalEvaluator": - signature = inspect.signature(self.open_compass_evaluator.score) + @root_validator + def _check_open_compass_evaluator( + cls, values: Dict[str, Any] + ) -> Dict[str, Any]: + open_compass_evaluator = values["open_compass_evaluator"] + signature = inspect.signature(open_compass_evaluator.score) params = list(signature.parameters.keys()) params.sort() if params != ["predictions", "references"]: raise ValueError( - f"unsupported opencompass evaluator {self.open_compass_evaluator}" + f"unsupported opencompass evaluator {type(open_compass_evaluator)}" ) - return self + return values def evaluate( self, input: Union[str, List[Dict[str, Any]]], reference: str, output: str diff --git a/src/qianfan/extensions/langchain/agents/baidu_qianfan_endpoint.py b/src/qianfan/extensions/langchain/agents/baidu_qianfan_endpoint.py index a8822e3b..340fcf0f 100644 --- a/src/qianfan/extensions/langchain/agents/baidu_qianfan_endpoint.py +++ b/src/qianfan/extensions/langchain/agents/baidu_qianfan_endpoint.py @@ -92,7 +92,7 @@ def _agent_input_keys() -> List[str]: def _agent_validate_logical_core(values: dict) -> dict: """check if llm is valid""" - if not isinstance(values["llm"], QianfanChatEndpoint): + if not isinstance(values["llm"], QianfanChatEndpoint): # type: ignore raise ValueError("Only supported with QianfanChatEndpoint models.") if not (values["llm"].model == "ERNIE-Bot" or values["llm"].model == "ERNIE-Bot-4"): raise ValueError( diff --git a/src/qianfan/model/configs.py b/src/qianfan/model/configs.py index 052ba52d..8ecf152f 100644 --- a/src/qianfan/model/configs.py +++ b/src/qianfan/model/configs.py @@ -1,8 +1,7 @@ from typing import Any -from pydantic import BaseModel - from qianfan.model.consts import ServiceType +from qianfan.pydantic import BaseModel from qianfan.resources.console import consts as console_consts diff --git a/src/qianfan/pydantic/__init__.py b/src/qianfan/pydantic/__init__.py new file mode 100644 index 00000000..69762618 --- /dev/null +++ b/src/qianfan/pydantic/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2023 Baidu, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Pydantic compatible layer +""" + +try: + from pydantic.v1 import * # noqa +except ImportError: + from pydantic import * # noqa diff --git a/src/qianfan/tests/config_test.py b/src/qianfan/tests/config_test.py index 4a38951e..8b271054 100644 --- a/src/qianfan/tests/config_test.py +++ b/src/qianfan/tests/config_test.py @@ -15,6 +15,8 @@ """ Unit test for config """ +import os + from qianfan import get_config from qianfan.consts import DefaultValue @@ -24,3 +26,12 @@ def test_rewrite_config_through_code(): assert get_config().AUTH_TIMEOUT == DefaultValue.AuthTimeout config_center.AUTH_TIMEOUT = 114514 assert get_config().AUTH_TIMEOUT == 114514 + + +def test_read_from_dot_env(): + try: + with open(".env", "w") as f: + f.write('QIANFAN_ACCESS_TOKEN="test_token"') + assert get_config().ACCESS_TOKEN == "test_token" + finally: + os.remove(".env") diff --git a/src/qianfan/tests/dataset/dataset_test.py b/src/qianfan/tests/dataset/dataset_test.py index eb927ae4..69c5e242 100644 --- a/src/qianfan/tests/dataset/dataset_test.py +++ b/src/qianfan/tests/dataset/dataset_test.py @@ -19,7 +19,6 @@ from unittest.mock import patch import pytest -from pydantic import BaseModel from qianfan.dataset.consts import QianfanDataGroupColumnName from qianfan.dataset.data_operator import FilterCheckNumberWords @@ -30,6 +29,7 @@ QianfanNonSortedConversation, QianfanSortedConversation, ) +from qianfan.pydantic import BaseModel from qianfan.resources.console.consts import DataTemplateType diff --git a/src/qianfan/tests/tool_test.py b/src/qianfan/tests/tool_test.py index 36ac4c56..ca0a7539 100644 --- a/src/qianfan/tests/tool_test.py +++ b/src/qianfan/tests/tool_test.py @@ -108,7 +108,8 @@ def test_tool_from_langchain_tool(): return from langchain.tools.base import BaseTool as LangchainBaseTool - from pydantic.v1 import BaseModel, Field + + from qianfan.pydantic import BaseModel, Field class CalculatorToolSchema(BaseModel): a: int = Field(description="a description") @@ -145,7 +146,8 @@ def test_tool_from_langchain_func_tool(): return from langchain.tools.base import Tool as LangchainTool - from pydantic.v1 import BaseModel, Field + + from qianfan.pydantic import BaseModel, Field def hello(a: str, b: str) -> str: return f"hello {a} {b}" diff --git a/src/qianfan/trainer/configs.py b/src/qianfan/trainer/configs.py index 7e1519c9..12be7819 100644 --- a/src/qianfan/trainer/configs.py +++ b/src/qianfan/trainer/configs.py @@ -13,8 +13,7 @@ # limitations under the License. from typing import Any, Dict, List, Optional, Tuple, Union -from pydantic import BaseModel - +from qianfan.pydantic import BaseModel from qianfan.trainer.consts import PeftType