feat evals trainer (#158)

* feat: support eval actions

* fix: refactor model to fix circular import &  fix evals

* fix: nbs import

* fix: move runnable -> common/

* fix: lint

* chore: update version

cookbook/finetune/trainer_finetune.ipynb

@@ -83,9 +83,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from qianfan.trainer.consts import ActionState, ServiceType\n",
+    "from qianfan.trainer.consts import ActionState\n",
+    "from qianfan.model.consts import ServiceType\n",
     "from qianfan.resources.console import consts as console_consts\n",
-    "from qianfan.trainer.configs import TrainConfig, DeployConfig\n",
+    "from qianfan.trainer.configs import TrainConfig\n",
+    "from qianfan.model.configs import DeployConfig\n",
     "from qianfan.resources import QfMessages\n",
     "from qianfan.trainer import LLMFinetune\n",
     "from qianfan.dataset import Dataset\n",
@@ -474,7 +476,7 @@
     }
    ],
    "source": [
-    "from qianfan.trainer import Model\n",
+    "from qianfan.model import Model\n",
     "from qianfan.dataset import Dataset\n",
     "\n",
     "# 首先需要先加载测试数据集，这里以加载平台预置数据集为例子：\n",
@@ -647,7 +649,7 @@
    ],
    "source": [
     "from qianfan.trainer.model import Model, Service, model_deploy\n",
-    "from qianfan.trainer.consts import ServiceType\n",
+    "from qianfan.model.consts import ServiceType\n",
     "from qianfan.resources.console.consts import DeployPoolType\n",
     "\n",
     "sft_svc: Service = m.deploy(DeployConfig(\n",

cookbook/finetune/trainer_finetune_event_resume.ipynb

@@ -80,9 +80,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from qianfan.trainer.consts import ActionState, ServiceType\n",
+    "from qianfan.trainer.consts import ActionState\n",
+    "from qianfan.model.consts import ServiceType\n",
     "from qianfan.resources.console import consts as console_consts\n",
-    "from qianfan.trainer.configs import TrainConfig, DeployConfig\n",
+    "from qianfan.trainer.configs import TrainConfig\n",
+    "from qianfan.model.configs import DeployConfig\n",
     "from qianfan.resources import QfMessages\n",
     "from qianfan.trainer import LLMFinetune, Service\n",
     "from qianfan.dataset import Dataset\n",
@@ -109,7 +111,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from qianfan.trainer import Model\n",
+    "from qianfan.model import Model\n",
     "from qianfan.dataset import Dataset\n",
     "\n",
     "# 首先需要先加载测试数据集，这里以加载平台预置数据集为例子：\n",

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "qianfan"
-version = "0.2.4"
+version = "0.2.5"
 description = "文心千帆大模型平台 Python SDK"
 authors = []
 license = "Apache-2.0"

src/qianfan/common/runnable/base.py

@@ -0,0 +1,74 @@
+# Copyright (c) 2023 Baidu, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from abc import ABC, abstractmethod
+from typing import (
+    Any,
+    Dict,
+    Generic,
+    Optional,
+    TypeVar,
+)
+
+Input = TypeVar("Input")
+Output = TypeVar("Output")
+
+
+class Executable(Generic[Input, Output], ABC):
+    """
+    generic abstraction class of executable
+
+    """
+
+    @abstractmethod
+    def exec(self, input: Optional[Input] = None, **kwargs: Dict) -> Output:
+        ...
+
+
+class Serializable(ABC):
+    """
+    generic abstraction class of serializable.
+    especially for the model, service, and trainer.
+    """
+
+    @abstractmethod
+    def dumps(self) -> Optional[bytes]:
+        """
+        dumps
+
+        Returns:
+            serialized bytes data
+        """
+        ...
+
+    @abstractmethod
+    def loads(self, data: bytes) -> Any:
+        """
+        loads
+
+        Parameters:
+            data (bytes): load
+
+        Returns:
+            Any: _description_
+        """
+        ...
+
+
+class ExecuteSerializable(Executable[Input, Output], Serializable):
+    """
+    set of executable and serializable. subclass implement it to support
+    exec and dumps/loads.
+    """
+
+    ...

src/qianfan/evaluation/evaluation_manager.py

@@ -33,9 +33,9 @@
     QianfanRefereeEvaluator,
     QianfanRuleEvaluator,
 )
+from qianfan.model import Model, Service
 from qianfan.resources import Model as ResourceModel
 from qianfan.resources.console.consts import EvaluationTaskStatus
-from qianfan.trainer import Model, Service
 from qianfan.utils import log_debug, log_error, log_info, log_warn
 from qianfan.utils.utils import generate_letter_num_random_id
 

src/qianfan/model/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2023 Baidu, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from qianfan.model.configs import DeployConfig
+from qianfan.model.model import Model, Service
+
+__all__ = ["Model", "Service", "DeployConfig"]

src/qianfan/model/configs.py

@@ -0,0 +1,37 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+from qianfan.model.consts import ServiceType
+from qianfan.resources.console import consts as console_consts
+
+
+class DeployConfig(BaseModel):
+    name: str = ""
+    """
+    Service name
+    """
+    endpoint_prefix: str = ""
+    """
+    Endpoint custom prefix, will be used to call resource api
+    """
+    description: str = ""
+    """
+    description of service
+    """
+    replicas: int = 1
+    """
+    replicas for model services, related to the capacity in QPS of model service.
+        default set to 1
+    """
+    pool_type: console_consts.DeployPoolType = (
+        console_consts.DeployPoolType.PrivateResource
+    )
+    """
+    resource pool type, public resource will be shared with others.
+    """
+    service_type: ServiceType
+    """
+    service type, after deploy, Service could behave like the specific type.
+    """
+    extras: Any = None

src/qianfan/model/consts.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2023 Baidu, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from enum import Enum
+
+
+class ServiceType(str, Enum):
+    Chat = "Chat"
+    """Corresponding to the `ChatCompletion`"""
+    Completion = "Completion"
+    """Corresponding to the `Completion`"""
+    Embedding = "Embedding"
+    """Corresponding to the `Embedding`"""
+    Text2Image = "Text2Image"
+    """Corresponding to the `Text2Image"""

src/qianfan/trainer/model.py → src/qianfan/model/model.py

@@ -17,9 +17,12 @@
 
 from qianfan import resources as api
 from qianfan.common import Prompt
+from qianfan.common.runnable.base import ExecuteSerializable
 from qianfan.config import get_config
 from qianfan.dataset import Dataset
 from qianfan.errors import InternalError, InvalidArgumentError
+from qianfan.model.configs import DeployConfig
+from qianfan.model.consts import ServiceType
 from qianfan.resources import (
     ChatCompletion,
     Completion,
@@ -29,9 +32,6 @@
 )
 from qianfan.resources.console import consts as console_const
 from qianfan.resources.console.model import Model as ResourceModel
-from qianfan.trainer.base import ExecuteSerializable
-from qianfan.trainer.configs import DeployConfig
-from qianfan.trainer.consts import ServiceType
 from qianfan.utils import log_error, log_info, log_warn