feat: 模块内部管控调用迁移到 V2

README.md

@@ -167,7 +167,7 @@ print(resp["result"])
 ```python
 from qianfan.dataset import Dataset
 
-ds = Dataset.load(qianfan_dataset_id="your_dataset_id")
+ds = Dataset.load(qianfan_dataset_version_id="your_dataset_id")
 ```
 
 且千帆 Python SDK 集成了一系列本地的数据处理功能，允许用户在本地对来自多个数据源的数据进行增删改查等操作，详见[Dataset 框架](./docs/dataset.md)。
@@ -192,34 +192,38 @@ from qianfan.dataset import Dataset
 # 从本地文件导入
 ds = Dataset.load(data_file="path/to/dataset_file.json")
 
+
 def filter_func(row: Dict[str, Any]) -> bool:
-  return "answer" in row.keys()
+    return "answer" in row.keys()
+
 
 def map_func(row: Dict[str, Any]) -> Dict[str, Any]:
-  return {
-      "prompt": row["question"],
-      "response": row["answer"],
-  }
+    return {
+        "prompt": row["question"],
+        "response": row["answer"],
+    }
+
 
 # 链式调用处理数据
 ds.filter(filter_func).map(map_func).pack()
 
 # 上传到千帆
 # 数据集只有上传到千帆后才可以用于训练
 # 请确保你的数据集格式符合要求
-ds.save(qianfan_dataset_id="your_dataset_id")
+ds.save(qianfan_dataset_version_id="your_dataset_id")
 ```
 
 #### Trainer
 
 千帆 Python SDK 以Pipeline为基础串联整个模型训练的流程，同时允许用户更好的把控训练流程状态 [Trainer 框架](./docs/trainer.md)。
 以下是一个快速实现ERNIE-Speed-8K fine-tuning的例子：
+
 ```python
 from qianfan.dataset import Dataset
 from qianfan.trainer import Finetune
 
 # 加载千帆平台上的数据集
-ds: Dataset = Dataset.load(qianfan_dataset_id="ds-xxx")
+ds: Dataset = Dataset.load(qianfan_dataset_version_id="ds-xxx")
 
 # 新建trainer LLMFinetune，最少传入train_type和dataset
 # 注意fine-tune任务需要指定的数据集类型要求为有标注的非排序对话数据集。
@@ -242,7 +246,7 @@ trainer.run()
 from qianfan.model import Model
 from qianfan.dataset import Dataset
 
-ds = Dataset.load(qianfan_dataset_id="ds-xx")
+ds = Dataset.load(qianfan_dataset_version_id="ds-xx")
 m = Model(version_id="amv-xx")
 
 m.batch_inference(dataset=ds)

cookbook/awesome_demo/customer_service_conversation/main.ipynb

@@ -247,7 +247,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -271,7 +271,7 @@
     }
    ],
    "source": [
-    "ds = Dataset.load(qianfan_dataset_id = \"ds-1j390abu4fv5abkf\", format = FormatType.Jsonl)\n",
+    "ds = Dataset.load(qianfan_dataset_version_id = \"ds-1j390abu4fv5abkf\", format = FormatType.Jsonl)\n",
     "print(ds[0])"
    ]
   },
@@ -475,9 +475,9 @@
     "\n",
     "sft_svc: Service = m.deploy(DeployConfig(\n",
     "    name=\"cusserv_1\",\n",
-    "    endpoint_prefix=\"customer\",\n",
+    "    endpoint_suffix=\"customer\",\n",
     "    replicas=1,\n",
-    "    pool_type=DeployPoolType.PrivateResource,\n",
+    "    months=1,\n",
     "    service_type=ServiceType.Completion,\n",
     "))"
    ]

cookbook/awesome_demo/dialogue_multi_tag_generation/main.ipynb

@@ -338,7 +338,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -362,7 +362,7 @@
     }
    ],
    "source": [
-    "ds = Dataset.load(qianfan_dataset_id = \"ds-scm8g98a7pv3zzf3\", format = FormatType.Jsonl)\n",
+    "ds = Dataset.load(qianfan_dataset_version_id = \"ds-scm8g98a7pv3zzf3\", format = FormatType.Jsonl)\n",
     "print(ds[0])"
    ]
   },
@@ -548,7 +548,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -562,7 +562,7 @@
     }
    ],
    "source": [
-    "eval_ds = Dataset.load(qianfan_dataset_id =\"ds-n1dg1czx3ciqrakr\",organize_data_as_group=False, input_columns=[\"prompt\"], reference_column=\"response\")"
+    "eval_ds = Dataset.load(qianfan_dataset_version_id =\"ds-n1dg1czx3ciqrakr\",organize_data_as_group=False, input_columns=[\"prompt\"], reference_column=\"response\")"
    ]
   },
   {

cookbook/awesome_demo/dpo_words_count_control/eval.py

@@ -5,7 +5,7 @@
 from qianfan.dataset import Dataset
 
 def eval(version_id, ds):
-    result_ds = ds.test_using_llm(model_version_id=version_id)
+    result_ds = ds.test_using_llm(model_id=version_id)
     res = []
     for i in result_ds:
 

cookbook/awesome_demo/dpo_words_count_control/main.ipynb

@@ -41,7 +41,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install 'qianfan>=0.3.16'"
+    "!pip install 'qianfan'"
    ]
   },
   {
@@ -96,7 +96,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -115,7 +115,7 @@
     }
    ],
    "source": [
-    "ds_test = Dataset.load(qianfan_dataset_id = \"ds-2hdewmq2w2yw8dz7\")\n",
+    "ds_test = Dataset.load(qianfan_dataset_version_id = \"ds-2hdewmq2w2yw8dz7\")\n",
     "ds_test = ds_test.save(data_file=\"data_file/dpo_test.jsonl\")"
    ]
   },
@@ -190,7 +190,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -204,14 +204,14 @@
    ],
    "source": [
     "#enable_log(logging.INFO)\n",
-    "ds_sft = Dataset.load(qianfan_dataset_id = \"ds-sjv3xchndftmg2fu\")#sft训练集\n",
+    "ds_sft = Dataset.load(qianfan_dataset_version_id = \"ds-sjv3xchndftmg2fu\")#sft训练集\n",
     "#ds_sft = ds_sft.save(data_file=\"data_file/sft_train.jsonl\")\n",
     "#print(new_ds[0])\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -224,7 +224,7 @@
     }
    ],
    "source": [
-    "ds_dpo = Dataset.load(qianfan_dataset_id = \"ds-ca94jxph35qp1ks3\")#dpo训练集\n",
+    "ds_dpo = Dataset.load(qianfan_dataset_version_id = \"ds-ca94jxph35qp1ks3\")#dpo训练集\n",
     "#ds_dpo = ds_dpo.save(data_file=\"data_file/dpo_train.jsonl\")"
    ]
   },
@@ -632,7 +632,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 70,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -645,7 +645,7 @@
     }
    ],
    "source": [
-    "dpo_test = Dataset.load(qianfan_dataset_id = \"ds-2hdewmq2w2yw8dz7\")#dpo评估集\n",
+    "dpo_test = Dataset.load(qianfan_dataset_version_id = \"ds-2hdewmq2w2yw8dz7\")#dpo评估集\n",
     "# dpo_test = dpo_test.save(data_file=\"data_file/dpo_test.jsonl\")"
    ]
   },

cookbook/awesome_demo/essay_scoring/main.ipynb

@@ -1862,14 +1862,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "# 加载训练用的预置数据集\n",
-    "qf_train_ds = Dataset.load(qianfan_dataset_id=\"ds-553hczysf3um4cc9\")\n",
+    "qf_train_ds = Dataset.load(qianfan_dataset_version_id=\"ds-553hczysf3um4cc9\")\n",
     "# 加载评估用的预置数据集\n",
-    "qf_eval_ds = Dataset.load(qianfan_dataset_id=\"ds-6ubasnsry5pa4azi\")"
+    "qf_eval_ds = Dataset.load(qianfan_dataset_version_id=\"ds-6ubasnsry5pa4azi\")"
    ]
   },
   {
@@ -2705,7 +2705,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 76,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -2731,9 +2731,8 @@
     "\n",
     "sft_svc: Service = m.deploy(DeployConfig(\n",
     "    name=\"essay_correct_3\",\n",
-    "    endpoint_prefix=\"essaycor\",\n",
+    "    endpoint_suffix=\"essaycor\",\n",
     "    replicas=1,\n",
-    "    pool_type=DeployPoolType.PrivateResource,\n",
     "    service_type=ServiceType.Chat,\n",
     "))"
    ]
@@ -2812,17 +2811,6 @@
     "for s in result:\n",
     "    print(s['result'])"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "\n",
-    "\n",
-    "Dataset.map_reduce"
-   ]
   }
  ],
  "metadata": {

cookbook/awesome_demo/role_play/main.ipynb

@@ -884,16 +884,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "bos_bucket_name = \"your_bos_bucket\"\n",
-    "bos_bucket_file_path = \"/your_path/\"\n",
+    "bos_bucket_file_path = \"bos://your_bos_bucket/your_path/\"\n",
     "qianfan_dataset_name = \"your_ds_name\"\n",
     "\n",
     "# 创建千帆数据集，并上传保存\n",
     "qianfan_data_source = QianfanDataSource.create_bare_dataset(\n",
     "    name=qianfan_dataset_name,\n",
-    "    template_type=console_consts.DataTemplateType.NonSortedConversation,\n",
-    "    storage_type=console_consts.DataStorageType.PrivateBos,\n",
-    "    storage_id=bos_bucket_name,\n",
+    "    dataset_format=console_consts.V2.DatasetFormat.PromptResponse,\n",
+    "    storage_type=console_consts.V2.StorageType.Bos,\n",
     "    storage_path=bos_bucket_file_path,\n",
     ")\n",
     "qf_ds = ds.save(qianfan_data_source, should_overwrite_existed_file=True)"
@@ -981,7 +979,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -1012,7 +1010,7 @@
     "\n",
     "# 导入平台上预置的测试集\n",
     "ds = Dataset.load(\n",
-    "    qianfan_dataset_id=\"ds-bimjvfatbnard1we\",\n",
+    "    qianfan_dataset_version_id=\"ds-bimjvfatbnard1we\",\n",
     "    organize_data_as_group=False,\n",
     "    input_columns=[\"prompt\"],\n",
     "    reference_column=\"response\",\n",
@@ -1054,7 +1052,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -1138,7 +1136,7 @@
    ],
    "source": [
     "#加载被评估模型，在version_id处填入模型版本ID\n",
-    "eb_turbo_model = Model(version_id=\"amv-3ytrunai0k0n\")\n",
+    "eb_turbo_model = Model(id=\"amv-3ytrunai0k0n\")\n",
     "#设置本地评估器\n",
     "em = EvaluationManager(local_evaluators=[local_evaluator])\n",
     "result = em.eval([eb_turbo_model], ds)\n",
@@ -1191,7 +1189,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -1209,9 +1207,8 @@
     "    name=\"role_play_sftfin\",\n",
     "    endpoint_suffix=\"sdkcqa1\",\n",
     "    replicas=1, # 副本数， 与qps强绑定\n",
-    "    pool_type=DeployPoolType.PrivateResource, # 私有资源池\n",
     "    service_type=ServiceType.Chat,\n",
-    "    hours=1,\n",
+    "    months=1,\n",
     "))\n"
    ]
   },

cookbook/dataset/batch_inference_using_dataset.ipynb

@@ -516,7 +516,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -621,9 +621,9 @@
     "#-# cell_skip\n",
     "cloud_dataset_id = \"dataset_id\"\n",
     "\n",
-    "qianfan_ds = Dataset.load(qianfan_dataset_id=cloud_dataset_id)\n",
+    "qianfan_ds = Dataset.load(qianfan_dataset_version_id=cloud_dataset_id)\n",
     "\n",
-    "result = qianfan_ds.test_using_llm(model_version_id=\"amv-qb8ijukaish3\")\n",
+    "result = qianfan_ds.test_using_llm(model_id=\"amv-qb8ijukaish3\")\n",
     "print(result[0])"
    ]
   },