Fix openssf

.github/workflows/workflow_finetune.yml

@@ -29,6 +29,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-ft
   cancel-in-progress: true
 
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
 jobs:
   finetune:
     name: finetune
@@ -63,7 +66,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Build Docker Image
         run: |
@@ -88,6 +91,12 @@ jobs:
           source dev/scripts/ci-functions.sh
           finetune_test ${{ matrix.model }}
 
+      - name: Run Finetune DPO Test
+        run: |
+          TARGET="finetune"
+          source dev/scripts/ci-functions.sh
+          finetune_dpo_test ${{ matrix.model }}
+
       - name: Run PEFT-LoRA Test
         run: |
           source dev/scripts/ci-functions.sh

.github/workflows/workflow_finetune_gpu.yml

@@ -13,6 +13,9 @@ on:
         type: string
         default: 'http://proxy-prc.intel.com:912'
 
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
 jobs:
   finetune-gpu:
     name: finetune-gpu
@@ -35,7 +38,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Running task on Intel GPU
         run: |

.github/workflows/workflow_inference.yml

@@ -29,6 +29,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-inf
   cancel-in-progress: true
 
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
 jobs:
   inference:
     name: inference
@@ -65,7 +68,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Determine Target
         id: "target"

.github/workflows/workflow_inference_gaudi2.yml

@@ -23,6 +23,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-inf-gaudi2
   cancel-in-progress: true
 
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
 jobs:
   inference:
     name: inference
@@ -81,7 +84,7 @@ jobs:
           echo "target=$target" >> $GITHUB_OUTPUT
 
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Build Docker Image
         run: |

.github/workflows/workflow_lint.yml

@@ -11,6 +11,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-lt
   cancel-in-progress: true
 
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
 jobs:
   lint:
     name: lint
@@ -22,7 +25,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Run Lint
         run: ./format.sh -a

.github/workflows/workflow_orders_nightly.yml

@@ -4,6 +4,9 @@ on: []
   # schedule:
   #   - cron: "0 16 * * *"
 
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
 jobs:
 
   call-inference:

.github/workflows/workflow_orders_on_merge.yml

@@ -8,6 +8,9 @@ on:
       - '**'
       - '!*.md'
 
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
 jobs:
   Lint:
     uses: ./.github/workflows/workflow_lint.yml

.github/workflows/workflow_orders_on_pr.yml

@@ -7,6 +7,10 @@ on:
     paths:
       - '**'
       - '!*.md'
+
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
 jobs:
 
   Lint:

.github/workflows/workflow_test_benchmark.yml

@@ -29,6 +29,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-bench
   cancel-in-progress: true
 
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
 jobs:
   setup-test:
 
@@ -51,7 +54,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Load environment variables
         run: cat /root/actions-runner-config/.env >> $GITHUB_ENV

.github/workflows/workflow_tests.yml

@@ -7,6 +7,9 @@ on:
         type: string
         default: 'pr'
 
+permissions:  # added using https://github.com/step-security/secure-repo
+  contents: read
+
 jobs:
   setup-test:
 
@@ -26,10 +29,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
         with:
           python-version: ${{matrix.python-version}}
           architecture: 'x64'
@@ -60,10 +63,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
         with:
           python-version: ${{matrix.python-version}}
           architecture: 'x64'
@@ -94,10 +97,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
         with:
           python-version: ${{matrix.python-version}}
           architecture: 'x64'
@@ -141,7 +144,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
       - name: Determine Target
         id: "target"
@@ -165,7 +168,7 @@ jobs:
           code_checkout_path=${{ github.workspace }}
           source dev/scripts/ci-functions.sh
           start_docker ${TARGET} ${code_checkout_path}
-      
+
       - name: Install Dependencies for Tests
         run: |
           TARGET=${{steps.target.outputs.target}}
@@ -189,4 +192,4 @@ jobs:
         run: |
           TARGET=${{steps.target.outputs.target}}
           source dev/scripts/ci-functions.sh
-          stop_container ${TARGET} 
+          stop_container ${TARGET}

dev/scripts/ci-functions.sh

@@ -216,6 +216,21 @@ finetune_test(){
     docker exec "finetune" bash -c "llm_on_ray-finetune --config_file llm_on_ray/finetune/finetune.yaml"
 }
 
+finetune_dpo_test(){
+    local model=$1
+    # Check if the model is 'EleutherAI/gpt-j-6b' or 'gpt2'
+    if [ "$model" == "EleutherAI/gpt-j-6b" ] || [ "$model" == "gpt2" ]; then
+        echo "Model '$model' is not supported for this operation."
+        return
+    fi
+    echo Set finetune source config :
+    docker exec "finetune" bash -c "source \$(python -c 'import oneccl_bindings_for_pytorch as torch_ccl;print(torch_ccl.cwd)')/env/setvars.sh; RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING=1 ray start --head --node-ip-address 127.0.0.1 --ray-debugger-external; RAY_SERVE_ENABLE_EXPERIMENTAL_STREAMING=1  ray start --address='127.0.0.1:6379' --ray-debugger-external"
+    echo Set "${model}" patch_yaml_config :
+    docker exec "finetune" bash -c "python dev/scripts/patch_yaml_config.py --conf_path "llm_on_ray/finetune/finetune.yaml" --models ${model} --dpo"
+    echo Stert "${model}" dpo finetune :
+    docker exec "finetune" bash -c "llm_on_ray-finetune --config_file llm_on_ray/finetune/finetune.yaml"
+}
+
 peft_lora_test(){
     local model=$1
     docker exec "finetune" bash -c "rm -rf /tmp/llm-ray/*"

dev/scripts/patch_yaml_config.py

@@ -24,6 +24,8 @@ def patch_yaml_config():
     parser.add_argument("--conf_path", type=str)
     parser.add_argument("--models", type=str)
     parser.add_argument("--peft_lora", action="store_true", default=False)
+    parser.add_argument("--dpo", action="store_true", default=False)
+
     args = parser.parse_args()
 
     conf_path = args.conf_path
@@ -69,6 +71,12 @@ def patch_yaml_config():
                 result["General"]["lora_config"]["target_modules"] = None
         else:
             result["General"]["lora_config"] = None
+        if args.dpo:
+            if "finetuning_model" not in result["Training"]:
+                result["Training"]["finetuning_model"] = {}
+            result["Dataset"]["train_file"] = "examples/data/sample_dpo_data.jsonl"
+            result["Training"]["beta"] = 0.1
+            result["Training"]["finetuning_model"]["dpo"] = True
 
     with open(conf_path, "w") as output:
         yaml.dump(result, output, sort_keys=False)

docs/finetune_parameters.md

@@ -38,6 +38,9 @@ The following are the parameters supported in the finetuning workflow.
 | mask_input                  |True| mask the input part in lables                                                                                                                                         |
 | mask_response               |True| mask the response part in lables                                                                                                                                         |
 | data_preprocess_type        |neural_chat| The type of the encode input                                                                                                                              |
+|pad_max|False|Whether to pad the data to the max length of the batch|
+|max_source_length|512|Maximum source sequence length. Sequences will be right padded|
+|torch_dtype|bfloat16|Override the default `torch.dtype` and load the model under this dtype|
 
 
 ## Training Parameters