Merge pull request #820 from kvcache-ai/develop-0.2.3

Develop 0.2.3 ready to release

ktransformers/ktransformers_ext/CMakeLists.txt

@@ -175,6 +175,7 @@ elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR CMAKE_GENERATOR_PLATFORM_LW
             list(APPEND ARCH_FLAGS -mavx512bw)
             list(APPEND ARCH_FLAGS -mavx512dq)
             list(APPEND ARCH_FLAGS -mavx512vnni)
+            list(APPEND ARCH_FLAGS -mavx512vpopcntdq)
         endif()
         if (LLAMA_AVX512_BF16)
             list(APPEND ARCH_FLAGS -mavx512bf16)

ktransformers/tests/AIME_2024/eval_api.py

@@ -78,13 +78,15 @@ def run_eval_api(
     format_tabs: bool = False,
     auth_token: str = None,
     problem_file: str = None,
-    append: bool = False
+    append: bool = False,
+    skip: int = 0
 ):
 
     data = load_data(problem_file)
     pbar = tqdm.tqdm(total=len(data) * 1)
-
+    pbar.update(skip)
     for i in range(len(data)):
+        i = i+skip
         data_item = data[i]
         question = data_item['Problem']
         # Start the timer for this evaluation
@@ -97,6 +99,7 @@ def run_eval_api(
             score = get_score(completion, answer)
             elapsed_time = time.time() - start_time
             result = {
+                "index": i,
                 "question_id": data_item["ID"],
                 "answer": answer,
                 "prediction": completion,
@@ -114,9 +117,9 @@ def run_eval_api(
         pbar.update(1)
 
 
-def main(output_path, api_url, model_name, auth_token, format_tabs,problem_file, append):
+def main(output_path, api_url, model_name, auth_token, format_tabs,problem_file, append,skip):
     os.makedirs(os.path.dirname(output_path), exist_ok=True)
-    run_eval_api(api_url, model_name, output_path, format_tabs, auth_token, problem_file,append)
+    run_eval_api(api_url, model_name, output_path, format_tabs, auth_token, problem_file,append,skip)
 
 
 if __name__ == "__main__":
@@ -128,6 +131,7 @@ def main(output_path, api_url, model_name, auth_token, format_tabs,problem_file,
     parser.add_argument("--format_tabs", action="store_true", help="Format Tabs")
     parser.add_argument("--problem_file", type=str, default="Maxwell-Jia/AIME_2024", help="Evalset File")
     parser.add_argument("--no_append", action="store_false", help="Append to existing file")
+    parser.add_argument("--skip", type=int, default=0, help="Skip some tasks")
     args = parser.parse_args()
     # api_url = "https://api.siliconflow.cn/v1/chat/completions"
-    main(args.out_path, args.api_url, args.model_name, args.auth_token, args.format_tabs, args.problem_file, args.no_append)
+    main(args.out_path, args.api_url, args.model_name, args.auth_token, args.format_tabs, args.problem_file, args.no_append, args.skip)

ktransformers/tests/humaneval/eval_api.py

@@ -39,16 +39,23 @@ def run_eval_api(
     format_tabs: bool = False,
     auth_token: str = None,
     problem_file: str = None,
-    append: bool = False
+    append: bool = False,
+    skip: int = 0
 ):
     if(problem_file is None):
         problems = read_problems()
     else:
         problems = read_problems(problem_file)
     samples = []
     pbar = tqdm.tqdm(total=len(problems) * 1)
+    pbar.update(skip)
     try:
         for task_id in problems:
+            # skip some tasks
+            if skip > 0:
+                skip -= 1
+                continue
+
             if format_tabs:
                 prompt = problems[task_id]["prompt"].replace("    ", "\t")
             else:
@@ -67,23 +74,25 @@ def run_eval_api(
         if not append:
             write_jsonl(out_path, samples,append=append)
     except Exception as e:
-        write_jsonl(out_path, samples,append=append)
+        if not append:
+            write_jsonl(out_path, samples,append=append)
         print(f"Error: {e}")
 
-def main(output_path, api_url, model_name, auth_token, format_tabs,problem_file, append):
+def main(output_path, api_url, model_name, auth_token, format_tabs,problem_file, append,skip):
     os.makedirs(os.path.dirname(output_path), exist_ok=True)
-    run_eval_api(api_url, model_name, output_path, format_tabs, auth_token, problem_file,append)
+    run_eval_api(api_url, model_name, output_path, format_tabs, auth_token, problem_file,append,skip)
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="API Generate Tester")
     parser.add_argument("--api_url", type=str, default="https://api.siliconflow.cn/v1/chat/completions", help="API URL")
     parser.add_argument("--model_name", type=str, default="Pro/deepseek-ai/DeepSeek-V3", help="Model Name")
-    parser.add_argument("--out_path", type=str, default="results/api/eval.jsonl", help="Output Path")
+    parser.add_argument("--out_path", type=str, default="results/api/eval_b.jsonl", help="Output Path")
     parser.add_argument("--auth_token", type=str, default=None, help="Auth Token")
     parser.add_argument("--format_tabs", action="store_true", help="Format Tabs")
     parser.add_argument("--problem_file", type=str, default=None, help="Evalset File")
     parser.add_argument("--no_append", action="store_false", help="Append to existing file")
+    parser.add_argument("--skip", type=int, default=0, help="Skip first n problems")
     args = parser.parse_args()
     # api_url = "https://api.siliconflow.cn/v1/chat/completions"
-    main(args.out_path, args.api_url, args.model_name, args.auth_token, args.format_tabs, args.problem_file, args.no_append)
+    main(args.out_path, args.api_url, args.model_name, args.auth_token, args.format_tabs, args.problem_file, args.no_append,args.skip)

ktransformers/tests/humaneval/evaluation.py

@@ -8,7 +8,7 @@ def filter_code(completion: str) -> str:
         completion = completion.split('if __name__ == "__main__":')[0]
     if "# Example usage" in completion:
         completion = completion.split("# Example usage")[0]
-    return completion.split("\n\n")[0]
+    return completion
 
 
 def fix_indents(text: str) -> str:

third_party/llamafile/iqk_mul_mat.inc

@@ -2388,7 +2388,8 @@ struct SimpleBits {
 
 
 struct EvenSignHelper {
-    #ifdef HAVE_FANCY_SIMD
+    #if defined HAVE_FANCY_SIMD
+    // #pragma message("Using AVX512VPOPCNTDQ in even sign helper")
         union sbits_t {
             __m128i vec;
             __mmask32 mask[4];