run gpt-4-turbo

analysis/batch_run.py

@@ -25,14 +25,16 @@ def main(config_yml):
         from_dir = config['repo_base_path']
         to_dir = config['response_path']
         repos = config['repo']
+        model = config['model']
 
         record = []
         for repo in tqdm(repos):
             for run in range(1, runs+1):
                 action.evaluate(
                     repo_path=os.path.join(from_dir, repo['path']),
-                    save_to=f"{to_dir}/{repo['name']}_{"{:02d}".format(run)}.json",
-                    checklist_path=checklist_path
+                    save_response_to=f"{to_dir}/{repo['name']}_{"{:02d}".format(run)}.json",
+                    checklist_path=checklist_path,
+                    model=model
                 )
 
                 record.append({
@@ -43,4 +45,4 @@ def main(config_yml):
                 with open(f"{to_dir}/record.yml", 'w') as file:
                     yaml.dump(record, file)
 
-    fire.Fire(main)
+    fire.Fire(main)

analysis/batch_run.yml

@@ -1,27 +1,28 @@
 runs: 30
 checklist_path: null
+model: 'gpt-4-turbo'
 repo_base_path: '../data/raw/openja/'
-response_path: '../data/processed/batch_run'
+response_path: '../data/processed/batch_run_4-turbo'
 repo:
   - name: lightfm
     path: './lightfm'
-  - name: qlib
-    path: './qlib'
-  - name: mmf
-    path: './mmf'
-  - name: nanodet
-    path: './nanodet'
-  - name: magenta
-    path: './magenta'
-  - name: nupic
-    path: './nupic'
-  - name: DeepSpeech
-    path: './DeepSpeech'
-  - name: apollo
-    path: './apollo'
-  - name: 'paperless-ng'
-    path: './paperless-ng'
-  - name: 'mycroft-core'
-    path: './mycroft-core'
-  - name: deepchem
-    path: './deepchem'
+      #  - name: qlib
+      #    path: './qlib'
+      #  - name: mmf
+      #    path: './mmf'
+      #  - name: nanodet
+      #    path: './nanodet'
+      #  - name: magenta
+      #    path: './magenta'
+      #  - name: nupic
+      #    path: './nupic'
+      #  - name: DeepSpeech
+      #    path: './DeepSpeech'
+      #  - name: apollo
+      #    path: './apollo'
+      #  - name: 'paperless-ng'
+      #    path: './paperless-ng'
+      #  - name: 'mycroft-core'
+      #    path: './mycroft-core'
+      #  - name: deepchem
+      #    path: './deepchem'