Merge branch 'cont-revise-report' of github.com:UBC-MDS/test-creation…

… into cont-revise-report
UBC-MDS · Jun 21, 2024 · cfc1d17 · cfc1d17
2 parents ba97433 + 6f6f81c
commit cfc1d17
Show file tree

Hide file tree

Showing 34 changed files with 1,524 additions and 9,060 deletions.
diff --git a/docs/02_finding-report.html b/docs/02_finding-report.html
diff --git a/docs/04_plots-for-presentations.html b/docs/04_plots-for-presentations.html
diff --git a/docs/final_report.html b/docs/final_report.html
diff --git a/img/checklist_sample.png → docs/img/checklist_sample.png b/img/checklist_sample.png → docs/img/checklist_sample.png
diff --git a/docs/logo.png → docs/img/logo.png b/docs/logo.png → docs/img/logo.png
diff --git a/docs/img/proposed_system_overview.png b/docs/img/proposed_system_overview.png
diff --git a/img/test_evaluation_report_sample.png → docs/img/test_evaluation_report_sample.png b/img/test_evaluation_report_sample.png → docs/img/test_evaluation_report_sample.png
diff --git a/img/test_spec_sample.png → docs/img/test_spec_sample.png b/img/test_spec_sample.png → docs/img/test_spec_sample.png
diff --git a/docs/proposal.html b/docs/proposal.html
diff --git a/docs/search.json b/docs/search.json
diff --git a/report/final_report/_quarto.yml b/report/final_report/_quarto.yml
@@ -1,19 +1,19 @@
 project:
   type: website
+  render:
+    - "*qmd"
   output-dir: docs
 
 website:
   sidebar:
     style: "docked"
-    logo: "logo.png"
+    logo: "img/logo.png"
     search: true
     contents:
-      - section: "Final Report"
-        contents:
-          - final_report.qmd
-      - section: "Proposal"
-        contents:
-          - proposal.ipynb
+      - text: "Capstone Final Report"
+        href: final_report.qmd
+      - text: "Capstone Proposal"
+        href: proposal.qmd
 
 format:
   html:

diff --git a/report/final_report/docs/01_preprocess.html b/report/final_report/docs/01_preprocess.html
diff --git a/report/final_report/docs/02_finding-report.html b/report/final_report/docs/02_finding-report.html
diff --git a/report/final_report/docs/02_plots-for-final-report.html b/report/final_report/docs/02_plots-for-final-report.html
diff --git a/report/final_report/docs/04_plots-for-presentations.html b/report/final_report/docs/04_plots-for-presentations.html
diff --git a/report/final_report/docs/final_report.html b/report/final_report/docs/final_report.html
diff --git a/report/final_report/docs/img/checklist_sample.png b/report/final_report/docs/img/checklist_sample.png
diff --git a/report/final_report/docs/logo.png → report/final_report/docs/img/logo.png b/report/final_report/docs/logo.png → report/final_report/docs/img/logo.png
diff --git a/report/final_report/docs/img/proposed_system_overview.png b/report/final_report/docs/img/proposed_system_overview.png
diff --git a/report/final_report/docs/img/test_evaluation_report_sample.png b/report/final_report/docs/img/test_evaluation_report_sample.png
diff --git a/report/final_report/docs/img/test_spec_sample.png b/report/final_report/docs/img/test_spec_sample.png
diff --git a/report/final_report/docs/index.html b/report/final_report/docs/index.html
@@ -1,7 +1,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml">    
   <head>      
-    <title>Redirect to 02_plots-for-final-report.html</title>      
-    <meta http-equiv="refresh" content="0;URL='02_plots-for-final-report.html'" />    
+    <title>Redirect to final_report.html</title>      
+    <meta http-equiv="refresh" content="0;URL='final_report.html'" />    
   </head>    
   <body> 
   </body>  

diff --git a/report/final_report/docs/proposal.html b/report/final_report/docs/proposal.html
@@ -7,7 +7,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
 
-<title>Proposal Report - Checklists and LLM prompts for efficient and effective test creation in data analysis</title>
+<title>proposal</title>
 <style>
 code{white-space: pre-wrap;}
 span.smallcaps{font-variant: small-caps;}
@@ -92,7 +92,7 @@
       <button type="button" class="quarto-btn-toggle btn" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar,#quarto-sidebar-glass" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
         <i class="bi bi-layout-text-sidebar-reverse"></i>
       </button>
-      <nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="./proposal.html">Proposal</a></li><li class="breadcrumb-item"><a href="./proposal.html">Proposal Report - Checklists and LLM prompts for efficient and effective test creation in data analysis</a></li></ol></nav>
+      <nav class="quarto-page-breadcrumbs" aria-label="breadcrumb"><ol class="breadcrumb"><li class="breadcrumb-item"><a href="./proposal.html">Capstone Proposal</a></li></ol></nav>
       <a class="flex-grow-1" role="button" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar,#quarto-sidebar-glass" aria-controls="quarto-sidebar" aria-expanded="false" aria-label="Toggle sidebar navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">      
       </a>
       <button type="button" class="btn quarto-search-button" aria-label="" onclick="window.quartoOpenSearch();">
@@ -107,7 +107,7 @@
   <nav id="quarto-sidebar" class="sidebar collapse collapse-horizontal sidebar-navigation docked overflow-auto">
     <div class="pt-lg-2 mt-2 text-left sidebar-header">
       <a href="./index.html" class="sidebar-logo-link">
-      <img src="./logo.png" alt="" class="sidebar-logo py-0 d-lg-inline d-none">
+      <img src="./img/logo.png" alt="" class="sidebar-logo py-0 d-lg-inline d-none">
       </a>
       </div>
         <div class="mt-2 flex-shrink-0 align-items-center">
@@ -117,40 +117,18 @@
         </div>
     <div class="sidebar-menu-container"> 
     <ul class="list-unstyled mt-1">
-        <li class="sidebar-item sidebar-item-section">
-      <div class="sidebar-item-container"> 
-            <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" aria-expanded="true">
- <span class="menu-text">Final Report</span></a>
-          <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-1" aria-expanded="true" aria-label="Toggle section">
-            <i class="bi bi-chevron-right ms-2"></i>
-          </a> 
-      </div>
-      <ul id="quarto-sidebar-section-1" class="collapse list-unstyled sidebar-section depth1 show">  
-          <li class="sidebar-item">
+        <li class="sidebar-item">
   <div class="sidebar-item-container"> 
   <a href="./final_report.html" class="sidebar-item-text sidebar-link">
- <span class="menu-text">DSCI591 Capstone Final Report</span></a>
+ <span class="menu-text">Capstone Final Report</span></a>
   </div>
 </li>
-      </ul>
-  </li>
-        <li class="sidebar-item sidebar-item-section">
-      <div class="sidebar-item-container"> 
-            <a class="sidebar-item-text sidebar-link text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" aria-expanded="true">
- <span class="menu-text">Proposal</span></a>
-          <a class="sidebar-item-toggle text-start" data-bs-toggle="collapse" data-bs-target="#quarto-sidebar-section-2" aria-expanded="true" aria-label="Toggle section">
-            <i class="bi bi-chevron-right ms-2"></i>
-          </a> 
-      </div>
-      <ul id="quarto-sidebar-section-2" class="collapse list-unstyled sidebar-section depth1 show">  
-          <li class="sidebar-item">
+        <li class="sidebar-item">
   <div class="sidebar-item-container"> 
   <a href="./proposal.html" class="sidebar-item-text sidebar-link active">
- <span class="menu-text">Proposal Report - Checklists and LLM prompts for efficient and effective test creation in data analysis</span></a>
+ <span class="menu-text">Capstone Proposal</span></a>
   </div>
 </li>
-      </ul>
-  </li>
     </ul>
     </div>
 </nav>
@@ -221,32 +199,17 @@ <h3 class="anchored" data-anchor-id="our-objectives">Our Objectives</h3>
 <section id="our-product" class="level2">
 <h2 class="anchored" data-anchor-id="our-product">Our Product</h2>
 <p>Our solution offers an end-to-end application for evaluating and enhancing the robustness of users’ ML systems.</p>
-<table class="table">
-<thead>
-<tr class="header">
-<th>```ngsvnkff ../../img/proposed_system_overview.png</th>
-</tr>
-</thead>
-<tbody>
-<tr class="odd">
-<td>name: overview-diagram</td>
-</tr>
-</tbody>
-</table>
-<p>Main components and workflow of the proposed system. The checklist would be written in <a href="https://yaml.org/">YAML</a> to maximize readability for both humans and machines. We hope this will encourage researchers/users to read, understand and modify the checklist items, while keeping the checklist closely integrated with other components in our system.</p>
-<pre><code>
-One big challenge in utilizing LLMs to reliably and consistently evaluate ML systems is their tendency to generate illogical and/or factually wrong information known as hallucination [@zhang2023sirens].
-
-To combat this, the proposed system will incorporate a checklist ([Fig. 1](overview-diagram)) which would be curated manually and incorporate best practices in software testing and identified areas to be tested inside ML pipeline from human experts and past research.
-
-This checklist will be our basis in evaluating the effectiveness and completeness of existing tests in a given codebase. Relevant information will be injected into a prompt template, which the LLMs would then be prompted to follow the checklist **exactly** during the evaluation.
-
-Here is an example of how the proposed checklist would be structured:
-
-
-```{toggle}
-```yaml
-%YAML 1.2
+<div class="quarto-figure quarto-figure-center">
+<figure class="figure">
+<p><img src="img/proposed_system_overview.png" class="img-fluid figure-img"></p>
+<figcaption class="figure-caption">Main components and workflow of the proposed system. The checklist would be written in <a href="https://yaml.org/">YAML</a> to maximize readability for both humans and machines. We hope this will encourage researchers/users to read, understand and modify the checklist items, while keeping the checklist closely integrated with other components in our system.</figcaption>
+</figure>
+</div>
+<p>One big challenge in utilizing LLMs to reliably and consistently evaluate ML systems is their tendency to generate illogical and/or factually wrong information known as hallucination <span class="citation" data-cites="zhang2023sirens">(<a href="#ref-zhang2023sirens" role="doc-biblioref">Zhang et al. 2023</a>)</span>.</p>
+<p>To combat this, the proposed system will incorporate a checklist (<a href="overview-diagram">Fig. 1</a>) which would be curated manually and incorporate best practices in software testing and identified areas to be tested inside ML pipeline from human experts and past research.</p>
+<p>This checklist will be our basis in evaluating the effectiveness and completeness of existing tests in a given codebase. Relevant information will be injected into a prompt template, which the LLMs would then be prompted to follow the checklist <strong>exactly</strong> during the evaluation.</p>
+<p>Here is an example of how the proposed checklist would be structured:</p>
+<pre class="{yaml}"><code>%YAML 1.2
 ---
 Title: Checklist for Tests in Machine Learning Projects
 Description: &gt;
@@ -457,8 +420,7 @@ <h2 class="anchored" data-anchor-id="delivery-timeline">Delivery Timeline</h2>
 </table>
 </section>
 <section id="references" class="level2">
-<h2 class="anchored" data-anchor-id="references">References</h2>
-<pre class="{bibliography}"><code></code></pre>
+
 
 
 
@@ -483,6 +445,9 @@ <h2 class="anchored" data-anchor-id="references">References</h2>
 <div id="ref-wattanakriengkrai2022github" class="csl-entry" role="listitem">
 Wattanakriengkrai, Supatsara, Bodin Chinthanet, Hideaki Hata, Raula Gaikovina Kula, Christoph Treude, Jin Guo, and Kenichi Matsumoto. 2022. <span>“GitHub Repositories with Links to Academic Papers: Public Access, Traceability, and Evolution.”</span> <em>Journal of Systems and Software</em> 183: 111117.
 </div>
+<div id="ref-zhang2023sirens" class="csl-entry" role="listitem">
+Zhang, Yue, Yafu Li, Leyang Cui, Deng Cai, Lemao Liu, Tingchen Fu, Xinting Huang, et al. 2023. <span>“Siren’s Song in the AI Ocean: A Survey on Hallucination in Large Language Models.”</span> <a href="https://arxiv.org/abs/2309.01219">https://arxiv.org/abs/2309.01219</a>.
+</div>
 </div></section></div></main> <!-- /main -->
 <script id="quarto-html-after-body" type="application/javascript">
 window.document.addEventListener("DOMContentLoaded", function (event) {

diff --git a/report/final_report/docs/search.json b/report/final_report/docs/search.json