sdgilley · sdgilley · Mar 27, 2024 · Feb 27, 2024 · Feb 29, 2024 · Feb 29, 2024
diff --git a/GitHub/DASHBOARD.md b/GitHub/DASHBOARD.md
diff --git a/GitHub/docs_dashboard.py → GitHub/docs_dashboard-h.py b/GitHub/docs_dashboard.py → GitHub/docs_dashboard-h.py
@@ -6,10 +6,8 @@ def get_notebooks(file_path):
         lines = [line.split('@')[0] for line in lines] # all we need is the file name, not the codeowners
     return lines
 
-def write_readme(notebooks):
-    script_dir = os.path.dirname(os.path.realpath(__file__))  # Get the directory that the script is in
-    readme_file = os.path.join(script_dir,"DASHBOARD.md")
-
+def write_html(notebooks):
+    html_file = os.path.join("docs","dashboard.html")
     wf_link = "https://github.com/Azure/azureml-examples/actions/workflows" # where to find the workflows
     gh_link = "https://github.com/Azure/azureml-examples/blob/main" # where to find the files
     rows_by_extension = {}
@@ -25,37 +23,36 @@ def write_readme(notebooks):
         if extension not in rows_by_extension:
             rows_by_extension[extension] = []
 
-        # print(workflow)
         file = notebook.split('/')[-1].replace('.', '&#46;') # last part of the path is the file name
         file_name = os.path.splitext(file)[0]
         # now put back the spaces and dots in the file name for better readability
         file = file.replace('%20', ' ').replace('&#46;', '.')
-        # print(f"FILE: {file}, File name: {file_name}")
-        status = f"[![{file_name}]({wf_link}/{workflow}/badge.svg?branch=main)]({wf_link}/{workflow})"
-        row = f"|{status} | [{file}]({gh_link}/{notebook})|\n"
-        # Add the row to the list for this extension
+        status = f'<a href="{wf_link}/{workflow}"><img src="{wf_link}/{workflow}/badge.svg?branch=main" alt="{file_name}"></a>'
+        row = f'<tr><td>{status}</td><td><a href="{gh_link}/{notebook}">{file}</a></td></tr>\n'
         rows_by_extension[extension].append(row)
 
-    # Now, iterate over the dictionary to write each list of rows to a separate table
-    with open(readme_file, 'w') as f:
-        prefix = "# Files referenced in docs \n\n These files are referenced in https://learn.microsoft.com/azure/machine-learning \n\n"
-        f.write(prefix)
-        # f.write('Jump to:\n')
-        # for extension in sorted(rows_by_extension.keys()):
-        #     # Write a link to each section
-        #     ext = extension.replace(".", "")
-        #     f.write(f'[{ext}](#{ext})\n')
-
-        for extension, rows in sorted(rows_by_extension.items()):
-            # Write a header for each extension
-            if extension != '': # ignore the files without extension
-                f.write(f'## {extension} files\n')
-                f.write('| Status | File |\n')
-                f.write('| --- | --- |\n')
+    # read the top part of the html file from top.html 
+    script_dir = os.path.dirname(os.path.realpath(__file__))  # Get the directory that the script is in
+    with open(os.path.join(script_dir,'top.html'), 'r') as top_file:
+        top_contents = top_file.read()
+    with open(os.path.join(script_dir,'jumps.html'), 'r') as jumps:
+        jumps = jumps.read()
+    for extension in sorted(rows_by_extension.keys()):
+        rows = rows_by_extension[extension]      
+        ext = extension[1:].strip() # get rid of the leading dot
+        if extension != '':
+            with open(f'docs/{ext}.html', 'w') as file:
+                file.write(f'<html>\n<head>\n<title>{extension} code snippets dashboard</title>\n')
+                file.write(top_contents)
+                file.write(f'<h1> {extension} code snippets dashboard</h1>\n')
+                file.write(jumps)
+                rows = rows_by_extension[extension]      
+                file.write(f'<a name={extension}></a><h2>{extension}</h2>\n')
+                file.write('<table>\n')
                 for row in rows:
-                    f.write(row)
-    print("finished writing DASHBOARD...")
-
+                    file.write(row)
+                file.write('</table>\n')
+                file.write('</body>\n</html>')
 
 # This is the main function
 if __name__=="__main__": 
@@ -67,4 +64,4 @@ def write_readme(notebooks):
     notebooks = get_notebooks(file_path)
     notebooks = [notebooks.replace('\\ ','%20') for notebooks in notebooks] # replace space with &nbsp;
     # print(notebooks)
-    write_readme(notebooks)
+    write_html(notebooks)
diff --git a/GitHub/find-snippets.py b/GitHub/find-snippets.py
@@ -30,6 +30,7 @@
 
 found = pd.DataFrame(columns=['ref_file', 'from_file'])
 dict_list = []
+dict_list2 = []
 branches = []
 # Record the start time
 start_time = datetime.now()
@@ -45,7 +46,10 @@
         # Get the file content
         file_content = content_file.decoded_content
         lines = file_content.decode().splitlines()
-
+        blocks = []
+        count = 0
+        code_type = None
+        inside_code_block = False
         for line in lines:
             # snippets have ~\azureml-examples in them.  Find all snippets in this file.
             match_snippet = re.findall(r'\(~\/azureml-examples[^)]*\)|source="~\/azureml-examples[^"]*"', line)
@@ -56,6 +60,19 @@
                     if branch == az_ml_branch: #PRs are merged into main, so only these files are relevant
                         row_dict = {'ref_file': ref_file, 'from_file': file}
                         dict_list.append(row_dict)
+            # count lines in code snippets
+            blocks, inside_code_block, count, code_type = h.count_code_lines(line, blocks, inside_code_block, count, code_type)
+        # done looking through lines of this file
+        if inside_code_block:
+            print(f"{file}: Warning: A code block started but did not end.")
+            print(f"  The last code block type was {code_type} and had {count} lines.")
+        if blocks:
+            # this file  has code blocks.  add info to the dictionary
+            for block in blocks:
+                dict_list2.append({'file': file, 'type': block[0], 'lines': block[1]})
+
+code_counts = pd.DataFrame.from_dict(dict_list2)                                            
+code_counts.to_csv("code-counts.csv", index=False)
 
 found = pd.DataFrame.from_dict(dict_list)
 branches = pd.DataFrame(branches)

diff --git a/GitHub/find-status.py b/GitHub/find-status.py
diff --git a/GitHub/jumps.html b/GitHub/jumps.html
@@ -0,0 +1,8 @@
+<div class="mini-banner">
+    <a href="ipynb.html"><i class="fa-brands fa-python"></i>.ipynb &nbsp;</a></ul>
+    <a href="json.html"><i class="fa-solid fa-sm">&#123;&#59;&#125;</i>.json &nbsp;</a>
+    <a href="py.html"><i class="fa-brands fa-python"></i>.py &nbsp;</a>
+    <a href="sh.html"><i class="fa-solid fa-terminal"></i>.sh &nbsp;</a>
+    <a href="yaml.html"><i class="fa fa-solid fa-2xs">&#89;&#77;&#76;</i>.yaml &nbsp;</a>
+    <a href="yml.html"><i class="fa fa-solid fa-2xs">&#89;&#77;&#76;</i>.yml &nbsp;</a>
+</div>
diff --git a/GitHub/refs-found.csv b/GitHub/refs-found.csv
@@ -13,8 +13,6 @@ cli/assets/data/local-folder.yml,reference-yaml-data.md
 cli/assets/environment/docker-context.yml,reference-yaml-environment.md
 cli/assets/environment/docker-context.yml,how-to-manage-environments-v2.md
 cli/assets/environment/docker-image-plus-conda.yaml,how-to-manage-environments-v2.md
-cli/assets/environment/docker-image-plus-conda.yaml,reference-yaml-environment.md
-cli/assets/environment/docker-image.yml,how-to-identity-based-service-authentication.md
 cli/assets/environment/docker-image.yml,how-to-manage-environments-v2.md
 cli/assets/environment/docker-image.yml,reference-yaml-environment.md
 cli/assets/model/local-file.yml,how-to-manage-models.md
@@ -66,6 +64,7 @@ cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/code/batch_d
 cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/deployment.yml,reference-yaml-deployment-batch.md
 cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/deployment.yml,how-to-use-batch-model-deployments.md
 cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/deployment.yml,how-to-batch-scoring-script.md
+cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/environment/conda.yaml,how-to-batch-scoring-script.md
 cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/environment/conda.yaml,how-to-use-batch-model-deployments.md
 cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/environment/conda.yaml,how-to-batch-scoring-script.md
 cli/endpoints/batch/deploy-models/mnist-classifier/endpoint.yml,reference-yaml-endpoint-batch.md
@@ -90,7 +89,6 @@ cli/endpoints/batch/deploy-pipelines/hello-batch/deployment-from-job.yml,how-to-
 cli/endpoints/batch/deploy-pipelines/hello-batch/deployment.yml,how-to-use-batch-pipeline-deployments.md
 cli/endpoints/batch/deploy-pipelines/hello-batch/deployment.yml,reference-yaml-deployment-batch.md
 cli/endpoints/batch/deploy-pipelines/hello-batch/endpoint.yml,how-to-use-batch-pipeline-deployments.md
-cli/endpoints/batch/deploy-pipelines/hello-batch/endpoint.yml,how-to-use-batch-pipeline-from-job.md
 cli/endpoints/batch/deploy-pipelines/hello-batch/hello-component/hello.yml,how-to-use-batch-pipeline-deployments.md
 cli/endpoints/batch/deploy-pipelines/hello-batch/pipeline-job.yml,how-to-use-batch-pipeline-from-job.md
 cli/endpoints/batch/deploy-pipelines/training-with-components/deploy-and-run.sh,how-to-use-batch-training-pipeline.md
@@ -112,21 +110,21 @@ cli/endpoints/online/deploy-with-packages/custom-model/environment/conda.yaml,ho
 cli/endpoints/online/deploy-with-packages/custom-model/environment/conda.yaml,how-to-package-models-moe.md
 cli/endpoints/online/deploy-with-packages/custom-model/environment/sklearn-regression-env.yml,how-to-package-models.md
 cli/endpoints/online/deploy-with-packages/custom-model/environment/sklearn-regression-env.yml,how-to-package-models-moe.md
+cli/endpoints/online/deploy-with-packages/custom-model/environment/sklearn-regression-env.yml,how-to-package-models.md
 cli/endpoints/online/deploy-with-packages/custom-model/package-external.yml,how-to-package-models.md
-cli/endpoints/online/deploy-with-packages/custom-model/package-moe.yml,how-to-package-models.md
 cli/endpoints/online/deploy-with-packages/custom-model/package-moe.yml,how-to-package-models-moe.md
+cli/endpoints/online/deploy-with-packages/custom-model/package-moe.yml,how-to-package-models.md
 cli/endpoints/online/deploy-with-packages/custom-model/sample-request.json,how-to-package-models-moe.md
 cli/endpoints/online/deploy-with-packages/mlflow-model/deploy.sh,concept-package-models.md
 cli/endpoints/online/deploy-with-packages/mlflow-model/deploy.sh,how-to-package-models-app-service.md
 cli/endpoints/online/deploy-with-packages/mlflow-model/package-external.yml,how-to-package-models-app-service.md
 cli/endpoints/online/deploy-with-packages/mlflow-model/sample-request.json,how-to-package-models-app-service.md
 cli/endpoints/online/deploy-with-packages/registry-model/deploy.sh,how-to-package-models.md
 cli/endpoints/online/deploy-with-packages/registry-model/package.yml,how-to-package-models.md
-cli/endpoints/online/managed/managed-identities/1-sai-create-endpoint.yml,how-to-access-resources-from-endpoints-managed-identities.md
 cli/endpoints/online/managed/managed-identities/1-sai-create-endpoint.yml,reference-yaml-endpoint-online.md
+cli/endpoints/online/managed/managed-identities/1-sai-create-endpoint.yml,how-to-access-resources-from-endpoints-managed-identities.md
 cli/endpoints/online/managed/managed-identities/1-uai-create-endpoint.yml,reference-yaml-endpoint-online.md
 cli/endpoints/online/managed/managed-identities/1-uai-create-endpoint.yml,how-to-access-resources-from-endpoints-managed-identities.md
-cli/endpoints/online/managed/managed-identities/2-sai-deployment.yml,reference-yaml-deployment-managed-online.md
 cli/endpoints/online/managed/managed-identities/2-sai-deployment.yml,how-to-access-resources-from-endpoints-managed-identities.md
 cli/endpoints/online/managed/managed-identities/2-uai-deployment.yml,how-to-access-resources-from-endpoints-managed-identities.md
 cli/endpoints/online/managed/managed-identities/2-uai-deployment.yml,reference-yaml-deployment-managed-online.md
@@ -139,8 +137,8 @@ cli/endpoints/online/managed/sample/endpoint.yml,how-to-deploy-automl-endpoint.m
 cli/endpoints/online/managed/sample/endpoint.yml,reference-yaml-endpoint-online.md
 cli/endpoints/online/managed/sample/endpoint.yml,how-to-safely-rollout-online-endpoints.md
 cli/endpoints/online/managed/sample/green-deployment.yml,reference-yaml-deployment-managed-online.md
-cli/endpoints/online/model-1/onlinescoring/score.py,how-to-inference-server-http.md
 cli/endpoints/online/model-1/onlinescoring/score.py,how-to-deploy-online-endpoints.md
+cli/endpoints/online/model-1/onlinescoring/score.py,how-to-inference-server-http.md
 cli/endpoints/online/model-1/onlinescoring/score_managedidentity.py,how-to-access-resources-from-endpoints-managed-identities.md
 cli/endpoints/online/model-1/sample-request.json,how-to-access-resources-from-endpoints-managed-identities.md
 cli/endpoints/online/ncd/create-endpoint.yaml,how-to-deploy-mlflow-models-online-endpoints.md
@@ -227,8 +225,8 @@ cli/resources/compute/cluster-minimal.yml,reference-yaml-compute-aml.md
 cli/resources/compute/cluster-ssh-password.yml,reference-yaml-compute-aml.md
 cli/resources/compute/cluster-system-identity.yml,how-to-identity-based-service-authentication.md
 cli/resources/compute/cluster-user-identity.yml,how-to-identity-based-service-authentication.md
-cli/resources/compute/instance-basic.yml,how-to-create-compute-instance.md
 cli/resources/compute/instance-basic.yml,reference-yaml-compute-instance.md
+cli/resources/compute/instance-basic.yml,how-to-create-compute-instance.md
 cli/resources/compute/instance-minimal.yml,reference-yaml-compute-instance.md
 cli/resources/compute/instance-schedule.yml,how-to-create-compute-instance.md
 cli/resources/compute/vm-attach.yml,reference-yaml-compute-vm.md
@@ -246,8 +244,8 @@ cli/resources/datastore/file.yml,reference-yaml-datastore-files.md
 cli/resources/registry/registry-storage-options.yml,reference-yaml-registry.md
 cli/resources/registry/registry.yml,reference-yaml-registry.md
 cli/resources/workspace/basic.yml,reference-yaml-workspace.md
-cli/resources/workspace/cmk.yml,reference-yaml-workspace.md
 cli/resources/workspace/cmk.yml,how-to-manage-workspace-cli.md
+cli/resources/workspace/cmk.yml,reference-yaml-workspace.md
 cli/resources/workspace/hbi.yml,reference-yaml-workspace.md
 cli/resources/workspace/mvnet-allow-internet-outbound.yml,reference-yaml-workspace.md
 cli/resources/workspace/mvnet-allow-only-approved-outbound.yml,reference-yaml-workspace.md

diff --git a/GitHub/top.html b/GitHub/top.html
@@ -0,0 +1,7 @@
+    <link rel="stylesheet" type="text/css" href="styles.css">
+    <script src="https://kit.fontawesome.com/5014c87d7e.js" crossorigin="anonymous"></script>
+    <!-- <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css"> -->
+    <body>
+    <div class="banner">
+        <a href="index.html"><i class="fa-solid fa-house"></i> Home</a> &nbsp; <a href="dashboards.html"><i class="fa-solid fa-table-columns"></i> Dashboard</a>
+    </div>
diff --git a/GitHub/utilities.py b/GitHub/utilities.py
@@ -118,4 +118,29 @@ def compare_branches(repo, file, branch1, branch2):
     if file_b1.sha == file_b2.sha:
         print(f"*azureml-examples {branch2} branch has the same version of this file as {branch1}\n")
     else:
-        print(f"*azureml-examples {branch2} branch has a DIFFERENT version of this file from {branch1}\n")
+        print(f"*azureml-examples {branch2} branch has a DIFFERENT version of this file from {branch1}\n")
+# call  for each line in the file, send in current info and get back updated values
+def count_code_lines(line, blocks, inside_code_block, count, code_type):
+    line = line.lstrip()
+    if line.startswith('```'):
+        if inside_code_block: # done - this is the end of the block
+            blocks.append((code_type, count))  # Add type and count to the list
+        else: # starting - get the type and reset the count
+            code_type = line[3:].strip()  # Get the rest of the line after ```
+            count = 0
+        inside_code_block = not inside_code_block
+    else:
+        count += 1
+    return blocks, inside_code_block, count, code_type
+
+def find_snippets(line, branches, az_ml_branch, file):
+    match_snippet = re.findall(r'\(~\/azureml-examples[^)]*\)|source="~\/azureml-examples[^"]*"', line)
+    if match_snippet:
+        for match in match_snippet:
+            path, ref_file, branch, match, name = cleanup_matches(match)
+            branches.append(branch)
+            if branch == az_ml_branch: #PRs are merged into main, so only these files are relevant
+                row_dict = {'ref_file': ref_file, 'from_file': file}
+                dict_list.append(row_dict)
+
+