Mkdocs (#8)

* SEC-4228 Security testing & hardening guide for C5 WIP

* SEC-4228 Security testing & hardening guide for C5 WIP (with MKDOCS)

* MKDOCS capabilities

* deleting RFIs from PDF

.vscode/launch.json

@@ -4,6 +4,7 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+
 
         {
             "name": "Python: Module createtickets",
@@ -138,7 +139,7 @@
         },
 
         {
-            "name": "Python: Directory fullBuild (html and diagrams/images)",
+            "name": "Python: Directory fullBuild MKDOCS (html and diagrams/images)",
             "type": "python",
             "request": "launch",
             "module": "r3threatmodeling.fullBuildDirectory",
@@ -153,7 +154,11 @@
                 "TM_templateFull",
                 "--formatYAML",
                 "--assetDir",
-                "../Corda5ThreatModels/threatModels/assets"
+                "../Corda5ThreatModels/threatModels/assets",
+                "--templateSiteFolderSRC", "../Corda5ThreatModels/security_site/mkdocs",
+                "--templateSiteFolderDST", "build/mkdocs/",
+                "--MKDocsSiteDir", "public",
+                "--MKDocsDir", "build/mkdocs"
             ]
         },
 

src/r3threatmodeling/MKDOCS_init/docs/css/mkdocs.css

@@ -0,0 +1,20 @@
+.wy-nav-content {
+    max-width: 100%
+}
+
+
+.wy-table-responsive table td, .wy-table-responsive table th td {
+    white-space: wrap;
+    word-wrap: break-word;
+}
+
+@media print {
+    .pagebreak {
+        clear: both;
+        min-height: 1px;
+        page-break-after: always;
+    }
+    body {
+        margin: 0.1in;  
+      }
+  }

src/r3threatmodeling/fullBuildDirectory.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os
+import shutil  
 import pathlib
 from tokenize import String
 import argparse
@@ -27,8 +28,8 @@ def dir_path(path):
     else:
         raise argparse.ArgumentTypeError(f"--TMDirectory:{path} is not a valid path")
 
-def generateIndexPage(tm_list, outputDir):
-    template = "index_tm_list"
+def generateFromTMLIst(tm_list, outputDir, outFile , template = "index_tm_list"):
+    # template = "index_tm_list"
     try:
         mdTemplate = Template(
             filename =  os.path.join(os.path.dirname(__file__),
@@ -40,7 +41,7 @@ def generateIndexPage(tm_list, outputDir):
                 ))
 
         outText = mdTemplate.render(tm_list=tm_list, outputDir=outputDir)
-        outputFilename = outMDPath = os.path.join(outputDir, "index.html")  
+        outputFilename = outMDPath = os.path.join(outputDir, outFile)  
         with open(outputFilename, 'w') as f:
             print(f"OUTPUT: {f.name}") 
             f.write(outText)
@@ -63,6 +64,17 @@ def main():
     required=False
     )
 
+    CLI.add_argument(
+    "--templateSiteFolderSRC",
+    type=dir_path,
+    required=False
+    )
+
+    CLI.add_argument(
+    "--templateSiteFolderDST",
+    required=False
+    )
+
     CLI.add_argument(
     "--outputDir", 
     default = "build",
@@ -118,11 +130,19 @@ def main():
 
     CLI.add_argument("--generatePDF", action='store_true')
 
+    CLI.add_argument('--MKDocsSiteDir',  required=False)
+    CLI.add_argument('--MKDocsDir',  required=False)
 
     args = CLI.parse_args()
     outputDir = args.outputDir
+    templateSiteFolderSRC = args.templateSiteFolderSRC
+    templateSiteFolderInitFromModule = os.path.join(os.path.dirname(__file__),'MKDOCS_init/')
+    templateSiteFolderDST = args.templateSiteFolderDST
     watchFiles = args.watchFiles
 
+    MKDocsSiteDir = args.MKDocsSiteDir
+    MKDocsDir = args.MKDocsDir
+
 
     template = args.template
     ancestorData = args.ancestorData
@@ -133,6 +153,15 @@ def main():
 
     TMDir = args.TMDirectory
 
+    os.makedirs(outputDir, exist_ok=True)
+    os.makedirs(templateSiteFolderDST, exist_ok=True)
+
+    if templateSiteFolderDST:
+        shutil.copytree(templateSiteFolderInitFromModule, templateSiteFolderDST, dirs_exist_ok=True)
+
+    if templateSiteFolderSRC and templateSiteFolderDST:
+        shutil.copytree(templateSiteFolderSRC, templateSiteFolderDST, dirs_exist_ok=True)
+
     tm_list = [{'name':f.stem, 'path':str(f)} for f in pathlib.Path(TMDir).glob("*/*.yaml") if pathlib.Path(f).parent.name == pathlib.Path(f).stem]
 
     tm_list = []#{'name':f.stem, 'path':str(f)} 
@@ -149,16 +178,30 @@ def main():
         pdfname = f'{title} Threat Model-{version}.pdf'
         pdfname = re.sub('[^\w_.)(_-]', '_', pdfname)   # replace invalid chars with underscore
 
-        tm_list.append({'name': name, 'path': path, 'title': title, 'pdf':pdfname})
+        tm_list.append({'name': name,'ID': tm['ID'], 'path': path, 'title': title, 'pdf':pdfname})
 
     print(tm_list)        
+
+    generateFromTMLIst(tm_list, outputDir+"/../", outFile="mkdocs.yml", template="conf_MKDOCS")
 
-    generateIndexPage(tm_list, outputDir)
+    generateFromTMLIst(tm_list, outputDir, outFile="index.md", template="index_MKDOCS")
 
     for tm in tm_list:
         rootTMYaml = tm['path']
         TMoutputDir = outputDir + "/" + tm['name']
-        fullBuildSingleTM.generateSingleTM(open(rootTMYaml), TMoutputDir, assetDir, template, ancestorData, browserSync , generatePDF=generatePDF, pdfHeaderNote=pdfHeaderNote)
+
+        fullBuildSingleTM.generateSingleTM(open(rootTMYaml), TMoutputDir, assetDir, template, ancestorData, browserSync , generatePDF=generatePDF, pdfHeaderNote=pdfHeaderNote, public=True)
+
+        # fullBuildSingleTM.generateSingleTM(open(rootTMYaml), TMoutputDir + "/full", assetDir, template, ancestorData, browserSync , generatePDF=generatePDF, pdfHeaderNote=pdfHeaderNote, public=False)
+
+    if MKDocsSiteDir:
+        os.makedirs(MKDocsSiteDir, exist_ok=True)
+        oldwd = os.getcwd()
+        os.chdir(MKDocsDir)
+        os.system(f"mkdocs build --clean --site-dir={MKDocsSiteDir}")
+        os.chdir(oldwd)
+
+
 
 
 if __name__ == "__main__":

src/r3threatmodeling/fullBuildSingleTM.py

@@ -33,8 +33,8 @@ def generateSingleTM(rootTMYaml, outputDir, assetDir, template, ancestorData=Tru
         shutil.copytree(assetDir[0], outputDir, dirs_exist_ok=True)
 
     tmo = ThreatModel(rootTMYaml, public=public, versionsFilterStr=versionsFilterStr)
-    tmID = tmo.id
-    tmTitle = tmo.title
+    # tmID = tmo.id
+    # tmTitle = tmo.title
 
     report_generator.generate(tmo, template, ancestorData, outputDir, browserSync, None, assetDir)
 

src/r3threatmodeling/requirements.txt

@@ -11,4 +11,5 @@ ruamel.yaml
 watchdog
 markdown-include
 importlib_resources
-semantic-version
+semantic-version
+shutils

src/r3threatmodeling/template/TM_templateFull.mako

@@ -6,22 +6,18 @@
 <%namespace name="lib" file="lib.mako"/> 
 
 
-${lib.renderTMReportPart(tmo, ancestorData, toc=True, summary=True)}
+${lib.renderTMReportPart(tmo, ancestorData, toc=True, summary=True, )}
 
 % for descendantTM in tmo.getDescendants():
   ${lib.renderTMReportPart(descendantTM, ancestorData=False)}
 % endfor
 
-${makeMarkdownLinkedHeader(2, 'Requests For Information')}
-
-__RFI_PLACEHOLDER__
-
 ${PAGEBREAK}
-${makeMarkdownLinkedHeader(1, 'Annex 1')}
+${makeMarkdownLinkedHeader(2, 'Annex 1 Operational Hardening')}
 <%include file="operationalHardeningGuide.mako" args="printTOC=False"/>
 
 ${PAGEBREAK}
-${makeMarkdownLinkedHeader(1, 'Annex 2')}
+${makeMarkdownLinkedHeader(2, 'Annex 2: Key Summary')}
 <%include file="keysSummary.mako" args="printTOC=False"/>
 
 

src/r3threatmodeling/template/TM_templateMKDOCS.mako

@@ -0,0 +1,33 @@
+<%! import html %>
+<%! from r3threatmodeling.template_utils import createTitleAnchorHash, makeMarkdownLinkedHeader, mermaid_escape, valueOr  %>
+
+<% PAGEBREAK = """<div class="pagebreak"></div>"""%>
+
+<%namespace name="lib" file="lib.mako"/> 
+
+
+${lib.renderTMReportPart(tmo, ancestorData, toc=False, summary=True, )}
+
+% for descendantTM in tmo.getDescendants():
+  ${lib.renderTMReportPart(descendantTM, ancestorData=False, headerLevel=2)}
+% endfor
+
+${makeMarkdownLinkedHeader(2, 'Requests For Information')}
+
+__RFI_PLACEHOLDER__
+
+${PAGEBREAK}
+
+<%include file="operationalHardeningGuide.mako" args="printTOC=False, headerLevel=2"/>
+
+${PAGEBREAK}
+
+<%include file="testingGuide.mako" args="printTOC=False, headerLevel=2"/>
+
+${PAGEBREAK}
+
+<%include file="keysSummary.mako" args="printTOC=False, headerLevel=2"/>
+
+
+
+

src/r3threatmodeling/template/conf_MKDOCS.mako

@@ -0,0 +1,44 @@
+<%! import html %>\
+<%! from r3threatmodeling.template_utils import createTitleAnchorHash, makeMarkdownLinkedHeader, mermaid_escape, valueOr  %>\
+<% PAGEBREAK = """<div class="pagebreak"></div>"""%>\
+<%namespace name="lib" file="lib.mako"/>\
+site_name: Threat Models
+use_directory_urls: false
+nav:
+  - Home: 'index.md'
+% for tm in tm_list:
+    <%
+        tmTitle = tm['title']
+        tmName = tm['name']
+        tmID   = tm['ID']
+    %>
+  - ${tmTitle}: '${tmID}/${tmID}.md'
+% endfor
+
+theme:
+  name: readthedocs
+  navigation_depth: 4
+  # features:
+  #   - navigation.tabs
+markdown_extensions:
+  # - toc:
+  #     baselevel: 1
+
+  - md_in_html
+# plugins:
+  # - mkdocstrings:
+  #     handlers:
+  #       python:
+  #         options:
+  #           heading_level: 1
+  # - search:
+  # - with-pdf
+  # - mkpdfs: broken
+  #   # - company: R3 
+  #   # - author: Security Team at R3
+  # - print-site
+  # - pdf-with-js:
+  #     enable: true
+
+extra_css:
+  - css/mkdocs.css

src/r3threatmodeling/template/index_MKDOCS.mako

@@ -0,0 +1,20 @@
+<%! import html %>
+<%! from r3threatmodeling.template_utils import createTitleAnchorHash, makeMarkdownLinkedHeader, mermaid_escape, valueOr  %>
+<% PAGEBREAK = """<div class="pagebreak"></div>"""%>
+<%namespace name="lib" file="lib.mako"/> 
+
+# Threat Models
+
+% for tm in tm_list:
+    <%
+        tmTitle = tm['title']
+        tmName = tm['name']
+        tmID   = tm['ID']
+    %>
+<a href="${tmID}/${tmID}.html">${tmTitle}</a>
+##   (${tmTitle})[${tmID}/${tmID}.html]
+% endfor
+
+## WARNING: this content is R3 private and confidential
+
+

src/r3threatmodeling/template/keysSummary.mako

@@ -1,35 +1,41 @@
 <%! import html %>
 <%! from r3threatmodeling.template_utils import createTitleAnchorHash, makeMarkdownLinkedHeader, mermaid_escape, valueOr  %>
-<%page args="printTOC=True"/>
+<%page args="printTOC=True, headerLevel=1"/>
 <% PAGEBREAK = """<div class="pagebreak"></div>"""%>
 <% H6 = "######" %>
 <%namespace name="lib" file="lib.mako"/> 
 
 
-${makeMarkdownLinkedHeader(1, 'Keys classification ', skipTOC = False)}
+ <%
+ try:
+  headerLevel
+except:
+  headerLevel=1
+ %>
+${makeMarkdownLinkedHeader(headerLevel, 'Keys classification ', skipTOC = False)}
 
 % if printTOC:
 __TOC_PLACEHOLDER__
 % endif
 
 <% appkeyassets=tmo.getAssetsByProps(applicationRelated=True, type='key') %>
 % if appkeyassets:
-  ${makeMarkdownLinkedHeader(2, 'Application-specific keys', skipTOC = False)}
+  ${makeMarkdownLinkedHeader(headerLevel+1, 'Application-specific keys', skipTOC = False)}
   Keys issued to processes to communicate in a secure manner, not linked to a specific business logic
   <%include file="keyTable.mako" args="assets=appkeyassets"/>
 % endif
 
 <% infrakeyassets=tmo.getAssetsByProps(infrastructureRelated=True, type = 'key') %>
 <% certassets=tmo.getAssetsByProps(type = 'certificate') %>
 % if infrakeyassets or certassets:
-  ${makeMarkdownLinkedHeader(2, 'Infrastructure Keys and PKI assets', skipTOC = False)}
+  ${makeMarkdownLinkedHeader(headerLevel+1, 'Infrastructure Keys and PKI assets', skipTOC = False)}
   <%include file="keyTable.mako" args="assets=infrakeyassets"/>
   <%include file="keyTable.mako" args="assets=certassets"/>
 % endif
 
 <% credassets=tmo.getAssetsByProps(type='credential') %>
 % if credassets:
-  ${makeMarkdownLinkedHeader(2, 'Credentials', skipTOC = False)}
+  ${makeMarkdownLinkedHeader(headerLevel+1, 'Credentials', skipTOC = False)}
   <%include file="keyTable.mako" args="assets=credassets"/>
 % endif