Create new screencast automatically on docs build (#339)

* Create new screencast automatically on docs build * dev reqs * Remove tree from replay * Update docs/scripts/generate-termynal.py Co-authored-by: Chris Kucharczyk <[email protected]> * Update docs/scripts/generate-termynal.py Co-authored-by: Chris Kucharczyk <[email protected]> * Apply suggestions from code review Co-authored-by: Chris Kucharczyk <[email protected]> * Apply suggestions from code review Co-authored-by: Chris Kucharczyk <[email protected]> --------- Co-authored-by: Chris Kucharczyk <[email protected]>
drivendataorg · Dec 29, 2023 · b0c3758 · b0c3758
1 parent e5b1183
commit b0c3758
Show file tree

Hide file tree

Showing 9 changed files with 232 additions and 6 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,7 @@
+# generated docs items
 docs/site/
+docs/docs/_partials/termynal.md
+docs/docs/_partials/*/*.html
 
 # test cache
 manual_test/

diff --git a/Makefile b/Makefile
@@ -26,13 +26,13 @@ requirements:
 
 ## Format the code using isort and black
 format:
-	isort --profile black ccds hooks tests "{{ cookiecutter.repo_name }}/{{ cookiecutter.module_name }}"
-	black ccds hooks tests "{{ cookiecutter.repo_name }}/{{ cookiecutter.module_name }}"
+	isort --profile black ccds hooks tests docs/scripts "{{ cookiecutter.repo_name }}/{{ cookiecutter.module_name }}"
+	black ccds hooks tests docs/scripts "{{ cookiecutter.repo_name }}/{{ cookiecutter.module_name }}"
 
 lint:
-	flake8 ccds hooks tests "{{ cookiecutter.repo_name }}/{{ cookiecutter.module_name }}"
-	isort --check --profile black ccds hooks tests "{{ cookiecutter.repo_name }}/{{ cookiecutter.module_name }}"
-	black --check ccds hooks tests "{{ cookiecutter.repo_name }}/{{ cookiecutter.module_name }}"
+	flake8 ccds hooks tests docs/scripts "{{ cookiecutter.repo_name }}/{{ cookiecutter.module_name }}"
+	isort --check --profile black ccds hooks tests docs/scripts "{{ cookiecutter.repo_name }}/{{ cookiecutter.module_name }}"
+	black --check ccds hooks tests docs/scripts "{{ cookiecutter.repo_name }}/{{ cookiecutter.module_name }}"
 
 
 ###     DOCS

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -1,12 +1,17 @@
 -e .
 
+ansi2html
 black
 chardet
 flake8
 isort
 mkdocs
 mkdocs-cinder
+mkdocs-gen-files
+mkdocs-include-markdown-plugin
+pexpect
 pipenv
 pytest
+termynal
 virtualenvwrapper; sys_platform != 'win32'
 virtualenvwrapper-win; sys_platform == 'win32'
diff --git a/docs/docs/_partials/.gitkeep b/docs/docs/_partials/.gitkeep
diff --git a/docs/docs/css/extra.css b/docs/docs/css/extra.css
@@ -1,3 +1,24 @@
 h1, h2, h3 {
     margin-top: 77px;
 }
+
+#termynal {
+    height: 80ex !important; /* 40 lines of 2ex */
+    min-height: 80ex !important;
+    max-height: 80ex !important;
+    overflow: scroll !important;
+    font-size: 1.5ex !important;
+}
+
+[data-ty] {
+    line-height: 2ex !important; 
+    white-space: pre;
+}
+
+.newline {
+    line-height: 0 !important;
+}
+
+.inline-input, .default-text {
+    display: inline-block !important;
+}
diff --git a/docs/docs/index.md b/docs/docs/index.md
@@ -83,7 +83,10 @@ ccds https://github.com/drivendata/cookiecutter-data-science
 
 ### Example
 
-<script id="asciicast-244658" src="https://asciinema.org/a/244658.js" async></script>
+{%
+   include-markdown "./_partials/termynal.md"
+%}
+
 
 ## Directory structure
 

diff --git a/docs/docs/js/extra.js b/docs/docs/js/extra.js
@@ -0,0 +1,28 @@
+/* Smooth scrolling for termynal replay */
+
+function scrollToBottomOfContainer(container, element) {
+    var positionToScroll = element.offsetTop + element.offsetHeight - container.offsetHeight;
+    container.scrollTo({
+        top: positionToScroll,
+        behavior: 'smooth'
+    });
+}
+
+// Select the node that will be observed for mutations
+const targetNode = document.getElementById("termynal");
+
+// Options for the observer (which mutations to observe)
+const config = { attributes: false, childList: true, subtree: false };
+
+// Callback function to execute when mutations are observed
+const callback = (mutationList, observer) => {
+  for (const mutation of mutationList) {
+    scrollToBottomOfContainer(targetNode, mutation.target);
+  }
+};
+
+// Create an observer instance linked to the callback function
+const observer = new MutationObserver(callback);
+
+// Start observing the target node for configured mutations
+observer.observe(targetNode, config);
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
@@ -8,5 +8,22 @@ google_analytics: ['UA-54096005-4', 'auto']
 theme: cinder
 extra_css:
   - css/extra.css
+extra_javascript:
+  - js/extra.js
 nav:
   - Home: index.md
+
+exclude_docs: |
+  _partials/termynal.md
+
+plugins:
+  - include-markdown
+  - termynal:
+      title: bash
+      buttons: macos
+      prompt_literal_start:
+        - "$"
+  - gen-files:
+      scripts:
+        - scripts/generate-termynal.py
+
diff --git a/docs/scripts/generate-termynal.py b/docs/scripts/generate-termynal.py
@@ -0,0 +1,149 @@
+import shutil
+from pathlib import Path
+
+import pexpect
+from ansi2html import Ansi2HTMLConverter
+
+CCDS_ROOT = Path(__file__).parents[2].resolve()
+
+
+def execute_command_and_get_output(command, input_script):
+    input_script = iter(input_script)
+    child = pexpect.spawn(command, encoding="utf-8")
+
+    interaction_history = [f"$ {command}\n"]
+
+    prompt, user_input = next(input_script)
+
+    try:
+        while True:
+            index = child.expect([prompt, pexpect.EOF, pexpect.TIMEOUT])
+
+            if index == 0:
+                output = child.before + child.after
+                interaction_history += [line.strip() for line in output.splitlines()]
+
+                child.sendline(user_input)
+
+                try:
+                    prompt, user_input = next(input_script)
+                except StopIteration:
+                    pass
+
+            elif index == 1:  # The subprocess has exited.
+                output = child.before
+                interaction_history += [line.strip() for line in output.splitlines()]
+                break
+            elif index == 2:  # Timeout waiting for new data.
+                print("\nTimeout waiting for subprocess response.")
+                continue
+
+    finally:
+        return interaction_history
+
+
+ccds_script = [
+    ("project_name", "My Analysis"),
+    ("repo_name", "my_analysis"),
+    ("module_name", ""),
+    ("author_name", "Dat A. Scientist"),
+    ("description", "This is my analysis of the data."),
+    ("python_version_number", "3.12"),
+    ("Choose from", "3"),
+    ("bucket", "s3://my-aws-bucket"),
+    ("aws_profile", ""),
+    ("Choose from", "2"),
+    ("Choose from", "1"),
+    ("Choose from", "2"),
+    ("Choose from", "2"),
+    ("Choose from", "1"),
+]
+
+
+def run_scripts():
+    try:
+        output = []
+        output += execute_command_and_get_output(f"ccds {CCDS_ROOT}", ccds_script)
+        return output
+
+    finally:
+        # always cleanup
+        if Path("my_analysis").exists():
+            shutil.rmtree("my_analysis")
+
+
+def render_termynal():
+    # actually execute the scripts and capture the output
+    results = run_scripts()
+
+    # watch for inputs and format them differently
+    script = iter(ccds_script)
+    _, user_input = next(script)
+
+    conv = Ansi2HTMLConverter(inline=True)
+    html_lines = [
+        '<div id="termynal" data-termynal class="termy" data-ty-macos data-ty-lineDelay="100" data-ty-typeDelay="50" title="Cookiecutter Data Science">'
+    ]
+    result_collector = []
+
+    for line_ix, result in enumerate(results):
+        # style bash user inputs
+        if result.startswith("$"):
+            result = conv.convert(result.strip("$"), full=False)
+            html_lines.append(
+                f'<span data-ty="input" data-ty-prompt="$">{result}</span>'
+            )
+
+        # style inline cookiecutter user inputs
+        elif ":" in result and user_input in result:
+            # treat all the options that were output as a single block
+            if len(result_collector) > 1:
+                prev_results = conv.convert(
+                    "\n".join(result_collector[:-1]), full=False
+                )
+                html_lines.append(f"<span data-ty>{prev_results}</span>")
+
+            # split the line up into the prompt text with options, the default, and the user input
+            prompt, user_input = result.strip().split(":", 1)
+            prompt = conv.convert(prompt, full=False)
+            prompt = f'<span data-ty class="inline-input">{result_collector[-1].strip()} {prompt}:</span>'
+            user_input = conv.convert(user_input.strip(), full=False)
+
+            # treat the cookiecutter prompt as a shell prompt
+            out_line = f"{prompt}"
+            out_line += f'<span class="inline-input" data-ty="input" data-ty-delay="500" data-ty-prompt="">{user_input}</span>'
+            html_lines.append(out_line)
+            html_lines.append('<span data-ty class="newline"></span>')
+            result_collector = []
+
+            try:
+                _, user_input = next(script)
+            except StopIteration:
+                user_input = "STOP ITER"  # never true so we just capture the remaining rows after the script
+
+        # collect all the other lines for a single output
+        else:
+            result_collector.append(result)
+
+    html_lines.append("</div>")
+    output = "\n".join(html_lines)
+
+    # replace local directory in ccds call with URL so it can be used for documentation
+    output = output.replace(
+        str(CCDS_ROOT), "https://github.com/drivendata/cookiecutter-data-science"
+    )
+    return output
+
+
+# script entry point for debugging
+if __name__ == "__main__":
+    print(render_termynal())
+
+# mkdocs build entry point
+else:
+    import mkdocs_gen_files
+
+    with mkdocs_gen_files.open(
+        Path(CCDS_ROOT / "docs" / "docs" / "_partials" / "termynal.md"), "w"
+    ) as f:
+        f.write(render_termynal())