Fix lightning and torch._dynamo (#5984)

Addresses #5879. I confirmed that the test fails before pinning
`typing-extensions`.

I haven't dug further into why this happens or how to fix it at the
level of `typing-extensions`. We may need to do that in future if we
encounter this again.

Note that this PR also updated a few other vendored packages. The
`pygments` update required a tiny change to one of our Help-related
tests.

### Release Notes

#### New Features

- N/A

#### Bug Fixes

- Fixed a recent regression causing an error when trying to import
`torch._dynamo`, and therefore `lightning` (#5879).

### QA Notes

Try to `import lightning` in a Python console. Note that you will have
to rerun `npm install` in the `positron-python` extension to install the
new vendored package versions.

extensions/positron-python/noxfile-positron.py

@@ -10,10 +10,12 @@
 @nox.session()
 @nox.parametrize('pandas', ['1.5.3'])
 @nox.parametrize('torch', ['1.12.1'])
-def test_minimum_reqs(session, pandas, torch):
+@nox.parametrize('lightning', ['2.1.4'])
+def test_minimum_reqs(session, pandas, torch, lightning):
     session.install("-r", "python_files/positron/pinned-test-requirements.txt")
     session.install('--force-reinstall', f'pandas=={pandas}')
     session.install('--force-reinstall', f'torch=={torch}')
+    session.install('--force-reinstall', f'lightning=={lightning}')
 
     if session.posargs:
         test_args = session.posargs

extensions/positron-python/python_files/jedilsp_requirements/requirements.in

@@ -15,4 +15,7 @@ markdown-it-py
 # but we could revisit it.
 pydantic==1.10.17
 pygments
+# typing-extensions>=4.11.0 causes torch._dynamo to fail to import. We're not yet sure why,
+# so we're pinning the version for now. See: https://github.com/posit-dev/positron/issues/5879.
+typing-extensions<4.11.0
 # --- End Positron ---

extensions/positron-python/python_files/jedilsp_requirements/requirements.txt

@@ -1,8 +1,8 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile --generate-hashes python_files/jedilsp_requirements/requirements.in
-attrs==24.2.0 \
-    --hash=sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346 \
-    --hash=sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2
+attrs==24.3.0 \
+    --hash=sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff \
+    --hash=sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308
     # via
     #   cattrs
     #   lsprotocol
@@ -19,6 +19,10 @@ docstring-to-markdown==0.13 \
     # via
     #   -r python_files/jedilsp_requirements/requirements.in
     #   jedi-language-server
+exceptiongroup==1.2.2 \
+    --hash=sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b \
+    --hash=sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc
+    # via cattrs
 jedi==0.19.2 \
     --hash=sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0 \
     --hash=sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9
@@ -96,11 +100,15 @@ pygls==1.3.1 \
     # via
     #   -r python_files/jedilsp_requirements/requirements.in
     #   jedi-language-server
-pygments==2.18.0 \
-    --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \
-    --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a
+pygments==2.19.1 \
+    --hash=sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f \
+    --hash=sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c
     # via -r python_files/jedilsp_requirements/requirements.in
-typing-extensions==4.12.2 \
-    --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \
-    --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8
-    # via pydantic
+typing-extensions==4.10.0 \
+    --hash=sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475 \
+    --hash=sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb
+    # via
+    #   -r python_files/jedilsp_requirements/requirements.in
+    #   cattrs
+    #   jedi-language-server
+    #   pydantic

extensions/positron-python/python_files/positron/pinned-test-requirements.txt

@@ -8,6 +8,7 @@ hvplot==0.8.0 ; python_version < '3.9'
 ibis-framework[duckdb]==9.5.0; python_version >= '3.10'
 ipykernel==6.29.5
 ipywidgets==8.1.5
+lightning==2.3.2
 matplotlib==3.9.2; python_version >= '3.9'
 matplotlib==3.7.4; python_version < '3.9'
 numpy==2.1.1; python_version >= '3.10'

extensions/positron-python/python_files/positron/positron_ipykernel/pydoc.py

@@ -773,7 +773,7 @@ def _getdoc(object: Any) -> str:
     """Override `pydoc.getdoc` to parse reStructuredText docstrings."""
     try:
         docstring = _pydoc_getdoc(object) or "No documentation found."
-        html = _rst_to_html(docstring, object)
+        html = _docstring_to_html(docstring, object)
     except Exception as exception:
         # This is caught somewhere above us in pydoc. Log the exception so we see it in Positron
         # logs.
@@ -941,11 +941,11 @@ def _highlight(code: str, name: str, attrs: str) -> str:
     return cast(str, result)
 
 
-def _rst_to_html(docstring: str, object: Any) -> str:
+def _docstring_to_html(docstring: str, object: Any) -> str:
     """
-    Parse a reStructuredText docstring to HTML.
+    Parse a docstring in one of the supported formats to HTML.
     """
-    logger.debug(f"Parsing rST to html for object: {object}")
+    logger.debug(f"Parsing docstring to html for object: {object}")
 
     markdown = convert_docstring(docstring)
 

extensions/positron-python/python_files/positron/positron_ipykernel/tests/test_positron_ipkernel.py

@@ -344,3 +344,9 @@ def test_console_warning_logger(shell: PositronShell, caplog, warning_kwargs):
     with caplog.at_level(logging.WARNING):
         shell.kernel._showwarning(filename=Path(__file__), **warning_kwargs)
         assert "this is a warning" in caplog.text
+
+
+def test_import_lightning_and_torch_dynamo(shell: PositronShell) -> None:
+    # See: https://github.com/posit-dev/positron/issues/5879
+    shell.run_cell("import lightning").raise_error()
+    shell.run_cell("import torch._dynamo").raise_error()

extensions/positron-python/python_files/positron/positron_ipykernel/tests/test_pydoc.py

@@ -504,7 +504,7 @@ def test_resolve(target: str, from_obj: Any, expected: Any) -> None:
         (
             _test_getdoc_code_blocks,
             """\
-<pre><code class="language-python"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
+<pre><code class="language-python"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">pandas</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pd</span>
 <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">()</span>
 </pre></div>
 </code></pre>

extensions/positron-python/python_files/positron/test-requirements.txt

@@ -5,6 +5,7 @@ holoviews
 ibis-framework[duckdb]; python_version >= '3.10'
 ipykernel
 ipywidgets
+lightning
 matplotlib
 numpy
 pandas