diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 48fbf7c..1c5397d 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -51,7 +51,7 @@ jobs: container: 'off' working-directory: ./python args: --extras devel - + - name: Run lints run: | source .venv/bin/activate @@ -64,6 +64,33 @@ jobs: source .venv/bin/activate pytest + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.13 + + - name: Create virtualenv + run: | + python3 -m venv .venv + + - uses: PyO3/maturin-action@v1 + with: + command: develop + sccache: 'true' + container: 'off' + working-directory: ./python + args: --extras docs + + - name: Build docs + run: | + source .venv/bin/activate + sphinx-build -M html docs/source/ docs/build/ + build: runs-on: ubuntu-latest steps: diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..3a7cf4d --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,31 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-24.04 + tools: + python: "3.13" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: python/docs/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - method: pip + path: python + extra_requirements: + - docs \ No newline at end of file diff --git a/python/docs/Makefile b/python/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/python/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/python/docs/make.bat b/python/docs/make.bat new file mode 100644 index 0000000..747ffb7 --- /dev/null +++ b/python/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/python/docs/source/api.rst b/python/docs/source/api.rst new file mode 100644 index 0000000..191b3de --- /dev/null +++ b/python/docs/source/api.rst @@ -0,0 +1,5 @@ +API +=== + +.. automodapi:: hdfs_native + :no-inheritance-diagram: \ No newline at end of file diff --git a/python/docs/source/api/hdfs_native.AclEntry.rst b/python/docs/source/api/hdfs_native.AclEntry.rst new file mode 100644 index 0000000..bf3b2ca --- /dev/null +++ b/python/docs/source/api/hdfs_native.AclEntry.rst @@ -0,0 +1,23 @@ +AclEntry +======== + +.. currentmodule:: hdfs_native + +.. autoclass:: AclEntry + :show-inheritance: + + .. rubric:: Attributes Summary + + .. autosummary:: + + ~AclEntry.name + ~AclEntry.permissions + ~AclEntry.scope + ~AclEntry.type + + .. rubric:: Attributes Documentation + + .. autoattribute:: name + .. autoattribute:: permissions + .. autoattribute:: scope + .. autoattribute:: type diff --git a/python/docs/source/api/hdfs_native.AclStatus.rst b/python/docs/source/api/hdfs_native.AclStatus.rst new file mode 100644 index 0000000..eb510f1 --- /dev/null +++ b/python/docs/source/api/hdfs_native.AclStatus.rst @@ -0,0 +1,25 @@ +AclStatus +========= + +.. currentmodule:: hdfs_native + +.. autoclass:: AclStatus + :show-inheritance: + + .. rubric:: Attributes Summary + + .. autosummary:: + + ~AclStatus.entries + ~AclStatus.group + ~AclStatus.owner + ~AclStatus.permission + ~AclStatus.sticky + + .. rubric:: Attributes Documentation + + .. autoattribute:: entries + .. autoattribute:: group + .. autoattribute:: owner + .. autoattribute:: permission + .. autoattribute:: sticky diff --git a/python/docs/source/api/hdfs_native.Client.rst b/python/docs/source/api/hdfs_native.Client.rst new file mode 100644 index 0000000..7ac4286 --- /dev/null +++ b/python/docs/source/api/hdfs_native.Client.rst @@ -0,0 +1,53 @@ +Client +====== + +.. currentmodule:: hdfs_native + +.. autoclass:: Client + :show-inheritance: + + .. rubric:: Methods Summary + + .. autosummary:: + + ~Client.append + ~Client.create + ~Client.delete + ~Client.get_acl_status + ~Client.get_content_summary + ~Client.get_file_info + ~Client.list_status + ~Client.mkdirs + ~Client.modify_acl_entries + ~Client.read + ~Client.remove_acl + ~Client.remove_acl_entries + ~Client.remove_default_acl + ~Client.rename + ~Client.set_acl + ~Client.set_owner + ~Client.set_permission + ~Client.set_replication + ~Client.set_times + + .. rubric:: Methods Documentation + + .. automethod:: append + .. automethod:: create + .. automethod:: delete + .. automethod:: get_acl_status + .. automethod:: get_content_summary + .. automethod:: get_file_info + .. automethod:: list_status + .. automethod:: mkdirs + .. automethod:: modify_acl_entries + .. automethod:: read + .. automethod:: remove_acl + .. automethod:: remove_acl_entries + .. automethod:: remove_default_acl + .. automethod:: rename + .. automethod:: set_acl + .. automethod:: set_owner + .. automethod:: set_permission + .. automethod:: set_replication + .. automethod:: set_times diff --git a/python/docs/source/api/hdfs_native.ContentSummary.rst b/python/docs/source/api/hdfs_native.ContentSummary.rst new file mode 100644 index 0000000..3f3f922 --- /dev/null +++ b/python/docs/source/api/hdfs_native.ContentSummary.rst @@ -0,0 +1,27 @@ +ContentSummary +============== + +.. currentmodule:: hdfs_native + +.. autoclass:: ContentSummary + :show-inheritance: + + .. rubric:: Attributes Summary + + .. autosummary:: + + ~ContentSummary.directory_count + ~ContentSummary.file_count + ~ContentSummary.length + ~ContentSummary.quota + ~ContentSummary.space_consumed + ~ContentSummary.space_quota + + .. rubric:: Attributes Documentation + + .. autoattribute:: directory_count + .. autoattribute:: file_count + .. autoattribute:: length + .. autoattribute:: quota + .. autoattribute:: space_consumed + .. autoattribute:: space_quota diff --git a/python/docs/source/api/hdfs_native.FileReader.rst b/python/docs/source/api/hdfs_native.FileReader.rst new file mode 100644 index 0000000..c1cb8f6 --- /dev/null +++ b/python/docs/source/api/hdfs_native.FileReader.rst @@ -0,0 +1,41 @@ +FileReader +========== + +.. currentmodule:: hdfs_native + +.. autoclass:: FileReader + :show-inheritance: + + .. rubric:: Attributes Summary + + .. autosummary:: + + ~FileReader.size + + .. rubric:: Methods Summary + + .. autosummary:: + + ~FileReader.close + ~FileReader.read + ~FileReader.read_range + ~FileReader.readable + ~FileReader.readall + ~FileReader.seek + ~FileReader.seekable + ~FileReader.tell + + .. rubric:: Attributes Documentation + + .. autoattribute:: size + + .. rubric:: Methods Documentation + + .. automethod:: close + .. automethod:: read + .. automethod:: read_range + .. automethod:: readable + .. automethod:: readall + .. automethod:: seek + .. automethod:: seekable + .. automethod:: tell diff --git a/python/docs/source/api/hdfs_native.FileStatus.rst b/python/docs/source/api/hdfs_native.FileStatus.rst new file mode 100644 index 0000000..4584d55 --- /dev/null +++ b/python/docs/source/api/hdfs_native.FileStatus.rst @@ -0,0 +1,35 @@ +FileStatus +========== + +.. currentmodule:: hdfs_native + +.. autoclass:: FileStatus + :show-inheritance: + + .. rubric:: Attributes Summary + + .. autosummary:: + + ~FileStatus.access_time + ~FileStatus.blocksize + ~FileStatus.group + ~FileStatus.isdir + ~FileStatus.length + ~FileStatus.modification_time + ~FileStatus.owner + ~FileStatus.path + ~FileStatus.permission + ~FileStatus.replication + + .. rubric:: Attributes Documentation + + .. autoattribute:: access_time + .. autoattribute:: blocksize + .. autoattribute:: group + .. autoattribute:: isdir + .. autoattribute:: length + .. autoattribute:: modification_time + .. autoattribute:: owner + .. autoattribute:: path + .. autoattribute:: permission + .. autoattribute:: replication diff --git a/python/docs/source/api/hdfs_native.FileWriter.rst b/python/docs/source/api/hdfs_native.FileWriter.rst new file mode 100644 index 0000000..ec27221 --- /dev/null +++ b/python/docs/source/api/hdfs_native.FileWriter.rst @@ -0,0 +1,21 @@ +FileWriter +========== + +.. currentmodule:: hdfs_native + +.. autoclass:: FileWriter + :show-inheritance: + + .. rubric:: Methods Summary + + .. autosummary:: + + ~FileWriter.close + ~FileWriter.writable + ~FileWriter.write + + .. rubric:: Methods Documentation + + .. automethod:: close + .. automethod:: writable + .. automethod:: write diff --git a/python/docs/source/api/hdfs_native.WriteOptions.rst b/python/docs/source/api/hdfs_native.WriteOptions.rst new file mode 100644 index 0000000..de210e6 --- /dev/null +++ b/python/docs/source/api/hdfs_native.WriteOptions.rst @@ -0,0 +1,25 @@ +WriteOptions +============ + +.. currentmodule:: hdfs_native + +.. autoclass:: WriteOptions + :show-inheritance: + + .. rubric:: Attributes Summary + + .. autosummary:: + + ~WriteOptions.block_size + ~WriteOptions.create_parent + ~WriteOptions.overwrite + ~WriteOptions.permission + ~WriteOptions.replication + + .. rubric:: Attributes Documentation + + .. autoattribute:: block_size + .. autoattribute:: create_parent + .. autoattribute:: overwrite + .. autoattribute:: permission + .. autoattribute:: replication diff --git a/python/docs/source/conf.py b/python/docs/source/conf.py new file mode 100644 index 0000000..6a39b8e --- /dev/null +++ b/python/docs/source/conf.py @@ -0,0 +1,35 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "hdfs-native" +copyright = "2024, Adam Binford" +author = "Adam Binford" +release = "0.11.0" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx_rtd_theme", + "sphinx_automodapi.automodapi", +] + + +templates_path = ["_templates"] +exclude_patterns = [] + +autosummary_imported_members = False + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "sphinx_rtd_theme" +html_static_path = ["_static"] diff --git a/python/docs/source/index.rst b/python/docs/source/index.rst new file mode 100644 index 0000000..a4327a5 --- /dev/null +++ b/python/docs/source/index.rst @@ -0,0 +1,17 @@ +.. hdfs-native documentation master file, created by + sphinx-quickstart on Mon Dec 16 14:57:12 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +hdfs-native documentation +========================= + +Python bindings for Rust-based `hdfs-native `_ library. + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + usage + api diff --git a/python/docs/source/usage.rst b/python/docs/source/usage.rst new file mode 100644 index 0000000..cf2df92 --- /dev/null +++ b/python/docs/source/usage.rst @@ -0,0 +1,19 @@ +Usage +===== + +Simply create a :py:class:`Client `. + +.. code-block:: + + from hdfs_native import Client + + # Load the fs.defaultFS from your core-site.xml config file + client = Client() + + # Connect to a specific NameNode + client = Client("hdfs://nn-1:9000") + + # Connect to a Name Service + client = Client("hdfs://ns") + +See :py:class:`Client ` for supported methods on a client. \ No newline at end of file diff --git a/python/hdfs_native/__init__.py b/python/hdfs_native/__init__.py index 44fa40a..b9d16ba 100644 --- a/python/hdfs_native/__init__.py +++ b/python/hdfs_native/__init__.py @@ -20,6 +20,17 @@ RawFileWriter, ) +__all__ = [ + "Client", + "FileReader", + "FileWriter", + "FileStatus", + "ContentSummary", + "WriteOptions", + "AclEntry", + "AclStatus", +] + class FileReader(io.RawIOBase): def __init__(self, inner: "RawFileReader"): diff --git a/python/pyproject.toml b/python/pyproject.toml index 2bb5aba..a99cca4 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -21,13 +21,20 @@ dependencies = [ [project.optional-dependencies] devel = [ "mypy~=1.13.0", - "ruff~=0.7.2", "pytest~=8.3", - "pytest-benchmark~=5.1" + "pytest-benchmark~=5.1", + "ruff~=0.7.2" +] + +docs = [ + "sphinx~=8.1", + "sphinx-automodapi~=0.18", + "sphinx-rtd-theme~=3.0" ] [project.urls] repository = "https://github.com/Kimahriman/hdfs-native" +documentation = "https://hdfs-native.readthedocs.io/" [project.scripts] hdfsn = "hdfs_native.cli:main"