diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..965538bd --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,3 @@ +python: + version: 3 + setup_py_install: true diff --git a/README.md b/README.md index 326f7e04..64353279 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ [![Build Status](https://travis-ci.org/MatterMiners/tardis.svg?branch=master)](https://travis-ci.org/MatterMiners/tardis) [![codecov](https://codecov.io/gh/MatterMiners/tardis/branch/master/graph/badge.svg)](https://codecov.io/gh/MatterMiners/tardis) +[![Documentation Status](https://readthedocs.org/projects/cobald-tardis/badge/?version=latest)](https://cobald-tardis.readthedocs.io/en/latest/?badge=latest) [![PyPI version](https://badge.fury.io/py/cobald-tardis.svg)](https://badge.fury.io/py/cobald-tardis) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/cobald-tardis.svg?style=flat-square) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/MatterMiners/tardis/blob/master/LICENSE.txt) [![DOI](https://zenodo.org/badge/132791417.svg)](https://zenodo.org/badge/latestdoi/132791417) diff --git a/docs/Makefile b/docs/Makefile index c6be491e..66ad88c2 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -2,11 +2,11 @@ # # You can set these variables from the command line. -SPHINXOPTS = +SPHINXOPTS = -d _build/doctrees SPHINXBUILD = sphinx-build SPHINXPROJ = tardis SOURCEDIR = source -BUILDDIR = build +BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: diff --git a/docs/generate_apidoc.sh b/docs/generate_apidoc.sh index 2b576002..acd58360 100755 --- a/docs/generate_apidoc.sh +++ b/docs/generate_apidoc.sh @@ -1,2 +1,7 @@ #!/bin/bash -sphinx-apidoc --force --module-first --separate -o source/ ../tardis +PROJECT=tardis + +source ../venv/bin/activate +sphinx-apidoc --force --module-first --separate -o source/api ../${PROJECT} && \ +make html && \ +open _build/html/index.html \ No newline at end of file diff --git a/docs/pics/TARDIS_logo.png b/docs/pics/TARDIS_logo.png new file mode 100644 index 00000000..5394179a Binary files /dev/null and b/docs/pics/TARDIS_logo.png differ diff --git a/docs/source/adapters/batchsystem.rst b/docs/source/adapters/batchsystem.rst new file mode 100644 index 00000000..0d1994fd --- /dev/null +++ b/docs/source/adapters/batchsystem.rst @@ -0,0 +1,115 @@ +.. _ref_batch_system_adapter: + +===================== +Batch System Adapters +===================== + +Fake Batch System Adapter +------------------------- + +.. |FakeBatchSystemAdapter.get_utilization| replace:: :py:meth:`~tardis.adapters.batchsystems.fakebatchsystem.FakeBatchSystemAdapter.get_utilization` + +.. |FakeBatchSystemAdapter.get_allocation| replace:: :py:meth:`~tardis.adapters.batchsystems.fakebatchsystem.FakeBatchSystemAdapter.get_allocation` + +.. |FakeBatchSystemAdapter.get_machine_status| replace:: :py:meth:`~tardis.adapters.batchsystems.fakebatchsystem.FakeBatchSystemAdapter.get_machine_status` + +The :py:class:`~tardis.adapters.batchsystems.fakebatchsystem.FakeBatchSystemAdapter` +implements a batch system adapter that mocks the response of hypothetical batch system. +It can be used for testing purposes and as a demonstrator in workshops and tutorials. + +The mocked response to the |FakeBatchSystemAdapter.get_allocation|, |FakeBatchSystemAdapter.get_utilization| and +|FakeBatchSystemAdapter.get_machine_status| API calls is configurable statically in the adapter configuration. + +Available configuration options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------+---------------------------------------------------------------------+-----------------+ +| Option | Short Description | Requirement | ++================+=====================================================================+=================+ +| adapter | Name of the adapter (FakeBatchSystem) | **Required** | ++----------------+---------------------------------------------------------------------+-----------------+ +| allocation | Mocked response to |FakeBatchSystemAdapter.get_allocation| call | **Required** | ++----------------+---------------------------------------------------------------------+-----------------+ +| utilization | Mocked response to |FakeBatchSystemAdapter.get_utilization| call | **Required** | ++----------------+---------------------------------------------------------------------+-----------------+ +| machine_status | Mocked response to |FakeBatchSystemAdapter.get_machine_status| call | **Required** | ++----------------+---------------------------------------------------------------------+-----------------+ + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + BatchSystem: + adapter: FakeBatchSystem + allocation: 1.0 + utilization: 1.0 + machine_status: Available + +HTCondor Batch System Adapter +----------------------------- +.. |HTCondorAdapter.get_utilization| replace:: :py:meth:`~tardis.adapters.batchsystems.htcondor.HTCondorAdapter.get_utilization` + +.. |HTCondorAdapter.get_allocation| replace:: :py:meth:`~tardis.adapters.batchsystems.htcondor.HTCondorAdapter.get_allocation` + +.. |HTCondorAdapter.get_machine_status| replace:: :py:meth:`~tardis.adapters.batchsystems.htcondor.HTCondorAdapter.get_machine_status` + +The :py:class:`~tardis.adapters.batchsystems.htcondor.HTCondorAdapter` implements the TARDIS interface to dynamically +integrate and manage opportunistic resources with the HTCondor batch system. + +Information provider for the API calls |HTCondorAdapter.get_utilization|, |HTCondorAdapter.get_allocation| and +|HTCondorAdapter.get_machine_status| is the HTCondor ``condor_status`` command, which is called asynchronously and its +output is cached for a configurable time ``max_age``. + +|HTCondorAdapter.get_machine_status| returns the status of the worker node by taking into account the HTCondor +ClassAds ``State`` and ``Activity``. It can take the states ``Available``, ``Draining``, ``Drained`` and +``NotAvailable``. + +The allocation and utilization of a worker node is defined as maximum and minimum of the relative ratio of requested +over total resources such as CPU, Memory, Disk, respectively. Which resource ratios to take into account can be +configured via the ``ratios`` option. Any valid HTCondor expression that returns a floating point number is accepted. + +Additional options for the condor_status call can be added by using the ``options`` option. + +For example + +.. code-block:: yaml + + options: + pool: htcondor.example + +translates into ``condor_status ... -pool htcondor.example``. + +Available configuration options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------+-------------------------------------------------------------------------+-----------------+ +| Option | Short Description | Requirement | ++================+=========================================================================+=================+ +| adapter | Name of the adapter (HTCondor) | **Required** | ++----------------+-------------------------------------------------------------------------+-----------------+ +| max_age | Maximum age of the cached ``condor_status`` information in minutes | **Required** | ++----------------+-------------------------------------------------------------------------+-----------------+ +| ratios | HTCondor expressions used to determine allocation and utilization | **Required** | ++----------------+-------------------------------------------------------------------------+-----------------+ +| options | Additional command line options to add to the ``condor_status`` command | **Optional** | ++----------------+-------------------------------------------------------------------------+-----------------+ + + +Example configuration +~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + BatchSystem: + adapter: HTCondor + max_age: 1 + ratios: + cpu_ratio: Real(TotalSlotCpus-Cpus)/TotalSlotCpus + memory_ratio: Real(TotalSlotMemory-Memory)/TotalSlotMemory + options: + pool: my-htcondor.mysite.mydomain + +Your favorite batch system is currently not supported? +Please, have a look at +:ref:`how to contribute.` diff --git a/docs/source/adapters/site.rst b/docs/source/adapters/site.rst new file mode 100644 index 00000000..0319171e --- /dev/null +++ b/docs/source/adapters/site.rst @@ -0,0 +1,383 @@ +============ +Site Adapter +============ + +The site adapters provide interfaces to various Cloud APIs and batch systems in order to allow a on-demand provisioning +of resources and a dynamic orchestration of pre-built VM images and containers. + +Sites are generally configured in the `Sites` configuration block. One has to specify a site name, the adapter to use +and a site quota in units of cores. Negative values for the site quota are interpreted as infinity. Multiple sites are +supported by using SequenceNodes. + +Generic Site Adapter Configuration +---------------------------------- + +Available configuration options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++---------+----------------------------------------------------------------------------------+-----------------+ +| Option | Short Description | Requirement | ++=========+==================================================================================+=================+ +| name | Name of the site | **Required** | ++---------+----------------------------------------------------------------------------------+-----------------+ +| adapter | Site adapter to use. Adapter will be auto-imported (class name without Adapter) | **Required** | ++---------+----------------------------------------------------------------------------------+-----------------+ +| quota | Core quota to be used for this site. Negative values are interpreted as infinity | **Required** | ++---------+----------------------------------------------------------------------------------+-----------------+ + +For each site in the `Sites` configuration block. A site specific configuration block carrying the site name +has to be added to the configuration as well. + +The site specific MappingNode contains site adapter specific configuration options that you can find below in +the particular site adapter documentation. + +In addition, it is required to add the following MappingNodes: + +* `MachineTypes` containing a SequenceNode of available machine types to be supported at the given site. +* `MachineTypeConfiguration` a MappingNode for each machine type containing machine type specific configurations, + details can be found below in the particular site adapter documentation. +* `MachineTypeMetaData` containing a MappingNode for each machine type specifying the amount of Cores, Memory and Disk + available + + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + Sites: + - name: MySiteName_1 + adapter: MyAdapter2Use + quota: 123 + - name: MySiteName_2 + adapter: OtherAdapter2Use + quota: 987 + + MySiteName_1: + general_adapter_option: something + MachineTypes: + - Micro + - Fat + MachineTypeConfiguration: + Micro: + machine_type_specific_option_1: 124234-1245-1345-15 + machine_type_specific_option_2: 4583453-3245-345-2345 + Fat: + machine_type_specific_option_1: 0034532-345-234-2341 + machine_type_specific_option_2: 1345-134-1345-134-1 + MachineMetaData: + Micro: + Cores: 1 + Memory: 2 + Disk: 30 + Fat: + Cores: 32 + Memory: 128 + Disk: 256 + + MySiteName_2: + general_adapter_option: something_else + MachineTypes: + - XL + MachineTypeConfiguration: + XL: + machine_type_specific_option_1: 9847867-467846-468748BC + MachineMetaData: + XL: + Cores: 128 + Memory: 256 + Disk: 1024 + +Cloud Stack Site Adapter +------------------------ +The :py:class:`~tardis.adapters.sites.cloudstack.CloudStackAdapter` implements an interface to the CloudStack API. +The following general adapter configuration options are available. + +Available adapter configuration options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------+---------------------------------------------------------------------+-----------------+ +| Option | Short Description | Requirement | ++================+=====================================================================+=================+ +| api_key | Your CloudStack API Key to authenticate yourself. | **Required** | ++----------------+---------------------------------------------------------------------+-----------------+ +| api_secret | Your CloudStack API secret to authenticate yourself. | **Required** | ++----------------+---------------------------------------------------------------------+-----------------+ +| end_point | The end point of the CloudStack API to contact. | **Required** | ++----------------+---------------------------------------------------------------------+-----------------+ + +All configuration entries in the `MachineTypeConfiguration` section of the machine types are +directly added as keyword arguments to the CloudStack API `deployVirtualMachine` call. All available options are +described in the `CloudStack documentation`_ + +.. _CloudStack documentation: https://cloudstack.apache.org/api/apidocs-4.12/apis/deployVirtualMachine.html + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + Sites: + - name: Hexascale + adapter: CloudStack + quota: 300 + + Hexascale: + api_key: BlaBlubbFooBar123456 + api_secret: AKshflajsdfjnASJFkajsfd + end_point: https://api.hexascale.com/compute + MachineTypes: + - Micro + - Tiny + MachineTypeConfiguration: + Micro: + templateid: 909ce5b7-2132-4ff0-9bf8-aadbb423f7d9 + serviceofferingid: 71004023-bb72-4a97-b1e9-bc66dfce9470 + zoneid: 35eb7739-d19e-45f7-a581-4687c54d6d02 + securitygroupnames: "secgrp-WN,NFS-access,Squid-access" + userdata: ini/hexascale.ini + keypair: MG + rootdisksize: 70 + Tiny: + templateid: 909ce5b7-2132-4ff0-9bf8-aadbb423f7d9 + serviceofferingid: b6cd1ff5-3a2f-4e9d-a4d1-8988c1191fe8 + zoneid: 35eb7739-d19e-45f7-a581-4687c54d6d02 + securitygroupnames: "secgrp-WN,NFS-access,Squid-access" + userdata: ini/hexascale.ini + keypair: MG + rootdisksize: 70 + +HTCondor Site Adapter +--------------------- +The :py:class:`~tardis.adapters.sites.htcondor.HTCondorAdapter` implements an interface to the HTCondor batch system. +Regular batch jobs are submitted that start the actual Drone, which than is integrated itself in overlay batch system +using the chosen :ref:`BatchSystemAdapter.` + +.. |executor| replace:: :ref:`executor` + +Available adapter configuration options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ++----------------+-----------------------------------------------------------------------------------+-----------------+ +| Option | Short Description | Requirement | ++================+===================================================================================+=================+ +| max_age | The result of the `condor_status` call is cached for `max_age` in minutes. | **Required** | ++----------------+-----------------------------------------------------------------------------------+-----------------+ +| executor | The |executor| used to run submission and further calls to the Moab batch system. | **Optional** | ++ + + + +| | Default: ShellExecutor is used! | | ++----------------+-----------------------------------------------------------------------------------+-----------------+ + +The only available option in the `MachineTypeConfiguration` section is a template jdl used to submit drones to the +HTCondor batch system. The template jdl is using the `Python template string`_ syntax. + +.. _Python template string: https://docs.python.org/3.4/library/string.html#template-strings + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + Sites: + - name: TOPAS + adapter: HTCondor + quota: 462 + + TOPAS: + max_age: 1 + MachineTypes: + - wholenode + MachineTypeConfiguration: + wholenode: + jdl: pilot_wholenode.jdl + MachineMetaData: + wholenode: + Cores: 42 + Memory: 256 + Disk: 840 + +Moab Site Adapter +----------------- +The :py:class:`~tardis.adapters.sites.moab.MoabAdapter` implements an interface to the Moab batch system. Regular batch +jobs are submitted that start the actual Drone, which than is integrated itself in overlay batch system +using the chosen :ref:`BatchSystemAdapter.`. + +Available adapter configuration options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ++----------------+-----------------------------------------------------------------------------------+-----------------+ +| Option | Short Description | Requirement | ++================+===================================================================================+=================+ +| StatusUpdate | The result of the status call is cached for `StatusUpdate` in minutes. | **Required** | ++----------------+-----------------------------------------------------------------------------------+-----------------+ +| StartUpCommand | The command executed in the batch job. | **Required** | ++----------------+-----------------------------------------------------------------------------------+-----------------+ +| executor | The |executor| used to run submission and further calls to the Moab batch system. | **Optional** | ++ + + + +| | Default: ShellExecutor is used! | | ++----------------+-----------------------------------------------------------------------------------+-----------------+ + +The available options in the `MachineTypeConfiguration` section are the expected `WallTime` of the placeholder jobs and +the requested `NodeType`. For details see the Moab documentation. + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + Sites: + - name: moab-site + adapter: Moab + quota: 2000 + + moab-site: + executor: !SSHExecutor + host: login.dorie.somewherein.de + username: clown + client_keys: + - /opt/tardis/ssh/tardis + StartupCommand: startVM.py + StatusUpdate: 2 + MachineTypes: + - singularity_d2.large + - singularity_d1.large + MachineTypeConfiguration: + singularity_d2.large: + Walltime: '02:00:00:00' + NodeType: '1:ppn=20' + singularity_d1.large: + Walltime: '01:00:00:00' + NodeType: '1:ppn=20' + MachineMetaData: + singularity_d2.large: + Cores: 20 + Memory: 120 + Disk: 196 + singularity_d1.large: + Cores: 20 + Memory: 120 + Disk: 196 + +OpenStack Site Adapter +---------------------- +The :py:class:`~tardis.adapters.sites.openstack.OpenStackAdapter` implements an interface to the OpenStack Cloud API. +The following general adapter configuration options are available. + +Available adapter configuration options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++---------------------+---------------------------------------------------------------------+-----------------+ +| Option | Short Description | Requirement | ++=====================+=====================================================================+=================+ +| auth_url | The end point of the OpenStack API to contact. | **Required** | ++---------------------+---------------------------------------------------------------------+-----------------+ +| username | Your OpenStack API username to authenticate yourself. | **Required** | ++---------------------+---------------------------------------------------------------------+-----------------+ +| password | Your OpenStack API password to authenticate yourself. | **Required** | ++---------------------+---------------------------------------------------------------------+-----------------+ +| user_domain_name | The name of the OpenStack user domain. | **Required** | ++---------------------+---------------------------------------------------------------------+-----------------+ +| project_domain_name | The name of the OpenStack project domain. | **Required** | ++---------------------+---------------------------------------------------------------------+-----------------+ + + +All configuration entries in the `MachineTypeConfiguration` section of the machine types are +directly added as keyword arguments to the OpenStack API `create-server` call. All available options are +described in the `OpenStack documentation`_ + +.. _OpenStack documentation: https://developer.openstack.org/api-ref/compute/#create-server + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + Sites: + - name: Woohoo + adapter: OpenStack + quota: 10 # CPU core quota + + Woohoo: + auth_url: https://whoowhoo:13000/v3 + username: woohoo + password: Woohoo123 + project_name: WooHoo + user_domain_name: Default + project_domain_name: Default + MachineTypes: + - m1.xlarge + MachineTypeConfiguration: + m1.xlarge: + flavorRef: 5 # ID of m1.xlarge + networks: + - uuid: fe0317c6-0bed-488b-9108-13726656a0ea + imageRef: bc613271-6a54-48ca-9222-47e009dc0c29 + key_name: MG + user_data: tardis/cloudinit/woohoo.ini + MachineMetaData: + m1.xlarge: + Cores: 8 + Memory: 16 + Disk: 160 + +Slurm Site Adapter +------------------ +The :py:class:`~tardis.adapters.sites.slurm.SlurmAdapter` implements an interface to the SLURM batch system. Regular +batch jobs are submitted that start the actual Drone, which than is integrated itself in overlay batch system +using the chosen :ref:`BatchSystemAdapter.`. + +Available adapter configuration options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------+-----------------------------------------------------------------------------------+-----------------+ +| Option | Short Description | Requirement | ++================+===================================================================================+=================+ +| StatusUpdate | The result of the status call is cached for `StatusUpdate` in minutes. | **Required** | ++----------------+-----------------------------------------------------------------------------------+-----------------+ +| StartUpCommand | The command executed in the batch job. | **Required** | ++----------------+-----------------------------------------------------------------------------------+-----------------+ +| executor | The |executor| used to run submission and further calls to the Moab batch system. | **Optional** | ++ + + + +| | Default: ShellExecutor is used! | | ++----------------+-----------------------------------------------------------------------------------+-----------------+ + + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + Sites: + - name: hpc2000 + adapter: Slurm + quota: 100 + + hpc2000: + executor: !SSHExecutor + host: hpc2000.hpc.org + username: billy + client_keys: + - /opt/tardis/ssh/tardis + StartupCommand: pilot_clean.sh + StatusUpdate: 2 + MachineTypes: + - one_day + - twelve_hours + MachineTypeConfiguration: + one_day: + Walltime: '1440' + Partition: normal + twelve_hours: + Walltime: '600' + Partition: normal + MachineMetaData: + one_day: + Cores: 20 + Memory: 62 + Disk: 480 + twelve_hours: + Cores: 20 + Memory: 62 + Disk: 480 + +Your favorite site is currently not supported? +Please, have a look at +:ref:`how to contribute.` \ No newline at end of file diff --git a/docs/source/modules.rst b/docs/source/api/modules.rst similarity index 100% rename from docs/source/modules.rst rename to docs/source/api/modules.rst diff --git a/docs/source/api/tardis.adapters.batchsystems.fakebatchsystem.rst b/docs/source/api/tardis.adapters.batchsystems.fakebatchsystem.rst new file mode 100644 index 00000000..15c02ad3 --- /dev/null +++ b/docs/source/api/tardis.adapters.batchsystems.fakebatchsystem.rst @@ -0,0 +1,7 @@ +tardis.adapters.batchsystems.fakebatchsystem module +=================================================== + +.. automodule:: tardis.adapters.batchsystems.fakebatchsystem + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.adapters.batchsystems.htcondor.rst b/docs/source/api/tardis.adapters.batchsystems.htcondor.rst new file mode 100644 index 00000000..0995a2df --- /dev/null +++ b/docs/source/api/tardis.adapters.batchsystems.htcondor.rst @@ -0,0 +1,7 @@ +tardis.adapters.batchsystems.htcondor module +============================================ + +.. automodule:: tardis.adapters.batchsystems.htcondor + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.adapters.batchsystems.rst b/docs/source/api/tardis.adapters.batchsystems.rst new file mode 100644 index 00000000..1ca8201a --- /dev/null +++ b/docs/source/api/tardis.adapters.batchsystems.rst @@ -0,0 +1,16 @@ +tardis.adapters.batchsystems package +==================================== + +.. automodule:: tardis.adapters.batchsystems + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + + tardis.adapters.batchsystems.fakebatchsystem + tardis.adapters.batchsystems.htcondor + diff --git a/docs/source/api/tardis.adapters.rst b/docs/source/api/tardis.adapters.rst new file mode 100644 index 00000000..acf21c51 --- /dev/null +++ b/docs/source/api/tardis.adapters.rst @@ -0,0 +1,16 @@ +tardis.adapters package +======================= + +.. automodule:: tardis.adapters + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + tardis.adapters.batchsystems + tardis.adapters.sites + diff --git a/docs/source/api/tardis.adapters.sites.cloudstack.rst b/docs/source/api/tardis.adapters.sites.cloudstack.rst new file mode 100644 index 00000000..0cdf76d5 --- /dev/null +++ b/docs/source/api/tardis.adapters.sites.cloudstack.rst @@ -0,0 +1,7 @@ +tardis.adapters.sites.cloudstack module +======================================= + +.. automodule:: tardis.adapters.sites.cloudstack + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.adapters.sites.fakesite.rst b/docs/source/api/tardis.adapters.sites.fakesite.rst new file mode 100644 index 00000000..5d4c44f1 --- /dev/null +++ b/docs/source/api/tardis.adapters.sites.fakesite.rst @@ -0,0 +1,7 @@ +tardis.adapters.sites.fakesite module +===================================== + +.. automodule:: tardis.adapters.sites.fakesite + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.adapters.sites.htcondor.rst b/docs/source/api/tardis.adapters.sites.htcondor.rst new file mode 100644 index 00000000..131ffe09 --- /dev/null +++ b/docs/source/api/tardis.adapters.sites.htcondor.rst @@ -0,0 +1,7 @@ +tardis.adapters.sites.htcondor module +===================================== + +.. automodule:: tardis.adapters.sites.htcondor + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.adapters.sites.moab.rst b/docs/source/api/tardis.adapters.sites.moab.rst new file mode 100644 index 00000000..f5bafb3e --- /dev/null +++ b/docs/source/api/tardis.adapters.sites.moab.rst @@ -0,0 +1,7 @@ +tardis.adapters.sites.moab module +================================= + +.. automodule:: tardis.adapters.sites.moab + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.adapters.sites.openstack.rst b/docs/source/api/tardis.adapters.sites.openstack.rst new file mode 100644 index 00000000..379b193b --- /dev/null +++ b/docs/source/api/tardis.adapters.sites.openstack.rst @@ -0,0 +1,7 @@ +tardis.adapters.sites.openstack module +====================================== + +.. automodule:: tardis.adapters.sites.openstack + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.adapters.sites.rst b/docs/source/api/tardis.adapters.sites.rst new file mode 100644 index 00000000..7306da08 --- /dev/null +++ b/docs/source/api/tardis.adapters.sites.rst @@ -0,0 +1,20 @@ +tardis.adapters.sites package +============================= + +.. automodule:: tardis.adapters.sites + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + + tardis.adapters.sites.cloudstack + tardis.adapters.sites.fakesite + tardis.adapters.sites.htcondor + tardis.adapters.sites.moab + tardis.adapters.sites.openstack + tardis.adapters.sites.slurm + diff --git a/docs/source/api/tardis.adapters.sites.slurm.rst b/docs/source/api/tardis.adapters.sites.slurm.rst new file mode 100644 index 00000000..7b1c8bdf --- /dev/null +++ b/docs/source/api/tardis.adapters.sites.slurm.rst @@ -0,0 +1,7 @@ +tardis.adapters.sites.slurm module +================================== + +.. automodule:: tardis.adapters.sites.slurm + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/tardis.agents.batchsystemagent.rst b/docs/source/api/tardis.agents.batchsystemagent.rst similarity index 100% rename from docs/source/tardis.agents.batchsystemagent.rst rename to docs/source/api/tardis.agents.batchsystemagent.rst diff --git a/docs/source/tardis.agents.rst b/docs/source/api/tardis.agents.rst similarity index 100% rename from docs/source/tardis.agents.rst rename to docs/source/api/tardis.agents.rst diff --git a/docs/source/tardis.agents.siteagent.rst b/docs/source/api/tardis.agents.siteagent.rst similarity index 100% rename from docs/source/tardis.agents.siteagent.rst rename to docs/source/api/tardis.agents.siteagent.rst diff --git a/docs/source/tardis.configuration.configuration.rst b/docs/source/api/tardis.configuration.configuration.rst similarity index 100% rename from docs/source/tardis.configuration.configuration.rst rename to docs/source/api/tardis.configuration.configuration.rst diff --git a/docs/source/tardis.configuration.rst b/docs/source/api/tardis.configuration.rst similarity index 87% rename from docs/source/tardis.configuration.rst rename to docs/source/api/tardis.configuration.rst index f92e9aa5..e70d4555 100644 --- a/docs/source/tardis.configuration.rst +++ b/docs/source/api/tardis.configuration.rst @@ -12,4 +12,5 @@ Submodules .. toctree:: tardis.configuration.configuration + tardis.configuration.utilities diff --git a/docs/source/api/tardis.configuration.utilities.rst b/docs/source/api/tardis.configuration.utilities.rst new file mode 100644 index 00000000..84525255 --- /dev/null +++ b/docs/source/api/tardis.configuration.utilities.rst @@ -0,0 +1,7 @@ +tardis.configuration.utilities module +===================================== + +.. automodule:: tardis.configuration.utilities + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.exceptions.executorexceptions.rst b/docs/source/api/tardis.exceptions.executorexceptions.rst new file mode 100644 index 00000000..5106fd1a --- /dev/null +++ b/docs/source/api/tardis.exceptions.executorexceptions.rst @@ -0,0 +1,7 @@ +tardis.exceptions.executorexceptions module +=========================================== + +.. automodule:: tardis.exceptions.executorexceptions + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.exceptions.rst b/docs/source/api/tardis.exceptions.rst new file mode 100644 index 00000000..f6d73d6f --- /dev/null +++ b/docs/source/api/tardis.exceptions.rst @@ -0,0 +1,16 @@ +tardis.exceptions package +========================= + +.. automodule:: tardis.exceptions + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + + tardis.exceptions.executorexceptions + tardis.exceptions.tardisexceptions + diff --git a/docs/source/api/tardis.exceptions.tardisexceptions.rst b/docs/source/api/tardis.exceptions.tardisexceptions.rst new file mode 100644 index 00000000..9b335612 --- /dev/null +++ b/docs/source/api/tardis.exceptions.tardisexceptions.rst @@ -0,0 +1,7 @@ +tardis.exceptions.tardisexceptions module +========================================= + +.. automodule:: tardis.exceptions.tardisexceptions + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.interfaces.batchsystemadapter.rst b/docs/source/api/tardis.interfaces.batchsystemadapter.rst new file mode 100644 index 00000000..abbf62c3 --- /dev/null +++ b/docs/source/api/tardis.interfaces.batchsystemadapter.rst @@ -0,0 +1,7 @@ +tardis.interfaces.batchsystemadapter module +=========================================== + +.. automodule:: tardis.interfaces.batchsystemadapter + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/tardis.interfaces.borg.rst b/docs/source/api/tardis.interfaces.borg.rst similarity index 100% rename from docs/source/tardis.interfaces.borg.rst rename to docs/source/api/tardis.interfaces.borg.rst diff --git a/docs/source/api/tardis.interfaces.executor.rst b/docs/source/api/tardis.interfaces.executor.rst new file mode 100644 index 00000000..1b1cd50b --- /dev/null +++ b/docs/source/api/tardis.interfaces.executor.rst @@ -0,0 +1,7 @@ +tardis.interfaces.executor module +================================= + +.. automodule:: tardis.interfaces.executor + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.interfaces.plugin.rst b/docs/source/api/tardis.interfaces.plugin.rst new file mode 100644 index 00000000..95489228 --- /dev/null +++ b/docs/source/api/tardis.interfaces.plugin.rst @@ -0,0 +1,7 @@ +tardis.interfaces.plugin module +=============================== + +.. automodule:: tardis.interfaces.plugin + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/tardis.interfaces.rst b/docs/source/api/tardis.interfaces.rst similarity index 67% rename from docs/source/tardis.interfaces.rst rename to docs/source/api/tardis.interfaces.rst index 1bd97080..17ab9fbe 100644 --- a/docs/source/tardis.interfaces.rst +++ b/docs/source/api/tardis.interfaces.rst @@ -11,7 +11,11 @@ Submodules .. toctree:: + tardis.interfaces.batchsystemadapter tardis.interfaces.borg + tardis.interfaces.executor + tardis.interfaces.plugin + tardis.interfaces.simulator tardis.interfaces.siteadapter tardis.interfaces.state diff --git a/docs/source/api/tardis.interfaces.simulator.rst b/docs/source/api/tardis.interfaces.simulator.rst new file mode 100644 index 00000000..b84c8ba5 --- /dev/null +++ b/docs/source/api/tardis.interfaces.simulator.rst @@ -0,0 +1,7 @@ +tardis.interfaces.simulator module +================================== + +.. automodule:: tardis.interfaces.simulator + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/tardis.interfaces.siteadapter.rst b/docs/source/api/tardis.interfaces.siteadapter.rst similarity index 100% rename from docs/source/tardis.interfaces.siteadapter.rst rename to docs/source/api/tardis.interfaces.siteadapter.rst diff --git a/docs/source/tardis.interfaces.state.rst b/docs/source/api/tardis.interfaces.state.rst similarity index 100% rename from docs/source/tardis.interfaces.state.rst rename to docs/source/api/tardis.interfaces.state.rst diff --git a/docs/source/api/tardis.plugins.rst b/docs/source/api/tardis.plugins.rst new file mode 100644 index 00000000..bcdca0d2 --- /dev/null +++ b/docs/source/api/tardis.plugins.rst @@ -0,0 +1,16 @@ +tardis.plugins package +====================== + +.. automodule:: tardis.plugins + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + + tardis.plugins.sqliteregistry + tardis.plugins.telegrafmonitoring + diff --git a/docs/source/api/tardis.plugins.sqliteregistry.rst b/docs/source/api/tardis.plugins.sqliteregistry.rst new file mode 100644 index 00000000..b57b5bf2 --- /dev/null +++ b/docs/source/api/tardis.plugins.sqliteregistry.rst @@ -0,0 +1,7 @@ +tardis.plugins.sqliteregistry module +==================================== + +.. automodule:: tardis.plugins.sqliteregistry + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.plugins.telegrafmonitoring.rst b/docs/source/api/tardis.plugins.telegrafmonitoring.rst new file mode 100644 index 00000000..54a6bc03 --- /dev/null +++ b/docs/source/api/tardis.plugins.telegrafmonitoring.rst @@ -0,0 +1,7 @@ +tardis.plugins.telegrafmonitoring module +======================================== + +.. automodule:: tardis.plugins.telegrafmonitoring + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/tardis.resources.drone.rst b/docs/source/api/tardis.resources.drone.rst similarity index 100% rename from docs/source/tardis.resources.drone.rst rename to docs/source/api/tardis.resources.drone.rst diff --git a/docs/source/tardis.resources.dronestates.rst b/docs/source/api/tardis.resources.dronestates.rst similarity index 100% rename from docs/source/tardis.resources.dronestates.rst rename to docs/source/api/tardis.resources.dronestates.rst diff --git a/docs/source/api/tardis.resources.poolfactory.rst b/docs/source/api/tardis.resources.poolfactory.rst new file mode 100644 index 00000000..76ac7adb --- /dev/null +++ b/docs/source/api/tardis.resources.poolfactory.rst @@ -0,0 +1,7 @@ +tardis.resources.poolfactory module +=================================== + +.. automodule:: tardis.resources.poolfactory + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/tardis.resources.rst b/docs/source/api/tardis.resources.rst similarity index 88% rename from docs/source/tardis.resources.rst rename to docs/source/api/tardis.resources.rst index f8e5b665..58fdc106 100644 --- a/docs/source/tardis.resources.rst +++ b/docs/source/api/tardis.resources.rst @@ -13,4 +13,5 @@ Submodules tardis.resources.drone tardis.resources.dronestates + tardis.resources.poolfactory diff --git a/docs/source/tardis.rst b/docs/source/api/tardis.rst similarity index 80% rename from docs/source/tardis.rst rename to docs/source/api/tardis.rst index 79cbc392..8b01dc0c 100644 --- a/docs/source/tardis.rst +++ b/docs/source/api/tardis.rst @@ -11,10 +11,12 @@ Subpackages .. toctree:: - tardis.adapter + tardis.adapters tardis.agents tardis.configuration + tardis.exceptions tardis.interfaces + tardis.plugins tardis.resources tardis.utilities diff --git a/docs/source/api/tardis.utilities.asynccachemap.rst b/docs/source/api/tardis.utilities.asynccachemap.rst new file mode 100644 index 00000000..2d29815a --- /dev/null +++ b/docs/source/api/tardis.utilities.asynccachemap.rst @@ -0,0 +1,7 @@ +tardis.utilities.asynccachemap module +===================================== + +.. automodule:: tardis.utilities.asynccachemap + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.utilities.attributedict.rst b/docs/source/api/tardis.utilities.attributedict.rst new file mode 100644 index 00000000..29e9e5e6 --- /dev/null +++ b/docs/source/api/tardis.utilities.attributedict.rst @@ -0,0 +1,7 @@ +tardis.utilities.attributedict module +===================================== + +.. automodule:: tardis.utilities.attributedict + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.utilities.executors.rst b/docs/source/api/tardis.utilities.executors.rst new file mode 100644 index 00000000..9c88ffea --- /dev/null +++ b/docs/source/api/tardis.utilities.executors.rst @@ -0,0 +1,16 @@ +tardis.utilities.executors package +================================== + +.. automodule:: tardis.utilities.executors + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + + tardis.utilities.executors.shellexecutor + tardis.utilities.executors.sshexecutor + diff --git a/docs/source/api/tardis.utilities.executors.shellexecutor.rst b/docs/source/api/tardis.utilities.executors.shellexecutor.rst new file mode 100644 index 00000000..6b303e72 --- /dev/null +++ b/docs/source/api/tardis.utilities.executors.shellexecutor.rst @@ -0,0 +1,7 @@ +tardis.utilities.executors.shellexecutor module +=============================================== + +.. automodule:: tardis.utilities.executors.shellexecutor + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.utilities.executors.sshexecutor.rst b/docs/source/api/tardis.utilities.executors.sshexecutor.rst new file mode 100644 index 00000000..ad9202f5 --- /dev/null +++ b/docs/source/api/tardis.utilities.executors.sshexecutor.rst @@ -0,0 +1,7 @@ +tardis.utilities.executors.sshexecutor module +============================================= + +.. automodule:: tardis.utilities.executors.sshexecutor + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.utilities.pipeline.rst b/docs/source/api/tardis.utilities.pipeline.rst new file mode 100644 index 00000000..c4a97c4d --- /dev/null +++ b/docs/source/api/tardis.utilities.pipeline.rst @@ -0,0 +1,7 @@ +tardis.utilities.pipeline module +================================ + +.. automodule:: tardis.utilities.pipeline + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.utilities.rst b/docs/source/api/tardis.utilities.rst new file mode 100644 index 00000000..8738cedf --- /dev/null +++ b/docs/source/api/tardis.utilities.rst @@ -0,0 +1,27 @@ +tardis.utilities package +======================== + +.. automodule:: tardis.utilities + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + tardis.utilities.executors + tardis.utilities.simulators + +Submodules +---------- + +.. toctree:: + + tardis.utilities.asynccachemap + tardis.utilities.attributedict + tardis.utilities.pipeline + tardis.utilities.staticmapping + tardis.utilities.utils + diff --git a/docs/source/api/tardis.utilities.simulators.periodicvalue.rst b/docs/source/api/tardis.utilities.simulators.periodicvalue.rst new file mode 100644 index 00000000..8ada337d --- /dev/null +++ b/docs/source/api/tardis.utilities.simulators.periodicvalue.rst @@ -0,0 +1,7 @@ +tardis.utilities.simulators.periodicvalue module +================================================ + +.. automodule:: tardis.utilities.simulators.periodicvalue + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.utilities.simulators.randomgauss.rst b/docs/source/api/tardis.utilities.simulators.randomgauss.rst new file mode 100644 index 00000000..44434ca9 --- /dev/null +++ b/docs/source/api/tardis.utilities.simulators.randomgauss.rst @@ -0,0 +1,7 @@ +tardis.utilities.simulators.randomgauss module +============================================== + +.. automodule:: tardis.utilities.simulators.randomgauss + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.utilities.simulators.rst b/docs/source/api/tardis.utilities.simulators.rst new file mode 100644 index 00000000..fd3eefc1 --- /dev/null +++ b/docs/source/api/tardis.utilities.simulators.rst @@ -0,0 +1,16 @@ +tardis.utilities.simulators package +=================================== + +.. automodule:: tardis.utilities.simulators + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + + tardis.utilities.simulators.periodicvalue + tardis.utilities.simulators.randomgauss + diff --git a/docs/source/api/tardis.utilities.staticmapping.rst b/docs/source/api/tardis.utilities.staticmapping.rst new file mode 100644 index 00000000..8357503b --- /dev/null +++ b/docs/source/api/tardis.utilities.staticmapping.rst @@ -0,0 +1,7 @@ +tardis.utilities.staticmapping module +===================================== + +.. automodule:: tardis.utilities.staticmapping + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api/tardis.utilities.utils.rst b/docs/source/api/tardis.utilities.utils.rst new file mode 100644 index 00000000..a1cd998c --- /dev/null +++ b/docs/source/api/tardis.utilities.utils.rst @@ -0,0 +1,7 @@ +tardis.utilities.utils module +============================= + +.. automodule:: tardis.utilities.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/conf.py b/docs/source/conf.py index 9bc5bc96..ca00d4d4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,21 +12,21 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys +sys.path.insert(0, os.path.abspath('../..')) # -- Project information ----------------------------------------------------- project = 'tardis' -copyright = '2018, Manuel Giffels' +copyright = '2019, Manuel Giffels' author = 'Manuel Giffels' # The short X.Y version -version = '' +version = '0.1.0' # The full version, including alpha/beta/rc tags -release = '0.0' +release = '0.1.0' # -- General configuration --------------------------------------------------- @@ -40,9 +40,7 @@ # ones. extensions = [ 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', - 'sphinx.ext.coverage', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', @@ -71,7 +69,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path . -exclude_patterns = [] +exclude_patterns = ['_build'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' @@ -82,7 +80,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -168,4 +166,4 @@ # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} \ No newline at end of file +intersphinx_mapping = {'https://docs.python.org/': None} diff --git a/docs/source/contribute/contribute.rst b/docs/source/contribute/contribute.rst new file mode 100644 index 00000000..d25761ec --- /dev/null +++ b/docs/source/contribute/contribute.rst @@ -0,0 +1,27 @@ +================== +How to Contribute? +================== +In progress ... + +.. _ref_contribute_batch_system_adapter: + +Batch System Adapter +-------------------- +Implement adapter according to the following abstract base class interface +:py:class:`~tardis.interfaces.batchsystemadapter.BatchSystemAdapter`. + +.. _ref_contribute_site_adapter: + +Site Adapter +------------ +Implement adapter according to the following abstract base class interface +:py:class:`~tardis.interfaces.siteadapter.SiteAdapter`. + +.. _ref_contribute_plugin: + +Plugin +------- +Implement plugin according to the following abstract base class interface +:py:class:`~tardis.interfaces.plugin.Plugin`. + + diff --git a/docs/source/executors/executors.rst b/docs/source/executors/executors.rst new file mode 100644 index 00000000..73542029 --- /dev/null +++ b/docs/source/executors/executors.rst @@ -0,0 +1,43 @@ +.. _ref_executors: + +========= +Executors +========= + +All executors can be loaded using YAML tags using the (`!Tag`) syntax. More details are available in the +`PyYAML documentation`_ + +.. _PyYAML documentation: https://pyyaml.org/wiki/PyYAMLDocumentation + +Shell Executor +-------------- + +The shell executor is used to execute shell commands asynchronously. + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + !ShellExecutor + +SSH Executor +------------ + +The ssh executor is used to asynchronously execute shell commands remotely via ssh. All parameters specified in the +configuration are directly passed as keyword arguments to `asyncssh` `connect` call. You can find all available +parameters in the `asyncssh documentation`_ + +.. _asyncssh documentation: https://asyncssh.readthedocs.io/en/latest/api.html#connect + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + !SSHExecutor + host: login.dorie.somewherein.de + username: clown + client_keys: + - /opt/tardis/ssh/tardis + diff --git a/docs/source/index.rst b/docs/source/index.rst index 280d3eb3..5266d0d9 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,16 +1,28 @@ -.. tardis documentation master file, created by - sphinx-quickstart on Fri Jun 8 14:53:15 2018. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +====================================================================== +TARDIS - The Transparent Adaptive Resource Dynamic Integration System +====================================================================== -Welcome to tardis's documentation! -================================== +Welcome to the TARDIS documentation! +==================================== .. toctree:: + :hidden: :maxdepth: 2 :caption: Contents: + adapters/batchsystem + adapters/site + executors/executors + plugins/plugins + contribute/contribute + Module Index +.. image:: ../pics/TARDIS_logo.svg + :alt: TARDIS Logo + :height: 150 + :align: left + +The ``TARDIS`` package Indices and tables ================== diff --git a/docs/source/plugins/plugins.rst b/docs/source/plugins/plugins.rst new file mode 100644 index 00000000..ab4d8813 --- /dev/null +++ b/docs/source/plugins/plugins.rst @@ -0,0 +1,72 @@ +======= +Plugins +======= + +All plugins are configured within the `plugins` section of the TARDIS configuration. Using multiple plugins is``` +supported by using a separate MappingNode per plugin. + +.. code-block:: yaml + + Plugins: + Plugin_1: + option_1: my_option_1 + Plugin_2: + option_123: my_option_123 + +SQLite Registry +--------------- +The :py:class:`~tardis.plugins.sqliteregistry.SqliteRegistry` implements a persistent storage of all Drone states in a +SQLite database. The usage of this module is recommended in order to recover the last state of TARDIS in case the +service has to be restarted. + +Available configuration options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------+-----------------------------------+-----------------+ +| Option | Short Description | Requirement | ++================+===================================+=================+ +| db_file | Location of the SQLite database. | **Required** | ++----------------+-----------------------------------+-----------------+ + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + Plugins: + SqliteRegistry: + db_file: drone_registry.db + +Telegraf Monitoring +------------------- +The :py:class:`~tardis.plugins.telegrafmonitoring.TelegrafMonitoring` implements an interface to monitor state changes +of the Drones in a telegraf service running a UDP input module. + +Available configuration options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ++----------------+---------------------------------------------------------------------------+-----------------+ +| Option | Short Description | Requirement | ++================+===========================================================================+=================+ +| host | Hostname or IP address the telegraf UDP input module is listening to. | **Required** | ++----------------+---------------------------------------------------------------------------+-----------------+ +| port | Port the telegraf UDP input module is listening on. | **Required** | ++----------------+---------------------------------------------------------------------------+-----------------+ +| default_tags | Tags that should be included by default for all entries sent to telegraf. | **Optional** | ++----------------+---------------------------------------------------------------------------+-----------------+ + +Example configuration +~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: yaml + + Plugins: + TelegrafMonitoring: + host: der_telegraf.foo.bar + port: 8092 + default_tags: + something_default: 'The Default Tag' + +Your favorite monitoring is currently not supported? +Please, have a look at +:ref:`how to contribute.` diff --git a/docs/source/tardis.adapter.exoscale.rst b/docs/source/tardis.adapter.exoscale.rst deleted file mode 100644 index 7f9c0faf..00000000 --- a/docs/source/tardis.adapter.exoscale.rst +++ /dev/null @@ -1,7 +0,0 @@ -tardis.adapter.exoscale module -============================== - -.. automodule:: tardis.adapter.exoscale - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/tardis.adapter.rst b/docs/source/tardis.adapter.rst deleted file mode 100644 index 8aa4ee6d..00000000 --- a/docs/source/tardis.adapter.rst +++ /dev/null @@ -1,15 +0,0 @@ -tardis.adapter package -====================== - -.. automodule:: tardis.adapter - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -.. toctree:: - - tardis.adapter.exoscale - diff --git a/docs/source/tardis.utilities.looper.rst b/docs/source/tardis.utilities.looper.rst deleted file mode 100644 index 9c60696b..00000000 --- a/docs/source/tardis.utilities.looper.rst +++ /dev/null @@ -1,7 +0,0 @@ -tardis.utilities.looper module -============================== - -.. automodule:: tardis.utilities.looper - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/tardis.utilities.rst b/docs/source/tardis.utilities.rst deleted file mode 100644 index 4a3ff78c..00000000 --- a/docs/source/tardis.utilities.rst +++ /dev/null @@ -1,15 +0,0 @@ -tardis.utilities package -======================== - -.. automodule:: tardis.utilities - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -.. toctree:: - - tardis.utilities.looper - diff --git a/setup.cfg b/setup.cfg index 4a89401d..15d3f347 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [build_sphinx] -project = tarids -version = 0.0.1 -release = 0.0.1 +project = cobald-tardis +version = 0.2.0 +release = 0.2.0 source-dir = docs/source build-dir = docs/build \ No newline at end of file diff --git a/setup.py b/setup.py index c0f168b4..ea520956 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name='cobald-tardis', - version='0.0.1', + version='0.2.0', description='Transparent Adaptive Resource Dynamic Integration System', long_description=long_description, long_description_content_type='text/markdown', diff --git a/tardis/adapters/batchsystems/fakebatchsystem.py b/tardis/adapters/batchsystems/fakebatchsystem.py index a0a92cd2..2dbc15e0 100644 --- a/tardis/adapters/batchsystems/fakebatchsystem.py +++ b/tardis/adapters/batchsystems/fakebatchsystem.py @@ -4,22 +4,59 @@ class FakeBatchSystemAdapter(BatchSystemAdapter): + """ + :py:class:`~tardis.adapters.batchsystems.fakebatchsystem.FakeBatchSystemAdapter` implements a batch system adapter + that mocks the response of a hypothetical batch system. It can be used for testing purposes and as a demonstrator + in workshops and tutorials. + + The mocked response to the + :py:meth:`~tardis.adapters.batchsystems.fakebatchsystem.FakeBatchSystemAdapter.get_utilization`, + :py:meth:`~tardis.adapters.batchsystems.fakebatchsystem.FakeBatchSystemAdapter.get_allocation` and + :py:meth:`~tardis.adapters.batchsystems.fakebatchsystem.FakeBatchSystemAdapter.get_machine_status` API calls + is configurable statically. + """ def __init__(self): config = Configuration() self.fake_config = config.BatchSystem self._drained_machines = {} - async def disintegrate_machine(self, drone_uuid): + async def disintegrate_machine(self, drone_uuid: str) -> None: + """ + FakeBatchSystemAdapter's do nothing disintegrate_machine implementation + + :param drone_uuid: Unique identifier of the drone + :type drone_uuid: str + """ return - async def drain_machine(self, drone_uuid): + async def drain_machine(self, drone_uuid: str) -> None: + """ + FakeBatchSystemAdapter's do nothing drain_machine implementation + + :param drone_uuid: Unique identifier of the drone + :type drone_uuid: str + """ self._drained_machines[drone_uuid] = MachineStatus.Drained return - async def integrate_machine(self, drone_uuid): + async def integrate_machine(self, drone_uuid: str) -> None: + """ + FakeBatchSystemAdapter's do nothing integrate_machine implementation + + :param drone_uuid: Unique identifier of the drone + :type drone_uuid: str + """ return - async def get_allocation(self, drone_uuid): + async def get_allocation(self, drone_uuid: str) -> float: + """ + Returns the fake allocation according to the configuration of the FakeBatchSystem + + :param drone_uuid: Unique identifier of the drone + :type drone_uuid: str + :return: Allocation value specified in the FakeBatchSystem configuration + :rtype: float + """ try: allocation = self.fake_config.allocation.get_value() except AttributeError: @@ -27,7 +64,15 @@ async def get_allocation(self, drone_uuid): else: return allocation - async def get_machine_status(self, drone_uuid): + async def get_machine_status(self, drone_uuid: str) -> MachineStatus: + """ + Returns a fake machine status according to the parameter set in the configuration of the FakeBatchSystem + + :param drone_uuid: Unique identifier of the drone + :type drone_uuid: str + :return: Machine status specified in the FakeBatchSystem configuration + :rtype: MachineStatus + """ try: machine_status = self._drained_machines[drone_uuid] except KeyError: @@ -35,7 +80,15 @@ async def get_machine_status(self, drone_uuid): else: return machine_status - async def get_utilization(self, drone_uuid): + async def get_utilization(self, drone_uuid: str) -> float: + """ + Returns the fake utilization according to the configuration of the FakeBatchSystem + + :param drone_uuid: Unique identifier of the drone + :type drone_uuid: str + :return: Utilization value specified in the FakeBatchSystem configuration + :rtype: float + """ try: utilization = self.fake_config.utilization.get_value() except AttributeError: diff --git a/tardis/adapters/batchsystems/htcondor.py b/tardis/adapters/batchsystems/htcondor.py index ead8d962..c37728c2 100644 --- a/tardis/adapters/batchsystems/htcondor.py +++ b/tardis/adapters/batchsystems/htcondor.py @@ -6,13 +6,26 @@ from ...utilities.utils import htcondor_cmd_option_formatter from ...utilities.utils import htcondor_csv_parser from ...utilities.asynccachemap import AsyncCacheMap +from ...utilities.attributedict import AttributeDict from functools import partial from shlex import quote +from typing import Iterable import logging -async def htcondor_status_updater(options, attributes): +async def htcondor_status_updater(options: AttributeDict, attributes: AttributeDict) -> dict: + """ + Helper function to call ``condor_status -af`` asynchronously and to translate the output into a dictionary + + :param options: Additional options for the condor_status call. For example ``{'pool': 'htcondor.example'}`` \ + will be translated into ``condor_status -af ... -pool htcondor.example`` + :type options: AttributeDict + :param attributes: Additional fields to add to output of the ``condor_status -af`` response. + :type attributes: AttributeDict + :return: Dictionary containing the output of the ``condor_status`` command + :rtype: dict + """ attributes_string = f'-af:t {" ".join(attributes.values())}' options_string = htcondor_cmd_option_formatter(options) @@ -41,6 +54,10 @@ async def htcondor_status_updater(options, attributes): class HTCondorAdapter(BatchSystemAdapter): + """ + :py:class:`~tardis.adapters.batchsystems.htcondor.HTCondorAdapter` implements the TARDIS interface to dynamically + integrate and manage opportunistic resources with the HTCondor Batch System. + """ def __init__(self): config = Configuration() self.ratios = config.BatchSystem.ratios @@ -58,15 +75,24 @@ def __init__(self): attributes), max_age=config.BatchSystem.max_age * 60) - async def disintegrate_machine(self, drone_uuid): + async def disintegrate_machine(self, drone_uuid: str) -> None: """ - HTCondor does not require any specific disintegration procedure. Other batchsystems probably do. - :param drone_uuid: Uuid of the worker node, for some sites corresponding to the host name of drone + HTCondor does not require any specific disintegration procedure. + + :param drone_uuid: Uuid of the worker node, for some sites corresponding to the host name of the drone. + :type drone_uuid: str :return: None """ - return None + return + + async def drain_machine(self, drone_uuid: str) -> None: + """ + Drain a machine in the HTCondor batch system, which means that no new jobs will be accepted - async def drain_machine(self, drone_uuid): + :param drone_uuid: Uuid of the worker node, for some sites corresponding to the host name of the drone. + :type drone_uuid: str + :return: None + """ await self._htcondor_status.update_status() try: machine = self._htcondor_status[drone_uuid]['Machine'] @@ -90,15 +116,26 @@ async def drain_machine(self, drone_uuid): return raise ex - async def integrate_machine(self, drone_uuid): + async def integrate_machine(self, drone_uuid: str) -> None: """ - HTCondor does not require any specific integration procedure. Other batchsystems probably do. - :param drone_uuid: DNS name of the worker node + HTCondor does not require any specific integration procedure + + :param drone_uuid: Uuid of the worker node, for some sites corresponding to the host name of the drone. + :type drone_uuid: str :return: None """ return None - async def get_resource_ratios(self, drone_uuid): + async def get_resource_ratios(self, drone_uuid: str) -> Iterable[float]: + """ + Get the ratio of requested over total resources (CPU, Memory, Disk, etc. ) for a worker node in HTCondor + according to the HTCondor expressions defined in the adapter configuration. + + :param drone_uuid: Uuid of the worker node, for some sites corresponding to the host name of the drone. + :type drone_uuid: str + :return: Iterable of float containing the ratios + :rtype: Iterable[float] + """ await self._htcondor_status.update_status() try: htcondor_status = self._htcondor_status[drone_uuid] @@ -107,10 +144,27 @@ async def get_resource_ratios(self, drone_uuid): else: return (float(value) for key, value in htcondor_status.items() if key in self.ratios.keys()) - async def get_allocation(self, drone_uuid): + async def get_allocation(self, drone_uuid: str) -> float: + """ + Get the allocation of a worker node in HTCondor, which is defined as maximum of the ratios of requested over + total resources (CPU, Memory, Disk, etc.). + + :param drone_uuid: Uuid of the worker node, for some sites corresponding to the host name of the drone. + :type drone_uuid: str + :return: The allocation of a worker node as described above. + :rtype: float + """ return max(await self.get_resource_ratios(drone_uuid), default=0.) - async def get_machine_status(self, drone_uuid): + async def get_machine_status(self, drone_uuid: str) -> MachineStatus: + """ + Get the status of a worker node in HTCondor (Available, Draining, Drained, NotAvailable) + + :param drone_uuid: Uuid of the worker node, for some sites corresponding to the host name of the drone. + :type drone_uuid: str + :return: The machine status in HTCondor (Available, Draining, Drained, NotAvailable) + :rtype: MachineStatus + """ status_mapping = {('Unclaimed', 'Idle'): MachineStatus.Available, ('Drained', 'Retiring'): MachineStatus.Draining, ('Drained', 'Idle'): MachineStatus.Drained, @@ -124,5 +178,14 @@ async def get_machine_status(self, drone_uuid): else: return status_mapping.get((machine_status['State'], machine_status['Activity']), MachineStatus.NotAvailable) - async def get_utilization(self, drone_uuid): + async def get_utilization(self, drone_uuid: str) -> float: + """ + Get the utilization of a worker node in HTCondor, which is defined as minimum of the ratios of requested over + total resources (CPU, Memory, Disk, etc.). + + :param drone_uuid: Uuid of the worker node, for some sites corresponding to the host name of the drone. + :type drone_uuid: str + :return: The utilization of a worker node as described above. + :rtype: float + """ return min(await self.get_resource_ratios(drone_uuid), default=0.) diff --git a/tardis/plugins/sqliteregistry.py b/tardis/plugins/sqliteregistry.py index a49ca027..29575e77 100644 --- a/tardis/plugins/sqliteregistry.py +++ b/tardis/plugins/sqliteregistry.py @@ -10,6 +10,11 @@ class SqliteRegistry(Plugin): def __init__(self): + """ + The :py:class:`~tardis.plugins.sqliteregistry.SqliteRegistry` implements a persistent storage of all Drone + states in a SQLite database. The usage of this module is recommended in order to recover the last state of + TARDIS in case the service has to be restarted. + """ self.logger = logging.getLogger("sqliteregistry") self.logger.setLevel(logging.DEBUG) configuration = Configuration() @@ -24,7 +29,7 @@ def __init__(self): for machine_type in getattr(configuration, site.name).MachineTypes: self.add_machine_types(site.name, machine_type) - def add_machine_types(self, site_name, machine_type): + def add_machine_types(self, site_name: str, machine_type: str): sql_query = """INSERT OR IGNORE INTO MachineTypes(machine_type, site_id) SELECT :machine_type, Sites.site_id FROM Sites WHERE Sites.site_name = :site_name""" self.execute(sql_query, dict(site_name=site_name, machine_type=machine_type)) diff --git a/tardis/plugins/telegrafmonitoring.py b/tardis/plugins/telegrafmonitoring.py index 26062e35..bae7b19f 100644 --- a/tardis/plugins/telegrafmonitoring.py +++ b/tardis/plugins/telegrafmonitoring.py @@ -1,5 +1,7 @@ from ..configuration.configuration import Configuration from ..interfaces.plugin import Plugin +from ..interfaces.state import State +from ..utilities.attributedict import AttributeDict import aiotelegraf from datetime import datetime @@ -7,6 +9,10 @@ class TelegrafMonitoring(Plugin): + """ + The :py:class:`~tardis.plugins.telegrafmonitoring.TelegrafMonitoring` implements an interface to monitor state + changes of the Drones in a telegraf service running a UDP input module. + """ def __init__(self): self.logger = logging.getLogger("telegrafmonitoring") self.logger.setLevel(logging.DEBUG) @@ -19,7 +25,17 @@ def __init__(self): self.client = aiotelegraf.Client(host=host, port=port, tags=default_tags) - async def notify(self, state, resource_attributes): + async def notify(self, state: State, resource_attributes: AttributeDict) -> None: + """ + Push changed state and updated meta-data of the drone into the telegraf server + + :param state: New state of the Drone + :type state: State + :param resource_attributes: Contains all meta-data of the Drone (created and updated timestamps, dns name, \ + unique id, site_name, machine_type, etc.) + :type resource_attributes: AttributeDict + :return: None + """ self.logger.debug(f"Drone: {str(resource_attributes)} has changed state to {state}") await self.client.connect() data = dict(state=str(state), created=datetime.timestamp(resource_attributes.created),