Skip to content

Commit

Permalink
Merge pull request #29 from lsst-dm/tickets/DM-48282
Browse files Browse the repository at this point in the history
DM-48282: Use daf_butler subclass for SIAv2Handler with entry point
  • Loading branch information
timj authored Jan 21, 2025
2 parents ebbe7df + 6c616e2 commit d1a357d
Show file tree
Hide file tree
Showing 9 changed files with 270 additions and 106 deletions.
3 changes: 0 additions & 3 deletions .github/workflows/build_docs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,6 @@ jobs:
cache: "pip"
cache-dependency-path: "setup.cfg"

- name: Install sqlite
run: sudo apt-get install sqlite libyaml-dev

- name: Update pip/wheel infrastructure
run: |
python -m pip install --upgrade pip
Expand Down
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ repos:
name: isort (python)
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: v0.8.0
rev: v0.9.1
hooks:
- id: ruff
- repo: https://github.com/numpy/numpydoc
Expand Down
10 changes: 9 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
# dax_obscore

Tools to generate ObsCore data for LSST processed images using Butler Gen 3.
[![pypi](https://img.shields.io/pypi/v/lsst-dax-obscore.svg)](https://pypi.org/project/lsst-dax-obscore/)
[![codecov](https://codecov.io/gh/lsst-dm/dax_obscore/graph/badge.svg?token=TI73D2SG4S)](https://codecov.io/gh/lsst-dm/dax_obscore)

Tools to generate [IVOA ObsCore](https://www.ivoa.net/documents/ObsCore/) data and process [IVOA SIAv2](https://www.ivoa.net/documents/SIA/) queries for data stored in a Butler repository.

* [Rubin Observatory Data Butler](https://github.com/lsst/daf_butler)
* [Overview of SIAv2 implementation](https://doi.org/10.48550/arXiv.2501.00544)

PyPI: [lsst-dax-obscore](https://pypi.org/project/lsst-dax-obscore/)
16 changes: 16 additions & 0 deletions doc/lsst.dax.obscore/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,22 @@ If the attribute in the template string does not exist for a particular record t
``obscore-export`` will read datasets from Registry using collections specified in ``collections`` configuration attribute and dataset types that appear in ``dataset_types`` attribute (indexed by dataset type names).


.. _lsst.dax.obscore-entry_points:

Entry Points
============

By default the SIAv2 query handler works with the default Butler dimension universe that is named ``daf_butler``.
Other dimension universes can be supported by providing a subclass to ``lsst.dax.obscore.siav2.SIAv2Handler`` specified in an entry point group named ``dax_obscore.siav2``.
The entry point label should be the dimension universe namespace.

For example, the default namespace entry point is defined as:

.. code:: toml
[project.entry-points.'dax_obscore.siav2']
daf_butler = "lsst.dax.obscore.siav2:get_daf_butler_siav2_handler"
.. _lsst.dax.obscore-pyapi:

Expand Down
15 changes: 12 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ dynamic = ["version"]

[project.entry-points.'butler.cli']
dax_obscore = "lsst.dax.obscore.cli:get_cli_subcommands"
[project.entry-points.'dax_obscore.siav2']
daf_butler = "lsst.dax.obscore.siav2:get_daf_butler_siav2_handler"

[project.urls]
"Homepage" = "https://github.com/lsst-dm/dax_obscore"
Expand Down Expand Up @@ -118,9 +120,6 @@ select = [
"D", # pydocstyle
"UP", # pyupgrade
]
extend-select = [
"RUF100", # Warn about unused noqa
]

[tool.ruff.lint.pycodestyle]
max-doc-length = 79
Expand All @@ -146,4 +145,14 @@ exclude = [
'^commands\.', # Click docstrings, not numpydoc
'^__init__$',
'\._[a-zA-Z_]+$', # Private methods.
'siav2.SIAv2DafButlerHandler.get_band_information', # inheritDoc
]

[tool.coverage.report]
exclude_lines = [
"pragma: no cover",
"raise AssertionError",
"raise NotImplementedError",
"if __name__ == .__main__.:",
"if TYPE_CHECKING:",
]
31 changes: 14 additions & 17 deletions python/lsst/dax/obscore/obscore_exporter.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
from lsst.daf.butler import Butler, DataCoordinate, ddl
from lsst.daf.butler.formatters.parquet import arrow_to_numpy
from lsst.daf.butler.registry.obscore import (
ExposureRegionFactory,
DerivedRegionFactory,
ObsCoreSchema,
RecordFactory,
SpatialObsCorePlugin,
Expand Down Expand Up @@ -197,8 +197,8 @@ def close(self) -> None:
super().close()


class _ExposureRegionFactory(ExposureRegionFactory):
"""Exposure region factory that returns an existing region, region is
class _DerivedRegionFactory(DerivedRegionFactory):
"""Region factory that returns an existing region, region is
specified via `set` method, which should be called before calling
record factory.
"""
Expand All @@ -214,7 +214,7 @@ def set(self, data_id: DataCoordinate, region: Region) -> None:
----------
data_id : `~lsst.daf.butler.DataCoordinate`
Data ID that will be matched against parameter of
`exposure_region`.
`derived_region`.
region : `Region`
Corresponding region.
"""
Expand All @@ -226,7 +226,7 @@ def reset(self) -> None:
self._data_id = None
self._region = None

def exposure_region(self, dataId: DataCoordinate) -> Region | None:
def derived_region(self, dataId: DataCoordinate) -> Region | None:
# Docstring inherited.
if dataId == self._data_id:
return self._region
Expand Down Expand Up @@ -254,10 +254,10 @@ def __init__(self, butler: Butler, config: ExporterConfig):

self.schema = self._make_schema(schema.table_spec)

self._exposure_region_factory = _ExposureRegionFactory()
self._derived_region_factory = _DerivedRegionFactory()
universe = self.butler.dimensions
self.record_factory = RecordFactory(
config, schema, universe, spatial_plugins, self._exposure_region_factory
self.record_factory = RecordFactory.get_record_type_from_universe(universe)(
config, schema, universe, spatial_plugins, self._derived_region_factory
)

def to_parquet(self, output: str) -> None:
Expand Down Expand Up @@ -458,16 +458,13 @@ def _make_record_batches(
# Want an empty default to match everything.
where_clauses = [WhereBind(where="")]

# Region can come from either visit or visit_detector_region. If we
# are looking at exposure then visit will be joined by the query
# system.
# Determine the relevant dimension for the region that can be
# joined by the query system.
dataset_type = self.butler.get_dataset_type(dataset_type_name)
region_dim, region_metadata_name = self.record_factory.region_dimension(dataset_type.dimensions)
region_key: str | None = None
if "exposure" in dataset_type.dimensions or "visit" in dataset_type.dimensions:
if "detector" in dataset_type.dimensions:
region_key = "visit_detector_region.region"
else:
region_key = "visit.region"
if region_dim is not None:
region_key = f"{region_dim}.{region_metadata_name}"

with self.butler.query() as query:
for where_clause in where_clauses:
Expand Down Expand Up @@ -503,7 +500,7 @@ def _make_record_batches(
_LOG.debug("New record, dataId=%s region=%s", dataId.mapping, region)
# _LOG.debug("New record, records=%s", dataId.records)

self._exposure_region_factory.set(dataId, region)
self._derived_region_factory.set(dataId, region)
record = self.record_factory(ref)
if record is None:
continue
Expand Down
73 changes: 73 additions & 0 deletions python/lsst/dax/obscore/plugins.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
# This file is part of dax_obscore.
#
# Developed for the LSST Data Management System.
# This product includes software developed by the LSST Project
# (http://www.lsst.org).
# See the COPYRIGHT file at the top-level directory of this distribution
# for details of code ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from __future__ import annotations

__all__ = ["get_siav2_handler"]

from importlib.metadata import EntryPoint, entry_points
from typing import TYPE_CHECKING

from lsst.utils.introspection import get_full_type_name

if TYPE_CHECKING:
from .siav2 import SIAv2Handler


def _get_siav2_entry_points() -> dict[str, EntryPoint]:
plugins = entry_points(group="dax_obscore.siav2")
return {p.name: p for p in plugins}


def get_siav2_handler(namespace: str) -> type[SIAv2Handler]:
"""Select the correct handler for this universe namespace.
Parameters
----------
namespace : `str`
The butler dimension universe namespace.
Returns
-------
handler : `type` [ `lsst.dax.obscore.siav2.SIAv2Handler` ]
The class of handler suitable for this namespace.
"""
plugins = _get_siav2_entry_points()
entry_point = plugins.get(namespace)
if entry_point is None:
known = ", ".join(plugins.keys())
raise RuntimeError(
f"Unable to find suitable SIAv2 Handler for namespace {namespace} [do understand: {known}]"
)
func = entry_point.load()
handler_type = func()

# Check that we have the right type. This has to be a deferred load but
# the code should already be loaded at this point.
from .siav2 import SIAv2Handler

if not issubclass(handler_type, SIAv2Handler):
raise TypeError(
f"Entry point for universe {namespace} did not return SIAv2Handler. "
f"Returned {get_full_type_name(handler_type)}"
)

return func()
Loading

0 comments on commit d1a357d

Please sign in to comment.