Skip to content

Commit

Permalink
Add two helper models for SIAv2
Browse files Browse the repository at this point in the history
* SIAv2Parameters parses the standard parameters into Python form.
* Interval represents a floating point interval such as that used
  by BAND and TIME.
  • Loading branch information
timj committed Aug 30, 2024
1 parent 9ebe3a0 commit 96ae5c3
Showing 1 changed file with 110 additions and 82 deletions.
192 changes: 110 additions & 82 deletions python/lsst/dax/obscore/siav2.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,40 +26,106 @@
import logging
import math
from collections import defaultdict
from collections.abc import Iterable
from typing import Any
from collections.abc import Iterable, Iterator
from typing import Any, Self

import astropy.io.votable
import astropy.time
from lsst.daf.butler import Butler, DimensionGroup, Timespan
from lsst.daf.butler.pydantic_utils import SerializableRegion, SerializableTime
from lsst.sphgeom import Region
from pydantic import BaseModel

from .config import ExporterConfig, WhereBind
from .obscore_exporter import ObscoreExporter

_LOG = logging.getLogger(__name__)


def _overlaps(start1: float, end1: float, start2: float, end2: float) -> bool:
"""Return whether range (start1, end1) overlaps with (start2, end2)
class Interval(BaseModel):
"""Representation of a simple interval."""

Parameters
----------
start1 : `float`
Start of first range.
end1 : `float`
End of first range.
start2 : `float`
Start of second range.
end2 : `float`
End of second range.
start: float
"""Start of the interval."""
end: float
"""End of the interval."""

Returns
-------
overlaps : `bool`
`True` if range 1 overlaps range 2.
"""
return end1 >= start2 and end2 >= start1
@classmethod
def from_string(cls, string: str) -> Self:
"""Create interval from string of form 'START END'.
Parameters
----------
string : `str`
String representing the interval. +Inf and -Inf are allowed.
If there is only one number the start and end interval are the
same.
Returns
-------
interval : `Interval`
The derived interval.
"""
interval = [float(b) for b in string.split()]
if len(interval) == 1:
interval.append(interval[0])
return cls(start=interval[0], end=interval[1])

Check warning on line 72 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L71-L72

Added lines #L71 - L72 were not covered by tests

def overlaps(self, other: Interval) -> bool:
"""Return whether two intervals overlap.
Parameters
----------
other : `Interval`
Interval to check against.
Returns
-------
overlaps : `bool`
`True` if this interval overlaps the other.
"""
return self.end >= other.start and other.end >= self.start

Check warning on line 87 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L87

Added line #L87 was not covered by tests

def __iter__(self) -> Iterator[float]: # type: ignore
return iter((self.start, self.end))

Check warning on line 90 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L90

Added line #L90 was not covered by tests


class SIAv2Parameters(BaseModel):
"""Parsed versions of SIAv2 parameters."""

instrument: str | None = None
pos: SerializableRegion | None = None
time: Timespan | SerializableTime | None = None
band: Interval | None = None
exptime: Interval | None = None

@classmethod
def from_siav2(
cls, instrument: str = "", pos: str = "", time: str = "", band: str = "", exptime: str = ""
) -> Self:
parsed: dict[str, Any] = {}

Check warning on line 106 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L106

Added line #L106 was not covered by tests
if instrument:
parsed["instrument"] = instrument

Check warning on line 108 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L108

Added line #L108 was not covered by tests
if pos:
parsed["pos"] = Region.from_ivoa_pos(pos)

Check warning on line 110 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L110

Added line #L110 was not covered by tests
if band:
parsed["band"] = Interval.from_string(band)

Check warning on line 112 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L112

Added line #L112 was not covered by tests
if exptime:
parsed["exptime"] = Interval.from_string(exptime)

Check warning on line 114 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L114

Added line #L114 was not covered by tests
if time:
time_interval = Interval.from_string(time)

Check warning on line 116 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L116

Added line #L116 was not covered by tests
if time_interval.start == time_interval.end:
parsed["time"] = astropy.time.Time(time_interval.start, scale="utc", format="mjd")

Check warning on line 118 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L118

Added line #L118 was not covered by tests
else:
times: list[astropy.time.Time | None] = []

Check warning on line 120 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L120

Added line #L120 was not covered by tests
for t in time_interval:
if not math.isfinite(t):
# Timespan uses None to indicate unbounded.
times.append(None)

Check warning on line 124 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L124

Added line #L124 was not covered by tests
else:
times.append(astropy.time.Time(float(t), scale="utc", format="mjd"))
parsed["time"] = Timespan(times[0], times[1])
return cls.model_validate(parsed)

Check warning on line 128 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L126-L128

Added lines #L126 - L128 were not covered by tests


class SIAv2Handler:
Expand All @@ -78,25 +144,6 @@ def __init__(self, butler: Butler, config: ExporterConfig):
self.butler = butler
self.config = config

Check warning on line 145 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L144-L145

Added lines #L144 - L145 were not covered by tests

@staticmethod
def _process_interval(param: str) -> tuple[float, float]:
"""Extract interval of two floats from string.
Parameters
----------
param : `str`
String of form "N1 N2" or "N1" of numbers.
Returns
-------
interval : `tuple` [ `float`, `float` ]
Two numbers. If only one is found it will be duplicated.
"""
interval = [float(b) for b in param.split()]
if len(interval) == 1:
interval.append(interval[0])
return interval[0], interval[1] # For mypy.

def get_all_instruments(self) -> list[str]:
"""Query butler for all known instruments.
Expand All @@ -109,16 +156,14 @@ def get_all_instruments(self) -> list[str]:
records = query.dimension_records("instrument")

Check warning on line 156 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L156

Added line #L156 was not covered by tests
return [rec.name for rec in records]

def get_band_information(
self, instruments: list[str], band_interval: tuple[float, float]
) -> dict[str, Any]:
def get_band_information(self, instruments: list[str], band_interval: Interval) -> dict[str, Any]:
"""Read all information from butler necessary to form a band query.
Parameters
----------
instruments : `list` [ `str` ]
Instruments that could be involved in the band query.
band_interval : `tuple` [ `float`, `float` ]
band_interval : `Interval`
The band constraints.
Returns
Expand Down Expand Up @@ -146,9 +191,10 @@ def get_band_information(
records = records.where(f"instrument IN ({instrs})")

Check warning on line 191 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L191

Added line #L191 was not covered by tests
for rec in records:
if (spec_range := self.config.spectral_ranges.get(rec.name)) is not None:
spec_interval = Interval(start=spec_range[0], end=spec_range[1])
assert spec_range[0] is not None # for mypy
assert spec_range[1] is not None

Check warning on line 196 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L194-L196

Added lines #L194 - L196 were not covered by tests
if _overlaps(band_interval[0], band_interval[1], spec_range[0], spec_range[1]):
if band_interval.overlaps(spec_interval):
matching_filters[rec.instrument].add(rec.name)
matching_bands.add(rec.band)

Check warning on line 199 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L198-L199

Added lines #L198 - L199 were not covered by tests
else:
Expand Down Expand Up @@ -188,60 +234,44 @@ def process_query(
present since some queries are incompatible with some dataset
types.
"""
# Store parsed versions of input parameters when not simple strings.
parsed: dict[str, Region | tuple[float, float] | astropy.time.Time | Timespan] = {}
# Parse the SIAv2 parameters
parsed = SIAv2Parameters.from_siav2(

Check warning on line 238 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L238

Added line #L238 was not covered by tests
instrument=instrument, pos=pos, band=band, time=time, exptime=exptime
)

if instrument:
parsed["instrument"] = [instrument]
if parsed.instrument:
instruments = [instrument]

Check warning on line 243 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L243

Added line #L243 was not covered by tests
else:
# If no explicit instrument, then all instruments could be valid.
# Some queries like band require the instrument so ask the butler
# for all values up front.
parsed["instruments"] = self.get_all_instruments()
if pos:
parsed["region"] = Region.from_ivoa_pos(pos)
if band:
parsed["band"] = self._process_interval(band)
parsed["band_info"] = self.get_band_information(parsed["instruments"], parsed["band"])
if exptime:
parsed["exptime"] = self._process_interval(exptime)
if time:
time_interval = self._process_interval(time)
if time_interval[0] == time_interval[1]:
parsed["time"] = astropy.time.Time(time_interval[0], scale="utc", format="mjd")
else:
times: list[astropy.time.Time | None] = []
for t in time_interval:
if not math.isfinite(t):
# Timespan uses None to indicate unbounded.
times.append(None)
else:
times.append(astropy.time.Time(float(t), scale="utc", format="mjd"))
parsed["time"] = Timespan(times[0], times[1])
instruments = self.get_all_instruments()

Check warning on line 248 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L248

Added line #L248 was not covered by tests

band_info: dict[str, Any] = {}

Check warning on line 250 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L250

Added line #L250 was not covered by tests
if parsed.band:
band_info = self.get_band_information(instruments, parsed.band)

Check warning on line 252 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L252

Added line #L252 was not covered by tests

# Loop over each dataset type calculating custom query parameters.
dataset_type_wheres = {}

Check warning on line 255 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L255

Added line #L255 was not covered by tests
for dataset_type_name in list(self.config.dataset_types):
wheres = []
dataset_type = self.butler.get_dataset_type(dataset_type_name)
dims = dataset_type.dimensions
instrument_wheres, where = self.from_instrument_or_band(
parsed["instruments"], parsed.get("band_info", {}), dims
)
instrument_wheres, where = self.from_instrument_or_band(instruments, band_info, dims)

Check warning on line 260 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L257-L260

Added lines #L257 - L260 were not covered by tests
if where is not None:
wheres.append(where)

Check warning on line 262 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L262

Added line #L262 was not covered by tests

if pos:
where = self.from_pos(parsed["region"], dims)
if parsed.pos:
where = self.from_pos(parsed.pos, dims)

Check warning on line 265 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L265

Added line #L265 was not covered by tests
if not where:
_LOG.warning(

Check warning on line 267 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L267

Added line #L267 was not covered by tests
"Can not support POS query for dataset type %s. Skipping it.", dataset_type_name
)
continue
wheres.append(where)

Check warning on line 271 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L270-L271

Added lines #L270 - L271 were not covered by tests

if time:
where = self.from_time(parsed["time"], dims)
if parsed.time:
where = self.from_time(parsed.time, dims)

Check warning on line 274 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L274

Added line #L274 was not covered by tests
if not where:
_LOG.warning(

Check warning on line 276 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L276

Added line #L276 was not covered by tests
"Dataset type %s has no timespan defined so assuming all datasets match.",
Expand All @@ -250,8 +280,8 @@ def process_query(
else:
wheres.append(where)

Check warning on line 281 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L281

Added line #L281 was not covered by tests

if exptime:
exptime_wheres = self.from_exptime(parsed["exptime"], dims)
if parsed.exptime:
exptime_wheres = self.from_exptime(parsed.exptime, dims)

Check warning on line 284 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L284

Added line #L284 was not covered by tests
if exptime_wheres is None:
_LOG.warning(

Check warning on line 286 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L286

Added line #L286 was not covered by tests
"Can not support EXPTIME query for dataset type %s. Skipping it.", dataset_type_name
Expand Down Expand Up @@ -371,14 +401,12 @@ def from_time(self, ts: astropy.time.Time | Timespan, dimensions: DimensionGroup
return None
return WhereBind(where=f"{time_dim}.timespan OVERLAPS(ts)", bind={"ts": ts})

Check warning on line 402 in python/lsst/dax/obscore/siav2.py

View check run for this annotation

Codecov / codecov/patch

python/lsst/dax/obscore/siav2.py#L401-L402

Added lines #L401 - L402 were not covered by tests

def from_exptime(
self, exptime_interval: tuple[float, float], dimensions: DimensionGroup
) -> list[WhereBind] | None:
def from_exptime(self, exptime_interval: Interval, dimensions: DimensionGroup) -> list[WhereBind] | None:
"""Convert an exposure time interval to a butler where clause.
Parameters
----------
exptime_interval : `tuple` [`float`, `float` ]
exptime_interval : `Interval`
The exposure time interval of interest as two floating point UTC
MJDs.
dimensions : `lsst.daf.butler.DimensionGroup`
Expand Down

0 comments on commit 96ae5c3

Please sign in to comment.