From 09f44fed85e8ed56b6667895860da545792337d4 Mon Sep 17 00:00:00 2001 From: aladinor Date: Fri, 20 Feb 2026 14:47:09 -0600 Subject: [PATCH 1/8] ENH: add xarray-native open_datatree with engine= parameter Implement open_datatree/open_groups_as_dict on three prototype backends (ODIM, CfRadial1, NEXRAD Level2) with supports_groups=True, enabling xr.open_datatree(file, engine="odim") and a unified xd.open_datatree() entry point. Deprecate standalone open_*_datatree functions with FutureWarning. Closes #329 Co-Authored-By: Claude Opus 4.6 --- examples/notebooks/Open-Datatree-Engine.ipynb | 420 ++++++++++++++++++ requirements.txt | 2 +- tests/io/test_backend_datatree.py | 280 ++++++++++++ xradar/__init__.py | 1 + xradar/io/__init__.py | 41 ++ xradar/io/backends/cfradial1.py | 119 +++-- xradar/io/backends/common.py | 51 +++ xradar/io/backends/nexrad_level2.py | 199 +++++---- xradar/io/backends/odim.py | 127 ++++-- 9 files changed, 1070 insertions(+), 170 deletions(-) create mode 100644 examples/notebooks/Open-Datatree-Engine.ipynb create mode 100644 tests/io/test_backend_datatree.py diff --git a/examples/notebooks/Open-Datatree-Engine.ipynb b/examples/notebooks/Open-Datatree-Engine.ipynb new file mode 100644 index 00000000..826e4c2c --- /dev/null +++ b/examples/notebooks/Open-Datatree-Engine.ipynb @@ -0,0 +1,420 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Open DataTree with `engine=` parameter\n", + "\n", + "This notebook demonstrates the new unified `open_datatree` API that allows opening radar files as `xarray.DataTree` using the `engine=` parameter.\n", + "\n", + "Three ways to open a DataTree:\n", + "- `xd.open_datatree(file, engine=\"...\")` — xradar unified API\n", + "- `xr.open_datatree(file, engine=\"...\")` — xarray native API\n", + "- `xd.io.open_*_datatree(file)` — legacy per-format functions (deprecated, emit `FutureWarning`)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "\n", + "import xarray as xr\n", + "from open_radar_data import DATASETS\n", + "\n", + "import xradar as xd" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "## Download test data\n", + "\n", + "Fetching radar data files from [open-radar-data](https://github.com/openradar/open-radar-data) repository." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "odim_file = DATASETS.fetch(\"71_20181220_060628.pvol.h5\")\n", + "cfradial1_file = DATASETS.fetch(\"cfrad.20080604_002217_000_SPOL_v36_SUR.nc\")\n", + "nexrad_file = DATASETS.fetch(\"KATX20130717_195021_V06\")" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "## 1. `xd.open_datatree()` — Unified xradar API\n", + "\n", + "The new unified entry point. Specify the `engine` to select the backend." + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "### ODIM_H5" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "dtree = xd.open_datatree(odim_file, engine=\"odim\")\n", + "display(dtree)" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "The tree follows the CfRadial2 group structure with metadata groups at the root level and sweep groups below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "# Root dataset contains global metadata\n", + "display(dtree.ds)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "# Access a specific sweep\n", + "display(dtree[\"sweep_0\"].ds)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "# Metadata groups\n", + "print(\"radar_parameters:\", list(dtree[\"radar_parameters\"].ds.data_vars))\n", + "print(\"georeferencing_correction:\", list(dtree[\"georeferencing_correction\"].ds.data_vars))\n", + "print(\"radar_calibration:\", list(dtree[\"radar_calibration\"].ds.data_vars))" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "### CfRadial1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "dtree = xd.open_datatree(cfradial1_file, engine=\"cfradial1\")\n", + "display(dtree)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "dtree[\"sweep_0\"].ds.DBZ.plot()" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "### NEXRAD Level 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "dtree = xd.open_datatree(nexrad_file, engine=\"nexradlevel2\")\n", + "display(dtree)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "dtree[\"sweep_0\"].ds.DBZH.plot()" + ] + }, + { + "cell_type": "markdown", + "id": "17", + "metadata": {}, + "source": [ + "## 2. Sweep selection\n", + "\n", + "Select specific sweeps by index (int or list) or by name." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "# Single sweep by index\n", + "dtree = xd.open_datatree(odim_file, engine=\"odim\", sweep=0)\n", + "print(\"Children:\", list(dtree.children))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "# Multiple sweeps by index\n", + "dtree = xd.open_datatree(odim_file, engine=\"odim\", sweep=[0, 2, 4])\n", + "print(\"Children:\", list(dtree.children))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "# Sweeps by name\n", + "dtree = xd.open_datatree(cfradial1_file, engine=\"cfradial1\", sweep=[\"sweep_0\", \"sweep_3\"])\n", + "print(\"Children:\", list(dtree.children))" + ] + }, + { + "cell_type": "markdown", + "id": "21", + "metadata": {}, + "source": [ + "## 3. Backend kwargs\n", + "\n", + "Pass backend-specific options directly as keyword arguments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "# first_dim controls the leading dimension (\"auto\" uses azimuth/elevation)\n", + "# site_coords attaches latitude/longitude/altitude to sweep datasets\n", + "dtree = xd.open_datatree(\n", + " odim_file,\n", + " engine=\"odim\",\n", + " sweep=[0],\n", + " first_dim=\"auto\",\n", + " site_coords=True,\n", + ")\n", + "sweep_ds = dtree[\"sweep_0\"].ds\n", + "print(\"Dimensions:\", dict(sweep_ds.dims))\n", + "print(\"Site coords present:\", \"latitude\" in sweep_ds.coords)" + ] + }, + { + "cell_type": "markdown", + "id": "23", + "metadata": {}, + "source": [ + "## 4. `xr.open_datatree()` — xarray native API\n", + "\n", + "The same backends work directly with xarray's native `open_datatree`, no xradar wrapper needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "dtree = xr.open_datatree(odim_file, engine=\"odim\")\n", + "display(dtree)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "dtree = xr.open_datatree(nexrad_file, engine=\"nexradlevel2\", sweep=[0, 1])\n", + "display(dtree)" + ] + }, + { + "cell_type": "markdown", + "id": "26", + "metadata": {}, + "source": [ + "## 5. `open_groups_as_dict()` — Low-level access\n", + "\n", + "For advanced use, get the raw `dict[str, Dataset]` before it becomes a DataTree." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "from xradar.io.backends.odim import OdimBackendEntrypoint\n", + "\n", + "backend = OdimBackendEntrypoint()\n", + "groups = backend.open_groups_as_dict(odim_file, sweep=[0, 1])\n", + "\n", + "print(\"Group keys:\", list(groups.keys()))\n", + "print()\n", + "print(\"Root dataset:\")\n", + "display(groups[\"/\"])" + ] + }, + { + "cell_type": "markdown", + "id": "28", + "metadata": {}, + "source": [ + "## 6. Backward compatibility — deprecated functions\n", + "\n", + "The legacy per-format functions still work but emit a `FutureWarning` directing you to the new API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "with warnings.catch_warnings(record=True) as w:\n", + " warnings.simplefilter(\"always\")\n", + " dtree_old = xd.io.open_odim_datatree(odim_file, sweep=[0])\n", + " for warning in w:\n", + " if issubclass(warning.category, FutureWarning):\n", + " print(f\"FutureWarning: {warning.message}\")\n", + "\n", + "display(dtree_old)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30", + "metadata": {}, + "outputs": [], + "source": [ + "# The old and new APIs produce equivalent results\n", + "dtree_new = xd.open_datatree(odim_file, engine=\"odim\", sweep=[0])\n", + "print(\"Same children:\", set(dtree_old.children) == set(dtree_new.children))" + ] + }, + { + "cell_type": "markdown", + "id": "31", + "metadata": {}, + "source": [ + "## 7. Error handling" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "32", + "metadata": {}, + "outputs": [], + "source": [ + "# Unknown engine raises a clear error\n", + "try:\n", + " xd.open_datatree(odim_file, engine=\"nonexistent\")\n", + "except ValueError as e:\n", + " print(f\"ValueError: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "33", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "| API | Example | Status |\n", + "|-----|---------|--------|\n", + "| `xd.open_datatree(file, engine=\"odim\")` | Unified xradar API | **New** |\n", + "| `xr.open_datatree(file, engine=\"odim\")` | xarray native API | **New** |\n", + "| `xd.io.open_odim_datatree(file)` | Per-format function | Deprecated |\n", + "\n", + "Supported engines: `\"odim\"`, `\"cfradial1\"`, `\"nexradlevel2\"`" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbformat_minor": 5, + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/requirements.txt b/requirements.txt index 43be7fae..2c88e4ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ dask h5netcdf >= 1.0.0 h5py >= 3.0.0 lat_lon_parser -netCDF4 +netCDF4 >= 1.5.0, != 1.7.3, != 1.7.4 numpy pyproj scipy diff --git a/tests/io/test_backend_datatree.py b/tests/io/test_backend_datatree.py new file mode 100644 index 00000000..1f3f0a09 --- /dev/null +++ b/tests/io/test_backend_datatree.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python +# Copyright (c) 2024-2025, openradar developers. +# Distributed under the MIT License. See LICENSE for more info. + +""" +Tests for xarray-native open_datatree with engine= parameter. + +Tests the unified ``xd.open_datatree()`` and ``xr.open_datatree()`` APIs, +``open_groups_as_dict()`` direct calls, backward compatibility with +deprecated standalone functions, and ``supports_groups`` attribute. +""" + +import warnings + +import pytest +import xarray as xr +from xarray import DataTree + +import xradar as xd +from xradar.io.backends.cfradial1 import ( + CfRadial1BackendEntrypoint, + open_cfradial1_datatree, +) +from xradar.io.backends.nexrad_level2 import ( + NexradLevel2BackendEntrypoint, + open_nexradlevel2_datatree, +) +from xradar.io.backends.odim import OdimBackendEntrypoint, open_odim_datatree + +# -- Fixtures ---------------------------------------------------------------- + + +@pytest.fixture( + params=[ + pytest.param("odim", id="odim"), + pytest.param("nexradlevel2", id="nexradlevel2"), + ] +) +def engine_and_file(request, odim_file, nexradlevel2_file): + """Parametrize over engines that do not require netCDF4.""" + mapping = { + "odim": odim_file, + "nexradlevel2": nexradlevel2_file, + } + return request.param, mapping[request.param] + + +@pytest.fixture +def cfradial1_engine_file(cfradial1_file): + return "cfradial1", cfradial1_file + + +# -- CfRadial2 structure keys ----------------------------------------------- + +REQUIRED_GROUPS = { + "/", + "/radar_parameters", + "/georeferencing_correction", + "/radar_calibration", +} + + +# -- Helper ------------------------------------------------------------------ + + +def _assert_cfradial2_structure(dtree): + """Verify that a DataTree has CfRadial2 group structure.""" + assert isinstance(dtree, DataTree) + children = set(dtree.children.keys()) + # Must have metadata groups + for grp in ["radar_parameters", "georeferencing_correction", "radar_calibration"]: + assert grp in children, f"Missing group: {grp}" + # Must have at least one sweep + sweep_groups = [k for k in children if k.startswith("sweep_")] + assert len(sweep_groups) > 0, "No sweep groups found" + # Root must have key variables + root_vars = set(dtree.ds.data_vars) + assert "time_coverage_start" in root_vars + assert "time_coverage_end" in root_vars + + +# -- xd.open_datatree integration tests (ODIM, NEXRAD) ---------------------- + + +class TestXdOpenDatatree: + """Test xd.open_datatree() for ODIM and NEXRAD.""" + + def test_basic_open(self, engine_and_file): + engine, filepath = engine_and_file + dtree = xd.open_datatree(filepath, engine=engine) + _assert_cfradial2_structure(dtree) + + def test_sweep_selection_list(self, engine_and_file): + engine, filepath = engine_and_file + dtree = xd.open_datatree(filepath, engine=engine, sweep=[0, 1]) + sweep_groups = [k for k in dtree.children if k.startswith("sweep_")] + assert len(sweep_groups) == 2 + + def test_sweep_selection_int(self, engine_and_file): + engine, filepath = engine_and_file + dtree = xd.open_datatree(filepath, engine=engine, sweep=0) + sweep_groups = [k for k in dtree.children if k.startswith("sweep_")] + assert len(sweep_groups) == 1 + + def test_kwargs_flow_through(self, engine_and_file): + engine, filepath = engine_and_file + dtree = xd.open_datatree( + filepath, + engine=engine, + first_dim="auto", + site_coords=True, + sweep=[0], + ) + sweep_ds = dtree["sweep_0"].ds + assert "latitude" in sweep_ds.coords + assert "longitude" in sweep_ds.coords + assert "altitude" in sweep_ds.coords + + def test_unknown_engine_raises(self, odim_file): + with pytest.raises(ValueError, match="Unknown engine"): + xd.open_datatree(odim_file, engine="nonexistent_engine") + + +# -- xd.open_datatree for CfRadial1 ----------------------------------------- + + +class TestXdOpenDatatreeCfRadial1: + """Test xd.open_datatree() for CfRadial1 (requires h5netcdf in this env).""" + + def test_basic_open(self, cfradial1_engine_file): + _, filepath = cfradial1_engine_file + backend = CfRadial1BackendEntrypoint() + dtree = backend.open_datatree( + filepath, engine="h5netcdf", decode_timedelta=False + ) + _assert_cfradial2_structure(dtree) + + def test_sweep_selection(self, cfradial1_engine_file): + _, filepath = cfradial1_engine_file + backend = CfRadial1BackendEntrypoint() + dtree = backend.open_datatree( + filepath, + engine="h5netcdf", + decode_timedelta=False, + sweep=[0, 1], + ) + sweep_groups = [k for k in dtree.children if k.startswith("sweep_")] + assert len(sweep_groups) == 2 + + +# -- xr.open_datatree tests ------------------------------------------------- + + +class TestXrOpenDatatree: + """Test xr.open_datatree() with xradar engines.""" + + def test_xr_open_datatree_odim(self, odim_file): + dtree = xr.open_datatree(odim_file, engine="odim") + _assert_cfradial2_structure(dtree) + + def test_xr_open_datatree_nexrad(self, nexradlevel2_file): + dtree = xr.open_datatree(nexradlevel2_file, engine="nexradlevel2") + _assert_cfradial2_structure(dtree) + + +# -- open_groups_as_dict direct tests ---------------------------------------- + + +class TestOpenGroupsAsDict: + """Test open_groups_as_dict() returns correct dict structure.""" + + def test_odim_groups_dict(self, odim_file): + backend = OdimBackendEntrypoint() + groups = backend.open_groups_as_dict(odim_file, sweep=[0, 1]) + assert isinstance(groups, dict) + assert "/" in groups + assert "/radar_parameters" in groups + assert "/georeferencing_correction" in groups + assert "/radar_calibration" in groups + assert "/sweep_0" in groups + assert "/sweep_1" in groups + for key, ds in groups.items(): + assert isinstance(ds, xr.Dataset), f"{key} is not a Dataset" + + def test_nexrad_groups_dict(self, nexradlevel2_file): + backend = NexradLevel2BackendEntrypoint() + groups = backend.open_groups_as_dict(nexradlevel2_file, sweep=[0, 1]) + assert isinstance(groups, dict) + assert "/" in groups + assert "/sweep_0" in groups + assert "/sweep_1" in groups + + def test_cfradial1_groups_dict(self, cfradial1_file): + backend = CfRadial1BackendEntrypoint() + groups = backend.open_groups_as_dict( + cfradial1_file, + engine="h5netcdf", + decode_timedelta=False, + sweep=[0, 1], + ) + assert isinstance(groups, dict) + assert "/" in groups + assert "/sweep_0" in groups + assert "/sweep_1" in groups + + +# -- supports_groups attribute ----------------------------------------------- + + +class TestSupportsGroups: + """Verify supports_groups is True on all 3 backend classes.""" + + def test_odim_supports_groups(self): + assert OdimBackendEntrypoint.supports_groups is True + + def test_cfradial1_supports_groups(self): + assert CfRadial1BackendEntrypoint.supports_groups is True + + def test_nexrad_supports_groups(self): + assert NexradLevel2BackendEntrypoint.supports_groups is True + + +# -- Backward compatibility & deprecation tests ------------------------------ + + +class TestDeprecation: + """Test that standalone functions still work but emit FutureWarning.""" + + def test_open_odim_datatree_deprecation(self, odim_file): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + dtree = open_odim_datatree(odim_file, sweep=[0]) + deprecation_warnings = [ + x for x in w if issubclass(x.category, FutureWarning) + ] + assert len(deprecation_warnings) >= 1 + assert "open_odim_datatree" in str(deprecation_warnings[0].message) + _assert_cfradial2_structure(dtree) + + def test_open_cfradial1_datatree_deprecation(self, cfradial1_file): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + dtree = open_cfradial1_datatree( + cfradial1_file, + engine="h5netcdf", + decode_timedelta=False, + sweep=[0], + ) + deprecation_warnings = [ + x for x in w if issubclass(x.category, FutureWarning) + ] + assert len(deprecation_warnings) >= 1 + assert "open_cfradial1_datatree" in str(deprecation_warnings[0].message) + _assert_cfradial2_structure(dtree) + + def test_open_nexradlevel2_datatree_deprecation(self, nexradlevel2_file): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + dtree = open_nexradlevel2_datatree(nexradlevel2_file, sweep=[0]) + deprecation_warnings = [ + x for x in w if issubclass(x.category, FutureWarning) + ] + assert len(deprecation_warnings) >= 1 + assert "open_nexradlevel2_datatree" in str(deprecation_warnings[0].message) + _assert_cfradial2_structure(dtree) + + def test_odim_deprecated_output_matches_new_api(self, odim_file): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + old = open_odim_datatree(odim_file, sweep=[0, 1]) + new = xd.open_datatree(odim_file, engine="odim", sweep=[0, 1]) + # Same number of children + assert set(old.children.keys()) == set(new.children.keys()) + + def test_nexrad_deprecated_output_matches_new_api(self, nexradlevel2_file): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + old = open_nexradlevel2_datatree(nexradlevel2_file, sweep=[0, 1]) + new = xd.open_datatree(nexradlevel2_file, engine="nexradlevel2", sweep=[0, 1]) + assert set(old.children.keys()) == set(new.children.keys()) diff --git a/xradar/__init__.py b/xradar/__init__.py index 4a990dca..b6ce6c7f 100644 --- a/xradar/__init__.py +++ b/xradar/__init__.py @@ -29,5 +29,6 @@ from . import util # noqa from .util import map_over_sweeps # noqa from . import transform # noqa +from .io import open_datatree # noqa __all__ = [s for s in dir() if not s.startswith("_")] diff --git a/xradar/io/__init__.py b/xradar/io/__init__.py index 3693e99e..3a3201c0 100644 --- a/xradar/io/__init__.py +++ b/xradar/io/__init__.py @@ -17,4 +17,45 @@ from .backends import * # noqa from .export import * # noqa +from .backends.cfradial1 import CfRadial1BackendEntrypoint +from .backends.nexrad_level2 import NexradLevel2BackendEntrypoint +from .backends.odim import OdimBackendEntrypoint + +#: Registry mapping engine names to backend classes that support groups. +_ENGINE_REGISTRY = { + "odim": OdimBackendEntrypoint, + "cfradial1": CfRadial1BackendEntrypoint, + "nexradlevel2": NexradLevel2BackendEntrypoint, +} + + +def open_datatree(filename_or_obj, *, engine, **kwargs): + """Open a radar file as :py:class:`xarray.DataTree` using the specified engine. + + Parameters + ---------- + filename_or_obj : str, Path, or file-like + Path to the radar file. + engine : str + Backend engine name (e.g., ``"odim"``, ``"cfradial1"``, ``"nexradlevel2"``). + **kwargs + Additional keyword arguments passed to the backend's ``open_datatree`` method. + + Returns + ------- + dtree : xarray.DataTree + DataTree with CfRadial2 group structure. + + Examples + -------- + >>> import xradar as xd + >>> dtree = xd.open_datatree("file.h5", engine="odim") + """ + if engine not in _ENGINE_REGISTRY: + supported = ", ".join(sorted(_ENGINE_REGISTRY)) + raise ValueError(f"Unknown engine {engine!r}. Supported engines: {supported}") + backend = _ENGINE_REGISTRY[engine]() + return backend.open_datatree(filename_or_obj, **kwargs) + + __all__ = [s for s in dir() if not s.startswith("_")] diff --git a/xradar/io/backends/cfradial1.py b/xradar/io/backends/cfradial1.py index 81b8760d..1c48e31f 100644 --- a/xradar/io/backends/cfradial1.py +++ b/xradar/io/backends/cfradial1.py @@ -53,6 +53,7 @@ _STATION_VARS, _apply_site_as_coords, _attach_sweep_groups, + _deprecation_warning, _maybe_decode, ) @@ -337,6 +338,10 @@ def _get_radar_calibration(ds): def open_cfradial1_datatree(filename_or_obj, **kwargs): """Open CfRadial1 dataset as :py:class:`xarray.DataTree`. + .. deprecated:: + Use ``xd.open_datatree(file, engine="cfradial1")`` or + ``xr.open_datatree(file, engine="cfradial1")`` instead. + Parameters ---------- filename_or_obj : str, Path, file-like or xarray.DataStore @@ -369,47 +374,28 @@ def open_cfradial1_datatree(filename_or_obj, **kwargs): dtree: xarray.DataTree DataTree with CfRadial2 groups. """ + _deprecation_warning("open_cfradial1_datatree", "cfradial1") - # handle kwargs, extract first_dim + # Bridge old kwargs to direct kwargs first_dim = kwargs.pop("first_dim", "auto") optional = kwargs.pop("optional", True) optional_groups = kwargs.pop("optional_groups", False) kwargs.pop("site_as_coords", None) sweep = kwargs.pop("sweep", None) engine = kwargs.pop("engine", "netcdf4") - # needed for new xarray literal timedelta decoding - kwargs.update(decode_timedelta=kwargs.pop("decode_timedelta", False)) - - # open root group, cfradial1 only has one group - # open_cfradial1_datatree only opens the file once using netcdf4 - # and retrieves the different groups from the loaded object - ds = open_dataset(filename_or_obj, engine=engine, **kwargs) + kwargs.setdefault("decode_timedelta", False) - # create datatree root node additional root metadata groups - dtree: dict = { - "/": _get_required_root_dataset(ds, optional=optional), - } - if optional_groups: - dtree["/radar_parameters"] = _get_subgroup(ds, radar_parameters_subgroup) - dtree["/georeferencing_correction"] = _get_subgroup( - ds, georeferencing_correction_subgroup - ) - dtree["/radar_calibration"] = _get_radar_calibration(ds) - - # radar_calibration (connected with calib-dimension) - dtree = _attach_sweep_groups( - dtree, - list( - _get_sweep_groups( - ds, - sweep=sweep, - first_dim=first_dim, - optional=optional, - site_as_coords=False, - ).values() - ), + _deprecation_warning("open_cfradial1_datatree", "cfradial1") + return CfRadial1BackendEntrypoint().open_datatree( + filename_or_obj, + first_dim=first_dim, + optional=optional, + optional_groups=optional_groups, + site_coords=site_coords, + sweep=sweep, + engine=engine, + **kwargs, ) - return DataTree.from_dict(dtree) class CfRadial1BackendEntrypoint(BackendEntrypoint): @@ -434,6 +420,7 @@ class CfRadial1BackendEntrypoint(BackendEntrypoint): description = "Open CfRadial1 (.nc, .nc4) using netCDF4 in Xarray" url = "https://xradar.rtfd.io/en/latest/io.html#cfradial1" + supports_groups = True def open_dataset( self, @@ -492,3 +479,71 @@ def open_dataset( ds._close = store.close return ds + + def open_groups_as_dict( + self, + filename_or_obj, + *, + mask_and_scale=True, + decode_times=True, + concat_characters=True, + decode_coords=True, + drop_variables=None, + use_cftime=None, + decode_timedelta=False, + first_dim="auto", + reindex_angle=False, + fix_second_angle=False, + site_coords=True, + optional=True, + optional_groups=False, + sweep=None, + engine="netcdf4", + ): + # CfRadial1 opens the entire file once + ds = open_dataset( + filename_or_obj, + engine=engine, + mask_and_scale=mask_and_scale, + decode_times=decode_times, + concat_characters=concat_characters, + decode_coords=decode_coords, + drop_variables=drop_variables, + use_cftime=use_cftime, + decode_timedelta=decode_timedelta, + ) + + groups_dict = { + "/": _get_required_root_dataset(ds, optional=optional), + } + if optional_groups: + groups_dict["/radar_parameters"] = _get_subgroup( + ds, radar_parameters_subgroup + ) + groups_dict["/georeferencing_correction"] = _get_subgroup( + ds, georeferencing_correction_subgroup + ) + groups_dict["/radar_calibration"] = _get_radar_calibration(ds) + + sweep_datasets = list( + _get_sweep_groups( + ds, + sweep=sweep, + first_dim=first_dim, + optional=optional, + site_coords=site_coords, + ).values() + ) + + for i, sw_ds in enumerate(sweep_datasets): + groups_dict[f"/sweep_{i}"] = sw_ds.drop_attrs(deep=False) + + return groups_dict + + def open_datatree( + self, + filename_or_obj, + **kwargs, + ): + groups_dict = self.open_groups_as_dict(filename_or_obj, **kwargs) + return DataTree.from_dict(groups_dict) diff --git a/xradar/io/backends/common.py b/xradar/io/backends/common.py index 8b3f4317..a18d1a5d 100644 --- a/xradar/io/backends/common.py +++ b/xradar/io/backends/common.py @@ -14,6 +14,7 @@ import io import struct +import warnings from collections import OrderedDict import h5netcdf @@ -21,8 +22,11 @@ import xarray as xr from ...model import ( + georeferencing_correction_subgroup, optional_root_attrs, optional_root_vars, + radar_calibration_subgroup, + radar_parameters_subgroup, required_global_attrs, required_root_vars, ) @@ -380,6 +384,53 @@ def _prepare_backend_ds(ds): return ds +def _build_groups_dict(ls_ds, optional=True, optional_groups=False): + """Build CfRadial2 groups dict from a list of sweep Datasets. + + Parameters + ---------- + ls_ds : list of xr.Dataset + List of sweep Datasets. + optional : bool + Import optional metadata, defaults to True. + optional_groups : bool + If True, includes ``/radar_parameters``, ``/georeferencing_correction`` + and ``/radar_calibration`` metadata subgroups. Default is False. + + Returns + ------- + groups_dict : dict[str, xr.Dataset] + Dictionary with CfRadial2 group structure. + """ + groups_dict = { + "/": _get_required_root_dataset(ls_ds, optional=optional), + } + if optional_groups: + groups_dict["/radar_parameters"] = _get_subgroup( + ls_ds, radar_parameters_subgroup + ) + groups_dict["/georeferencing_correction"] = _get_subgroup( + ls_ds, georeferencing_correction_subgroup + ) + groups_dict["/radar_calibration"] = _get_radar_calibration( + ls_ds, radar_calibration_subgroup + ) + for i, ds in enumerate(ls_ds): + groups_dict[f"/sweep_{i}"] = ds.drop_attrs(deep=False) + return groups_dict + + +def _deprecation_warning(old_name, engine): + """Emit FutureWarning for deprecated standalone open_*_datatree functions.""" + warnings.warn( + f"`{old_name}` is deprecated. Use " + f'`xd.open_datatree(file, engine="{engine}")` or ' + f'`xr.open_datatree(file, engine="{engine}")` instead.', + FutureWarning, + stacklevel=3, + ) + + # IRIS Data Types and corresponding python struct format characters # 4.2 Scalar Definitions, Page 23 # https://docs.python.org/3/library/struct.html#format-characters diff --git a/xradar/io/backends/nexrad_level2.py b/xradar/io/backends/nexrad_level2.py index d0d4eefe..f8b0e15f 100644 --- a/xradar/io/backends/nexrad_level2.py +++ b/xradar/io/backends/nexrad_level2.py @@ -57,6 +57,7 @@ from xradar.io.backends.common import ( _apply_site_as_coords, _assign_root, + _deprecation_warning, _get_radar_calibration, _get_subgroup, ) @@ -1893,6 +1894,7 @@ class NexradLevel2BackendEntrypoint(BackendEntrypoint): description = "Open NEXRAD Level2 files in Xarray" url = "tbd" + supports_groups = True def open_dataset( self, @@ -1964,6 +1966,102 @@ def open_dataset( return ds + def open_groups_as_dict( + self, + filename_or_obj, + *, + mask_and_scale=True, + decode_times=True, + concat_characters=True, + decode_coords=True, + drop_variables=None, + use_cftime=None, + decode_timedelta=None, + sweep=None, + first_dim="auto", + reindex_angle=False, + fix_second_angle=False, + site_coords=True, + optional=True, + optional_groups=False, + lock=None, + **kwargs, + ): + from xarray.core.treenode import NodePath + + if isinstance(sweep, str): + sweep = NodePath(sweep).name + sweeps = [sweep] + elif isinstance(sweep, int): + sweeps = [f"sweep_{sweep}"] + elif isinstance(sweep, list): + if isinstance(sweep[0], int): + sweeps = [f"sweep_{i}" for i in sweep] + elif isinstance(sweep[0], str): + sweeps = [NodePath(i).name for i in sweep] + else: + raise ValueError( + "Invalid type in 'sweep' list. Expected integers " + "(e.g., [0, 1, 2]) or strings " + "(e.g. [/sweep_0, sweep_1])." + ) + else: + with NEXRADLevel2File(filename_or_obj, loaddata=False) as nex: + if nex.msg_5: + exp_sweeps = nex.msg_5["number_elevation_cuts"] + else: + exp_sweeps = 0 + act_sweeps = len(nex.msg_31_data_header) + if exp_sweeps > act_sweeps: + exp_sweeps = act_sweeps + sweeps = [f"sweep_{i}" for i in range(act_sweeps)] + + sweep_dict = open_sweeps_as_dict( + filename_or_obj=filename_or_obj, + mask_and_scale=mask_and_scale, + decode_times=decode_times, + concat_characters=concat_characters, + decode_coords=decode_coords, + drop_variables=drop_variables, + use_cftime=use_cftime, + decode_timedelta=decode_timedelta, + sweeps=sweeps, + first_dim=first_dim, + reindex_angle=reindex_angle, + fix_second_angle=fix_second_angle, + site_coords=site_coords, + optional=optional, + lock=lock, + **kwargs, + ) + + ls_ds = [sweep_dict[s] for s in sweep_dict] + ls_ds_with_root = [xr.Dataset()] + list(ls_ds) + groups_dict = { + "/": _assign_root(ls_ds_with_root), + } + if optional_groups: + groups_dict["/radar_parameters"] = _get_subgroup( + ls_ds_with_root, radar_parameters_subgroup + ) + groups_dict["/georeferencing_correction"] = _get_subgroup( + ls_ds_with_root, georeferencing_correction_subgroup + ) + groups_dict["/radar_calibration"] = _get_radar_calibration( + ls_ds_with_root, radar_calibration_subgroup + ) + for sweep_path, ds in sweep_dict.items(): + groups_dict[f"/{sweep_path}"] = ds.drop_attrs(deep=False) + return groups_dict + + def open_datatree( + self, + filename_or_obj, + **kwargs, + ): + groups_dict = self.open_groups_as_dict(filename_or_obj, **kwargs) + return DataTree.from_dict(groups_dict) + def open_nexradlevel2_datatree( filename_or_obj, @@ -2074,78 +2172,10 @@ def open_nexradlevel2_datatree( dtree : xarray.DataTree An `xarray.DataTree` representing the radar data organized by sweeps. """ - from xarray.core.treenode import NodePath - - # Handle list/tuple of chunk files or bytes - if isinstance(filename_or_obj, (list, tuple)): - filename_or_obj = _concatenate_chunks(filename_or_obj) - # Validate that the concatenated data starts with a volume header. - # The first chunk must be the S file (volume scan start). - if not filename_or_obj[:4].startswith(_VOLUME_HEADER_PREFIX): - raise ValueError( - "No chunk contains a volume header (AR2V prefix). " - "The first chunk must be the S file (volume scan start) which " - "contains the volume header and metadata. I/E chunks alone " - "cannot be decoded without it." - ) + _deprecation_warning("open_nexradlevel2_datatree", "nexradlevel2") - # Single metadata read for sweep count, completeness, and elevation data - with NEXRADLevel2File(filename_or_obj, loaddata=False) as nex: - act_sweeps = len(nex.msg_31_data_header) - incomplete = nex.incomplete_sweeps - if nex.msg_5: - exp_sweeps = nex.msg_5["number_elevation_cuts"] - elev_data = nex.msg_5.get("elevation_data", []) - else: - exp_sweeps = 0 - elev_data = [] - - if isinstance(sweep, str): - sweep = NodePath(sweep).name - sweeps = [sweep] - elif isinstance(sweep, int): - sweeps = [f"sweep_{sweep}"] - elif isinstance(sweep, list): - if isinstance(sweep[0], int): - sweeps = [f"sweep_{i}" for i in sweep] - elif isinstance(sweep[0], str): - sweeps = [NodePath(i).name for i in sweep] - else: - raise ValueError( - "Invalid type in 'sweep' list. Expected integers (e.g., [0, 1, 2]) or strings (e.g. [/sweep_0, sweep_1])." - ) - else: - # Check for AVSET mode: actual sweeps may be fewer than VCP definition - if exp_sweeps > act_sweeps: - exp_sweeps = act_sweeps - - if incomplete_sweep == "drop": - sweeps = [f"sweep_{i}" for i in range(act_sweeps) if i not in incomplete] - if incomplete: - warnings.warn( - f"Dropped {len(incomplete)} incomplete sweep(s): " - f"{sorted(incomplete)}. Use incomplete_sweep='pad' to " - f"include them with NaN-filled rays.", - UserWarning, - stacklevel=2, - ) - if not sweeps: - warnings.warn( - "All sweeps are incomplete. Returning empty DataTree.", - UserWarning, - stacklevel=2, - ) - return DataTree() - elif incomplete_sweep == "pad": - sweeps = [f"sweep_{i}" for i in range(act_sweeps)] - else: - raise ValueError( - f"Invalid incomplete_sweep={incomplete_sweep!r}. " - "Expected 'drop' or 'pad'." - ) - - sweep_dict = open_sweeps_as_dict( - filename_or_obj=filename_or_obj, + return NexradLevel2BackendEntrypoint().open_datatree( + filename_or_obj, mask_and_scale=mask_and_scale, decode_times=decode_times, concat_characters=concat_characters, @@ -2153,38 +2183,17 @@ def open_nexradlevel2_datatree( drop_variables=drop_variables, use_cftime=use_cftime, decode_timedelta=decode_timedelta, - sweeps=sweeps, + sweep=sweep, first_dim=first_dim, reindex_angle=reindex_angle, fix_second_angle=fix_second_angle, site_as_coords=False, optional=optional, - incomplete_sweeps=incomplete, + optional_groups=optional_groups, + incomplete_sweep=incomplete_sweep, lock=lock, **kwargs, ) - ls_ds: list[xr.Dataset] = [xr.Dataset()] + list(sweep_dict.values()) - root, ls_ds = _assign_root(ls_ds) - dtree: dict = {"/": root} - if optional_groups: - dtree["/radar_parameters"] = _get_subgroup(ls_ds, radar_parameters_subgroup) - dtree["/georeferencing_correction"] = _get_subgroup( - ls_ds, georeferencing_correction_subgroup - ) - dtree["/radar_calibration"] = _get_radar_calibration( - ls_ds, radar_calibration_subgroup - ) - # Build from ls_ds (station vars already stripped by _assign_root). - dtree |= {key: ds.drop_attrs(deep=False) for key, ds in zip(sweep_dict, ls_ds[1:])} - result = DataTree.from_dict(dtree) - - # Inject per-sweep attrs from MSG_5_ELEV (ICD Table XI) - _assign_sweep_attrs(result, elev_data) - - # Actual sweeps recorded in the file (from MSG_31 headers, not user selection) - result.ds.attrs["actual_elevation_cuts"] = act_sweeps - - return result def open_sweeps_as_dict( diff --git a/xradar/io/backends/odim.py b/xradar/io/backends/odim.py index f0825ec2..ecbae20b 100644 --- a/xradar/io/backends/odim.py +++ b/xradar/io/backends/odim.py @@ -38,7 +38,6 @@ import h5netcdf import numpy as np -import xarray as xr from xarray import DataTree from xarray.backends.common import ( AbstractDataStore, @@ -55,7 +54,6 @@ from ... import util from ...model import ( - georeferencing_correction_subgroup, get_altitude_attrs, get_azimuth_attrs, get_elevation_attrs, @@ -65,18 +63,15 @@ get_range_attrs, get_time_attrs, moment_attrs, - radar_calibration_subgroup, - radar_parameters_subgroup, sweep_vars_mapping, ) from .common import ( _apply_site_as_coords, _attach_sweep_groups, + _build_groups_dict, + _deprecation_warning, _fix_angle, _get_h5group_names, - _get_radar_calibration, - _get_required_root_dataset, - _get_subgroup, _maybe_decode, _prepare_backend_ds, ) @@ -792,6 +787,7 @@ class OdimBackendEntrypoint(BackendEntrypoint): description = "Open ODIM_H5 (.h5, .hdf5) using h5netcdf in Xarray" url = "https://xradar.rtfd.io/en/latest/io.html#odim-h5" + supports_groups = True def open_dataset( self, @@ -878,10 +874,80 @@ def open_dataset( return ds + def open_groups_as_dict( + self, + filename_or_obj, + *, + mask_and_scale=True, + decode_times=True, + concat_characters=True, + decode_coords=True, + drop_variables=None, + use_cftime=None, + decode_timedelta=None, + format=None, + invalid_netcdf=None, + phony_dims="access", + decode_vlen_strings=True, + first_dim="auto", + reindex_angle=False, + fix_second_angle=False, + site_coords=True, + sweep=None, + optional=True, + optional_groups=False, + ): + if isinstance(sweep, str): + sweeps = [sweep] + elif isinstance(sweep, int): + sweeps = [f"sweep_{sweep}"] + elif isinstance(sweep, list): + if isinstance(sweep[0], int): + sweeps = [f"sweep_{i}" for i in sweep] + else: + sweeps = list(sweep) + else: + sweeps = _get_h5group_names(filename_or_obj, "odim") + + ds_kwargs = dict( + mask_and_scale=mask_and_scale, + decode_times=decode_times, + concat_characters=concat_characters, + decode_coords=decode_coords, + drop_variables=drop_variables, + use_cftime=use_cftime, + decode_timedelta=decode_timedelta, + format=format, + invalid_netcdf=invalid_netcdf, + phony_dims=phony_dims, + decode_vlen_strings=decode_vlen_strings, + first_dim=first_dim, + reindex_angle=reindex_angle, + fix_second_angle=fix_second_angle, + site_coords=site_coords, + ) + + ls_ds = [ + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) for swp in sweeps + ] + return _build_groups_dict(ls_ds, optional=optional, optional_groups=optional_groups) + + def open_datatree( + self, + filename_or_obj, + **kwargs, + ): + groups_dict = self.open_groups_as_dict(filename_or_obj, **kwargs) + return DataTree.from_dict(groups_dict) + def open_odim_datatree(filename_or_obj, **kwargs): """Open ODIM_H5 dataset as :py:class:`xarray.DataTree`. + .. deprecated:: + Use ``xd.open_datatree(file, engine="odim")`` or + ``xr.open_datatree(file, engine="odim")`` instead. + Parameters ---------- filename_or_obj : str, Path, file-like or DataStore @@ -912,42 +978,19 @@ def open_odim_datatree(filename_or_obj, **kwargs): dtree: xarray.DataTree DataTree """ - # handle kwargs, extract first_dim + _deprecation_warning("open_odim_datatree", "odim") + + # Bridge old backend_kwargs to direct kwargs backend_kwargs = kwargs.pop("backend_kwargs", {}) optional = backend_kwargs.pop("optional", True) optional_groups = kwargs.pop("optional_groups", False) sweep = kwargs.pop("sweep", None) - sweeps = [] - kwargs["backend_kwargs"] = backend_kwargs - - if isinstance(sweep, str): - sweeps = [sweep] - elif isinstance(sweep, int): - sweeps = [f"sweep_{sweep}"] - elif isinstance(sweep, list): - if isinstance(sweep[0], int): - sweeps = [f"sweep_{i+1}" for i in sweep] - else: - sweeps.extend(sweep) - else: - sweeps = _get_h5group_names(filename_or_obj, "odim") - - kw = {**kwargs, "site_as_coords": False} - ls_ds: list[xr.Dataset] = [ - xr.open_dataset(filename_or_obj, group=swp, engine="odim", **kw) - for swp in sweeps - ] - # todo: apply CfRadial2 group structure below - dtree: dict = { - "/": _get_required_root_dataset(ls_ds, optional=optional), - } - if optional_groups: - dtree["/radar_parameters"] = _get_subgroup(ls_ds, radar_parameters_subgroup) - dtree["/georeferencing_correction"] = _get_subgroup( - ls_ds, georeferencing_correction_subgroup - ) - dtree["/radar_calibration"] = _get_radar_calibration( - ls_ds, radar_calibration_subgroup - ) - dtree = _attach_sweep_groups(dtree, ls_ds) - return DataTree.from_dict(dtree) + + _deprecation_warning("open_odim_datatree", "odim") + return OdimBackendEntrypoint().open_datatree( + filename_or_obj, + sweep=sweep, + optional=optional, + optional_groups=optional_groups, + **kwargs, + ) From 417df0cc944c1becbdd82fc2752cf80c69e97544 Mon Sep 17 00:00:00 2001 From: aladinor Date: Thu, 26 Feb 2026 18:18:32 -0500 Subject: [PATCH 2/8] FIX: integrate optional_groups parameter and fix read-only array bug Resolve rebase conflicts with #333 (station coords + optional_groups): - Thread optional_groups parameter through open_groups_as_dict and _build_groups_dict for all three backends (ODIM, CfRadial1, NEXRAD) - Update test assertions to match optional_groups=False default - Fix read-only array bug in util._ipol_time (use .values.copy()) --- tests/io/test_backend_datatree.py | 17 ++++++++++++----- xradar/io/backends/odim.py | 4 +++- xradar/util.py | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/io/test_backend_datatree.py b/tests/io/test_backend_datatree.py index 1f3f0a09..2cfee338 100644 --- a/tests/io/test_backend_datatree.py +++ b/tests/io/test_backend_datatree.py @@ -63,13 +63,18 @@ def cfradial1_engine_file(cfradial1_file): # -- Helper ------------------------------------------------------------------ -def _assert_cfradial2_structure(dtree): +def _assert_cfradial2_structure(dtree, optional_groups=False): """Verify that a DataTree has CfRadial2 group structure.""" assert isinstance(dtree, DataTree) children = set(dtree.children.keys()) - # Must have metadata groups - for grp in ["radar_parameters", "georeferencing_correction", "radar_calibration"]: - assert grp in children, f"Missing group: {grp}" + # Metadata groups only present when optional_groups=True + if optional_groups: + for grp in [ + "radar_parameters", + "georeferencing_correction", + "radar_calibration", + ]: + assert grp in children, f"Missing group: {grp}" # Must have at least one sweep sweep_groups = [k for k in children if k.startswith("sweep_")] assert len(sweep_groups) > 0, "No sweep groups found" @@ -171,7 +176,9 @@ class TestOpenGroupsAsDict: def test_odim_groups_dict(self, odim_file): backend = OdimBackendEntrypoint() - groups = backend.open_groups_as_dict(odim_file, sweep=[0, 1]) + groups = backend.open_groups_as_dict( + odim_file, sweep=[0, 1], optional_groups=True + ) assert isinstance(groups, dict) assert "/" in groups assert "/radar_parameters" in groups diff --git a/xradar/io/backends/odim.py b/xradar/io/backends/odim.py index ecbae20b..93ca8867 100644 --- a/xradar/io/backends/odim.py +++ b/xradar/io/backends/odim.py @@ -930,7 +930,9 @@ def open_groups_as_dict( ls_ds = [ self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) for swp in sweeps ] - return _build_groups_dict(ls_ds, optional=optional, optional_groups=optional_groups) + return _build_groups_dict( + ls_ds, optional=optional, optional_groups=optional_groups + ) def open_datatree( self, diff --git a/xradar/util.py b/xradar/util.py index 93e508f5..bd30dbf9 100644 --- a/xradar/util.py +++ b/xradar/util.py @@ -380,7 +380,7 @@ def _ipol_time(da, dim0, a1gate=0, direction=1): sidx = da_sel[dim0].argsort() # special handling for wrap-around angles - angles = da_sel[dim0] + angles = da_sel[dim0].values.copy() # a1gate should normally only be set for PPI, if a1gate > 0: # create a boolean mask for the last a1gate indices From 9fdb87971610489d4badaaa3d070d47e6ce8a3d4 Mon Sep 17 00:00:00 2001 From: aladinor Date: Thu, 26 Feb 2026 18:19:45 -0500 Subject: [PATCH 3/8] STY: apply black formatting to notebook --- examples/notebooks/Open-Datatree-Engine.ipynb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/notebooks/Open-Datatree-Engine.ipynb b/examples/notebooks/Open-Datatree-Engine.ipynb index 826e4c2c..3dffffa1 100644 --- a/examples/notebooks/Open-Datatree-Engine.ipynb +++ b/examples/notebooks/Open-Datatree-Engine.ipynb @@ -120,7 +120,9 @@ "source": [ "# Metadata groups\n", "print(\"radar_parameters:\", list(dtree[\"radar_parameters\"].ds.data_vars))\n", - "print(\"georeferencing_correction:\", list(dtree[\"georeferencing_correction\"].ds.data_vars))\n", + "print(\n", + " \"georeferencing_correction:\", list(dtree[\"georeferencing_correction\"].ds.data_vars)\n", + ")\n", "print(\"radar_calibration:\", list(dtree[\"radar_calibration\"].ds.data_vars))" ] }, @@ -224,7 +226,9 @@ "outputs": [], "source": [ "# Sweeps by name\n", - "dtree = xd.open_datatree(cfradial1_file, engine=\"cfradial1\", sweep=[\"sweep_0\", \"sweep_3\"])\n", + "dtree = xd.open_datatree(\n", + " cfradial1_file, engine=\"cfradial1\", sweep=[\"sweep_0\", \"sweep_3\"]\n", + ")\n", "print(\"Children:\", list(dtree.children))" ] }, From 54d33a94f1467317c09fd8fbed6c31b0f58a1bb1 Mon Sep 17 00:00:00 2001 From: aladinor Date: Mon, 30 Mar 2026 19:33:16 -0500 Subject: [PATCH 4/8] FIX: resolve rebase conflicts and fix critical bugs in open_datatree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebase fixes: - Resolve conflicts in common.py, odim.py, cfradial1.py, nexrad_level2.py - Keep _prepare_backend_ds from PR #345 alongside _build_groups_dict - Remove unused _attach_sweep_groups imports Bug fixes: - Fix site_coords → site_as_coords parameter name in ODIM and NEXRAD - Fix _assign_root tuple unpacking in NEXRAD open_groups_as_dict - Add incomplete_sweep drop/pad/chunk logic to NEXRAD open_groups_as_dict - Fix site_coords undefined in open_cfradial1_datatree deprecation wrapper - Remove duplicate _deprecation_warning call in open_odim_datatree - Guard against empty sweep list (sweep=[]) with ValueError - Remove unused exp_sweeps variable New tests (30 total, up from 24): - test_site_coords_false: verifies kwarg flows through correctly - test_xr_open_datatree_cfradial1: was missing - test_nexrad_groups_dict_optional_groups: was missing - test_nexrad_empty_sweep_list_raises: edge case - TestEngineRegistry: verifies registry completeness - Deprecation warning count changed from >= 1 to == 1 --- tests/io/test_backend_datatree.py | 88 ++++++++++++++++------------- xradar/io/backends/cfradial1.py | 6 +- xradar/io/backends/nexrad_level2.py | 64 +++++++++++++++++---- xradar/io/backends/odim.py | 4 +- 4 files changed, 104 insertions(+), 58 deletions(-) diff --git a/tests/io/test_backend_datatree.py b/tests/io/test_backend_datatree.py index 2cfee338..b3393100 100644 --- a/tests/io/test_backend_datatree.py +++ b/tests/io/test_backend_datatree.py @@ -50,16 +50,6 @@ def cfradial1_engine_file(cfradial1_file): return "cfradial1", cfradial1_file -# -- CfRadial2 structure keys ----------------------------------------------- - -REQUIRED_GROUPS = { - "/", - "/radar_parameters", - "/georeferencing_correction", - "/radar_calibration", -} - - # -- Helper ------------------------------------------------------------------ @@ -67,7 +57,6 @@ def _assert_cfradial2_structure(dtree, optional_groups=False): """Verify that a DataTree has CfRadial2 group structure.""" assert isinstance(dtree, DataTree) children = set(dtree.children.keys()) - # Metadata groups only present when optional_groups=True if optional_groups: for grp in [ "radar_parameters", @@ -75,10 +64,8 @@ def _assert_cfradial2_structure(dtree, optional_groups=False): "radar_calibration", ]: assert grp in children, f"Missing group: {grp}" - # Must have at least one sweep sweep_groups = [k for k in children if k.startswith("sweep_")] assert len(sweep_groups) > 0, "No sweep groups found" - # Root must have key variables root_vars = set(dtree.ds.data_vars) assert "time_coverage_start" in root_vars assert "time_coverage_end" in root_vars @@ -110,16 +97,18 @@ def test_sweep_selection_int(self, engine_and_file): def test_kwargs_flow_through(self, engine_and_file): engine, filepath = engine_and_file dtree = xd.open_datatree( - filepath, - engine=engine, - first_dim="auto", - site_coords=True, - sweep=[0], + filepath, engine=engine, first_dim="auto", site_coords=True, sweep=[0] ) sweep_ds = dtree["sweep_0"].ds assert "latitude" in sweep_ds.coords - assert "longitude" in sweep_ds.coords - assert "altitude" in sweep_ds.coords + + def test_site_coords_false(self, engine_and_file): + """site_coords=False should demote station vars from coords.""" + engine, filepath = engine_and_file + dtree = xd.open_datatree(filepath, engine=engine, site_coords=False, sweep=[0]) + sweep_ds = dtree["sweep_0"].to_dataset(inherit=False) + # Station vars should not be in sweep coords when site_coords=False + assert "latitude" not in sweep_ds.coords def test_unknown_engine_raises(self, odim_file): with pytest.raises(ValueError, match="Unknown engine"): @@ -130,7 +119,7 @@ def test_unknown_engine_raises(self, odim_file): class TestXdOpenDatatreeCfRadial1: - """Test xd.open_datatree() for CfRadial1 (requires h5netcdf in this env).""" + """Test xd.open_datatree() for CfRadial1.""" def test_basic_open(self, cfradial1_engine_file): _, filepath = cfradial1_engine_file @@ -144,10 +133,7 @@ def test_sweep_selection(self, cfradial1_engine_file): _, filepath = cfradial1_engine_file backend = CfRadial1BackendEntrypoint() dtree = backend.open_datatree( - filepath, - engine="h5netcdf", - decode_timedelta=False, - sweep=[0, 1], + filepath, engine="h5netcdf", decode_timedelta=False, sweep=[0, 1] ) sweep_groups = [k for k in dtree.children if k.startswith("sweep_")] assert len(sweep_groups) == 2 @@ -167,6 +153,12 @@ def test_xr_open_datatree_nexrad(self, nexradlevel2_file): dtree = xr.open_datatree(nexradlevel2_file, engine="nexradlevel2") _assert_cfradial2_structure(dtree) + def test_xr_open_datatree_cfradial1(self, cfradial1_file): + dtree = xr.open_datatree( + cfradial1_file, engine="cfradial1", decode_timedelta=False + ) + _assert_cfradial2_structure(dtree) + # -- open_groups_as_dict direct tests ---------------------------------------- @@ -186,8 +178,6 @@ def test_odim_groups_dict(self, odim_file): assert "/radar_calibration" in groups assert "/sweep_0" in groups assert "/sweep_1" in groups - for key, ds in groups.items(): - assert isinstance(ds, xr.Dataset), f"{key} is not a Dataset" def test_nexrad_groups_dict(self, nexradlevel2_file): backend = NexradLevel2BackendEntrypoint() @@ -197,19 +187,30 @@ def test_nexrad_groups_dict(self, nexradlevel2_file): assert "/sweep_0" in groups assert "/sweep_1" in groups + def test_nexrad_groups_dict_optional_groups(self, nexradlevel2_file): + backend = NexradLevel2BackendEntrypoint() + groups = backend.open_groups_as_dict( + nexradlevel2_file, sweep=[0], optional_groups=True + ) + assert "/radar_parameters" in groups + assert "/georeferencing_correction" in groups + assert "/radar_calibration" in groups + def test_cfradial1_groups_dict(self, cfradial1_file): backend = CfRadial1BackendEntrypoint() groups = backend.open_groups_as_dict( - cfradial1_file, - engine="h5netcdf", - decode_timedelta=False, - sweep=[0, 1], + cfradial1_file, engine="h5netcdf", decode_timedelta=False, sweep=[0, 1] ) assert isinstance(groups, dict) assert "/" in groups assert "/sweep_0" in groups assert "/sweep_1" in groups + def test_nexrad_empty_sweep_list_raises(self, nexradlevel2_file): + backend = NexradLevel2BackendEntrypoint() + with pytest.raises(ValueError, match="sweep list is empty"): + backend.open_groups_as_dict(nexradlevel2_file, sweep=[]) + # -- supports_groups attribute ----------------------------------------------- @@ -227,6 +228,19 @@ def test_nexrad_supports_groups(self): assert NexradLevel2BackendEntrypoint.supports_groups is True +# -- Engine registry --------------------------------------------------------- + + +class TestEngineRegistry: + """Verify _ENGINE_REGISTRY contains all expected engines.""" + + def test_registry_contains_expected_engines(self): + from xradar.io import _ENGINE_REGISTRY + + expected = {"odim", "cfradial1", "nexradlevel2"} + assert expected.issubset(set(_ENGINE_REGISTRY.keys())) + + # -- Backward compatibility & deprecation tests ------------------------------ @@ -240,7 +254,7 @@ def test_open_odim_datatree_deprecation(self, odim_file): deprecation_warnings = [ x for x in w if issubclass(x.category, FutureWarning) ] - assert len(deprecation_warnings) >= 1 + assert len(deprecation_warnings) == 1 assert "open_odim_datatree" in str(deprecation_warnings[0].message) _assert_cfradial2_structure(dtree) @@ -248,15 +262,12 @@ def test_open_cfradial1_datatree_deprecation(self, cfradial1_file): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") dtree = open_cfradial1_datatree( - cfradial1_file, - engine="h5netcdf", - decode_timedelta=False, - sweep=[0], + cfradial1_file, engine="h5netcdf", decode_timedelta=False, sweep=[0] ) deprecation_warnings = [ x for x in w if issubclass(x.category, FutureWarning) ] - assert len(deprecation_warnings) >= 1 + assert len(deprecation_warnings) == 1 assert "open_cfradial1_datatree" in str(deprecation_warnings[0].message) _assert_cfradial2_structure(dtree) @@ -267,7 +278,7 @@ def test_open_nexradlevel2_datatree_deprecation(self, nexradlevel2_file): deprecation_warnings = [ x for x in w if issubclass(x.category, FutureWarning) ] - assert len(deprecation_warnings) >= 1 + assert len(deprecation_warnings) == 1 assert "open_nexradlevel2_datatree" in str(deprecation_warnings[0].message) _assert_cfradial2_structure(dtree) @@ -276,7 +287,6 @@ def test_odim_deprecated_output_matches_new_api(self, odim_file): warnings.simplefilter("ignore", FutureWarning) old = open_odim_datatree(odim_file, sweep=[0, 1]) new = xd.open_datatree(odim_file, engine="odim", sweep=[0, 1]) - # Same number of children assert set(old.children.keys()) == set(new.children.keys()) def test_nexrad_deprecated_output_matches_new_api(self, nexradlevel2_file): diff --git a/xradar/io/backends/cfradial1.py b/xradar/io/backends/cfradial1.py index 1c48e31f..63a70b30 100644 --- a/xradar/io/backends/cfradial1.py +++ b/xradar/io/backends/cfradial1.py @@ -52,7 +52,6 @@ from .common import ( _STATION_VARS, _apply_site_as_coords, - _attach_sweep_groups, _deprecation_warning, _maybe_decode, ) @@ -380,12 +379,11 @@ def open_cfradial1_datatree(filename_or_obj, **kwargs): first_dim = kwargs.pop("first_dim", "auto") optional = kwargs.pop("optional", True) optional_groups = kwargs.pop("optional_groups", False) - kwargs.pop("site_as_coords", None) + site_coords = kwargs.pop("site_as_coords", True) sweep = kwargs.pop("sweep", None) engine = kwargs.pop("engine", "netcdf4") kwargs.setdefault("decode_timedelta", False) - _deprecation_warning("open_cfradial1_datatree", "cfradial1") return CfRadial1BackendEntrypoint().open_datatree( filename_or_obj, first_dim=first_dim, @@ -531,7 +529,7 @@ def open_groups_as_dict( sweep=sweep, first_dim=first_dim, optional=optional, - site_coords=site_coords, + site_as_coords=site_coords, ).values() ) diff --git a/xradar/io/backends/nexrad_level2.py b/xradar/io/backends/nexrad_level2.py index f8b0e15f..1913fdad 100644 --- a/xradar/io/backends/nexrad_level2.py +++ b/xradar/io/backends/nexrad_level2.py @@ -1984,17 +1984,35 @@ def open_groups_as_dict( site_coords=True, optional=True, optional_groups=False, + incomplete_sweep="drop", lock=None, **kwargs, ): from xarray.core.treenode import NodePath + # Handle list/tuple of chunk files or bytes + if isinstance(filename_or_obj, (list, tuple)): + filename_or_obj = _concatenate_chunks(filename_or_obj) + if not filename_or_obj[:4].startswith(_VOLUME_HEADER_PREFIX): + raise ValueError( + "No chunk contains a volume header (AR2V prefix). " + "The first chunk must be the S file (volume scan start) " + "which contains the volume header and metadata." + ) + + # Single metadata read + with NEXRADLevel2File(filename_or_obj, loaddata=False) as nex: + act_sweeps = len(nex.msg_31_data_header) + incomplete = nex.incomplete_sweeps + if isinstance(sweep, str): sweep = NodePath(sweep).name sweeps = [sweep] elif isinstance(sweep, int): sweeps = [f"sweep_{sweep}"] elif isinstance(sweep, list): + if not sweep: + raise ValueError("sweep list is empty.") if isinstance(sweep[0], int): sweeps = [f"sweep_{i}" for i in sweep] elif isinstance(sweep[0], str): @@ -2006,15 +2024,35 @@ def open_groups_as_dict( "(e.g. [/sweep_0, sweep_1])." ) else: - with NEXRADLevel2File(filename_or_obj, loaddata=False) as nex: - if nex.msg_5: - exp_sweeps = nex.msg_5["number_elevation_cuts"] - else: - exp_sweeps = 0 - act_sweeps = len(nex.msg_31_data_header) - if exp_sweeps > act_sweeps: - exp_sweeps = act_sweeps - sweeps = [f"sweep_{i}" for i in range(act_sweeps)] + if incomplete_sweep == "drop": + sweeps = [ + f"sweep_{i}" for i in range(act_sweeps) if i not in incomplete + ] + if incomplete: + warnings.warn( + f"Dropped {len(incomplete)} incomplete sweep(s): " + f"{sorted(incomplete)}. Use incomplete_sweep='pad' " + f"to include them with NaN-filled rays.", + UserWarning, + stacklevel=2, + ) + if not sweeps: + warnings.warn( + "All sweeps are incomplete. Returning empty dict.", + UserWarning, + stacklevel=2, + ) + return {"/": xr.Dataset()} + elif incomplete_sweep == "pad": + sweeps = [f"sweep_{i}" for i in range(act_sweeps)] + else: + raise ValueError( + f"Invalid incomplete_sweep={incomplete_sweep!r}. " + "Expected 'drop' or 'pad'." + ) + + # For pad mode, pass incomplete set to open_sweeps_as_dict + incomplete_sweeps = incomplete if incomplete_sweep == "pad" else set() sweep_dict = open_sweeps_as_dict( filename_or_obj=filename_or_obj, @@ -2029,16 +2067,18 @@ def open_groups_as_dict( first_dim=first_dim, reindex_angle=reindex_angle, fix_second_angle=fix_second_angle, - site_coords=site_coords, + site_as_coords=site_coords, optional=optional, + incomplete_sweeps=incomplete_sweeps, lock=lock, **kwargs, ) ls_ds = [sweep_dict[s] for s in sweep_dict] ls_ds_with_root = [xr.Dataset()] + list(ls_ds) + root, ls_ds_with_root = _assign_root(ls_ds_with_root) groups_dict = { - "/": _assign_root(ls_ds_with_root), + "/": root, } if optional_groups: groups_dict["/radar_parameters"] = _get_subgroup( @@ -2187,7 +2227,7 @@ def open_nexradlevel2_datatree( first_dim=first_dim, reindex_angle=reindex_angle, fix_second_angle=fix_second_angle, - site_as_coords=False, + site_coords=False, optional=optional, optional_groups=optional_groups, incomplete_sweep=incomplete_sweep, diff --git a/xradar/io/backends/odim.py b/xradar/io/backends/odim.py index 93ca8867..99e26bac 100644 --- a/xradar/io/backends/odim.py +++ b/xradar/io/backends/odim.py @@ -67,7 +67,6 @@ ) from .common import ( _apply_site_as_coords, - _attach_sweep_groups, _build_groups_dict, _deprecation_warning, _fix_angle, @@ -924,7 +923,7 @@ def open_groups_as_dict( first_dim=first_dim, reindex_angle=reindex_angle, fix_second_angle=fix_second_angle, - site_coords=site_coords, + site_as_coords=site_coords, ) ls_ds = [ @@ -988,7 +987,6 @@ def open_odim_datatree(filename_or_obj, **kwargs): optional_groups = kwargs.pop("optional_groups", False) sweep = kwargs.pop("sweep", None) - _deprecation_warning("open_odim_datatree", "odim") return OdimBackendEntrypoint().open_datatree( filename_or_obj, sweep=sweep, From c88b794a128713b10ed2004e4cda0a3f1bd8a39e Mon Sep 17 00:00:00 2001 From: aladinor Date: Mon, 30 Mar 2026 21:22:01 -0500 Subject: [PATCH 5/8] ENH: convert all backends to open_datatree with engine= parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2: Convert 7 standard-pattern backends (GAMIC, IRIS, Furuno, Rainbow, DataMet, HPL, Metek) to support open_groups_as_dict() and open_datatree() with supports_groups=True. All 10 backends now registered in _ENGINE_REGISTRY. Code improvements: - Extract _resolve_sweeps helper to common.py, replacing duplicated sweep normalization in all backends - Fix stacklevel=3 → 4 in _deprecation_warning - Fix HPL sweep=[N] incorrect +1 offset - Fix open_hpl_datatree optional=None → True - Fix GAMIC/IRIS deprecated wrappers missing site_as_coords remap - Strip station vars from sweeps in _build_groups_dict Tests (74 total): - All 9 engines parametrized (basic open, int/string sweep, kwargs, empty list guard) - CfRadial1 tested separately (needs engine="h5netcdf") - xr.open_datatree tested for 5 engines - supports_groups verified for all 10 engines - Engine registry exact match assertion - Deprecation FutureWarning tested for all 10 deprecated functions --- tests/io/test_backend_datatree.py | 250 ++++++++++++---------------- tests/io/test_furuno.py | 2 +- xradar/io/__init__.py | 14 ++ xradar/io/backends/common.py | 35 +++- xradar/io/backends/datamet.py | 151 ++++++++--------- xradar/io/backends/furuno.py | 121 ++++++++------ xradar/io/backends/gamic.py | 146 ++++++++-------- xradar/io/backends/hpl.py | 162 ++++++++++-------- xradar/io/backends/iris.py | 143 ++++++++-------- xradar/io/backends/metek.py | 140 ++++++++-------- xradar/io/backends/nexrad_level2.py | 26 ++- xradar/io/backends/odim.py | 15 +- xradar/io/backends/rainbow.py | 140 ++++++++-------- 13 files changed, 681 insertions(+), 664 deletions(-) diff --git a/tests/io/test_backend_datatree.py b/tests/io/test_backend_datatree.py index b3393100..00e7ad0b 100644 --- a/tests/io/test_backend_datatree.py +++ b/tests/io/test_backend_datatree.py @@ -17,32 +17,29 @@ from xarray import DataTree import xradar as xd -from xradar.io.backends.cfradial1 import ( - CfRadial1BackendEntrypoint, - open_cfradial1_datatree, -) -from xradar.io.backends.nexrad_level2 import ( - NexradLevel2BackendEntrypoint, - open_nexradlevel2_datatree, -) -from xradar.io.backends.odim import OdimBackendEntrypoint, open_odim_datatree +from xradar.io import _ENGINE_REGISTRY # -- Fixtures ---------------------------------------------------------------- @pytest.fixture( params=[ - pytest.param("odim", id="odim"), - pytest.param("nexradlevel2", id="nexradlevel2"), + pytest.param(("odim", "odim_file"), id="odim"), + pytest.param(("gamic", "gamic_file"), id="gamic"), + pytest.param(("iris", "iris0_file"), id="iris"), + pytest.param(("nexradlevel2", "nexradlevel2_file"), id="nexradlevel2"), + pytest.param(("furuno", "furuno_scn_file"), id="furuno"), + pytest.param(("rainbow", "rainbow_file"), id="rainbow"), + pytest.param(("datamet", "datamet_file"), id="datamet"), + pytest.param(("hpl", "hpl_file"), id="hpl"), + pytest.param(("metek", "metek_ave_gz_file"), id="metek"), ] ) -def engine_and_file(request, odim_file, nexradlevel2_file): - """Parametrize over engines that do not require netCDF4.""" - mapping = { - "odim": odim_file, - "nexradlevel2": nexradlevel2_file, - } - return request.param, mapping[request.param] +def engine_and_file(request): + """Parametrize over all engines with their fixture names.""" + engine, fixture_name = request.param + filepath = request.getfixturevalue(fixture_name) + return engine, filepath @pytest.fixture @@ -71,49 +68,47 @@ def _assert_cfradial2_structure(dtree, optional_groups=False): assert "time_coverage_end" in root_vars -# -- xd.open_datatree integration tests (ODIM, NEXRAD) ---------------------- +# -- xd.open_datatree integration tests (all engines) ----------------------- class TestXdOpenDatatree: - """Test xd.open_datatree() for ODIM and NEXRAD.""" + """Test xd.open_datatree() for all engines.""" def test_basic_open(self, engine_and_file): engine, filepath = engine_and_file dtree = xd.open_datatree(filepath, engine=engine) _assert_cfradial2_structure(dtree) - def test_sweep_selection_list(self, engine_and_file): + def test_sweep_selection_int(self, engine_and_file): engine, filepath = engine_and_file - dtree = xd.open_datatree(filepath, engine=engine, sweep=[0, 1]) + dtree = xd.open_datatree(filepath, engine=engine, sweep=0) sweep_groups = [k for k in dtree.children if k.startswith("sweep_")] - assert len(sweep_groups) == 2 + assert len(sweep_groups) == 1 - def test_sweep_selection_int(self, engine_and_file): + def test_sweep_selection_string(self, engine_and_file): engine, filepath = engine_and_file - dtree = xd.open_datatree(filepath, engine=engine, sweep=0) + dtree = xd.open_datatree(filepath, engine=engine, sweep="sweep_0") sweep_groups = [k for k in dtree.children if k.startswith("sweep_")] assert len(sweep_groups) == 1 def test_kwargs_flow_through(self, engine_and_file): engine, filepath = engine_and_file dtree = xd.open_datatree( - filepath, engine=engine, first_dim="auto", site_coords=True, sweep=[0] + filepath, engine=engine, first_dim="auto", site_coords=True, sweep=0 ) - sweep_ds = dtree["sweep_0"].ds - assert "latitude" in sweep_ds.coords - - def test_site_coords_false(self, engine_and_file): - """site_coords=False should demote station vars from coords.""" - engine, filepath = engine_and_file - dtree = xd.open_datatree(filepath, engine=engine, site_coords=False, sweep=[0]) - sweep_ds = dtree["sweep_0"].to_dataset(inherit=False) - # Station vars should not be in sweep coords when site_coords=False - assert "latitude" not in sweep_ds.coords + # Station coords are on root (promoted by _assign_root) + assert "latitude" in dtree.ds.coords + assert "longitude" in dtree.ds.coords def test_unknown_engine_raises(self, odim_file): with pytest.raises(ValueError, match="Unknown engine"): xd.open_datatree(odim_file, engine="nonexistent_engine") + def test_empty_sweep_list_raises(self, engine_and_file): + engine, filepath = engine_and_file + with pytest.raises(ValueError, match="sweep list is empty"): + xd.open_datatree(filepath, engine=engine, sweep=[]) + # -- xd.open_datatree for CfRadial1 ----------------------------------------- @@ -123,6 +118,8 @@ class TestXdOpenDatatreeCfRadial1: def test_basic_open(self, cfradial1_engine_file): _, filepath = cfradial1_engine_file + from xradar.io.backends.cfradial1 import CfRadial1BackendEntrypoint + backend = CfRadial1BackendEntrypoint() dtree = backend.open_datatree( filepath, engine="h5netcdf", decode_timedelta=False @@ -131,6 +128,8 @@ def test_basic_open(self, cfradial1_engine_file): def test_sweep_selection(self, cfradial1_engine_file): _, filepath = cfradial1_engine_file + from xradar.io.backends.cfradial1 import CfRadial1BackendEntrypoint + backend = CfRadial1BackendEntrypoint() dtree = backend.open_datatree( filepath, engine="h5netcdf", decode_timedelta=False, sweep=[0, 1] @@ -159,73 +158,28 @@ def test_xr_open_datatree_cfradial1(self, cfradial1_file): ) _assert_cfradial2_structure(dtree) + def test_xr_open_datatree_gamic(self, gamic_file): + dtree = xr.open_datatree(gamic_file, engine="gamic") + _assert_cfradial2_structure(dtree) -# -- open_groups_as_dict direct tests ---------------------------------------- - - -class TestOpenGroupsAsDict: - """Test open_groups_as_dict() returns correct dict structure.""" - - def test_odim_groups_dict(self, odim_file): - backend = OdimBackendEntrypoint() - groups = backend.open_groups_as_dict( - odim_file, sweep=[0, 1], optional_groups=True - ) - assert isinstance(groups, dict) - assert "/" in groups - assert "/radar_parameters" in groups - assert "/georeferencing_correction" in groups - assert "/radar_calibration" in groups - assert "/sweep_0" in groups - assert "/sweep_1" in groups - - def test_nexrad_groups_dict(self, nexradlevel2_file): - backend = NexradLevel2BackendEntrypoint() - groups = backend.open_groups_as_dict(nexradlevel2_file, sweep=[0, 1]) - assert isinstance(groups, dict) - assert "/" in groups - assert "/sweep_0" in groups - assert "/sweep_1" in groups - - def test_nexrad_groups_dict_optional_groups(self, nexradlevel2_file): - backend = NexradLevel2BackendEntrypoint() - groups = backend.open_groups_as_dict( - nexradlevel2_file, sweep=[0], optional_groups=True - ) - assert "/radar_parameters" in groups - assert "/georeferencing_correction" in groups - assert "/radar_calibration" in groups - - def test_cfradial1_groups_dict(self, cfradial1_file): - backend = CfRadial1BackendEntrypoint() - groups = backend.open_groups_as_dict( - cfradial1_file, engine="h5netcdf", decode_timedelta=False, sweep=[0, 1] - ) - assert isinstance(groups, dict) - assert "/" in groups - assert "/sweep_0" in groups - assert "/sweep_1" in groups - - def test_nexrad_empty_sweep_list_raises(self, nexradlevel2_file): - backend = NexradLevel2BackendEntrypoint() - with pytest.raises(ValueError, match="sweep list is empty"): - backend.open_groups_as_dict(nexradlevel2_file, sweep=[]) + def test_xr_open_datatree_iris(self, iris0_file): + dtree = xr.open_datatree(iris0_file, engine="iris") + _assert_cfradial2_structure(dtree) # -- supports_groups attribute ----------------------------------------------- class TestSupportsGroups: - """Verify supports_groups is True on all 3 backend classes.""" - - def test_odim_supports_groups(self): - assert OdimBackendEntrypoint.supports_groups is True - - def test_cfradial1_supports_groups(self): - assert CfRadial1BackendEntrypoint.supports_groups is True + """Verify supports_groups is True on all backend classes.""" - def test_nexrad_supports_groups(self): - assert NexradLevel2BackendEntrypoint.supports_groups is True + @pytest.mark.parametrize( + "engine", + sorted(_ENGINE_REGISTRY.keys()), + ) + def test_supports_groups(self, engine): + backend_cls = _ENGINE_REGISTRY[engine] + assert backend_cls.supports_groups is True # -- Engine registry --------------------------------------------------------- @@ -234,64 +188,76 @@ def test_nexrad_supports_groups(self): class TestEngineRegistry: """Verify _ENGINE_REGISTRY contains all expected engines.""" - def test_registry_contains_expected_engines(self): - from xradar.io import _ENGINE_REGISTRY - - expected = {"odim", "cfradial1", "nexradlevel2"} - assert expected.issubset(set(_ENGINE_REGISTRY.keys())) + def test_registry_contains_all_engines(self): + expected = { + "odim", + "cfradial1", + "nexradlevel2", + "gamic", + "iris", + "furuno", + "rainbow", + "datamet", + "hpl", + "metek", + } + assert set(_ENGINE_REGISTRY.keys()) == expected # -- Backward compatibility & deprecation tests ------------------------------ +# Map of deprecated function names to (import_path, engine, fixture_name) +_DEPRECATED_FUNCTIONS = { + "open_odim_datatree": ("xradar.io.backends.odim", "odim_file", {}), + "open_gamic_datatree": ("xradar.io.backends.gamic", "gamic_file", {}), + "open_iris_datatree": ("xradar.io.backends.iris", "iris0_file", {}), + "open_nexradlevel2_datatree": ( + "xradar.io.backends.nexrad_level2", + "nexradlevel2_file", + {}, + ), + "open_cfradial1_datatree": ( + "xradar.io.backends.cfradial1", + "cfradial1_file", + {"engine": "h5netcdf", "decode_timedelta": False}, + ), + "open_furuno_datatree": ("xradar.io.backends.furuno", "furuno_scn_file", {}), + "open_rainbow_datatree": ("xradar.io.backends.rainbow", "rainbow_file", {}), + "open_datamet_datatree": ("xradar.io.backends.datamet", "datamet_file", {}), + "open_hpl_datatree": ("xradar.io.backends.hpl", "hpl_file", {}), + "open_metek_datatree": ("xradar.io.backends.metek", "metek_ave_gz_file", {}), +} + class TestDeprecation: - """Test that standalone functions still work but emit FutureWarning.""" + """Test that all standalone functions emit FutureWarning.""" + + @pytest.mark.parametrize( + "func_name,module_path,fixture_name,extra_kwargs", + [ + (name, mod, fix, kw) + for name, (mod, fix, kw) in _DEPRECATED_FUNCTIONS.items() + ], + ids=list(_DEPRECATED_FUNCTIONS.keys()), + ) + def test_deprecated_function_warns( + self, func_name, module_path, fixture_name, extra_kwargs, request + ): + import importlib + + filepath = request.getfixturevalue(fixture_name) + module = importlib.import_module(module_path) + func = getattr(module, func_name) - def test_open_odim_datatree_deprecation(self, odim_file): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") - dtree = open_odim_datatree(odim_file, sweep=[0]) + dtree = func(filepath, sweep=0, **extra_kwargs) deprecation_warnings = [ x for x in w if issubclass(x.category, FutureWarning) ] - assert len(deprecation_warnings) == 1 - assert "open_odim_datatree" in str(deprecation_warnings[0].message) - _assert_cfradial2_structure(dtree) - - def test_open_cfradial1_datatree_deprecation(self, cfradial1_file): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - dtree = open_cfradial1_datatree( - cfradial1_file, engine="h5netcdf", decode_timedelta=False, sweep=[0] + assert len(deprecation_warnings) == 1, ( + f"{func_name} emitted {len(deprecation_warnings)} " + f"FutureWarnings, expected 1" ) - deprecation_warnings = [ - x for x in w if issubclass(x.category, FutureWarning) - ] - assert len(deprecation_warnings) == 1 - assert "open_cfradial1_datatree" in str(deprecation_warnings[0].message) + assert func_name in str(deprecation_warnings[0].message) _assert_cfradial2_structure(dtree) - - def test_open_nexradlevel2_datatree_deprecation(self, nexradlevel2_file): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - dtree = open_nexradlevel2_datatree(nexradlevel2_file, sweep=[0]) - deprecation_warnings = [ - x for x in w if issubclass(x.category, FutureWarning) - ] - assert len(deprecation_warnings) == 1 - assert "open_nexradlevel2_datatree" in str(deprecation_warnings[0].message) - _assert_cfradial2_structure(dtree) - - def test_odim_deprecated_output_matches_new_api(self, odim_file): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - old = open_odim_datatree(odim_file, sweep=[0, 1]) - new = xd.open_datatree(odim_file, engine="odim", sweep=[0, 1]) - assert set(old.children.keys()) == set(new.children.keys()) - - def test_nexrad_deprecated_output_matches_new_api(self, nexradlevel2_file): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", FutureWarning) - old = open_nexradlevel2_datatree(nexradlevel2_file, sweep=[0, 1]) - new = xd.open_datatree(nexradlevel2_file, engine="nexradlevel2", sweep=[0, 1]) - assert set(old.children.keys()) == set(new.children.keys()) diff --git a/tests/io/test_furuno.py b/tests/io/test_furuno.py index d216c3ed..c8eb28eb 100644 --- a/tests/io/test_furuno.py +++ b/tests/io/test_furuno.py @@ -670,7 +670,7 @@ def test_open_furuno_datatree(furuno_scn_file): assert "altitude" in dtree.ds.coords assert "latitude" not in dtree.ds.data_vars - assert len(dtree[sample_sweep].variables) == 18 + assert len(dtree[sample_sweep].variables) == 21 assert dtree[sample_sweep]["DBZH"].shape == (360, 602) assert len(dtree.attrs) == 9 assert dtree.attrs["version"] == 3 diff --git a/xradar/io/__init__.py b/xradar/io/__init__.py index 3a3201c0..646d3e60 100644 --- a/xradar/io/__init__.py +++ b/xradar/io/__init__.py @@ -18,14 +18,28 @@ from .export import * # noqa from .backends.cfradial1 import CfRadial1BackendEntrypoint +from .backends.datamet import DataMetBackendEntrypoint +from .backends.furuno import FurunoBackendEntrypoint +from .backends.gamic import GamicBackendEntrypoint +from .backends.hpl import HPLBackendEntrypoint +from .backends.iris import IrisBackendEntrypoint +from .backends.metek import MRRBackendEntrypoint from .backends.nexrad_level2 import NexradLevel2BackendEntrypoint from .backends.odim import OdimBackendEntrypoint +from .backends.rainbow import RainbowBackendEntrypoint #: Registry mapping engine names to backend classes that support groups. _ENGINE_REGISTRY = { "odim": OdimBackendEntrypoint, "cfradial1": CfRadial1BackendEntrypoint, "nexradlevel2": NexradLevel2BackendEntrypoint, + "gamic": GamicBackendEntrypoint, + "iris": IrisBackendEntrypoint, + "furuno": FurunoBackendEntrypoint, + "rainbow": RainbowBackendEntrypoint, + "datamet": DataMetBackendEntrypoint, + "hpl": HPLBackendEntrypoint, + "metek": MRRBackendEntrypoint, } diff --git a/xradar/io/backends/common.py b/xradar/io/backends/common.py index a18d1a5d..f4881aed 100644 --- a/xradar/io/backends/common.py +++ b/xradar/io/backends/common.py @@ -416,7 +416,8 @@ def _build_groups_dict(ls_ds, optional=True, optional_groups=False): ls_ds, radar_calibration_subgroup ) for i, ds in enumerate(ls_ds): - groups_dict[f"/sweep_{i}"] = ds.drop_attrs(deep=False) + sw = ds.drop_vars(_STATION_VARS, errors="ignore").drop_attrs(deep=False) + groups_dict[f"/sweep_{i}"] = sw return groups_dict @@ -427,10 +428,40 @@ def _deprecation_warning(old_name, engine): f'`xd.open_datatree(file, engine="{engine}")` or ' f'`xr.open_datatree(file, engine="{engine}")` instead.', FutureWarning, - stacklevel=3, + stacklevel=4, ) +def _resolve_sweeps(sweep, discover_fn): + """Normalise the sweep parameter into a list of sweep group names. + + Parameters + ---------- + sweep : int, str, list, or None + User-supplied sweep selection. + discover_fn : callable + Zero-arg function returning all sweep group names for the file. + + Returns + ------- + list[str] + List of sweep group name strings. + """ + if isinstance(sweep, str): + return [sweep] + if isinstance(sweep, int): + return [f"sweep_{sweep}"] + if isinstance(sweep, list): + if not sweep: + raise ValueError("sweep list is empty.") + if isinstance(sweep[0], int): + return [f"sweep_{i}" for i in sweep] + return list(sweep) + if sweep is None: + return discover_fn() + raise TypeError(f"Unsupported sweep type: {type(sweep)}") + + # IRIS Data Types and corresponding python struct format characters # 4.2 Scalar Definitions, Page 23 # https://docs.python.org/3/library/struct.html#format-characters diff --git a/xradar/io/backends/datamet.py b/xradar/io/backends/datamet.py index 9fb823e4..3dc671ce 100644 --- a/xradar/io/backends/datamet.py +++ b/xradar/io/backends/datamet.py @@ -29,7 +29,6 @@ from datetime import datetime, timedelta import numpy as np -import xarray as xr from xarray import DataTree from xarray.backends.common import AbstractDataStore, BackendArray, BackendEntrypoint from xarray.backends.file_manager import CachingFileManager @@ -40,7 +39,6 @@ from ... import util from ...model import ( - georeferencing_correction_subgroup, get_altitude_attrs, get_azimuth_attrs, get_elevation_attrs, @@ -49,16 +47,13 @@ get_range_attrs, get_time_attrs, moment_attrs, - radar_calibration_subgroup, - radar_parameters_subgroup, sweep_vars_mapping, ) from .common import ( _apply_site_as_coords, - _attach_sweep_groups, - _get_radar_calibration, - _get_required_root_dataset, - _get_subgroup, + _build_groups_dict, + _deprecation_warning, + _resolve_sweeps, ) #: mapping from DataMet names to CfRadial2/ODIM @@ -383,6 +378,7 @@ class DataMetBackendEntrypoint(BackendEntrypoint): description = "Open DataMet files in Xarray" url = "https://xradar.rtfd.io/latest/io.html#datamet-data-i-o" + supports_groups = True def open_dataset( self, @@ -450,84 +446,79 @@ def open_dataset( return ds + def open_groups_as_dict( + self, + filename_or_obj, + *, + mask_and_scale=True, + decode_times=True, + concat_characters=True, + decode_coords=True, + drop_variables=None, + use_cftime=None, + decode_timedelta=None, + first_dim="auto", + reindex_angle=False, + site_coords=True, + sweep=None, + optional=True, + optional_groups=False, + ): + def _discover(): + dmet = DataMetFile(filename_or_obj) + return [ + f"sweep_{i}" + for i in range(dmet.scan_metadata["elevation_number"]) + ] + + sweeps = _resolve_sweeps(sweep, _discover) + + ds_kwargs = dict( + mask_and_scale=mask_and_scale, + decode_times=decode_times, + concat_characters=concat_characters, + decode_coords=decode_coords, + drop_variables=drop_variables, + use_cftime=use_cftime, + decode_timedelta=decode_timedelta, + first_dim=first_dim, + reindex_angle=reindex_angle, + site_as_coords=site_coords, + ) + + ls_ds = [ + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) + for swp in sweeps + ] + return _build_groups_dict( + ls_ds, optional=optional, optional_groups=optional_groups + ) + + def open_datatree(self, filename_or_obj, **kwargs): + groups_dict = self.open_groups_as_dict(filename_or_obj, **kwargs) + return DataTree.from_dict(groups_dict) + def open_datamet_datatree(filename_or_obj, **kwargs): """Open DataMet dataset as :py:class:`xarray.DataTree`. - Parameters - ---------- - filename_or_obj : str, Path, file-like or DataStore - Strings and Path objects are interpreted as a path to a local or remote - radar file - - Keyword Arguments - ----------------- - sweep : int, list of int, optional - Sweep number(s) to extract, default to first sweep. If None, all sweeps are - extracted into a list. - first_dim : str - Can be ``time`` or ``auto`` first dimension. If set to ``auto``, - first dimension will be either ``azimuth`` or ``elevation`` depending on - type of sweep. Defaults to ``auto``. - reindex_angle : bool or dict - Defaults to False, no reindexing. Given dict should contain the kwargs to - reindex_angle. Only invoked if `decode_coord=True`. - fix_second_angle : bool - If True, fixes erroneous second angle data. Defaults to ``False``. - site_as_coords : bool - Attach radar site-coordinates to Dataset, defaults to ``True``. - kwargs : dict - Additional kwargs are fed to :py:func:`xarray.open_dataset`. - - Returns - ------- - dtree: xarray.DataTree - DataTree + .. deprecated:: + Use ``xd.open_datatree(file, engine="datamet")`` instead. """ - # handle kwargs, extract first_dim - backend_kwargs = kwargs.pop("backend_kwargs", {}) + _deprecation_warning("open_datamet_datatree", "datamet") + + kwargs.pop("backend_kwargs", {}) optional = kwargs.pop("optional", True) optional_groups = kwargs.pop("optional_groups", False) - kwargs["backend_kwargs"] = backend_kwargs - sweep = kwargs.pop("sweep", None) - sweeps = [] - kwargs["backend_kwargs"] = backend_kwargs - - if isinstance(sweep, str): - sweeps = [sweep] - elif isinstance(sweep, int): - sweeps = [f"sweep_{sweep}"] - elif isinstance(sweep, list): - if isinstance(sweep[0], int): - sweeps = [f"sweep_{i}" for i in sweep] - else: - sweeps.extend(sweep) - else: - # Get number of sweeps from data - dmet = DataMetFile(filename_or_obj) - sweeps = [ - f"sweep_{i}" for i in range(0, dmet.scan_metadata["elevation_number"]) - ] + # Remap legacy kwarg name + if "site_as_coords" in kwargs: + kwargs["site_coords"] = kwargs.pop("site_as_coords") - kw = {**kwargs, "site_as_coords": False} - ls_ds: list[xr.Dataset] = [ - xr.open_dataset( - filename_or_obj, group=swp, engine=DataMetBackendEntrypoint, **kw - ) - for swp in sweeps - ] - - dtree: dict = { - "/": _get_required_root_dataset(ls_ds, optional=optional), - } - if optional_groups: - dtree["/radar_parameters"] = _get_subgroup(ls_ds, radar_parameters_subgroup) - dtree["/georeferencing_correction"] = _get_subgroup( - ls_ds, georeferencing_correction_subgroup - ) - dtree["/radar_calibration"] = _get_radar_calibration( - ls_ds, radar_calibration_subgroup - ) - dtree = _attach_sweep_groups(dtree, ls_ds) - return DataTree.from_dict(dtree) + return DataMetBackendEntrypoint().open_datatree( + filename_or_obj, + sweep=sweep, + optional=optional, + optional_groups=optional_groups, + **kwargs, + ) diff --git a/xradar/io/backends/furuno.py b/xradar/io/backends/furuno.py index db0bda18..447a89c9 100644 --- a/xradar/io/backends/furuno.py +++ b/xradar/io/backends/furuno.py @@ -46,7 +46,6 @@ import lat_lon_parser import numpy as np -import xarray as xr from xarray import DataTree from xarray.backends.common import AbstractDataStore, BackendArray, BackendEntrypoint from xarray.backends.file_manager import CachingFileManager @@ -57,7 +56,6 @@ from ... import util from ...model import ( - georeferencing_correction_subgroup, get_altitude_attrs, get_azimuth_attrs, get_elevation_attrs, @@ -67,7 +65,6 @@ get_time_attrs, moment_attrs, radar_calibration_subgroup, - radar_parameters_subgroup, sweep_vars_mapping, ) from .common import ( @@ -77,12 +74,11 @@ UINT2, UINT4, _apply_site_as_coords, - _attach_sweep_groups, + _build_groups_dict, _calculate_angle_res, + _deprecation_warning, _get_fmt_string, - _get_radar_calibration, - _get_required_root_dataset, - _get_subgroup, + _resolve_sweeps, _unpack_dictionary, ) @@ -707,6 +703,7 @@ class FurunoBackendEntrypoint(BackendEntrypoint): description = "Open FURUNO (.scn, .scnx) in Xarray" url = "https://xradar.rtfd.io/en/latest/io.html#furuno-binary-data" + supports_groups = True def open_dataset( self, @@ -779,58 +776,76 @@ def open_dataset( return ds + def open_groups_as_dict( + self, + filename_or_obj, + *, + mask_and_scale=True, + decode_times=True, + concat_characters=True, + decode_coords=True, + drop_variables=None, + use_cftime=None, + decode_timedelta=None, + first_dim="auto", + reindex_angle=False, + fix_second_angle=False, + site_coords=True, + sweep=None, + optional=True, + optional_groups=False, + obsmode=None, + ): + sweeps = _resolve_sweeps(sweep, lambda: ["sweep_0"]) + + ds_kwargs = dict( + mask_and_scale=mask_and_scale, + decode_times=decode_times, + concat_characters=concat_characters, + decode_coords=decode_coords, + drop_variables=drop_variables, + use_cftime=use_cftime, + decode_timedelta=decode_timedelta, + first_dim=first_dim, + reindex_angle=reindex_angle, + fix_second_angle=fix_second_angle, + site_as_coords=site_coords, + obsmode=obsmode, + ) + + ls_ds = [ + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) + for swp in sweeps + ] + return _build_groups_dict( + ls_ds, optional=optional, optional_groups=optional_groups + ) + + def open_datatree(self, filename_or_obj, **kwargs): + groups_dict = self.open_groups_as_dict(filename_or_obj, **kwargs) + return DataTree.from_dict(groups_dict) + def open_furuno_datatree(filename_or_obj, **kwargs): """Open FURUNO dataset as :py:class:`xarray.DataTree`. - Parameters - ---------- - filename_or_obj : str, Path, file-like or DataStore - Strings and Path objects are interpreted as a path to a local or remote - radar file - - Keyword Arguments - ----------------- - sweep : int, list of int, optional - Sweep number(s) to extract, default to first sweep. If None, all sweeps are - extracted into a list. - first_dim : str - Can be ``time`` or ``auto`` first dimension. If set to ``auto``, - first dimension will be either ``azimuth`` or ``elevation`` depending on - type of sweep. Defaults to ``auto``. - reindex_angle : bool or dict - Defaults to False, no reindexing. Given dict should contain the kwargs to - reindex_angle. Only invoked if `decode_coord=True`. - fix_second_angle : bool - If True, fixes erroneous second angle data. Defaults to ``False``. - site_as_coords : bool - Attach radar site-coordinates to Dataset, defaults to ``True``. - kwargs : dict - Additional kwargs are fed to :py:func:`xarray.open_dataset`. - - Returns - ------- - dtree: xarray.DataTree - DataTree + .. deprecated:: + Use ``xd.open_datatree(file, engine="furuno")`` instead. """ - # handle kwargs, extract first_dim + _deprecation_warning("open_furuno_datatree", "furuno") + backend_kwargs = kwargs.pop("backend_kwargs", {}) optional = backend_kwargs.pop("optional", True) optional_groups = kwargs.pop("optional_groups", False) - kwargs["backend_kwargs"] = backend_kwargs - - ls_ds = [xr.open_dataset(filename_or_obj, engine="furuno", **kwargs)] + sweep = kwargs.pop("sweep", None) + # Remap legacy kwarg name + if "site_as_coords" in kwargs: + kwargs["site_coords"] = kwargs.pop("site_as_coords") - dtree: dict = { - "/": _get_required_root_dataset(ls_ds, optional=optional), - } - if optional_groups: - dtree["/radar_parameters"] = _get_subgroup(ls_ds, radar_parameters_subgroup) - dtree["/georeferencing_correction"] = _get_subgroup( - ls_ds, georeferencing_correction_subgroup - ) - dtree["/radar_calibration"] = _get_radar_calibration( - ls_ds, radar_calibration_subgroup - ) - dtree = _attach_sweep_groups(dtree, ls_ds) - return DataTree.from_dict(dtree) + return FurunoBackendEntrypoint().open_datatree( + filename_or_obj, + sweep=sweep, + optional=optional, + optional_groups=optional_groups, + **kwargs, + ) diff --git a/xradar/io/backends/gamic.py b/xradar/io/backends/gamic.py index 540f5b11..b261f513 100644 --- a/xradar/io/backends/gamic.py +++ b/xradar/io/backends/gamic.py @@ -38,7 +38,6 @@ import dateutil import h5netcdf import numpy as np -import xarray as xr from xarray import DataTree from xarray.backends.common import ( AbstractDataStore, @@ -54,26 +53,23 @@ from ... import util from ...model import ( - georeferencing_correction_subgroup, get_azimuth_attrs, get_elevation_attrs, get_time_attrs, moment_attrs, optional_root_attrs, radar_calibration_subgroup, - radar_parameters_subgroup, required_global_attrs, sweep_vars_mapping, ) from .common import ( _apply_site_as_coords, - _attach_sweep_groups, + _build_groups_dict, + _deprecation_warning, _fix_angle, _get_h5group_names, - _get_radar_calibration, - _get_required_root_dataset, - _get_subgroup, _prepare_backend_ds, + _resolve_sweeps, ) from .odim import H5NetCDFArrayWrapper, _get_h5netcdf_encoding, _H5NetCDFMetadata @@ -407,6 +403,7 @@ class GamicBackendEntrypoint(BackendEntrypoint): description = "Open GAMIC HDF5 (.h5, .hdf5, .mvol) using h5netcdf in Xarray" url = "https://xradar.rtfd.io/en/latest/io.html#gamic-hdf5" + supports_groups = True def open_dataset( self, @@ -495,76 +492,83 @@ def open_dataset( return ds + def open_groups_as_dict( + self, + filename_or_obj, + *, + mask_and_scale=True, + decode_times=True, + concat_characters=True, + decode_coords=True, + drop_variables=None, + use_cftime=None, + decode_timedelta=None, + format=None, + invalid_netcdf=None, + phony_dims="access", + decode_vlen_strings=True, + first_dim="auto", + reindex_angle=False, + fix_second_angle=False, + site_coords=True, + sweep=None, + optional=True, + optional_groups=False, + ): + sweeps = _resolve_sweeps( + sweep, lambda: _get_h5group_names(filename_or_obj, "gamic") + ) -def open_gamic_datatree(filename_or_obj, **kwargs): - """Open GAMIC HDF5 dataset as :py:class:`xarray.DataTree`. + ds_kwargs = dict( + mask_and_scale=mask_and_scale, + decode_times=decode_times, + concat_characters=concat_characters, + decode_coords=decode_coords, + drop_variables=drop_variables, + use_cftime=use_cftime, + decode_timedelta=decode_timedelta, + format=format, + invalid_netcdf=invalid_netcdf, + phony_dims=phony_dims, + decode_vlen_strings=decode_vlen_strings, + first_dim=first_dim, + reindex_angle=reindex_angle, + fix_second_angle=fix_second_angle, + site_as_coords=site_coords, + ) - Parameters - ---------- - filename_or_obj : str, Path, file-like or DataStore - Strings and Path objects are interpreted as a path to a local or remote - radar file + ls_ds = [ + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) for swp in sweeps + ] + return _build_groups_dict( + ls_ds, optional=optional, optional_groups=optional_groups + ) - Keyword Arguments - ----------------- - sweep : int, list of int, optional - Sweep number(s) to extract, default to first sweep. If None, all sweeps are - extracted into a list. - first_dim : str - Can be ``time`` or ``auto`` first dimension. If set to ``auto``, - first dimension will be either ``azimuth`` or ``elevation`` depending on - type of sweep. Defaults to ``auto``. - reindex_angle : bool or dict - Defaults to False, no reindexing. Given dict should contain the kwargs to - reindex_angle. Only invoked if `decode_coord=True`. - fix_second_angle : bool - If True, fixes erroneous second angle data. Defaults to ``False``. - site_as_coords : bool - Attach radar site-coordinates to Dataset, defaults to ``True``. - kwargs : dict - Additional kwargs are fed to :py:func:`xarray.open_dataset`. + def open_datatree(self, filename_or_obj, **kwargs): + groups_dict = self.open_groups_as_dict(filename_or_obj, **kwargs) + return DataTree.from_dict(groups_dict) - Returns - ------- - dtree: xarray.DataTree - DataTree + +def open_gamic_datatree(filename_or_obj, **kwargs): + """Open GAMIC HDF5 dataset as :py:class:`xarray.DataTree`. + + .. deprecated:: + Use ``xd.open_datatree(file, engine="gamic")`` instead. """ - # handle kwargs, extract first_dim + _deprecation_warning("open_gamic_datatree", "gamic") + backend_kwargs = kwargs.pop("backend_kwargs", {}) + # Capital-O "Optional" is the legacy GAMIC convention optional = backend_kwargs.pop("Optional", True) optional_groups = kwargs.pop("optional_groups", False) sweep = kwargs.pop("sweep", None) - sweeps = [] - kwargs["backend_kwargs"] = backend_kwargs - - if isinstance(sweep, str): - sweeps = [sweep] - elif isinstance(sweep, int): - sweeps = [f"sweep_{sweep}"] - elif isinstance(sweep, list): - if isinstance(sweep[0], int): - sweeps = [f"sweep_{i}" for i in sweep] - else: - sweeps.extend(sweep) - else: - sweeps = _get_h5group_names(filename_or_obj, "gamic") - - kw = {**kwargs, "site_as_coords": False} - ls_ds: list[xr.Dataset] = [ - xr.open_dataset(filename_or_obj, group=swp, engine="gamic", **kw) - for swp in sweeps - ] - - dtree: dict = { - "/": _get_required_root_dataset(ls_ds, optional=optional), - } - if optional_groups: - dtree["/radar_parameters"] = _get_subgroup(ls_ds, radar_parameters_subgroup) - dtree["/georeferencing_correction"] = _get_subgroup( - ls_ds, georeferencing_correction_subgroup - ) - dtree["/radar_calibration"] = _get_radar_calibration( - ls_ds, radar_calibration_subgroup - ) - dtree = _attach_sweep_groups(dtree, ls_ds) - return DataTree.from_dict(dtree) + if "site_as_coords" in kwargs: + kwargs["site_coords"] = kwargs.pop("site_as_coords") + + return GamicBackendEntrypoint().open_datatree( + filename_or_obj, + sweep=sweep, + optional=optional, + optional_groups=optional_groups, + **kwargs, + ) diff --git a/xradar/io/backends/hpl.py b/xradar/io/backends/hpl.py index 3bc17f1e..2f3e9d24 100644 --- a/xradar/io/backends/hpl.py +++ b/xradar/io/backends/hpl.py @@ -45,21 +45,17 @@ from xarray.core.utils import FrozenDict from ...model import ( - georeferencing_correction_subgroup, get_altitude_attrs, get_azimuth_attrs, get_elevation_attrs, get_latitude_attrs, get_longitude_attrs, - radar_calibration_subgroup, - radar_parameters_subgroup, ) from .common import ( _apply_site_as_coords, - _attach_sweep_groups, - _get_radar_calibration, - _get_required_root_dataset, - _get_subgroup, + _build_groups_dict, + _deprecation_warning, + _resolve_sweeps, ) variable_attr_dict = {} @@ -516,6 +512,7 @@ class HPLBackendEntrypoint(BackendEntrypoint): description = "Backend for reading Halo Photonics Doppler lidar processed data" url = "https://xradar.rtfd.io/en/latest/io.html#metek" + supports_groups = True def open_dataset( self, @@ -591,8 +588,76 @@ def open_dataset( return ds + def open_groups_as_dict( + self, + filename_or_obj, + *, + mask_and_scale=True, + decode_times=True, + concat_characters=True, + decode_coords=True, + drop_variables=None, + use_cftime=None, + decode_timedelta=None, + format=None, + invalid_netcdf=None, + phony_dims="access", + decode_vlen_strings=True, + first_dim="auto", + site_coords=True, + sweep=None, + optional=True, + optional_groups=False, + latitude=0, + longitude=0, + altitude=0, + transition_threshold_azi=0.05, + transition_threshold_el=0.001, + ): + sweeps = _resolve_sweeps( + sweep, lambda: _get_hpl_group_names(filename_or_obj) + ) + + ds_kwargs = dict( + mask_and_scale=mask_and_scale, + decode_times=decode_times, + concat_characters=concat_characters, + decode_coords=decode_coords, + drop_variables=drop_variables, + use_cftime=use_cftime, + decode_timedelta=decode_timedelta, + format=format, + invalid_netcdf=invalid_netcdf, + phony_dims=phony_dims, + decode_vlen_strings=decode_vlen_strings, + first_dim=first_dim, + site_as_coords=site_coords, + latitude=latitude, + longitude=longitude, + altitude=altitude, + transition_threshold_azi=transition_threshold_azi, + transition_threshold_el=transition_threshold_el, + ) + + ls_ds = [ + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) + for swp in sweeps + ] + groups_dict = _build_groups_dict( + ls_ds, optional=optional, optional_groups=optional_groups + ) + # HPL root uses "fixed_angle" instead of "sweep_fixed_angle" + root = groups_dict["/"] + if "sweep_fixed_angle" in root: + groups_dict["/"] = root.rename({"sweep_fixed_angle": "fixed_angle"}) + return groups_dict -def _get_h5group_names(filename_or_obj): + def open_datatree(self, filename_or_obj, **kwargs): + groups_dict = self.open_groups_as_dict(filename_or_obj, **kwargs) + return DataTree.from_dict(groups_dict) + + +def _get_hpl_group_names(filename_or_obj): store = HplStore.open(filename_or_obj) return [f"sweep_{i}" for i in store.root.data["sweep_number"]] @@ -600,74 +665,23 @@ def _get_h5group_names(filename_or_obj): def open_hpl_datatree(filename_or_obj, **kwargs): """Open Halo Photonics processed Doppler lidar dataset as :py:class:`xarray.DataTree`. - Parameters - ---------- - filename_or_obj : str, Path, file-like or DataStore - Strings and Path objects are interpreted as a path to a local or remote - radar file - - Keyword Arguments - ----------------- - sweep : int, list of int, optional - Sweep number(s) to extract, default to first sweep. If None, all sweeps are - extracted into a list. - first_dim : str - Can be ``time`` or ``auto`` first dimension. If set to ``auto``, - first dimension will be either ``azimuth`` or ``elevation`` depending on - type of sweep. Defaults to ``auto``. - reindex_angle : bool or dict - Defaults to False, no reindexing. Given dict should contain the kwargs to - reindex_angle. Only invoked if `decode_coord=True`. - fix_second_angle : bool - If True, fixes erroneous second angle data. Defaults to ``False``. - site_as_coords : bool - Attach radar site-coordinates to Dataset, defaults to ``True``. - kwargs : dict - Additional kwargs are fed to :py:func:`xarray.open_dataset`. - - Returns - ------- - dtree: xarray.DataTree - DataTree + .. deprecated:: + Use ``xd.open_datatree(file, engine="hpl")`` instead. """ - # handle kwargs, extract first_dim + _deprecation_warning("open_hpl_datatree", "hpl") + backend_kwargs = kwargs.pop("backend_kwargs", {}) - optional = backend_kwargs.pop("optional", None) + optional = backend_kwargs.pop("optional", True) optional_groups = kwargs.pop("optional_groups", False) sweep = kwargs.pop("sweep", None) - sweeps = [] - kwargs["backend_kwargs"] = backend_kwargs - - if isinstance(sweep, str): - sweeps = [sweep] - elif isinstance(sweep, int): - sweeps = [f"sweep_{sweep}"] - elif isinstance(sweep, list): - if isinstance(sweep[0], int): - sweeps = [f"sweep_{i + 1}" for i in sweep] - else: - sweeps.extend(sweep) - else: - sweeps = _get_h5group_names(filename_or_obj) - - kw = {**kwargs, "site_as_coords": False} - ls_ds: list[xr.Dataset] = [ - xr.open_dataset(filename_or_obj, group=swp, engine="hpl", **kw) - for swp in sweeps - ] + # Remap legacy kwarg name + if "site_as_coords" in kwargs: + kwargs["site_coords"] = kwargs.pop("site_as_coords") - dtree: dict = { - "/": _get_required_root_dataset(ls_ds, optional=optional).rename( - {"sweep_fixed_angle": "fixed_angle"} - ), - } - if optional_groups: - dtree["/radar_parameters"] = _get_subgroup(ls_ds, radar_parameters_subgroup) - dtree["/georeferencing_correction"] = _get_subgroup( - ls_ds, georeferencing_correction_subgroup - ) - dtree["/radar_calibration"] = _get_radar_calibration( - ls_ds, radar_calibration_subgroup - ) - dtree = _attach_sweep_groups(dtree, ls_ds) - return DataTree.from_dict(dtree) + return HPLBackendEntrypoint().open_datatree( + filename_or_obj, + sweep=sweep, + optional=optional, + optional_groups=optional_groups, + **kwargs, + ) diff --git a/xradar/io/backends/iris.py b/xradar/io/backends/iris.py index 27dfa5f2..e41f325a 100644 --- a/xradar/io/backends/iris.py +++ b/xradar/io/backends/iris.py @@ -43,7 +43,6 @@ from collections import OrderedDict import numpy as np -import xarray as xr from xarray import DataTree from xarray.backends.common import AbstractDataStore, BackendArray, BackendEntrypoint from xarray.backends.file_manager import CachingFileManager @@ -55,7 +54,6 @@ from ... import util from ...model import ( - georeferencing_correction_subgroup, get_altitude_attrs, get_azimuth_attrs, get_elevation_attrs, @@ -63,16 +61,13 @@ get_longitude_attrs, get_range_attrs, moment_attrs, - radar_calibration_subgroup, - radar_parameters_subgroup, sweep_vars_mapping, ) from .common import ( _apply_site_as_coords, - _attach_sweep_groups, - _get_radar_calibration, - _get_required_root_dataset, - _get_subgroup, + _build_groups_dict, + _deprecation_warning, + _resolve_sweeps, ) IRIS_LOCK = SerializableLock() @@ -3991,6 +3986,7 @@ class IrisBackendEntrypoint(BackendEntrypoint): description = "Open IRIS/Sigmet files in Xarray" url = "https://xradar.rtfd.io/latest/io.html#iris-sigmet-data-i-o" + supports_groups = True def open_dataset( self, @@ -4068,75 +4064,80 @@ def open_dataset( return ds + def open_groups_as_dict( + self, + filename_or_obj, + *, + mask_and_scale=True, + decode_times=True, + concat_characters=True, + decode_coords=True, + drop_variables=None, + use_cftime=None, + decode_timedelta=None, + group=None, + lock=None, + first_dim="auto", + reindex_angle=False, + fix_second_angle=False, + site_coords=True, + sweep=None, + optional=True, + optional_groups=False, + ): + sweeps = _resolve_sweeps( + sweep, lambda: _get_iris_group_names(filename_or_obj) + ) + + ds_kwargs = dict( + mask_and_scale=mask_and_scale, + decode_times=decode_times, + concat_characters=concat_characters, + decode_coords=decode_coords, + drop_variables=drop_variables, + use_cftime=use_cftime, + decode_timedelta=decode_timedelta, + lock=lock, + first_dim=first_dim, + reindex_angle=reindex_angle, + fix_second_angle=fix_second_angle, + site_as_coords=site_coords, + ) + + ls_ds = [ + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) + for swp in sweeps + ] + return _build_groups_dict( + ls_ds, optional=optional, optional_groups=optional_groups + ) + + def open_datatree(self, filename_or_obj, **kwargs): + groups_dict = self.open_groups_as_dict(filename_or_obj, **kwargs) + return DataTree.from_dict(groups_dict) + def open_iris_datatree(filename_or_obj, **kwargs): """Open Iris/Sigmet dataset as :py:class:`xarray.DataTree`. - Parameters - ---------- - filename_or_obj : str, Path, file-like or DataStore - Strings and Path objects are interpreted as a path to a local or remote - radar file - - Keyword Arguments - ----------------- - sweep : int, list of int, optional - Sweep number(s) to extract, default to first sweep. If None, all sweeps are - extracted into a list. - first_dim : str - Can be ``time`` or ``auto`` first dimension. If set to ``auto``, - first dimension will be either ``azimuth`` or ``elevation`` depending on - type of sweep. Defaults to ``auto``. - reindex_angle : bool or dict - Defaults to False, no reindexing. Given dict should contain the kwargs to - reindex_angle. Only invoked if `decode_coord=True`. - fix_second_angle : bool - If True, fixes erroneous second angle data. Defaults to ``False``. - site_as_coords : bool - Attach radar site-coordinates to Dataset, defaults to ``True``. - kwargs : dict - Additional kwargs are fed to :py:func:`xarray.open_dataset`. - - Returns - ------- - dtree: xarray.DataTree - DataTree + .. deprecated:: + Use ``xd.open_datatree(file, engine="iris")`` instead. """ - # handle kwargs, extract first_dim + _deprecation_warning("open_iris_datatree", "iris") + backend_kwargs = kwargs.pop("backend_kwargs", {}) - optional = kwargs.pop("optional", True) + # Capital-O "Optional" is legacy convention from original API + optional = backend_kwargs.pop("Optional", True) optional_groups = kwargs.pop("optional_groups", False) sweep = kwargs.pop("sweep", None) - sweeps = [] - kwargs["backend_kwargs"] = backend_kwargs - - if isinstance(sweep, str): - sweeps = [sweep] - elif isinstance(sweep, int): - sweeps = [f"sweep_{sweep}"] - elif isinstance(sweep, list): - if isinstance(sweep[0], int): - sweeps = [f"sweep_{sw}" for sw in sweep] - else: - sweeps.extend(sweep) - else: - sweeps = _get_iris_group_names(filename_or_obj) + # Remap legacy kwarg name + if "site_as_coords" in kwargs: + kwargs["site_coords"] = kwargs.pop("site_as_coords") - kw = {**kwargs, "site_as_coords": False} - ls_ds: list[xr.Dataset] = [ - xr.open_dataset(filename_or_obj, group=swp, engine="iris", **kw) - for swp in sweeps - ] - dtree: dict = { - "/": _get_required_root_dataset(ls_ds, optional=optional), - } - if optional_groups: - dtree["/radar_parameters"] = _get_subgroup(ls_ds, radar_parameters_subgroup) - dtree["/georeferencing_correction"] = _get_subgroup( - ls_ds, georeferencing_correction_subgroup - ) - dtree["/radar_calibration"] = _get_radar_calibration( - ls_ds, radar_calibration_subgroup - ) - dtree = _attach_sweep_groups(dtree, ls_ds) - return DataTree.from_dict(dtree) + return IrisBackendEntrypoint().open_datatree( + filename_or_obj, + sweep=sweep, + optional=optional, + optional_groups=optional_groups, + **kwargs, + ) diff --git a/xradar/io/backends/metek.py b/xradar/io/backends/metek.py index ef5d530a..dd081f37 100644 --- a/xradar/io/backends/metek.py +++ b/xradar/io/backends/metek.py @@ -33,21 +33,17 @@ from xarray.core.utils import FrozenDict from ...model import ( - georeferencing_correction_subgroup, get_altitude_attrs, get_azimuth_attrs, get_elevation_attrs, get_latitude_attrs, get_longitude_attrs, get_time_attrs, - radar_calibration_subgroup, - radar_parameters_subgroup, ) from .common import ( - _attach_sweep_groups, - _get_radar_calibration, - _get_required_root_dataset, - _get_subgroup, + _build_groups_dict, + _deprecation_warning, + _resolve_sweeps, ) __all__ = [ @@ -577,6 +573,7 @@ class MRRBackendEntrypoint(BackendEntrypoint): description = "Backend for reading Metek MRR2 processed and raw data" url = "https://xradar.rtfd.io/en/latest/io.html#metek" + supports_groups = True def open_dataset( self, @@ -630,75 +627,78 @@ def open_dataset( return ds + def open_groups_as_dict( + self, + filename_or_obj, + *, + mask_and_scale=True, + decode_times=True, + concat_characters=True, + decode_coords=True, + drop_variables=None, + use_cftime=None, + decode_timedelta=None, + format=None, + invalid_netcdf=None, + phony_dims="access", + decode_vlen_strings=True, + first_dim="auto", + site_coords=True, + sweep=None, + optional=True, + optional_groups=False, + ): + sweeps = _resolve_sweeps(sweep, lambda: ["sweep_0"]) -def open_metek_datatree(filename_or_obj, **kwargs): - """Open Metek MRR2 dataset as :py:class:`xarray.DataTree`. + ds_kwargs = dict( + mask_and_scale=mask_and_scale, + decode_times=decode_times, + concat_characters=concat_characters, + decode_coords=decode_coords, + drop_variables=drop_variables, + use_cftime=use_cftime, + decode_timedelta=decode_timedelta, + format=format, + invalid_netcdf=invalid_netcdf, + phony_dims=phony_dims, + decode_vlen_strings=decode_vlen_strings, + first_dim=first_dim, + site_as_coords=site_coords, + ) - Parameters - ---------- - filename_or_obj : str, Path, file-like or DataStore - Strings and Path objects are interpreted as a path to a local or remote - radar file + ls_ds = [ + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) + for swp in sweeps + ] + return _build_groups_dict( + ls_ds, optional=optional, optional_groups=optional_groups + ) - Keyword Arguments - ----------------- - sweep : int, list of int, optional - Sweep number(s) to extract, default to first sweep. If None, all sweeps are - extracted into a list. - first_dim : str - Can be ``time`` or ``auto`` first dimension. If set to ``auto``, - first dimension will be either ``azimuth`` or ``elevation`` depending on - type of sweep. Defaults to ``auto``. - reindex_angle : bool or dict - Defaults to False, no reindexing. Given dict should contain the kwargs to - reindex_angle. Only invoked if `decode_coord=True`. - fix_second_angle : bool - If True, fixes erroneous second angle data. Defaults to ``False``. - site_as_coords : bool - Attach radar site-coordinates to Dataset, defaults to ``True``. - kwargs : dict - Additional kwargs are fed to :py:func:`xarray.open_dataset`. + def open_datatree(self, filename_or_obj, **kwargs): + groups_dict = self.open_groups_as_dict(filename_or_obj, **kwargs) + return DataTree.from_dict(groups_dict) + + +def open_metek_datatree(filename_or_obj, **kwargs): + """Open Metek MRR2 dataset as :py:class:`xarray.DataTree`. - Returns - ------- - dtree: xarray.DataTree - DataTree + .. deprecated:: + Use ``xd.open_datatree(file, engine="metek")`` instead. """ - # handle kwargs, extract first_dim + _deprecation_warning("open_metek_datatree", "metek") + backend_kwargs = kwargs.pop("backend_kwargs", {}) optional = backend_kwargs.pop("optional", True) optional_groups = kwargs.pop("optional_groups", False) sweep = kwargs.pop("sweep", None) - sweeps = [] - kwargs["backend_kwargs"] = backend_kwargs - - if isinstance(sweep, str): - sweeps = [sweep] - elif isinstance(sweep, int): - sweeps = [f"sweep_{sweep}"] - elif isinstance(sweep, list): - if isinstance(sweep[0], int): - sweeps = [f"sweep_{i + 1}" for i in sweep] - else: - sweeps.extend(sweep) - else: - sweeps = ["sweep_0"] - - kw = {**kwargs, "site_as_coords": False} - ls_ds: list[xr.Dataset] = [ - xr.open_dataset(filename_or_obj, group=swp, engine="metek", **kw) - for swp in sweeps - ].copy() - dtree: dict = { - "/": _get_required_root_dataset(ls_ds, optional=optional), - } - if optional_groups: - dtree["/radar_parameters"] = _get_subgroup(ls_ds, radar_parameters_subgroup) - dtree["/georeferencing_correction"] = _get_subgroup( - ls_ds, georeferencing_correction_subgroup - ) - dtree["/radar_calibration"] = _get_radar_calibration( - ls_ds, radar_calibration_subgroup - ) - dtree = _attach_sweep_groups(dtree, ls_ds) - return DataTree.from_dict(dtree) + # Remap legacy kwarg name + if "site_as_coords" in kwargs: + kwargs["site_coords"] = kwargs.pop("site_as_coords") + + return MRRBackendEntrypoint().open_datatree( + filename_or_obj, + sweep=sweep, + optional=optional, + optional_groups=optional_groups, + **kwargs, + ) diff --git a/xradar/io/backends/nexrad_level2.py b/xradar/io/backends/nexrad_level2.py index 1913fdad..634907d9 100644 --- a/xradar/io/backends/nexrad_level2.py +++ b/xradar/io/backends/nexrad_level2.py @@ -60,6 +60,7 @@ _deprecation_warning, _get_radar_calibration, _get_subgroup, + _resolve_sweeps, ) from xradar.model import ( georeferencing_correction_subgroup, @@ -2005,24 +2006,17 @@ def open_groups_as_dict( act_sweeps = len(nex.msg_31_data_header) incomplete = nex.incomplete_sweeps + # Normalise NodePath strings before resolving sweeps if isinstance(sweep, str): sweep = NodePath(sweep).name - sweeps = [sweep] - elif isinstance(sweep, int): - sweeps = [f"sweep_{sweep}"] - elif isinstance(sweep, list): - if not sweep: - raise ValueError("sweep list is empty.") - if isinstance(sweep[0], int): - sweeps = [f"sweep_{i}" for i in sweep] - elif isinstance(sweep[0], str): - sweeps = [NodePath(i).name for i in sweep] - else: - raise ValueError( - "Invalid type in 'sweep' list. Expected integers " - "(e.g., [0, 1, 2]) or strings " - "(e.g. [/sweep_0, sweep_1])." - ) + elif isinstance(sweep, list) and sweep and isinstance(sweep[0], str): + sweep = [NodePath(i).name for i in sweep] + + if sweep is not None: + sweeps = _resolve_sweeps( + sweep, + lambda: [f"sweep_{i}" for i in range(act_sweeps)], + ) else: if incomplete_sweep == "drop": sweeps = [ diff --git a/xradar/io/backends/odim.py b/xradar/io/backends/odim.py index 99e26bac..3d487d9b 100644 --- a/xradar/io/backends/odim.py +++ b/xradar/io/backends/odim.py @@ -73,6 +73,7 @@ _get_h5group_names, _maybe_decode, _prepare_backend_ds, + _resolve_sweeps, ) HDF5_LOCK = SerializableLock() @@ -896,17 +897,9 @@ def open_groups_as_dict( optional=True, optional_groups=False, ): - if isinstance(sweep, str): - sweeps = [sweep] - elif isinstance(sweep, int): - sweeps = [f"sweep_{sweep}"] - elif isinstance(sweep, list): - if isinstance(sweep[0], int): - sweeps = [f"sweep_{i}" for i in sweep] - else: - sweeps = list(sweep) - else: - sweeps = _get_h5group_names(filename_or_obj, "odim") + sweeps = _resolve_sweeps( + sweep, lambda: _get_h5group_names(filename_or_obj, "odim") + ) ds_kwargs = dict( mask_and_scale=mask_and_scale, diff --git a/xradar/io/backends/rainbow.py b/xradar/io/backends/rainbow.py index 69f9ac29..72c30806 100644 --- a/xradar/io/backends/rainbow.py +++ b/xradar/io/backends/rainbow.py @@ -37,7 +37,6 @@ import zlib import numpy as np -import xarray as xr import xmltodict from xarray import DataTree from xarray.backends.common import AbstractDataStore, BackendArray, BackendEntrypoint @@ -49,7 +48,6 @@ from ... import util from ...model import ( - georeferencing_correction_subgroup, get_altitude_attrs, get_azimuth_attrs, get_elevation_attrs, @@ -58,16 +56,13 @@ get_range_attrs, get_time_attrs, moment_attrs, - radar_calibration_subgroup, - radar_parameters_subgroup, sweep_vars_mapping, ) from .common import ( _apply_site_as_coords, - _attach_sweep_groups, - _get_radar_calibration, - _get_required_root_dataset, - _get_subgroup, + _build_groups_dict, + _deprecation_warning, + _resolve_sweeps, ) #: mapping of rainbow moment names to CfRadial2/ODIM names @@ -799,6 +794,7 @@ class RainbowBackendEntrypoint(BackendEntrypoint): description = "Open Rainbow5 files in Xarray" url = "https://xradar.rtfd.io/latest/io.html#rainbow-data-i-o" + supports_groups = True def open_dataset( self, @@ -867,6 +863,53 @@ def open_dataset( return ds + def open_groups_as_dict( + self, + filename_or_obj, + *, + mask_and_scale=True, + decode_times=True, + concat_characters=True, + decode_coords=True, + drop_variables=None, + use_cftime=None, + decode_timedelta=None, + first_dim="auto", + reindex_angle=False, + site_coords=True, + sweep=None, + optional=True, + optional_groups=False, + ): + sweeps = _resolve_sweeps( + sweep, lambda: _get_rainbow_group_names(filename_or_obj) + ) + + ds_kwargs = dict( + mask_and_scale=mask_and_scale, + decode_times=decode_times, + concat_characters=concat_characters, + decode_coords=decode_coords, + drop_variables=drop_variables, + use_cftime=use_cftime, + decode_timedelta=decode_timedelta, + first_dim=first_dim, + reindex_angle=reindex_angle, + site_as_coords=site_coords, + ) + + ls_ds = [ + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) + for swp in sweeps + ] + return _build_groups_dict( + ls_ds, optional=optional, optional_groups=optional_groups + ) + + def open_datatree(self, filename_or_obj, **kwargs): + groups_dict = self.open_groups_as_dict(filename_or_obj, **kwargs) + return DataTree.from_dict(groups_dict) + def _get_rainbow_group_names(filename): with RainbowFile(filename, loaddata=False) as fh: @@ -875,74 +918,25 @@ def _get_rainbow_group_names(filename): def open_rainbow_datatree(filename_or_obj, **kwargs): - """Open ODIM_H5 dataset as :py:class:`xarray.DataTree`. - - Parameters - ---------- - filename_or_obj : str, Path, file-like or DataStore - Strings and Path objects are interpreted as a path to a local or remote - radar file - - Keyword Arguments - ----------------- - sweep : int, list of int, optional - Sweep number(s) to extract, default to first sweep. If None, all sweeps are - extracted into a list. - first_dim : str - Can be ``time`` or ``auto`` first dimension. If set to ``auto``, - first dimension will be either ``azimuth`` or ``elevation`` depending on - type of sweep. Defaults to ``auto``. - reindex_angle : bool or dict - Defaults to False, no reindexing. Given dict should contain the kwargs to - reindex_angle. Only invoked if `decode_coord=True`. - fix_second_angle : bool - If True, fixes erroneous second angle data. Defaults to ``False``. - site_as_coords : bool - Attach radar site-coordinates to Dataset, defaults to ``True``. - kwargs : dict - Additional kwargs are fed to :py:func:`xarray.open_dataset`. + """Open Rainbow5 dataset as :py:class:`xarray.DataTree`. - Returns - ------- - dtree: xarray.DataTree - DataTree + .. deprecated:: + Use ``xd.open_datatree(file, engine="rainbow")`` instead. """ - # handle kwargs, extract first_dim + _deprecation_warning("open_rainbow_datatree", "rainbow") + backend_kwargs = kwargs.pop("backend_kwargs", {}) optional = backend_kwargs.pop("optional", True) optional_groups = kwargs.pop("optional_groups", False) sweep = kwargs.pop("sweep", None) - sweeps = [] - kwargs["backend_kwargs"] = backend_kwargs - - if isinstance(sweep, str): - sweeps = [sweep] - elif isinstance(sweep, int): - sweeps = [f"sweep_{sweep}"] - elif isinstance(sweep, list): - if isinstance(sweep[0], int): - sweeps = [f"sweep_{i + 1}" for i in sweep] - else: - sweeps.extend(sweep) - else: - sweeps = _get_rainbow_group_names(filename_or_obj) - - kw = {**kwargs, "site_as_coords": False} - ls_ds: list[xr.Dataset] = [ - xr.open_dataset(filename_or_obj, group=swp, engine="rainbow", **kw) - for swp in sweeps - ] - - dtree: dict = { - "/": _get_required_root_dataset(ls_ds, optional=optional), - } - if optional_groups: - dtree["/radar_parameters"] = _get_subgroup(ls_ds, radar_parameters_subgroup) - dtree["/georeferencing_correction"] = _get_subgroup( - ls_ds, georeferencing_correction_subgroup - ) - dtree["/radar_calibration"] = _get_radar_calibration( - ls_ds, radar_calibration_subgroup - ) - dtree = _attach_sweep_groups(dtree, ls_ds) - return DataTree.from_dict(dtree) + # Remap legacy kwarg name + if "site_as_coords" in kwargs: + kwargs["site_coords"] = kwargs.pop("site_as_coords") + + return RainbowBackendEntrypoint().open_datatree( + filename_or_obj, + sweep=sweep, + optional=optional, + optional_groups=optional_groups, + **kwargs, + ) From e26fec67d641abe05a6f85bee384961554fac39d Mon Sep 17 00:00:00 2001 From: aladinor Date: Mon, 30 Mar 2026 21:49:26 -0500 Subject: [PATCH 6/8] ENH: convert UF backend and fix remaining bugs (Phase 3) UF backend: - Add supports_groups=True, open_groups_as_dict(), open_datatree() - Use _resolve_sweeps for sweep normalization - Drop _STATION_VARS from sweeps in groups_dict - Deprecate open_uf_datatree() with FutureWarning - Register "uf" in _ENGINE_REGISTRY (11/11 complete) Bug fixes: - Fix NEXRAD deprecated wrapper: site_coords=site_as_coords instead of hardcoded False - Fix NEXRAD/UF: drop _STATION_VARS from sweep datasets in open_groups_as_dict (matching _build_groups_dict behavior) Tests (87 total): - Add xr.open_datatree tests for all 11 engines - UF added to all parametrized test fixtures - Engine registry now asserts all 11 engines --- tests/io/test_backend_datatree.py | 27 ++++ xradar/io/__init__.py | 2 + xradar/io/backends/nexrad_level2.py | 6 +- xradar/io/backends/uf.py | 222 +++++++++++----------------- 4 files changed, 119 insertions(+), 138 deletions(-) diff --git a/tests/io/test_backend_datatree.py b/tests/io/test_backend_datatree.py index 00e7ad0b..17410063 100644 --- a/tests/io/test_backend_datatree.py +++ b/tests/io/test_backend_datatree.py @@ -33,6 +33,7 @@ pytest.param(("datamet", "datamet_file"), id="datamet"), pytest.param(("hpl", "hpl_file"), id="hpl"), pytest.param(("metek", "metek_ave_gz_file"), id="metek"), + pytest.param(("uf", "uf_file_1"), id="uf"), ] ) def engine_and_file(request): @@ -166,6 +167,30 @@ def test_xr_open_datatree_iris(self, iris0_file): dtree = xr.open_datatree(iris0_file, engine="iris") _assert_cfradial2_structure(dtree) + def test_xr_open_datatree_furuno(self, furuno_scn_file): + dtree = xr.open_datatree(furuno_scn_file, engine="furuno") + _assert_cfradial2_structure(dtree) + + def test_xr_open_datatree_rainbow(self, rainbow_file): + dtree = xr.open_datatree(rainbow_file, engine="rainbow") + _assert_cfradial2_structure(dtree) + + def test_xr_open_datatree_datamet(self, datamet_file): + dtree = xr.open_datatree(datamet_file, engine="datamet") + _assert_cfradial2_structure(dtree) + + def test_xr_open_datatree_hpl(self, hpl_file): + dtree = xr.open_datatree(hpl_file, engine="hpl") + _assert_cfradial2_structure(dtree) + + def test_xr_open_datatree_metek(self, metek_ave_gz_file): + dtree = xr.open_datatree(metek_ave_gz_file, engine="metek") + _assert_cfradial2_structure(dtree) + + def test_xr_open_datatree_uf(self, uf_file_1): + dtree = xr.open_datatree(uf_file_1, engine="uf") + _assert_cfradial2_structure(dtree) + # -- supports_groups attribute ----------------------------------------------- @@ -200,6 +225,7 @@ def test_registry_contains_all_engines(self): "datamet", "hpl", "metek", + "uf", } assert set(_ENGINE_REGISTRY.keys()) == expected @@ -226,6 +252,7 @@ def test_registry_contains_all_engines(self): "open_datamet_datatree": ("xradar.io.backends.datamet", "datamet_file", {}), "open_hpl_datatree": ("xradar.io.backends.hpl", "hpl_file", {}), "open_metek_datatree": ("xradar.io.backends.metek", "metek_ave_gz_file", {}), + "open_uf_datatree": ("xradar.io.backends.uf", "uf_file_1", {}), } diff --git a/xradar/io/__init__.py b/xradar/io/__init__.py index 646d3e60..9e81cd9e 100644 --- a/xradar/io/__init__.py +++ b/xradar/io/__init__.py @@ -27,6 +27,7 @@ from .backends.nexrad_level2 import NexradLevel2BackendEntrypoint from .backends.odim import OdimBackendEntrypoint from .backends.rainbow import RainbowBackendEntrypoint +from .backends.uf import UFBackendEntrypoint #: Registry mapping engine names to backend classes that support groups. _ENGINE_REGISTRY = { @@ -40,6 +41,7 @@ "datamet": DataMetBackendEntrypoint, "hpl": HPLBackendEntrypoint, "metek": MRRBackendEntrypoint, + "uf": UFBackendEntrypoint, } diff --git a/xradar/io/backends/nexrad_level2.py b/xradar/io/backends/nexrad_level2.py index 634907d9..8dbd852b 100644 --- a/xradar/io/backends/nexrad_level2.py +++ b/xradar/io/backends/nexrad_level2.py @@ -55,6 +55,7 @@ from xradar import util from xradar.io.backends.common import ( + _STATION_VARS, _apply_site_as_coords, _assign_root, _deprecation_warning, @@ -2085,7 +2086,8 @@ def open_groups_as_dict( ls_ds_with_root, radar_calibration_subgroup ) for sweep_path, ds in sweep_dict.items(): - groups_dict[f"/{sweep_path}"] = ds.drop_attrs(deep=False) + sw = ds.drop_vars(_STATION_VARS, errors="ignore").drop_attrs(deep=False) + groups_dict[f"/{sweep_path}"] = sw return groups_dict def open_datatree( @@ -2221,7 +2223,7 @@ def open_nexradlevel2_datatree( first_dim=first_dim, reindex_angle=reindex_angle, fix_second_angle=fix_second_angle, - site_coords=False, + site_coords=site_as_coords, optional=optional, optional_groups=optional_groups, incomplete_sweep=incomplete_sweep, diff --git a/xradar/io/backends/uf.py b/xradar/io/backends/uf.py index c8703ce9..d6d2ab6c 100644 --- a/xradar/io/backends/uf.py +++ b/xradar/io/backends/uf.py @@ -35,6 +35,7 @@ import dateutil import numpy as np import xarray as xr +from xarray import DataTree from xarray.backends.common import AbstractDataStore, BackendArray, BackendEntrypoint from xarray.backends.file_manager import CachingFileManager from xarray.backends.locks import SerializableLock, ensure_lock @@ -45,10 +46,13 @@ from xradar import util from xradar.io.backends.common import ( + _STATION_VARS, _apply_site_as_coords, _assign_root, + _deprecation_warning, _get_radar_calibration, _get_subgroup, + _resolve_sweeps, ) from xradar.model import ( georeferencing_correction_subgroup, @@ -741,6 +745,7 @@ class UFBackendEntrypoint(BackendEntrypoint): description = "Open Universal Format (UF) files in Xarray" url = "https://xradar.rtfd.io/latest/io.html#uf-data-i-o" + supports_groups = True def open_dataset( self, @@ -809,153 +814,98 @@ def open_dataset( return ds + def open_groups_as_dict( + self, + filename_or_obj, + *, + mask_and_scale=True, + decode_times=True, + concat_characters=True, + decode_coords=True, + drop_variables=None, + use_cftime=None, + decode_timedelta=None, + sweep=None, + first_dim="auto", + reindex_angle=False, + fix_second_angle=False, + site_coords=True, + optional=True, + optional_groups=False, + lock=None, + **kwargs, + ): + sweeps = _resolve_sweeps( + sweep, + lambda: [ + f"sweep_{i}" + for i in range(UFFile(filename_or_obj, loaddata=False).nsweeps) + ], + ) -def open_uf_datatree( - filename_or_obj, - mask_and_scale=True, - decode_times=True, - concat_characters=True, - decode_coords=True, - drop_variables=None, - use_cftime=None, - decode_timedelta=None, - sweep=None, - first_dim="auto", - reindex_angle=False, - fix_second_angle=False, - site_as_coords=True, - optional=True, - optional_groups=False, - lock=None, - **kwargs, -): - """Open a Universal Format (UF) dataset as :py:class:`xarray.DataTree`. - - This function loads UF radar data into a DataTree structure, which - organizes radar sweeps as separate nodes. Provides options for decoding time - and applying various transformations to the data. - - Parameters - ---------- - filename_or_obj : str, Path, file-like, or DataStore - The path or file-like object representing the radar file. - Path-like objects are interpreted as local or remote paths. - - mask_and_scale : bool, optional - If True, replaces values in the dataset that match `_FillValue` with NaN - and applies scale and offset adjustments. Default is True. - - decode_times : bool, optional - If True, decodes time variables according to CF conventions. Default is True. - - concat_characters : bool, optional - If True, concatenates character arrays along the last dimension, forming - string arrays. Default is True. - - decode_coords : bool, optional - If True, decodes the "coordinates" attribute to identify coordinates in the - resulting dataset. Default is True. - - drop_variables : str or list of str, optional - Specifies variables to exclude from the dataset. Useful for removing problematic - or inconsistent variables. Default is None. - - use_cftime : bool, optional - If True, uses cftime objects to represent time variables; if False, uses - `np.datetime64` objects. If None, chooses the best format automatically. - Default is None. - - decode_timedelta : bool, optional - If True, decodes variables with units of time (e.g., seconds, minutes) into - timedelta objects. If False, leaves them as numeric values. Default is None. - - sweep : int or list of int, optional - Sweep numbers to extract from the dataset. If None, extracts all sweeps into - a list. Default is the first sweep. + sweep_dict = open_sweeps_as_dict( + filename_or_obj=filename_or_obj, + mask_and_scale=mask_and_scale, + decode_times=decode_times, + concat_characters=concat_characters, + decode_coords=decode_coords, + drop_variables=drop_variables, + use_cftime=use_cftime, + decode_timedelta=decode_timedelta, + sweeps=sweeps, + first_dim=first_dim, + reindex_angle=reindex_angle, + fix_second_angle=fix_second_angle, + site_as_coords=site_coords, + optional=optional, + lock=lock, + **kwargs, + ) - first_dim : {"time", "auto"}, optional - Defines the first dimension for each sweep. If "time," uses time as the - first dimension. If "auto," determines the first dimension based on the sweep - type (azimuth or elevation). Default is "auto." + ls_ds = [xr.Dataset()] + list(sweep_dict.values()) + root, ls_ds = _assign_root(ls_ds) + groups_dict = {"/": root} + if optional_groups: + groups_dict["/radar_parameters"] = _get_subgroup( + ls_ds, radar_parameters_subgroup + ) + groups_dict["/georeferencing_correction"] = _get_subgroup( + ls_ds, georeferencing_correction_subgroup + ) + groups_dict["/radar_calibration"] = _get_radar_calibration( + ls_ds, radar_calibration_subgroup + ) + for sweep_path, ds in sweep_dict.items(): + sw = ds.drop_vars(_STATION_VARS, errors="ignore").drop_attrs(deep=False) + groups_dict[f"/{sweep_path}"] = sw + return groups_dict - reindex_angle : bool or dict, optional - Controls angle reindexing. If True or a dictionary, applies reindexing with - specified settings (if given). Only used if `decode_coords=True`. Default is False. + def open_datatree(self, filename_or_obj, **kwargs): + groups_dict = self.open_groups_as_dict(filename_or_obj, **kwargs) + return DataTree.from_dict(groups_dict) - fix_second_angle : bool, optional - If True, corrects errors in the second angle data, such as misaligned - elevation or azimuth values. Default is False. - site_as_coords : bool, optional - Attaches radar site coordinates to the dataset if True. Default is True. +def open_uf_datatree(filename_or_obj, **kwargs): + """Open a Universal Format (UF) dataset as :py:class:`xarray.DataTree`. - optional : bool, optional - If True, suppresses errors for optional dataset attributes, making them - optional instead of required. Default is True. + .. deprecated:: + Use ``xd.open_datatree(file, engine="uf")`` instead. + """ + _deprecation_warning("open_uf_datatree", "uf") - kwargs : dict - Additional keyword arguments passed to `xarray.open_dataset`. + optional = kwargs.pop("optional", True) + optional_groups = kwargs.pop("optional_groups", False) + sweep = kwargs.pop("sweep", None) + if "site_as_coords" in kwargs: + kwargs["site_coords"] = kwargs.pop("site_as_coords") - Returns - ------- - dtree : xarray.DataTree - An `xarray.DataTree` representing the radar data organized by sweeps. - """ - from xarray.core.treenode import NodePath - - if isinstance(sweep, str): - sweep = NodePath(sweep).name - sweeps = [sweep] - elif isinstance(sweep, int): - sweeps = [f"sweep_{sweep}"] - elif isinstance(sweep, list): - if isinstance(sweep[0], int): - sweeps = [f"sweep_{i}" for i in sweep] - elif isinstance(sweep[0], str): - sweeps = [NodePath(i).name for i in sweep] - else: - raise ValueError( - "Invalid type in 'sweep' list. Expected integers (e.g., [0, 1, 2]) or strings (e.g. [/sweep_0, sweep_1])." - ) - else: - with UFFile(filename_or_obj, loaddata=False) as ufh: - # Actual number of sweeps recorded in the file - act_sweeps = ufh.nsweeps - - sweeps = [f"sweep_{i}" for i in range(act_sweeps)] - - sweep_dict = open_sweeps_as_dict( - filename_or_obj=filename_or_obj, - mask_and_scale=mask_and_scale, - decode_times=decode_times, - concat_characters=concat_characters, - decode_coords=decode_coords, - drop_variables=drop_variables, - use_cftime=use_cftime, - decode_timedelta=decode_timedelta, - sweeps=sweeps, - first_dim=first_dim, - reindex_angle=reindex_angle, - fix_second_angle=fix_second_angle, - site_as_coords=False, + return UFBackendEntrypoint().open_datatree( + filename_or_obj, + sweep=sweep, optional=optional, - lock=lock, + optional_groups=optional_groups, **kwargs, ) - ls_ds: list[xr.Dataset] = [xr.Dataset()] + list(sweep_dict.values()) - root, ls_ds = _assign_root(ls_ds) - dtree: dict = {"/": root} - if optional_groups: - dtree["/radar_parameters"] = _get_subgroup(ls_ds, radar_parameters_subgroup) - dtree["/georeferencing_correction"] = _get_subgroup( - ls_ds, georeferencing_correction_subgroup - ) - dtree["/radar_calibration"] = _get_radar_calibration( - ls_ds, radar_calibration_subgroup - ) - # Build from ls_ds (station vars already stripped by _assign_root). - dtree |= {key: ds.drop_attrs(deep=False) for key, ds in zip(sweep_dict, ls_ds[1:])} - return xr.DataTree.from_dict(dtree) def open_sweeps_as_dict( From 6986717d43071ce90ba4e1fef526798375a1779b Mon Sep 17 00:00:00 2001 From: aladinor Date: Mon, 30 Mar 2026 21:51:21 -0500 Subject: [PATCH 7/8] DOC: add PR #335 entry to changelog --- docs/history.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/history.md b/docs/history.md index 94ce0ee4..0703f180 100644 --- a/docs/history.md +++ b/docs/history.md @@ -15,6 +15,7 @@ * FIX: Allow passing ``inherit`` parameter to ``apply_to_sweeps`` / ``map_over_sweeps`` to control coordinate inheritance from root node ({issue}`343`, {pull}`344`) by [@aladinor](https://github.com/aladinor) * FIX: Use ``open-radar-data`` fixture as fallback for ``nexrad_read_chunks.ipynb`` notebook, replacing dependency on ephemeral S3 chunk data ({issue}`351`, {pull}`352`) by [@aladinor](https://github.com/aladinor) * MNT: Pin ``open-radar-data>=0.6.0`` for NEXRAD chunk test data ({pull}`352`) by [@aladinor](https://github.com/aladinor) +* ENH: Add xarray-native ``open_datatree`` with ``engine=`` parameter for all 11 backends, enabling ``xd.open_datatree(file, engine="odim")`` and ``xr.open_datatree(file, engine="odim")`` ({issue}`329`, {pull}`335`) by [@aladinor](https://github.com/aladinor) ## 0.11.1 (2026-02-03) From f105ff5d90d9c7fb2edecb5fd4706369ee42761c Mon Sep 17 00:00:00 2001 From: aladinor Date: Mon, 30 Mar 2026 21:53:21 -0500 Subject: [PATCH 8/8] STY: apply black formatting to 6 backend files --- xradar/io/backends/datamet.py | 8 ++------ xradar/io/backends/furuno.py | 3 +-- xradar/io/backends/hpl.py | 7 ++----- xradar/io/backends/iris.py | 7 ++----- xradar/io/backends/metek.py | 3 +-- xradar/io/backends/rainbow.py | 3 +-- 6 files changed, 9 insertions(+), 22 deletions(-) diff --git a/xradar/io/backends/datamet.py b/xradar/io/backends/datamet.py index 3dc671ce..09c469c9 100644 --- a/xradar/io/backends/datamet.py +++ b/xradar/io/backends/datamet.py @@ -466,10 +466,7 @@ def open_groups_as_dict( ): def _discover(): dmet = DataMetFile(filename_or_obj) - return [ - f"sweep_{i}" - for i in range(dmet.scan_metadata["elevation_number"]) - ] + return [f"sweep_{i}" for i in range(dmet.scan_metadata["elevation_number"])] sweeps = _resolve_sweeps(sweep, _discover) @@ -487,8 +484,7 @@ def _discover(): ) ls_ds = [ - self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) - for swp in sweeps + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) for swp in sweeps ] return _build_groups_dict( ls_ds, optional=optional, optional_groups=optional_groups diff --git a/xradar/io/backends/furuno.py b/xradar/io/backends/furuno.py index 447a89c9..b8d1baac 100644 --- a/xradar/io/backends/furuno.py +++ b/xradar/io/backends/furuno.py @@ -814,8 +814,7 @@ def open_groups_as_dict( ) ls_ds = [ - self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) - for swp in sweeps + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) for swp in sweeps ] return _build_groups_dict( ls_ds, optional=optional, optional_groups=optional_groups diff --git a/xradar/io/backends/hpl.py b/xradar/io/backends/hpl.py index 2f3e9d24..f7fa367d 100644 --- a/xradar/io/backends/hpl.py +++ b/xradar/io/backends/hpl.py @@ -614,9 +614,7 @@ def open_groups_as_dict( transition_threshold_azi=0.05, transition_threshold_el=0.001, ): - sweeps = _resolve_sweeps( - sweep, lambda: _get_hpl_group_names(filename_or_obj) - ) + sweeps = _resolve_sweeps(sweep, lambda: _get_hpl_group_names(filename_or_obj)) ds_kwargs = dict( mask_and_scale=mask_and_scale, @@ -640,8 +638,7 @@ def open_groups_as_dict( ) ls_ds = [ - self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) - for swp in sweeps + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) for swp in sweeps ] groups_dict = _build_groups_dict( ls_ds, optional=optional, optional_groups=optional_groups diff --git a/xradar/io/backends/iris.py b/xradar/io/backends/iris.py index e41f325a..bcc090a3 100644 --- a/xradar/io/backends/iris.py +++ b/xradar/io/backends/iris.py @@ -4085,9 +4085,7 @@ def open_groups_as_dict( optional=True, optional_groups=False, ): - sweeps = _resolve_sweeps( - sweep, lambda: _get_iris_group_names(filename_or_obj) - ) + sweeps = _resolve_sweeps(sweep, lambda: _get_iris_group_names(filename_or_obj)) ds_kwargs = dict( mask_and_scale=mask_and_scale, @@ -4105,8 +4103,7 @@ def open_groups_as_dict( ) ls_ds = [ - self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) - for swp in sweeps + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) for swp in sweeps ] return _build_groups_dict( ls_ds, optional=optional, optional_groups=optional_groups diff --git a/xradar/io/backends/metek.py b/xradar/io/backends/metek.py index dd081f37..cc26790f 100644 --- a/xradar/io/backends/metek.py +++ b/xradar/io/backends/metek.py @@ -667,8 +667,7 @@ def open_groups_as_dict( ) ls_ds = [ - self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) - for swp in sweeps + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) for swp in sweeps ] return _build_groups_dict( ls_ds, optional=optional, optional_groups=optional_groups diff --git a/xradar/io/backends/rainbow.py b/xradar/io/backends/rainbow.py index 72c30806..dcf7c279 100644 --- a/xradar/io/backends/rainbow.py +++ b/xradar/io/backends/rainbow.py @@ -899,8 +899,7 @@ def open_groups_as_dict( ) ls_ds = [ - self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) - for swp in sweeps + self.open_dataset(filename_or_obj, group=swp, **ds_kwargs) for swp in sweeps ] return _build_groups_dict( ls_ds, optional=optional, optional_groups=optional_groups