From 55be3ded7e6841179932aa2877927864f8dcb0e5 Mon Sep 17 00:00:00 2001 From: Simon Smart Date: Tue, 27 Feb 2024 23:05:10 +0000 Subject: [PATCH 01/23] FDB-321: Add axes() support --- src/fdb5/CMakeLists.txt | 5 + src/fdb5/api/FDB.cc | 10 ++ src/fdb5/api/FDB.h | 3 + src/fdb5/api/FDBFactory.h | 4 +- src/fdb5/api/LocalFDB.cc | 6 + src/fdb5/api/LocalFDB.h | 2 + src/fdb5/api/helpers/AxesIterator.cc | 41 +++++++ src/fdb5/api/helpers/AxesIterator.h | 74 +++++++++++++ src/fdb5/api/local/AxesVisitor.cc | 82 ++++++++++++++ src/fdb5/api/local/AxesVisitor.h | 57 ++++++++++ src/fdb5/database/IndexAxis.cc | 66 ++++++++++- src/fdb5/database/IndexAxis.h | 16 ++- src/fdb5/toc/TocCatalogue.cc | 4 +- src/fdb5/tools/fdb-axes.cc | 83 ++++++++++++++ tests/fdb/CMakeLists.txt | 1 + tests/fdb/database/CMakeLists.txt | 9 ++ tests/fdb/database/test_indexaxis.cc | 157 +++++++++++++++++++++++++++ tests/fdb/tools/CMakeLists.txt | 3 +- tests/fdb/tools/fdb_axes.sh.in | 68 ++++++++++++ tests/fdb/tools/local.yaml | 6 + tests/fdb/tools/x.grib | 1 + 21 files changed, 691 insertions(+), 7 deletions(-) create mode 100644 src/fdb5/api/helpers/AxesIterator.cc create mode 100644 src/fdb5/api/helpers/AxesIterator.h create mode 100644 src/fdb5/api/local/AxesVisitor.cc create mode 100644 src/fdb5/api/local/AxesVisitor.h create mode 100644 src/fdb5/tools/fdb-axes.cc create mode 100644 tests/fdb/database/CMakeLists.txt create mode 100644 tests/fdb/database/test_indexaxis.cc create mode 100755 tests/fdb/tools/fdb_axes.sh.in create mode 100644 tests/fdb/tools/local.yaml create mode 120000 tests/fdb/tools/x.grib diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 8ccf68784..525cf8837 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -32,6 +32,8 @@ list( APPEND fdb5_srcs api/SelectFDB.h api/helpers/APIIterator.h + api/helpers/Axesiterator.cc + api/helpers/AxesIterator.h api/helpers/ControlIterator.cc api/helpers/ControlIterator.h api/helpers/FDBToolRequest.cc @@ -50,6 +52,8 @@ list( APPEND fdb5_srcs api/local/QueryVisitor.h api/local/QueueStringLogTarget.h api/local/ListVisitor.h + api/local/AxesVisitor.cc + api/local/AxesVisitor.h api/local/ControlVisitor.cc api/local/ControlVisitor.h api/local/DumpVisitor.h @@ -408,6 +412,7 @@ if(HAVE_FDB_BUILD_TOOLS) endif() list( APPEND fdb5_tools + fdb-axes fdb-write fdb-copy fdb-dump diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc index f04547e36..7ff05a04c 100644 --- a/src/fdb5/api/FDB.cc +++ b/src/fdb5/api/FDB.cc @@ -294,6 +294,16 @@ void FDB::flush() { } } +IndexAxis FDB::axes(const FDBToolRequest& request, int level) { + IndexAxis axes; + AxesElement elem; + auto it = internal_->axes(request, level); + while (it.next(elem)) { + axes.merge(elem.axes()); + } + return axes; +} + bool FDB::dirty() const { return dirty_; } diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h index cb9879363..54dab0c52 100644 --- a/src/fdb5/api/FDB.h +++ b/src/fdb5/api/FDB.h @@ -109,6 +109,9 @@ class FDB { ControlIterator control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers); + + IndexAxis axes(const FDBToolRequest& request, int level=3); + bool enabled(const ControlIdentifier& controlIdentifier) const; bool dirty() const; diff --git a/src/fdb5/api/FDBFactory.h b/src/fdb5/api/FDBFactory.h index 1d9970d5f..2f0f6a62d 100644 --- a/src/fdb5/api/FDBFactory.h +++ b/src/fdb5/api/FDBFactory.h @@ -28,7 +28,7 @@ #include "fdb5/database/DB.h" #include "fdb5/config/Config.h" #include "fdb5/api/FDBStats.h" -#include "fdb5/api/helpers/ListIterator.h" +#include "fdb5/api/helpers/AxesIterator.h" #include "fdb5/api/helpers/ListIterator.h" #include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/api/helpers/DumpIterator.h" @@ -88,6 +88,8 @@ class FDBBase : private eckit::NonCopyable { virtual MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) = 0; + virtual AxesIterator axes(const FDBToolRequest& request, int axes) { NOTIMP; } + // -------------- API management ---------------------------- /// ID used for hashing in the Rendezvous hash. Should be unique amongst those used diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc index b2ea1a7ea..8705a6da4 100644 --- a/src/fdb5/api/LocalFDB.cc +++ b/src/fdb5/api/LocalFDB.cc @@ -29,6 +29,7 @@ #include "fdb5/rules/Schema.h" #include "fdb5/LibFdb5.h" +#include "fdb5/api/local/AxesVisitor.h" #include "fdb5/api/local/ControlVisitor.h" #include "fdb5/api/local/DumpVisitor.h" #include "fdb5/api/local/ListVisitor.h" @@ -123,6 +124,11 @@ ControlIterator LocalFDB::control(const FDBToolRequest& request, return queryInternal(request, action, identifiers); } +AxesIterator LocalFDB::axes(const FDBToolRequest& request, int level) { + LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::axes() : " << request << std::endl; + return queryInternal(request, config_, level); +} + void LocalFDB::flush() { if (archiver_) { archiver_->flush(); diff --git a/src/fdb5/api/LocalFDB.h b/src/fdb5/api/LocalFDB.h index e7b226c78..96f4a6914 100644 --- a/src/fdb5/api/LocalFDB.h +++ b/src/fdb5/api/LocalFDB.h @@ -59,6 +59,8 @@ class LocalFDB : public FDBBase { MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override; + AxesIterator axes(const FDBToolRequest& request, int axes) override; + void flush() override; private: // methods diff --git a/src/fdb5/api/helpers/AxesIterator.cc b/src/fdb5/api/helpers/AxesIterator.cc new file mode 100644 index 000000000..99cdfe89a --- /dev/null +++ b/src/fdb5/api/helpers/AxesIterator.cc @@ -0,0 +1,41 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project NextGenIO + * (Project ID: 671951) www.nextgenio.eu + */ + +#include "fdb5/api/helpers/AxesIterator.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +AxesElement::AxesElement(Key&& dbKey, IndexAxis&& axes) : + dbKey_(std::move(dbKey)), axes_(std::move(axes)) {} + +AxesElement::AxesElement(eckit::Stream& s) { + s >> dbKey_; + axes_ = IndexAxis(s, IndexAxis::currentVersion()); +} + +void AxesElement::print(std::ostream& out) const { + out << "Axes(db=" << dbKey_ << ", axes=" << axes_ << ")"; +} + +void AxesElement::encode(eckit::Stream& s) const { + s << dbKey_; + axes_.encode(s, IndexAxis::currentVersion()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/api/helpers/AxesIterator.h b/src/fdb5/api/helpers/AxesIterator.h new file mode 100644 index 000000000..9881358dd --- /dev/null +++ b/src/fdb5/api/helpers/AxesIterator.h @@ -0,0 +1,74 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Simon Smart +/// @date January 2024 + +#pragma once + +#include "fdb5/database/Key.h" +#include "fdb5/database/IndexAxis.h" +#include "fdb5/api/helpers/APIIterator.h" + +namespace eckit { +class Stream; +class JSON; +} + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class AxesElement { +public: // methods + + AxesElement() = default; + AxesElement(Key&& dbKey, IndexAxis&& axis); + explicit AxesElement(eckit::Stream& s); + + [[ nodiscard ]] + const Key& key() const { return dbKey_; } + + [[ nodiscard ]] + const IndexAxis& axes() const { return axes_; } + + void print(std::ostream& out) const; + +private: // methods + + void encode(eckit::Stream& s) const; + + friend std::ostream& operator<<(std::ostream& os, const AxesElement& e) { + e.print(os); + return os; + } + + friend eckit::Stream& operator<<(eckit::Stream& s, const AxesElement& r) { + r.encode(s); + return s; + } + +private: // members + + Key dbKey_; + IndexAxis axes_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +using AxesAggregateIterator = APIAggregateIterator; + +using AxesAsyncIterator = APIAsyncIterator; + +using AxesIterator = APIIterator; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/api/local/AxesVisitor.cc b/src/fdb5/api/local/AxesVisitor.cc new file mode 100644 index 000000000..a1a2a536a --- /dev/null +++ b/src/fdb5/api/local/AxesVisitor.cc @@ -0,0 +1,82 @@ +/* + * (C) Copyright 2018- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/api/local/AxesVisitor.h" + +#include "fdb5/database/Catalogue.h" + +namespace fdb5 { +namespace api { +namespace local { + +//---------------------------------------------------------------------------------------------------------------------- + +AxesVisitor::AxesVisitor(eckit::Queue& queue, + const metkit::mars::MarsRequest& request, + const Config& config, + int level) : + QueryVisitor(queue, request), + schema_(config.schema()), + level_(level) {} + +#if 0 + +// TODO: Here we can do nice tricks to make things go muuuuuuuuuuuuuch faster... +// See improvements to the EntryVisitMechanism... & the schema + +bool AxesVisitor::preVisitDatabase(const eckit::URI& uri) { + + // If level == 1, avoid constructing the Catalogue/Store objects, so just interrogate the URIs + if (level_ == 1 && uri.scheme() == "toc") { + // TODO: This is hacky, only works with the toc backend... + if (schema_.matchFirstLevel(uri.path().baseName(), dbKey_)) { + axes_.wipe(); + axes_.insert(dbKey_); + axes_.sort(); + queue_.emplace(AxesElement{std::move(dbKey_), std::move(axes_)}); + } + return false; + } + + return true; +} +#endif + +bool AxesVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) { + dbKey_ = catalogue.key(); + axes_.wipe(); + axes_.insert(dbKey_); + axes_.sort(); + return (level_ > 1); +} + +bool AxesVisitor::visitIndex(const Index& index) { + if (index.partialMatch(request_)) { + IndexAxis tmpAxis; + tmpAxis.insert(index.key()); + tmpAxis.sort(); + axes_.merge(tmpAxis); // avoid sorts on the (growing) main Axes object + + if (level_ > 2) { + axes_.merge(index.axes()); + } + } + return false; +} + +void AxesVisitor::catalogueComplete(const fdb5::Catalogue& catalogue) { + queue_.emplace(AxesElement{std::move(dbKey_), std::move(axes_)}); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace local +} // namespace api +} // namespace fdb5 diff --git a/src/fdb5/api/local/AxesVisitor.h b/src/fdb5/api/local/AxesVisitor.h new file mode 100644 index 000000000..e2a3d8987 --- /dev/null +++ b/src/fdb5/api/local/AxesVisitor.h @@ -0,0 +1,57 @@ +/* + * (C) Copyright 2018- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Simon Smart +/// @date January 2024 + +#pragma once + +#include "fdb5/api/local/QueryVisitor.h" +#include "fdb5/api/helpers/AxesIterator.h" + + +namespace fdb5 { +namespace api { +namespace local { + +/// @note Helper classes for LocalFDB + +//---------------------------------------------------------------------------------------------------------------------- + +class AxesVisitor : public QueryVisitor { +public: + + AxesVisitor(eckit::Queue& queue, + const metkit::mars::MarsRequest& request, + const Config& config, + int level); + + bool visitIndexes() override { return true; } + bool visitEntries() override { return false; } + void catalogueComplete(const fdb5::Catalogue& catalogue) override; + +// bool preVisitDatabase(const eckit::URI& uri) override; + bool visitDatabase(const Catalogue& catalogue, const Store& store) override; + bool visitIndex(const Index&) override; + void visitDatum(const Field&, const Key&) override { NOTIMP; } + +private: // members + + Key dbKey_; + IndexAxis axes_; + const Schema& schema_; + int level_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace local +} // namespace api +} // namespace fdb5 diff --git a/src/fdb5/database/IndexAxis.cc b/src/fdb5/database/IndexAxis.cc index 912b49be7..aec44d4e9 100755 --- a/src/fdb5/database/IndexAxis.cc +++ b/src/fdb5/database/IndexAxis.cc @@ -45,6 +45,35 @@ IndexAxis::IndexAxis(eckit::Stream &s, const int version) : decode(s, version); } +IndexAxis::IndexAxis(IndexAxis&& rhs) noexcept : + axis_(std::move(rhs.axis_)), + readOnly_(rhs.readOnly_), + dirty_(rhs.dirty_) {} + +IndexAxis& IndexAxis::operator=(IndexAxis&& rhs) noexcept { + axis_ = std::move(rhs.axis_); + readOnly_ = rhs.readOnly_; + dirty_ = rhs.dirty_; + return *this; +} + +bool IndexAxis::operator==(const IndexAxis& rhs) const { + + if (axis_.size() != rhs.axis_.size()) return false; + + for (const auto& kv : axis_) { + auto it = rhs.axis_.find(kv.first); + if (it == rhs.axis_.end()) return false; + if (*kv.second != *it->second) return false; + } + + return true; +} + +bool IndexAxis::operator!=(const IndexAxis& rhs) const { + return !(*this == rhs); +} + void IndexAxis::encode(eckit::Stream &s, const int version) const { if (version >= 3) { encodeCurrent(s, version); @@ -295,10 +324,45 @@ const eckit::DenseSet &IndexAxis::values(const std::string &keyword void IndexAxis::print(std::ostream &out) const { out << "IndexAxis[" << "axis="; - eckit::__print_container(out, axis_); + + const char* sep = ""; + out << "{"; + for (const auto& kv : axis_) { + out << sep << kv.first << "=("; + const char* sep2 = ""; + for (const auto& v : *kv.second) { + out << sep2 << v; + sep2 = ","; + } + out << ")"; + sep = ","; + } + out << "}"; out << "]"; } +void IndexAxis::json(eckit::JSON& json) const { + json.startObject(); + for (const auto& kv : axis_) { + json << kv.first << *kv.second; + } + json.endObject(); +} + +void IndexAxis::merge(const fdb5::IndexAxis& other) { + + ASSERT(!readOnly_); + for (const auto& kv : other.axis_) { + + auto it = axis_.find(kv.first); + if (it == axis_.end()) { + axis_.emplace(kv.first, kv.second); + } else { + it->second->merge(*kv.second); + }; + } +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/database/IndexAxis.h b/src/fdb5/database/IndexAxis.h index a83b92b6d..e581dbe3b 100644 --- a/src/fdb5/database/IndexAxis.h +++ b/src/fdb5/database/IndexAxis.h @@ -45,11 +45,20 @@ class IndexAxis : private eckit::NonCopyable { IndexAxis(); IndexAxis(eckit::Stream &s, const int version); + IndexAxis(IndexAxis&& rhs) noexcept; + + IndexAxis& operator=(IndexAxis&& rhs) noexcept; ~IndexAxis(); + bool operator==(const IndexAxis& rhs) const; + bool operator!=(const IndexAxis& rhs) const; + void insert(const Key &key); void encode(eckit::Stream &s, const int version) const; + static int currentVersion() { return 3; } + + void merge(const IndexAxis& other); // Decode can be used for two-stage initialisation (IndexAxis a; a.decode(s);) void decode(eckit::Stream& s, const int version); @@ -78,6 +87,11 @@ class IndexAxis : private eckit::NonCopyable { return s; } + friend eckit::JSON& operator<<(eckit::JSON& j, const IndexAxis& x) { + x.json(j); + return j; + } + private: // methods void encodeCurrent(eckit::Stream &s, const int version) const; @@ -87,7 +101,7 @@ class IndexAxis : private eckit::NonCopyable { void decodeLegacy(eckit::Stream& s, const int version); void print(std::ostream &out) const; - + void json(eckit::JSON& j) const; private: // members diff --git a/src/fdb5/toc/TocCatalogue.cc b/src/fdb5/toc/TocCatalogue.cc index 9241d1eb4..63188fd83 100644 --- a/src/fdb5/toc/TocCatalogue.cc +++ b/src/fdb5/toc/TocCatalogue.cc @@ -92,10 +92,8 @@ void TocCatalogue::visitEntries(EntryVisitor& visitor, const Store& store, bool } } } - - visitor.catalogueComplete(*this); } - + visitor.catalogueComplete(*this); } void TocCatalogue::loadSchema() { diff --git a/src/fdb5/tools/fdb-axes.cc b/src/fdb5/tools/fdb-axes.cc new file mode 100644 index 000000000..ed71f215e --- /dev/null +++ b/src/fdb5/tools/fdb-axes.cc @@ -0,0 +1,83 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "eckit/option/CmdArgs.h" +#include "eckit/option/SimpleOption.h" +#include "eckit/log/JSON.h" + +#include "fdb5/api/FDB.h" +#include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/tools/FDBVisitTool.h" + +using namespace eckit; +using namespace option; + + +namespace fdb5::tools { + +//---------------------------------------------------------------------------------------------------------------------- + +class FDBAxisTest : public FDBVisitTool { + +public: // methods + + FDBAxisTest(int argc, char **argv) : + FDBVisitTool(argc, argv, "class,expver"), + level_(3), + json_(false) { + options_.push_back(new SimpleOption("level", "Specify how many levels of the keys should be should be explored")); + options_.push_back(new SimpleOption("json", "Output available fields in JSON form")); + } + +private: // methods + + void execute(const CmdArgs& args) final; + void init(const CmdArgs &args) final; + +private: // members + + int level_; + bool json_; +}; + +void FDBAxisTest::init(const CmdArgs& args) { + FDBVisitTool::init(args); + json_ = args.getBool("json", json_); + level_ = args.getInt("level", level_); +} + +void FDBAxisTest::execute(const CmdArgs& args) { + + FDB fdb(config(args)); + + for (const FDBToolRequest& request : requests()) { +// Timer taxes("axes"); + auto r = fdb.axes(request, level_); +// taxes.stop(); + + if (json_) { + JSON json(Log::info()); + json << r; + } else { + Log::info() << r; + } + Log::info() << std::endl; + } +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5::tools + +int main(int argc, char **argv) { + fdb5::tools::FDBAxisTest app(argc, argv); + return app.start(); +} + diff --git a/tests/fdb/CMakeLists.txt b/tests/fdb/CMakeLists.txt index 3f9748011..3bb56caa6 100644 --- a/tests/fdb/CMakeLists.txt +++ b/tests/fdb/CMakeLists.txt @@ -72,5 +72,6 @@ endforeach() add_subdirectory( pmem ) add_subdirectory( api ) +add_subdirectory( database ) add_subdirectory( tools ) add_subdirectory( type ) diff --git a/tests/fdb/database/CMakeLists.txt b/tests/fdb/database/CMakeLists.txt new file mode 100644 index 000000000..1bc08100f --- /dev/null +++ b/tests/fdb/database/CMakeLists.txt @@ -0,0 +1,9 @@ + +list( APPEND _test_environment + FDB_HOME=${PROJECT_BINARY_DIR} ) + +ecbuild_add_test( TARGET test_fdb5_database_indexaxis + SOURCES test_indexaxis.cc + INCLUDES ${PMEM_INCLUDE_DIRS} + LIBS fdb5 + ENVIRONMENT "${_test_environment}") diff --git a/tests/fdb/database/test_indexaxis.cc b/tests/fdb/database/test_indexaxis.cc new file mode 100644 index 000000000..190d50d2a --- /dev/null +++ b/tests/fdb/database/test_indexaxis.cc @@ -0,0 +1,157 @@ + + +#include "fdb5/database/IndexAxis.h" +#include "fdb5/database/Key.h" + +#include "eckit/io/Buffer.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/serialisation/ResizableMemoryStream.h" +#include "eckit/testing/Test.h" + +namespace { + +fdb5::Key EXAMPLE_K1{{ + {"class", "od"}, + {"expver", "0001"}, + {"date", "20240223"} +}}; + +fdb5::Key EXAMPLE_K2{{ + {"class", "rd"}, + {"expver", "0001"}, + {"time", "1200"} +}}; + +fdb5::Key EXAMPLE_K3{{ + {"class", "rd"}, + {"expver", "gotx"}, + {"time", "0000"} +}}; + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("Insertion and comparison") { + + fdb5::IndexAxis ia1; + fdb5::IndexAxis ia2; + EXPECT(ia1 == ia2); + EXPECT(!(ia1 != ia2)); + + ia1.insert(EXAMPLE_K1); + EXPECT(!(ia1 == ia2)); + EXPECT(ia1 != ia2); + + ia2.insert(EXAMPLE_K2); + EXPECT(!(ia1 == ia2)); + EXPECT(ia1 != ia2); + + ia1.insert(EXAMPLE_K2); + EXPECT(!(ia1 == ia2)); + EXPECT(ia1 != ia2); + + ia2.insert(EXAMPLE_K1); + + EXPECT(!(ia1 == ia2)); + EXPECT(ia1 != ia2); + ia1.sort(); + ia2.sort(); + EXPECT(ia1 == ia2); + EXPECT(!(ia1 != ia2)); +} + +CASE("iostream and JSON output functions correctly") { + + fdb5::IndexAxis ia; + + { + std::ostringstream ss; + ss << ia; + EXPECT(ss.str() == "IndexAxis[axis={}]"); + } + + { + std::ostringstream ss; + eckit::JSON json(ss); + json << ia; + EXPECT(ss.str() == "{}"); + } + + ia.insert(EXAMPLE_K1); + ia.insert(EXAMPLE_K2); + ia.insert(EXAMPLE_K3); + ia.sort(); + + { + std::ostringstream ss; + ss << ia; + EXPECT(ss.str() == "IndexAxis[axis={class=(od,rd),date=(20240223),expver=(0001,gotx),time=(0000,1200)}]"); + } + + { + std::ostringstream ss; + eckit::JSON json(ss); + json << ia; + EXPECT(ss.str() == "{\"class\":[\"od\",\"rd\"],\"date\":[\"20240223\"],\"expver\":[\"0001\",\"gotx\"],\"time\":[\"0000\",\"1200\"]}"); + } +} + +CASE("serialiastion and deserialisation") { + + fdb5::IndexAxis ia; + ia.insert(EXAMPLE_K1); + ia.insert(EXAMPLE_K2); + ia.insert(EXAMPLE_K3); + ia.sort(); + + eckit::Buffer buf; + { + eckit::ResizableMemoryStream ms(buf); + ia.encode(ms, fdb5::IndexAxis::currentVersion()); + } + + { + eckit::MemoryStream ms(buf); + fdb5::IndexAxis newia(ms, fdb5::IndexAxis::currentVersion()); + EXPECT(ia == newia); + } +} + +CASE("Check that merging works correctly") { + + fdb5::IndexAxis ia1; + ia1.insert(EXAMPLE_K1); + ia1.insert(EXAMPLE_K2); + ia1.sort(); + + fdb5::IndexAxis ia2; + ia2.insert(EXAMPLE_K1); + ia2.insert(EXAMPLE_K3); + ia2.sort(); + + EXPECT(ia1 != ia2); + + fdb5::IndexAxis iatest; + iatest.insert(EXAMPLE_K1); + iatest.insert(EXAMPLE_K2); + iatest.insert(EXAMPLE_K3); + iatest.sort(); + + EXPECT(iatest != ia1); + EXPECT(iatest != ia2); + + ia1.merge(ia2); + + EXPECT(iatest == ia1); + EXPECT(iatest != ia2); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // anonymous namespace + +int main(int argc, char** argv) { + + eckit::Log::info() << ::getenv("FDB_HOME") << std::endl; + + return ::eckit::testing::run_tests(argc, argv); +} diff --git a/tests/fdb/tools/CMakeLists.txt b/tests/fdb/tools/CMakeLists.txt index 88ad0314d..939db703a 100644 --- a/tests/fdb/tools/CMakeLists.txt +++ b/tests/fdb/tools/CMakeLists.txt @@ -1,5 +1,6 @@ list( APPEND fdb_tools_tests - fdb_info ) + fdb_info + fdb_axes ) foreach( _t ${fdb_tools_tests} ) diff --git a/tests/fdb/tools/fdb_axes.sh.in b/tests/fdb/tools/fdb_axes.sh.in new file mode 100755 index 000000000..a9a3abeb2 --- /dev/null +++ b/tests/fdb/tools/fdb_axes.sh.in @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +set -eux + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB_HOME=@PROJECT_BINARY_DIR@ + +srcdir=@CMAKE_CURRENT_SOURCE_DIR@ +bindir=@CMAKE_CURRENT_BINARY_DIR@ +cd $bindir + +mkdir -p axes_test +cd axes_test + +rm -rf localroot || true +mkdir localroot + +for f in local.yaml x.grib +do + cp $srcdir/$f ./ +done + +export FDB5_CONFIG_FILE=local.yaml + +grib_set -s step=6 x.grib 6.grib +grib_set -s step=9 x.grib 9.grib +grib_set -s type=an 6.grib an6.grib +grib_set -s type=an 9.grib an9.grib + +fdb-write 6.grib +fdb-write 9.grib +fdb-write an6.grib +fdb-write an9.grib +grib_ls -m 9.grib + +####### Check lots of content + +fdb-axes class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g --json > axes.out +echo '{"class":["rd"],"date":["20201102"],"domain":["g"],"expver":["xxxx"],"levelist":[""],"levtype":["sfc"],"param":["166"],"step":["6","9"],"stream":["oper"],"time":["0000"],"type":["an","fc"]}' > expected.out +cat axes.out +cat expected.out +cmp axes.out expected.out + +fdb-axes class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g > axes.out +echo 'IndexAxis[axis={class=(rd),date=(20201102),domain=(g),expver=(xxxx),levelist=(),levtype=(sfc),param=(166),step=(6,9),stream=(oper),time=(0000),type=(an,fc)}]' > expected.out +cat axes.out +cat expected.out +cmp axes.out expected.out + +fdb-axes class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g --level=2 > axes.out +echo 'IndexAxis[axis={class=(rd),date=(20201102),domain=(g),expver=(xxxx),levtype=(sfc),stream=(oper),time=(0000),type=(an,fc)}]' > expected.out +cat axes.out +cat expected.out +cmp axes.out expected.out + +fdb-axes class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g --level=1 > axes.out +echo 'IndexAxis[axis={class=(rd),date=(20201102),domain=(g),expver=(xxxx),stream=(oper),time=(0000)}]' > expected.out +cat axes.out +cat expected.out +cmp axes.out expected.out + +####### Sub-selection by index + +fdb-axes class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g,step=6 > axes.out +echo 'IndexAxis[axis={class=(rd),date=(20201102),domain=(g),expver=(xxxx),levelist=(),levtype=(sfc),param=(166),step=(6),stream=(oper),time=(0000),type=(an,fc)}]' > expected.out +cat axes.out +cat expected.out +cmp axes.out expected.out diff --git a/tests/fdb/tools/local.yaml b/tests/fdb/tools/local.yaml new file mode 100644 index 000000000..8e43d2e8f --- /dev/null +++ b/tests/fdb/tools/local.yaml @@ -0,0 +1,6 @@ +--- +type: local +engine: toc +spaces: + - roots: + - path: ./localroot diff --git a/tests/fdb/tools/x.grib b/tests/fdb/tools/x.grib new file mode 120000 index 000000000..7d3306ed4 --- /dev/null +++ b/tests/fdb/tools/x.grib @@ -0,0 +1 @@ +../../regressions/FDB-307/x.grib \ No newline at end of file From 751a32eabb18f642a288929a82597f8c82233c0a Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Fri, 22 Mar 2024 14:13:54 +0000 Subject: [PATCH 02/23] FDB-303: Remove Rule from the Key, and replace with TypesRegistry information --- src/fdb5/api/FDB.cc | 9 ++ src/fdb5/config/Config.cc | 28 ------ src/fdb5/database/FieldLocation.cc | 2 +- src/fdb5/database/Key.cc | 137 ++++++++++++++++------------ src/fdb5/database/Key.h | 27 ++++-- src/fdb5/message/MessageArchiver.cc | 2 +- src/fdb5/rules/Rule.cc | 10 +- src/fdb5/rules/Schema.cc | 30 +++++- src/fdb5/rules/Schema.h | 14 +++ src/fdb5/toc/TocCatalogue.cc | 5 +- src/fdb5/toc/TocCatalogue.h | 3 +- tests/fdb/api/test_fdb_c.cc | 16 +++- tests/fdb/test_fdb5_service.cc | 3 +- tests/fdb/type/test_toKey.cc | 30 +++--- 14 files changed, 193 insertions(+), 123 deletions(-) diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc index 12ee0e857..54f1dd3c2 100644 --- a/src/fdb5/api/FDB.cc +++ b/src/fdb5/api/FDB.cc @@ -100,6 +100,7 @@ void FDB::archive(const Key& key, const void* data, size_t length) { eckit::Timer timer; timer.start(); +<<<<<<< HEAD auto stepunit = key.find("stepunits"); if (stepunit != key.end()) { Key k; @@ -118,6 +119,14 @@ void FDB::archive(const Key& key, const void* data, size_t length) { } else { internal_->archive(key, data, length); } +======= + // This is the API entrypoint. Keys supplied by the user may not have type registry info attached (so + // serialisation won't work properly...) + Key keyInternal(key); + keyInternal.registry(config().schema().registry()); + + internal_->archive(keyInternal, data, length); +>>>>>>> c062d99a (FDB-303: Remove Rule from the Key, and replace with TypesRegistry information) dirty_ = true; timer.stop(); diff --git a/src/fdb5/config/Config.cc b/src/fdb5/config/Config.cc index 22c022520..c6c02cb17 100644 --- a/src/fdb5/config/Config.cc +++ b/src/fdb5/config/Config.cc @@ -27,34 +27,6 @@ using namespace eckit; namespace fdb5 { - -/// Schemas are persisted in this registry -/// -class SchemaRegistry { -public: - static SchemaRegistry& instance() { - static SchemaRegistry me; - return me; - } - - const Schema& get(const PathName& path) { - std::lock_guard lock(m_); - auto it = schemas_.find(path); - if (it != schemas_.end()) { - return *it->second; - } - - Schema* p = new Schema(path); - ASSERT(p); - schemas_[path] = std::unique_ptr(p); - return *schemas_[path]; - } - -private: - std::mutex m_; - std::map> schemas_; -}; - //---------------------------------------------------------------------------------------------------------------------- Config::Config() : schemaPath_("") { diff --git a/src/fdb5/database/FieldLocation.cc b/src/fdb5/database/FieldLocation.cc index 437fdaac6..9aaaeb79a 100644 --- a/src/fdb5/database/FieldLocation.cc +++ b/src/fdb5/database/FieldLocation.cc @@ -127,7 +127,7 @@ FieldLocation::FieldLocation(const eckit::URI& uri) : uri_(uri) { std::string keyStr = uri.query("remapKey"); if (!keyStr.empty()) { - remapKey_ = Key(keyStr); + remapKey_ = Key::parseStringUntyped(keyStr); } else { remapKey_ = Key(); } diff --git a/src/fdb5/database/Key.cc b/src/fdb5/database/Key.cc index db9f9c7c6..e920a2090 100644 --- a/src/fdb5/database/Key.cc +++ b/src/fdb5/database/Key.cc @@ -26,13 +26,15 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -Key::Key() : +Key::Key(const TypesRegistry* reg) : keys_(), - rule_(0) {} + registry_(reg) {} + +Key::Key(const TypesRegistry& reg) : Key(®) {} Key::Key(const std::string &s, const Rule *rule) : keys_(), - rule_(0) { + registry_(rule ? &rule->registry() : nullptr) { eckit::Tokenizer parse(":", true); eckit::StringList values; parse(s, values); @@ -41,60 +43,91 @@ Key::Key(const std::string &s, const Rule *rule) : rule->fill(*this, values); } -Key::Key(const std::string &s) : - keys_(), - rule_(0) { - const TypesRegistry ®istry = this->registry(); +//Key::Key(const std::string &s, const TypesRegistry& reg) : Key(s, ®) {} +// +Key::Key(const eckit::StringDict &keys, const TypesRegistry* reg) : + keys_(keys), + registry_(reg) { + + eckit::StringDict::const_iterator it = keys.begin(); + eckit::StringDict::const_iterator end = keys.end(); + for (; it != end; ++it) { + names_.emplace_back(it->first); + } +} + +Key::Key(const eckit::StringDict &keys, const TypesRegistry& reg) : Key(keys, ®) {} + +Key::Key(eckit::Stream& s, const TypesRegistry* reg) : + registry_(reg) { + decode(s); +} + +Key::Key(std::initializer_list> l, const TypesRegistry* reg) : + keys_(l), + registry_(reg) { + + for (const auto& kv : keys_) { + names_.emplace_back(kv.first); + } +} + +Key::Key(std::initializer_list> l, const TypesRegistry& reg) : Key(l, ®) {} + +Key::Key(eckit::Stream& s, const TypesRegistry& reg) : Key(s, ®) {} + +Key Key::parseStringUntyped(const std::string& s) { eckit::Tokenizer parse1(","); - eckit::StringList v; + eckit::Tokenizer parse2("="); + eckit::StringDict keys; + eckit::StringList v; parse1(s, v); + for (const auto& bit : v) { + eckit::StringList kv; + parse2(bit, kv); + ASSERT(kv.size() == 2); + keys.emplace(std::move(kv[0]), std::move(kv[1])); + } + return Key{keys}; +} + +Key Key::parseString(const std::string &s, const TypesRegistry& registry) { + + eckit::Tokenizer parse1(","); eckit::Tokenizer parse2("="); - for (eckit::StringList::const_iterator i = v.begin(); i != v.end(); ++i) { + Key ret(registry); + + eckit::StringList vals; + parse1(s, vals); + + for (const auto& bit : vals) { eckit::StringList kv; - parse2(*i, kv); + parse2(bit, kv); ASSERT(kv.size() == 2); const Type &t = registry.lookupType(kv[0]); - std::string v = t.tidy(kv[0], kv[1]); - if (find(kv[0]) == end()) { - push(kv[0], v); + if (ret.find(kv[0]) == ret.end()) { + ret.push(kv[0], v); } else { - set(kv[0], v); + ret.set(kv[0], v); } } -} - -Key::Key(const eckit::StringDict &keys) : - keys_(keys), - rule_(0) { - - eckit::StringDict::const_iterator it = keys.begin(); - eckit::StringDict::const_iterator end = keys.end(); - for (; it != end; ++it) { - names_.emplace_back(it->first); - } -} - -Key::Key(eckit::Stream& s) : - rule_(nullptr) { - decode(s); + return ret; } void Key::decode(eckit::Stream& s) { - ASSERT(rule_ == nullptr); - + registry_ = nullptr; keys_.clear(); names_.clear(); - size_t n; s >> n; @@ -119,9 +152,7 @@ void Key::encode(eckit::Stream& s) const { s << keys_.size(); for (eckit::StringDict::const_iterator i = keys_.begin(); i != keys_.end(); ++i) { - const Type &t = registry.lookupType(i->first); s << i->first << canonicalise(i->first, i->second); - } s << names_.size(); @@ -143,16 +174,8 @@ std::set Key::keys() const { return k; } - -void Key::rule(const Rule *rule) { - rule_ = rule; -} - -const Rule *Key::rule() const { - return rule_; -} - void Key::clear() { + registry_ = nullptr; keys_.clear(); names_.clear(); } @@ -290,14 +313,20 @@ bool Key::partialMatch(const metkit::mars::MarsRequest& request) const { return true; } +void Key::registry(const TypesRegistry& reg) { + registry_ = ® +} const TypesRegistry& Key::registry() const { - if(rule_) { - return rule_->registry(); - } - else { - return LibFdb5::instance().defaultConfig().schema().registry(); + if (!registry_) { + throw eckit::SeriousBug("TypesRegistry has not been set for Key prior to use", Here()); } + + return *registry_; +} + +const void* Key::reg() const { + return registry_; } std::string Key::canonicalise(const std::string& keyword, const std::string& value) const { @@ -381,10 +410,6 @@ void Key::validateKeysOf(const Key& other, bool checkAlsoValues) const oss << "for key " << *this << " validating " << other; - if (rule()) { - oss << " " << *rule(); - } - throw eckit::SeriousBug(oss.str()); } } @@ -433,14 +458,8 @@ fdb5::Key::operator eckit::StringDict() const void Key::print(std::ostream &out) const { if (names_.size() == keys_.size()) { out << "{" << toString() << "}"; - if (rule_) { - out << " (" << *rule_ << ")"; - } } else { out << keys_; - if (rule_) { - out << " (" << *rule_ << ")"; - } } } diff --git a/src/fdb5/database/Key.h b/src/fdb5/database/Key.h index 5e889907f..3bec0079f 100644 --- a/src/fdb5/database/Key.h +++ b/src/fdb5/database/Key.h @@ -45,13 +45,23 @@ class Key { public: // methods - Key(); + explicit Key(const TypesRegistry* reg=nullptr); + explicit Key(const TypesRegistry& reg); - explicit Key(eckit::Stream &); - explicit Key(const std::string &request); + explicit Key(eckit::Stream &, const TypesRegistry* reg=nullptr); + explicit Key(eckit::Stream &, const TypesRegistry& reg); + /// @todo remove ?? explicit Key(const std::string &request, const TypesRegistry* reg=nullptr); + /// @todo remove ?? explicit Key(const std::string &request, const TypesRegistry& reg); explicit Key(const std::string &keys, const Rule* rule); - explicit Key(const eckit::StringDict &keys); + explicit Key(const eckit::StringDict &keys, const TypesRegistry* reg=nullptr); + explicit Key(const eckit::StringDict &keys, const TypesRegistry& reg); + Key(std::initializer_list>, const TypesRegistry* reg=nullptr); + Key(std::initializer_list>, const TypesRegistry& reg); + + static Key parseStringUntyped(const std::string& s); + /// @todo - this functionality should not be supported any more. + static Key parseString(const std::string&, const TypesRegistry& reg); std::set keys() const; @@ -104,9 +114,11 @@ class Key { return s; } - void rule(const Rule *rule); - const Rule *rule() const; + // Registry is needed before we can stringise/canonicalise. + void registry(const TypesRegistry& reg); + [[ nodiscard ]] const TypesRegistry& registry() const; + const void* reg() const; std::string valuesToString() const; @@ -155,8 +167,7 @@ class Key { eckit::StringDict keys_; eckit::StringList names_; - const Rule *rule_; - + const TypesRegistry* registry_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/message/MessageArchiver.cc b/src/fdb5/message/MessageArchiver.cc index 87a7040f9..59d700870 100644 --- a/src/fdb5/message/MessageArchiver.cc +++ b/src/fdb5/message/MessageArchiver.cc @@ -85,7 +85,7 @@ static std::vector make_filter_requests(const std::st if(str.empty()) return std::vector(); - std::set keys = fdb5::Key(str).keys(); //< keys to filter from that request + std::set keys = fdb5::Key::parseStringUntyped(str).keys(); //< keys to filter from that request std::vector v = str_to_requests(str); diff --git a/src/fdb5/rules/Rule.cc b/src/fdb5/rules/Rule.cc index 86974df5c..d87ea4b28 100644 --- a/src/fdb5/rules/Rule.cc +++ b/src/fdb5/rules/Rule.cc @@ -60,9 +60,9 @@ void Rule::expand( const metkit::mars::MarsRequest &request, if (cur == predicates_.end()) { - // TODO: join these 2 methods - keys[depth].rule(this); + keys[depth].registry(registry()); + // TODO: join these 2 methods if (rules_.empty()) { ASSERT(depth == 2); /// we have 3 levels ATM if (!visitor.selectDatum( keys[2], full)) { @@ -153,7 +153,7 @@ void Rule::expand( const Key &field, if (cur == predicates_.end()) { - keys[depth].rule(this); + keys[depth].registry(registry()); if (rules_.empty()) { ASSERT(depth == 2); /// we have 3 levels ATM @@ -231,7 +231,6 @@ void Rule::expandFirstLevel( const Key &dbKey, std::vector::const_i if (cur == predicates_.end()) { found = true; - result.rule(this); return; } @@ -260,7 +259,6 @@ void Rule::expandFirstLevel(const metkit::mars::MarsRequest& rq, std::vector
& result, const char* missing) const {
-    Key tmp;
+    Key tmp(registry());
     matchFirstLevel(request, predicates_.begin(), tmp, result, missing);
 }
 
diff --git a/src/fdb5/rules/Schema.cc b/src/fdb5/rules/Schema.cc
index a25616e7a..2affc165d 100644
--- a/src/fdb5/rules/Schema.cc
+++ b/src/fdb5/rules/Schema.cc
@@ -51,8 +51,9 @@ const Rule*  Schema::ruleFor(const Key& dbKey, const Key& idxKey) const {
 }
 
 void Schema::expand(const metkit::mars::MarsRequest &request, ReadVisitor &visitor) const {
-    Key full;
+    Key full(registry());
     std::vector keys(3);
+    for (auto& k : keys) k.registry(registry());
 
     for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) {
 		// eckit::Log::info() << "Rule " << **i <<  std::endl;
@@ -62,8 +63,9 @@ void Schema::expand(const metkit::mars::MarsRequest &request, ReadVisitor &visit
 }
 
 void Schema::expand(const Key &field, WriteVisitor &visitor) const {
-    Key full;
+    Key full(registry());
     std::vector keys(3);
+    for (auto& k : keys) k.registry(registry());
 
     visitor.rule(0); // reset to no rule so we verify that we pick at least one
 
@@ -86,6 +88,8 @@ void Schema::expandSecond(const metkit::mars::MarsRequest& request, ReadVisitor&
     Key full = dbKey;
     std::vector keys(3);
     keys[0] = dbKey;
+    keys[1].registry(registry());
+    keys[2].registry(registry());
 
     for (std::vector:: const_iterator i = dbRule->rules_.begin(); i != dbRule->rules_.end(); ++i) {
         (*i)->expand(request, visitor, 1, keys, full);
@@ -106,6 +110,8 @@ void Schema::expandSecond(const Key& field, WriteVisitor& visitor, const Key& db
     Key full = dbKey;
     std::vector keys(3);
     keys[0] = dbKey;
+    keys[1].registry(registry());
+    keys[2].registry(registry());
 
     for (std::vector:: const_iterator i = dbRule->rules_.begin(); i != dbRule->rules_.end(); ++i) {
         (*i)->expand(field, visitor, 1, keys, full);
@@ -219,4 +225,24 @@ std::ostream &operator<<(std::ostream &s, const Schema &x) {
 
 //----------------------------------------------------------------------------------------------------------------------
 
+SchemaRegistry& SchemaRegistry::instance() {
+    static SchemaRegistry me;
+    return me;
+}
+
+const Schema& SchemaRegistry::get(const eckit::PathName& path) {
+    std::lock_guard lock(m_);
+    auto it = schemas_.find(path);
+    if (it != schemas_.end()) {
+        return *it->second;
+    }
+
+    Schema* p = new Schema(path);
+    ASSERT(p);
+    schemas_[path] = std::unique_ptr(p);
+    return *schemas_[path];
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
 } // namespace fdb5
diff --git a/src/fdb5/rules/Schema.h b/src/fdb5/rules/Schema.h
index 43d565d97..44051cf8e 100644
--- a/src/fdb5/rules/Schema.h
+++ b/src/fdb5/rules/Schema.h
@@ -97,6 +97,20 @@ class Schema : private eckit::NonCopyable {
 
 //----------------------------------------------------------------------------------------------------------------------
 
+/// Schemas are persisted in this registry
+///
+class SchemaRegistry {
+public:
+    static SchemaRegistry& instance();
+    const Schema& get(const eckit::PathName& path);
+
+private:
+    std::mutex m_;
+    std::map> schemas_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
 } // namespace fdb5
 
 #endif
diff --git a/src/fdb5/toc/TocCatalogue.cc b/src/fdb5/toc/TocCatalogue.cc
index 9241d1eb4..5dfd4bdfb 100644
--- a/src/fdb5/toc/TocCatalogue.cc
+++ b/src/fdb5/toc/TocCatalogue.cc
@@ -57,7 +57,8 @@ eckit::URI TocCatalogue::uri() const {
 }
 
 const Schema& TocCatalogue::schema() const {
-    return schema_;
+    ASSERT(schema_);
+    return *schema_;
 }
 
 const eckit::PathName& TocCatalogue::basePath() const {
@@ -100,7 +101,7 @@ void TocCatalogue::visitEntries(EntryVisitor& visitor, const Store& store, bool
 
 void TocCatalogue::loadSchema() {
     Timer timer("TocCatalogue::loadSchema()", Log::debug());
-    schema_.load( schemaPath() );
+    schema_ = &SchemaRegistry::instance().get(schemaPath());
 }
 
 StatsReportVisitor* TocCatalogue::statsReportVisitor() const {
diff --git a/src/fdb5/toc/TocCatalogue.h b/src/fdb5/toc/TocCatalogue.h
index 2ad22112c..c4bbda046 100644
--- a/src/fdb5/toc/TocCatalogue.h
+++ b/src/fdb5/toc/TocCatalogue.h
@@ -88,7 +88,8 @@ class TocCatalogue : public Catalogue, public TocHandler {
     friend class TocWipeVisitor;
     friend class TocMoveVisitor;
 
-    Schema schema_;
+    // non-owning
+    const Schema* schema_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/tests/fdb/api/test_fdb_c.cc b/tests/fdb/api/test_fdb_c.cc
index e3c9d432f..04afc699b 100644
--- a/tests/fdb/api/test_fdb_c.cc
+++ b/tests/fdb/api/test_fdb_c.cc
@@ -120,7 +120,13 @@ CASE( "fdb_c - archive & list" ) {
     fdb_listiterator_attrs(it, &uri, &off, &attr_len);
     EXPECT(attr_len == 3280398);
 
-    std::vector k1test{fdb5::Key{"class=rd,expver=xxxx,stream=oper,date=20191110,time=0000,domain=g"},fdb5::Key{"type=an,levtype=pl"},fdb5::Key{"step=0,levelist=300,param=138"}};
+    std::vector k1test {
+        {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}},
+        {{"type", "an"}, {"levtype", "pl"}},
+        {{"step", "0"}, {"levelist", "300"}, {"param", "138"}},
+    };
+//    std::vector k1test{fdb5::Key{"class=rd,expver=xxxx,stream=oper,date=20191110,time=0000,domain=g"},
+//                                  fdb5::Key{"type=an,levtype=pl"},fdb5::Key{"step=0,levelist=300,param=138"}};
     key_compare(k1test, it);
 
     err = fdb_listiterator_next(it);
@@ -162,7 +168,13 @@ CASE( "fdb_c - archive & list" ) {
     fdb_listiterator_attrs(it, &uri, &off, &attr_len);
     EXPECT(attr_len == 3280398);
 
-    std::vector k2test{fdb5::Key{"class=rd,expver=xxxx,stream=oper,date=20191110,time=0000,domain=g"},fdb5::Key{"type=an,levtype=pl"},fdb5::Key{"step=0,levelist=400,param=138"}};
+    std::vector k2test {
+        {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}},
+        {{"type", "an"}, {"levtype", "pl"}},
+        {{"step", "0"}, {"levelist", "400"}, {"param", "138"}},
+    };
+//    std::vector k2test{fdb5::Key{"class=rd,expver=xxxx,stream=oper,date=20191110,time=0000,domain=g"},
+//                                  fdb5::Key{"type=an,levtype=pl"},fdb5::Key{"step=0,levelist=400,param=138"}};
     key_compare(k2test, it, false);
     key_compare(k2test, it);
 
diff --git a/tests/fdb/test_fdb5_service.cc b/tests/fdb/test_fdb5_service.cc
index be13dfaa0..d960eac57 100644
--- a/tests/fdb/test_fdb5_service.cc
+++ b/tests/fdb/test_fdb5_service.cc
@@ -52,6 +52,7 @@ struct FixtureService {
 
     metkit::mars::MarsRequest env;
     StringDict p;
+    fdb5::Config config;
 
 	std::vector modelParams_;
 
@@ -95,7 +96,7 @@ struct FixtureService {
 						 << std::endl;
 					std::string data_str = data.str();
 
-                    fdb5::Key k(p);
+                    fdb5::Key k(p, config.schema().registry());
                     ArchiveVisitor visitor(fdb, k, static_cast(data_str.c_str()), data_str.size());
                     fdb.archive(k, visitor);
 				}
diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc
index 892c75e20..1cb91f70e 100644
--- a/tests/fdb/type/test_toKey.cc
+++ b/tests/fdb/type/test_toKey.cc
@@ -31,7 +31,7 @@ char data[4];
 
 CASE( "ClimateDaily - no expansion" ) {
 
-    fdb5::Key key;
+    fdb5::Key key(config.schema().registry());
     EXPECT(key.valuesToString() == "");
     EXPECT_THROWS(key.canonicalValue("date"));
 
@@ -48,7 +48,7 @@ CASE( "ClimateDaily - no expansion" ) {
 
 CASE( "Step & ClimateDaily - expansion" ) {
 
-    fdb5::Key key;
+    fdb5::Key key(config.schema().registry());
     EXPECT(key.valuesToString() == "");
     EXPECT_THROWS(key.canonicalValue("date"));
 
@@ -127,7 +127,7 @@ CASE( "Step & ClimateDaily - expansion" ) {
     fdb5::Archiver archiver;
     fdb5::ArchiveVisitor visitor(archiver, key, data, 4);
     config.schema().expand(key, visitor);
-    key.rule(visitor.rule());
+    key.registry(visitor.rule()->registry());
 
     EXPECT(key.canonicalValue("date") == "0427");
     EXPECT(key.canonicalValue("time") == "0000");
@@ -177,7 +177,7 @@ CASE( "Levelist" ) {
     values.insert("0.333333");
     values.sort();
 
-    fdb5::Key key;
+    fdb5::Key key(config.schema().registry());
     EXPECT(key.valuesToString() == "");
     EXPECT_THROWS(key.canonicalValue("levelist"));
 
@@ -232,7 +232,9 @@ CASE( "Levelist" ) {
 
 CASE( "Expver, Time & ClimateDaily - string ctor - expansion" ) {
 
-    fdb5::Key key("class=ei,expver=1,stream=dacl,domain=g,type=pb,levtype=pl,date=20210427,time=6,step=0,quantile=99:100,levelist=50,param=129.128");
+    fdb5::Key key = fdb5::Key::parseString(
+        "class=ei,expver=1,stream=dacl,domain=g,type=pb,levtype=pl,date=20210427,time=6,step=0,quantile=99:100,levelist=50,param=129.128",
+        config.schema().registry());
 
     EXPECT(key.canonicalValue("date") == "20210427");
     EXPECT(key.canonicalValue("time") == "0600");
@@ -241,16 +243,17 @@ CASE( "Expver, Time & ClimateDaily - string ctor - expansion" ) {
     fdb5::Archiver archiver;
     fdb5::ArchiveVisitor visitor(archiver, key, data, 4);
     config.schema().expand(key, visitor);
-    key.rule(visitor.rule());
+    key.registry(visitor.rule()->registry());
 
     EXPECT(key.canonicalValue("date") == "0427");
     EXPECT(key.valuesToString() == "ei:0001:dacl:g:pb:pl:0427:0600:0:99:100:50:129.128");
-
 }
 
 CASE( "ClimateMonthly - string ctor - expansion" ) {
 
-    fdb5::Key key("class=op,expver=1,stream=mnth,domain=g,type=cl,levtype=pl,date=20210427,time=0000,levelist=50,param=129.128");
+    fdb5::Key key = fdb5::Key::parseString(
+        "class=op,expver=1,stream=mnth,domain=g,type=cl,levtype=pl,date=20210427,time=0000,levelist=50,param=129.128",
+        config.schema().registry());
 
     EXPECT(key.canonicalValue("date") == "20210427");
     EXPECT(key.valuesToString() == "op:0001:mnth:g:cl:pl:20210427:0000:50:129.128");
@@ -258,9 +261,10 @@ CASE( "ClimateMonthly - string ctor - expansion" ) {
     fdb5::Archiver archiver;
     fdb5::ArchiveVisitor visitor(archiver, key, data, 4);
     config.schema().expand(key, visitor);
-    key.rule(visitor.rule());
+    key.registry(visitor.rule()->registry());
 
-//    std::cout << key.valuesToString() << std::endl;
+    std::cout << key.valuesToString() << std::endl;
+    std::cout << key.canonicalValue("date");
     EXPECT(key.canonicalValue("date") == "4");
     EXPECT(key.valuesToString() == "op:0001:mnth:g:cl:pl:4:0000:50:129.128");
 
@@ -269,7 +273,9 @@ CASE( "ClimateMonthly - string ctor - expansion" ) {
 // do we need to keep this behaviour? should we rely on metkit for date expansion and remove it from Key?
 CASE( "Date - string ctor - expansion" ) {
 
-    fdb5::Key key("class=od,expver=1,stream=oper,type=ofb,date=-2,time=0000,obsgroup=MHS,reportype=3001");
+    fdb5::Key key = fdb5::Key::parseString(
+            "class=od,expver=1,stream=oper,type=ofb,date=-2,time=0000,obsgroup=MHS,reportype=3001",
+            config.schema().registry());
 
     eckit::Date now(-2);
     eckit::Translator t;
@@ -280,7 +286,7 @@ CASE( "Date - string ctor - expansion" ) {
     fdb5::Archiver archiver;
     fdb5::ArchiveVisitor visitor(archiver, key, data, 4);
     config.schema().expand(key, visitor);
-    key.rule(visitor.rule());
+    key.registry(visitor.rule()->registry());
 
 //    std::cout << key.valuesToString() << std::endl;
     EXPECT(key.canonicalValue("date") == t(now.yyyymmdd()));

From 10dc8f47faf6bfde4ec9e1c62cc511b64f29cfda Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Mon, 30 Jan 2023 16:01:38 +0000
Subject: [PATCH 03/23] FDB-303 fixed unit test

---
 tests/fdb/api/test_fdb_c.cc | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/fdb/api/test_fdb_c.cc b/tests/fdb/api/test_fdb_c.cc
index 04afc699b..cda453e04 100644
--- a/tests/fdb/api/test_fdb_c.cc
+++ b/tests/fdb/api/test_fdb_c.cc
@@ -226,8 +226,16 @@ CASE( "fdb_c - multiple archive & list" ) {
     fdb_handle_t* fdb;
     fdb_new_handle(&fdb);
 
-    std::vector k1{fdb5::Key{"class=rd,expver=xxxx,stream=oper,date=20191110,time=0000,domain=g"},fdb5::Key{"type=an,levtype=pl"},fdb5::Key{"step=0,levelist=300,param=138"}};
-    std::vector k2{fdb5::Key{"class=rd,expver=xxxx,stream=oper,date=20191110,time=0000,domain=g"},fdb5::Key{"type=an,levtype=pl"},fdb5::Key{"step=0,levelist=400,param=138"}};
+    std::vector k1 {
+        {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}},
+        {{"type", "an"}, {"levtype", "pl"}},
+        {{"step", "0"}, {"levelist", "300"}, {"param", "138"}},
+    };
+    std::vector k2 {
+        {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}},
+        {{"type", "an"}, {"levtype", "pl"}},
+        {{"step", "0"}, {"levelist", "400"}, {"param", "138"}},
+    };
 
     eckit::PathName grib1("x138-300.grib");
     length1 = grib1.size();

From 20488416bf85d327ad0e4242abdf2423fb4fd519 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 20 Mar 2024 10:07:11 +0000
Subject: [PATCH 04/23] avoid canonicalisation if key is decoded from stream

---
 src/fdb5/api/fdb_c.cc                   |  2 +-
 src/fdb5/api/helpers/ControlIterator.cc |  2 +-
 src/fdb5/api/helpers/ListIterator.cc    |  3 +--
 src/fdb5/database/DB.cc                 |  3 ++-
 src/fdb5/database/FieldLocation.h       |  2 +-
 src/fdb5/database/Key.cc                | 33 ++++++++++++-------------
 src/fdb5/database/Key.h                 |  1 +
 src/fdb5/rules/Schema.cc                |  5 +++-
 tests/fdb/test_fdb5_service.cc          |  6 +++++
 9 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc
index 9c2fa3363..fee215c7e 100644
--- a/src/fdb5/api/fdb_c.cc
+++ b/src/fdb5/api/fdb_c.cc
@@ -371,7 +371,7 @@ int fdb_delete_handle(fdb_handle_t* fdb) {
 
 int fdb_new_key(fdb_key_t** key) {
     return wrapApiFunction([key] {
-        *key = new fdb_key_t();
+        *key = new fdb_key_t(nullptr);
     });
 }
 int fdb_key_add(fdb_key_t* key, const char* param, const char* value) {
diff --git a/src/fdb5/api/helpers/ControlIterator.cc b/src/fdb5/api/helpers/ControlIterator.cc
index e0ea53602..cd2fe8673 100644
--- a/src/fdb5/api/helpers/ControlIterator.cc
+++ b/src/fdb5/api/helpers/ControlIterator.cc
@@ -141,7 +141,7 @@ std::ostream &operator<<(std::ostream &s, const ControlIdentifiers &x) {
 using value_type = typename std::underlying_type::type;
 
 ControlElement::ControlElement() : 
-    location() {}
+    key(nullptr), location() {}
 
 ControlElement::ControlElement(const Catalogue& catalogue) :
     key(catalogue.key()), location(catalogue.uri()) {
diff --git a/src/fdb5/api/helpers/ListIterator.cc b/src/fdb5/api/helpers/ListIterator.cc
index 8e14b3efd..15ca1bb2f 100644
--- a/src/fdb5/api/helpers/ListIterator.cc
+++ b/src/fdb5/api/helpers/ListIterator.cc
@@ -31,14 +31,13 @@ ListElement::ListElement(eckit::Stream &s) {
 }
 
 Key ListElement::combinedKey() const {
-    Key combined;
+    Key combined = keyParts_[2];
 
     for (const Key& partKey : keyParts_) {
         for (const auto& kv : partKey) {
             combined.set(kv.first, kv.second);
         }
     }
-
     return combined;
 }
 
diff --git a/src/fdb5/database/DB.cc b/src/fdb5/database/DB.cc
index c0ada8e76..2eeb46a08 100644
--- a/src/fdb5/database/DB.cc
+++ b/src/fdb5/database/DB.cc
@@ -121,8 +121,9 @@ bool DB::open() {
 }
 
 void DB::flush() {
-    if (store_ != nullptr)
+    if (store_ != nullptr) {
         store_->flush();
+    }
     catalogue_->flush();
 }
 
diff --git a/src/fdb5/database/FieldLocation.h b/src/fdb5/database/FieldLocation.h
index 09c9eb961..a6a684ad4 100644
--- a/src/fdb5/database/FieldLocation.h
+++ b/src/fdb5/database/FieldLocation.h
@@ -40,7 +40,7 @@ class FieldLocation : public eckit::OwnedLock, public eckit::Streamable {
 
 public: // methods
 
-    FieldLocation() : offset_(eckit::Offset(0)), length_(eckit::Length(0)), remapKey_(Key()) {}
+    FieldLocation() : offset_(eckit::Offset(0)), length_(eckit::Length(0)), remapKey_(Key{}) {}
     FieldLocation(const eckit::URI& uri);
     FieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey);
     FieldLocation(eckit::Stream&);
diff --git a/src/fdb5/database/Key.cc b/src/fdb5/database/Key.cc
index e920a2090..61eac48f5 100644
--- a/src/fdb5/database/Key.cc
+++ b/src/fdb5/database/Key.cc
@@ -28,13 +28,13 @@ namespace fdb5 {
 
 Key::Key(const TypesRegistry* reg) :
     keys_(),
-    registry_(reg) {}
+    registry_(reg), canonical_(false) {}
 
 Key::Key(const TypesRegistry& reg) : Key(®) {}
 
 Key::Key(const std::string &s, const Rule *rule) :
     keys_(),
-    registry_(rule ? &rule->registry() : nullptr) {
+    registry_(rule ? &rule->registry() : nullptr), canonical_(false) {
     eckit::Tokenizer parse(":", true);
     eckit::StringList values;
     parse(s, values);
@@ -43,12 +43,9 @@ Key::Key(const std::string &s, const Rule *rule) :
     rule->fill(*this, values);
 }
 
-
-//Key::Key(const std::string &s, const TypesRegistry& reg) : Key(s, ®) {}
-//
 Key::Key(const eckit::StringDict &keys, const TypesRegistry* reg) :
     keys_(keys),
-    registry_(reg) {
+    registry_(reg), canonical_(false) {
 
     eckit::StringDict::const_iterator it = keys.begin();
     eckit::StringDict::const_iterator end = keys.end();
@@ -60,13 +57,13 @@ Key::Key(const eckit::StringDict &keys, const TypesRegistry* reg) :
 Key::Key(const eckit::StringDict &keys, const TypesRegistry& reg) : Key(keys, ®) {}
 
 Key::Key(eckit::Stream& s, const TypesRegistry* reg) :
-    registry_(reg) {
+    registry_(reg), canonical_(reg==nullptr) {
     decode(s);
 }
 
 Key::Key(std::initializer_list> l, const TypesRegistry* reg) :
     keys_(l),
-    registry_(reg) {
+    registry_(reg), canonical_(false) {
 
     for (const auto& kv : keys_) {
         names_.emplace_back(kv.first);
@@ -125,6 +122,7 @@ Key Key::parseString(const std::string &s, const TypesRegistry& registry) {
 void Key::decode(eckit::Stream& s) {
 
     registry_ = nullptr;
+    canonical_ = true;
     keys_.clear();
     names_.clear();
 
@@ -148,7 +146,7 @@ void Key::decode(eckit::Stream& s) {
 }
 
 void Key::encode(eckit::Stream& s) const {
-    const TypesRegistry& registry = this->registry();
+    const TypesRegistry* registry = canonical_ ? nullptr : &this->registry();
 
     s << keys_.size();
     for (eckit::StringDict::const_iterator i = keys_.begin(); i != keys_.end(); ++i) {
@@ -157,9 +155,8 @@ void Key::encode(eckit::Stream& s) const {
 
     s << names_.size();
     for (eckit::StringList::const_iterator i = names_.begin(); i != names_.end(); ++i) {
-        const Type &t = registry.lookupType(*i);
         s << (*i);
-        s << t.type();
+        s << (canonical_ ? "" : registry->lookupType(*i).type());
     }
 }
 
@@ -319,7 +316,9 @@ void Key::registry(const TypesRegistry& reg) {
 
 const TypesRegistry& Key::registry() const {
     if (!registry_) {
-        throw eckit::SeriousBug("TypesRegistry has not been set for Key prior to use", Here());
+        std::stringstream ss;
+        ss << "TypesRegistry has not been set for Key " << (*this) << " prior to use";
+        throw eckit::SeriousBug(ss.str(), Here());
     }
 
     return *registry_;
@@ -330,7 +329,7 @@ const void* Key::reg() const {
 }
 
 std::string Key::canonicalise(const std::string& keyword, const std::string& value) const {
-    if (value.empty()) {
+    if (canonical_ || value.empty()) {
         return value;
     } else {
         return this->registry().lookupType(keyword).toKey(keyword, value);
@@ -382,7 +381,7 @@ void Key::validateKeysOf(const Key& other, bool checkAlsoValues) const
     eckit::StringSet missing;
     eckit::StringSet mismatch;
 
-    const TypesRegistry& registry = this->registry();
+    const TypesRegistry* registry = canonical_ ? nullptr : &this->registry();
 
     for (Key::const_iterator j = begin(); j != end(); ++j) {
         const std::string& keyword = (*j).first;
@@ -391,7 +390,7 @@ void Key::validateKeysOf(const Key& other, bool checkAlsoValues) const
             missing.insert(keyword);
         }
         else {
-            if(checkAlsoValues && !registry.lookupType(keyword).match(keyword, k->second, j->second) ) {
+            if(checkAlsoValues && !canonical_ && !registry->lookupType(keyword).match(keyword, k->second, j->second) ) {
                 mismatch.insert((*j).first + "=" + j->second + " and " + k->second);
             }
         }
@@ -440,7 +439,7 @@ fdb5::Key::operator eckit::StringDict() const
 
     ASSERT(names_.size() == keys_.size());
 
-    const TypesRegistry ®istry = this->registry();
+    const TypesRegistry* registry = canonical_ ? nullptr : &this->registry();
 
     for (eckit::StringList::const_iterator j = names_.begin(); j != names_.end(); ++j) {
 
@@ -449,7 +448,7 @@ fdb5::Key::operator eckit::StringDict() const
         ASSERT(i != keys_.end());
         ASSERT(!(*i).second.empty());
 
-        res[*j] = registry.lookupType(*j).tidy(*j, (*i).second);
+        res[*j] = canonical_ ? (*i).second : registry->lookupType(*j).tidy(*j, (*i).second);
     }
 
     return res;
diff --git a/src/fdb5/database/Key.h b/src/fdb5/database/Key.h
index 3bec0079f..ad0dfd95c 100644
--- a/src/fdb5/database/Key.h
+++ b/src/fdb5/database/Key.h
@@ -168,6 +168,7 @@ class Key {
     eckit::StringList names_;
 
     const TypesRegistry* registry_;
+    bool canonical_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/fdb5/rules/Schema.cc b/src/fdb5/rules/Schema.cc
index 2affc165d..4911c8970 100644
--- a/src/fdb5/rules/Schema.cc
+++ b/src/fdb5/rules/Schema.cc
@@ -130,7 +130,10 @@ bool Schema::expandFirstLevel(const metkit::mars::MarsRequest& request, Key &res
     bool found = false;
     for (const Rule* rule : rules_) {
         rule->expandFirstLevel(request, result, found);
-        if (found) break;
+        if (found) {
+            result.registry(rule->registry());
+            break;
+        }
     }
     return found;
 }
diff --git a/tests/fdb/test_fdb5_service.cc b/tests/fdb/test_fdb5_service.cc
index d960eac57..7c23287a4 100644
--- a/tests/fdb/test_fdb5_service.cc
+++ b/tests/fdb/test_fdb5_service.cc
@@ -313,6 +313,9 @@ CASE ( "test_fdb_service" ) {
             	while ((dp = ::readdir(dirp)) != nullptr) {
                 	EXPECT_NOT(strstr( dp->d_name, "toc."));
 				}
+
+				// consuming the rest of the queue
+				while (iter.next(el));
 			}
         }
 
@@ -497,6 +500,9 @@ CASE ( "test_fdb_service_subtoc" ) {
 					}
 				}
 				EXPECT(subtoc);
+
+				// consuming the rest of the queue
+				while (iter.next(el));
 			}
         }
 

From 4dd08c84e7be0552f2c2843aad1967d019deff62 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 20 Mar 2024 10:11:38 +0000
Subject: [PATCH 05/23] avoid canonicalisation if key is decoded from stream

---
 src/fdb5/api/fdb_c.cc                   | 2 +-
 src/fdb5/api/helpers/ControlIterator.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc
index fee215c7e..9c2fa3363 100644
--- a/src/fdb5/api/fdb_c.cc
+++ b/src/fdb5/api/fdb_c.cc
@@ -371,7 +371,7 @@ int fdb_delete_handle(fdb_handle_t* fdb) {
 
 int fdb_new_key(fdb_key_t** key) {
     return wrapApiFunction([key] {
-        *key = new fdb_key_t(nullptr);
+        *key = new fdb_key_t();
     });
 }
 int fdb_key_add(fdb_key_t* key, const char* param, const char* value) {
diff --git a/src/fdb5/api/helpers/ControlIterator.cc b/src/fdb5/api/helpers/ControlIterator.cc
index cd2fe8673..d566bcb44 100644
--- a/src/fdb5/api/helpers/ControlIterator.cc
+++ b/src/fdb5/api/helpers/ControlIterator.cc
@@ -141,7 +141,7 @@ std::ostream &operator<<(std::ostream &s, const ControlIdentifiers &x) {
 using value_type = typename std::underlying_type::type;
 
 ControlElement::ControlElement() : 
-    key(nullptr), location() {}
+    key(), location() {}
 
 ControlElement::ControlElement(const Catalogue& catalogue) :
     key(catalogue.key()), location(catalogue.uri()) {

From 3a22b6ad6a63805f6d2a8c93ba5df9abe3f67cc6 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 20 Mar 2024 11:00:15 +0000
Subject: [PATCH 06/23] small cleanup

---
 src/fdb5/api/helpers/ControlIterator.cc | 2 +-
 src/fdb5/database/DB.cc                 | 3 +--
 src/fdb5/database/FieldLocation.h       | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/fdb5/api/helpers/ControlIterator.cc b/src/fdb5/api/helpers/ControlIterator.cc
index d566bcb44..e0ea53602 100644
--- a/src/fdb5/api/helpers/ControlIterator.cc
+++ b/src/fdb5/api/helpers/ControlIterator.cc
@@ -141,7 +141,7 @@ std::ostream &operator<<(std::ostream &s, const ControlIdentifiers &x) {
 using value_type = typename std::underlying_type::type;
 
 ControlElement::ControlElement() : 
-    key(), location() {}
+    location() {}
 
 ControlElement::ControlElement(const Catalogue& catalogue) :
     key(catalogue.key()), location(catalogue.uri()) {
diff --git a/src/fdb5/database/DB.cc b/src/fdb5/database/DB.cc
index 2eeb46a08..c0ada8e76 100644
--- a/src/fdb5/database/DB.cc
+++ b/src/fdb5/database/DB.cc
@@ -121,9 +121,8 @@ bool DB::open() {
 }
 
 void DB::flush() {
-    if (store_ != nullptr) {
+    if (store_ != nullptr)
         store_->flush();
-    }
     catalogue_->flush();
 }
 
diff --git a/src/fdb5/database/FieldLocation.h b/src/fdb5/database/FieldLocation.h
index a6a684ad4..09c9eb961 100644
--- a/src/fdb5/database/FieldLocation.h
+++ b/src/fdb5/database/FieldLocation.h
@@ -40,7 +40,7 @@ class FieldLocation : public eckit::OwnedLock, public eckit::Streamable {
 
 public: // methods
 
-    FieldLocation() : offset_(eckit::Offset(0)), length_(eckit::Length(0)), remapKey_(Key{}) {}
+    FieldLocation() : offset_(eckit::Offset(0)), length_(eckit::Length(0)), remapKey_(Key()) {}
     FieldLocation(const eckit::URI& uri);
     FieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey);
     FieldLocation(eckit::Stream&);

From e22c9ac9aeb6325b41cf4827479e11ff3f0c6f0f Mon Sep 17 00:00:00 2001
From: Chris Bradley 
Date: Thu, 21 Mar 2024 15:59:03 +0000
Subject: [PATCH 07/23] Escape regex characters in root on list pathway

---
 src/fdb5/toc/TocEngine.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/toc/TocEngine.cc b/src/fdb5/toc/TocEngine.cc
index c2a65b121..4f3d83df6 100644
--- a/src/fdb5/toc/TocEngine.cc
+++ b/src/fdb5/toc/TocEngine.cc
@@ -164,7 +164,7 @@ std::set TocEngine::databases(const std::set& keys,
 
             for(std::vector::const_iterator dbpath = dbpaths.begin(); dbpath != dbpaths.end(); ++dbpath) {
 
-                Regex re("^" + *j + "/" + *dbpath + "$");
+                Regex re("^" + Regex::escape(j->asString()) + "/" + *dbpath + "$");
 
                 LOG_DEBUG_LIB(LibFdb5) << " -> key i " << *i
                                      << " dbpath " << *dbpath

From 9a39a6f56c7efc928ac4eb5b29935400d6f5de47 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 22 Mar 2024 12:20:38 +0000
Subject: [PATCH 08/23] FDB-303 fixed TypesRegistry lifecycle

---
 src/fdb5/database/Key.cc        | 34 +++++++++++++--------------------
 src/fdb5/database/Key.h         | 22 +++++++--------------
 src/fdb5/rules/Rule.cc          | 15 +++++++--------
 src/fdb5/rules/Rule.h           |  4 ++--
 src/fdb5/rules/Schema.cc        | 17 +++++++++--------
 src/fdb5/rules/Schema.h         |  4 ++--
 src/fdb5/types/TypesRegistry.cc |  6 +++---
 src/fdb5/types/TypesRegistry.h  |  4 ++--
 8 files changed, 45 insertions(+), 61 deletions(-)

diff --git a/src/fdb5/database/Key.cc b/src/fdb5/database/Key.cc
index 61eac48f5..fcf2ed7d6 100644
--- a/src/fdb5/database/Key.cc
+++ b/src/fdb5/database/Key.cc
@@ -26,15 +26,13 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-Key::Key(const TypesRegistry* reg) :
+Key::Key(const std::shared_ptr reg) :
     keys_(),
     registry_(reg), canonical_(false) {}
 
-Key::Key(const TypesRegistry& reg) : Key(®) {}
-
 Key::Key(const std::string &s, const Rule *rule) :
     keys_(),
-    registry_(rule ? &rule->registry() : nullptr), canonical_(false) {
+    registry_(rule ? rule->registry() : nullptr), canonical_(false) {
     eckit::Tokenizer parse(":", true);
     eckit::StringList values;
     parse(s, values);
@@ -43,7 +41,7 @@ Key::Key(const std::string &s, const Rule *rule) :
     rule->fill(*this, values);
 }
 
-Key::Key(const eckit::StringDict &keys, const TypesRegistry* reg) :
+Key::Key(const eckit::StringDict &keys, const std::shared_ptr reg) :
     keys_(keys),
     registry_(reg), canonical_(false) {
 
@@ -54,14 +52,12 @@ Key::Key(const eckit::StringDict &keys, const TypesRegistry* reg) :
     }
 }
 
-Key::Key(const eckit::StringDict &keys, const TypesRegistry& reg) : Key(keys, ®) {}
-
-Key::Key(eckit::Stream& s, const TypesRegistry* reg) :
+Key::Key(eckit::Stream& s, const std::shared_ptr reg) :
     registry_(reg), canonical_(reg==nullptr) {
     decode(s);
 }
 
-Key::Key(std::initializer_list> l, const TypesRegistry* reg) :
+Key::Key(std::initializer_list> l, const std::shared_ptr reg) :
     keys_(l),
     registry_(reg), canonical_(false) {
 
@@ -70,10 +66,6 @@ Key::Key(std::initializer_list> l, con
     }
 }
 
-Key::Key(std::initializer_list> l, const TypesRegistry& reg) : Key(l, ®) {}
-
-Key::Key(eckit::Stream& s, const TypesRegistry& reg) : Key(s, ®) {}
-
 Key Key::parseStringUntyped(const std::string& s) {
 
     eckit::Tokenizer parse1(",");
@@ -92,7 +84,7 @@ Key Key::parseStringUntyped(const std::string& s) {
     return Key{keys};
 }
 
-Key Key::parseString(const std::string &s, const TypesRegistry& registry) {
+Key Key::parseString(const std::string &s, const std::shared_ptr registry) {
 
     eckit::Tokenizer parse1(",");
     eckit::Tokenizer parse2("=");
@@ -106,7 +98,7 @@ Key Key::parseString(const std::string &s, const TypesRegistry& registry) {
         parse2(bit, kv);
         ASSERT(kv.size() == 2);
 
-        const Type &t = registry.lookupType(kv[0]);
+        const Type &t = registry->lookupType(kv[0]);
         std::string v = t.tidy(kv[0], kv[1]);
 
         if (ret.find(kv[0]) == ret.end()) {
@@ -150,13 +142,13 @@ void Key::encode(eckit::Stream& s) const {
 
     s << keys_.size();
     for (eckit::StringDict::const_iterator i = keys_.begin(); i != keys_.end(); ++i) {
-        s << i->first << canonicalise(i->first, i->second);
+        s << i->first << (registry ? canonicalise(i->first, i->second) : i->second);
     }
 
     s << names_.size();
     for (eckit::StringList::const_iterator i = names_.begin(); i != names_.end(); ++i) {
         s << (*i);
-        s << (canonical_ ? "" : registry->lookupType(*i).type());
+        s << (registry ? registry->lookupType(*i).type() : "");
     }
 }
 
@@ -310,8 +302,8 @@ bool Key::partialMatch(const metkit::mars::MarsRequest& request) const {
     return true;
 }
 
-void Key::registry(const TypesRegistry& reg) {
-    registry_ = ®
+void Key::registry(const std::shared_ptr reg) {
+    registry_ = reg;
 }
 
 const TypesRegistry& Key::registry() const {
@@ -325,11 +317,11 @@ const TypesRegistry& Key::registry() const {
 }
 
 const void* Key::reg() const {
-    return registry_;
+    return registry_.get();
 }
 
 std::string Key::canonicalise(const std::string& keyword, const std::string& value) const {
-    if (canonical_ || value.empty()) {
+    if (value.empty() || canonical_) {
         return value;
     } else {
         return this->registry().lookupType(keyword).toKey(keyword, value);
diff --git a/src/fdb5/database/Key.h b/src/fdb5/database/Key.h
index ad0dfd95c..54c94f88b 100644
--- a/src/fdb5/database/Key.h
+++ b/src/fdb5/database/Key.h
@@ -45,23 +45,15 @@ class Key {
 
 public: // methods
 
-    explicit Key(const TypesRegistry* reg=nullptr);
-    explicit Key(const TypesRegistry& reg);
-
-    explicit Key(eckit::Stream &, const TypesRegistry* reg=nullptr);
-    explicit Key(eckit::Stream &, const TypesRegistry& reg);
-    /// @todo remove ?? explicit Key(const std::string &request, const TypesRegistry* reg=nullptr);
-    /// @todo remove ?? explicit Key(const std::string &request, const TypesRegistry& reg);
+    explicit Key(const std::shared_ptr reg = nullptr);
+    explicit Key(eckit::Stream &, const std::shared_ptr reg = nullptr);
     explicit Key(const std::string &keys, const Rule* rule);
-
-    explicit Key(const eckit::StringDict &keys, const TypesRegistry* reg=nullptr);
-    explicit Key(const eckit::StringDict &keys, const TypesRegistry& reg);
-    Key(std::initializer_list>, const TypesRegistry* reg=nullptr);
-    Key(std::initializer_list>, const TypesRegistry& reg);
+    explicit Key(const eckit::StringDict &keys, const std::shared_ptr reg=nullptr);
+    Key(std::initializer_list>, const std::shared_ptr reg=nullptr);
 
     static Key parseStringUntyped(const std::string& s);
     /// @todo - this functionality should not be supported any more.
-    static Key parseString(const std::string&, const TypesRegistry& reg);
+    static Key parseString(const std::string&, const std::shared_ptr reg);
 
     std::set keys() const;
 
@@ -115,7 +107,7 @@ class Key {
     }
 
     // Registry is needed before we can stringise/canonicalise.
-    void registry(const TypesRegistry& reg);
+    void registry(const std::shared_ptr reg);
     [[ nodiscard ]]
     const TypesRegistry& registry() const;
     const void* reg() const;
@@ -167,7 +159,7 @@ class Key {
     eckit::StringDict keys_;
     eckit::StringList names_;
 
-    const TypesRegistry* registry_;
+    std::shared_ptr registry_;
     bool canonical_;
 };
 
diff --git a/src/fdb5/rules/Rule.cc b/src/fdb5/rules/Rule.cc
index d87ea4b28..6879ad3a9 100644
--- a/src/fdb5/rules/Rule.cc
+++ b/src/fdb5/rules/Rule.cc
@@ -30,12 +30,11 @@ Rule::Rule(const Schema &schema,
            size_t line,
            std::vector &predicates, std::vector &rules,
            const std::map &types):
-    schema_(schema),
-    line_(line) {
+    schema_(schema), registry_(new TypesRegistry()), line_(line) {
     std::swap(predicates, predicates_);
     std::swap(rules, rules_);
     for (std::map::const_iterator i = types.begin(); i != types.end(); ++i) {
-        registry_.addType(i->first, i->second);
+        registry_->addType(i->first, i->second);
     }
 }
 
@@ -105,7 +104,7 @@ void Rule::expand( const metkit::mars::MarsRequest &request,
     const std::string &keyword = (*cur)->keyword();
 
     eckit::StringList values;
-    visitor.values(request, keyword, registry_, values);
+    visitor.values(request, keyword, *registry_, values);
 
     // eckit::Log::info() << "keyword " << keyword << " values " << values << std::endl;
 
@@ -443,7 +442,7 @@ void Rule::dump(std::ostream &s, size_t depth) const {
     const char *sep = "";
     for (std::vector::const_iterator i = predicates_.begin(); i != predicates_.end(); ++i ) {
         s << sep;
-        (*i)->dump(s, registry_);
+        (*i)->dump(s, *registry_);
         sep = ",";
     }
 
@@ -464,14 +463,14 @@ size_t Rule::depth() const {
 void Rule::updateParent(const Rule *parent) {
     parent_ = parent;
     if (parent) {
-        registry_.updateParent(&parent_->registry_);
+        registry_->updateParent(parent_->registry_);
     }
     for (std::vector::iterator i = rules_.begin(); i != rules_.end(); ++i ) {
         (*i)->updateParent(this);
     }
 }
 
-const TypesRegistry &Rule::registry() const {
+const std::shared_ptr Rule::registry() const {
     return registry_;
 }
 
@@ -497,7 +496,7 @@ void Rule::check(const Key& key) const {
         auto k = key.find(pred->keyword());
         if (k != key.end()) {
             const std::string& value = (*k).second;
-            const Type& type = registry_.lookupType(pred->keyword());
+            const Type& type = registry_->lookupType(pred->keyword());
             if (value != type.tidy(pred->keyword(), value)) {
                 std::stringstream ss;
                 ss << "Rule check - metadata not valid (not in canonical form) - found: ";
diff --git a/src/fdb5/rules/Rule.h b/src/fdb5/rules/Rule.h
index b9f877ecf..beef3e5ea 100644
--- a/src/fdb5/rules/Rule.h
+++ b/src/fdb5/rules/Rule.h
@@ -81,7 +81,7 @@ class Rule : public eckit::NonCopyable {
     const Rule &topRule() const;
 
     const Schema &schema() const;
-    const TypesRegistry ®istry() const;
+    const std::shared_ptr registry() const;
 
     void check(const Key& key) const;
 
@@ -127,7 +127,7 @@ class Rule : public eckit::NonCopyable {
     std::vector predicates_;
     std::vector      rules_;
 
-    TypesRegistry registry_;
+    std::shared_ptr registry_;
 
     friend class Schema;
     size_t line_;
diff --git a/src/fdb5/rules/Schema.cc b/src/fdb5/rules/Schema.cc
index 4911c8970..8e8c0c46f 100644
--- a/src/fdb5/rules/Schema.cc
+++ b/src/fdb5/rules/Schema.cc
@@ -21,14 +21,15 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-Schema::Schema() {
+Schema::Schema() : registry_(new TypesRegistry()) {
+
 }
 
-Schema::Schema(const eckit::PathName &path) {
+Schema::Schema(const eckit::PathName &path) : registry_(new TypesRegistry()) {
     load(path);
 }
 
-Schema::Schema(std::istream& s) {
+Schema::Schema(std::istream& s) : registry_(new TypesRegistry()) {
     load(s);
 }
 
@@ -171,7 +172,7 @@ void Schema::load(std::istream& s, bool replace) {
 
     SchemaParser parser(s);
 
-    parser.parse(*this, rules_, registry_);
+    parser.parse(*this, rules_, *registry_);
 
     check();
 }
@@ -183,7 +184,7 @@ void Schema::clear() {
 }
 
 void Schema::dump(std::ostream &s) const {
-    registry_.dump(s);
+    registry_->dump(s);
     for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) {
         (*i)->dump(s);
         s << std::endl;
@@ -194,7 +195,7 @@ void Schema::check() {
     for (std::vector::iterator i = rules_.begin(); i != rules_.end(); ++i ) {
         /// @todo print offending rule in meaningful message
         ASSERT((*i)->depth() == 3);
-        (*i)->registry_.updateParent(®istry_);
+        (*i)->registry_->updateParent(registry_);
         (*i)->updateParent(0);
     }
 }
@@ -204,7 +205,7 @@ void Schema::print(std::ostream &out) const {
 }
 
 const Type &Schema::lookupType(const std::string &keyword) const {
-    return registry_.lookupType(keyword);
+    return registry_->lookupType(keyword);
 }
 
 
@@ -216,7 +217,7 @@ const std::string &Schema::path() const {
     return path_;
 }
 
-const TypesRegistry& Schema::registry() const {
+const std::shared_ptr Schema::registry() const {
     return registry_;
 }
 
diff --git a/src/fdb5/rules/Schema.h b/src/fdb5/rules/Schema.h
index 44051cf8e..b0cc9d1e4 100644
--- a/src/fdb5/rules/Schema.h
+++ b/src/fdb5/rules/Schema.h
@@ -75,7 +75,7 @@ class Schema : private eckit::NonCopyable {
 
     const std::string &path() const;
 
-    const TypesRegistry& registry() const;
+    const std::shared_ptr registry() const;
 
 
 private: // methods
@@ -89,7 +89,7 @@ class Schema : private eckit::NonCopyable {
 
 private: // members
 
-    TypesRegistry registry_;
+    std::shared_ptr registry_;
     std::vector  rules_;
     std::string path_;
 
diff --git a/src/fdb5/types/TypesRegistry.cc b/src/fdb5/types/TypesRegistry.cc
index 7f2e9ec74..a2bdd7074 100644
--- a/src/fdb5/types/TypesRegistry.cc
+++ b/src/fdb5/types/TypesRegistry.cc
@@ -19,7 +19,7 @@ namespace fdb5 {
 //----------------------------------------------------------------------------------------------------------------------
 
 TypesRegistry::TypesRegistry():
-    parent_(0) {
+    parent_(nullptr) {
 }
 
 TypesRegistry::~TypesRegistry() {
@@ -28,7 +28,7 @@ TypesRegistry::~TypesRegistry() {
     }
 }
 
-void TypesRegistry::updateParent(const TypesRegistry *parent) {
+void TypesRegistry::updateParent(std::shared_ptr parent) {
     parent_ = parent;
 }
 
@@ -56,7 +56,7 @@ const Type &TypesRegistry::lookupType(const std::string &keyword) const {
         }
 
         // eckit::Log::info() << "Type of " << keyword << " is " << type << std::endl;
-        Type *newKH = TypesFactory::build(type, keyword);
+        Type* newKH = TypesFactory::build(type, keyword);
         cache_[keyword] = newKH;
         return *newKH;
     }
diff --git a/src/fdb5/types/TypesRegistry.h b/src/fdb5/types/TypesRegistry.h
index 9fc082318..6cd143858 100644
--- a/src/fdb5/types/TypesRegistry.h
+++ b/src/fdb5/types/TypesRegistry.h
@@ -38,7 +38,7 @@ class TypesRegistry : private eckit::NonCopyable {
     const Type &lookupType(const std::string &keyword) const;
 
     void addType(const std::string &, const std::string &);
-    void updateParent(const TypesRegistry *);
+    void updateParent(std::shared_ptr parent);
     void dump( std::ostream &out ) const;
     void dump( std::ostream &out, const std::string &keyword ) const;
 
@@ -50,7 +50,7 @@ class TypesRegistry : private eckit::NonCopyable {
     mutable TypeMap cache_;
 
     std::map types_;
-    const TypesRegistry *parent_;
+    std::shared_ptr parent_;
 
     friend std::ostream &operator<<(std::ostream &s, const TypesRegistry &x);
 

From 781e9e0f88ac535d18f060204cb8b577b1abd5bd Mon Sep 17 00:00:00 2001
From: Simon Smart 
Date: Fri, 27 Jan 2023 09:54:13 +0000
Subject: [PATCH 09/23] FDB-303: Remove Rule from the Key, and replace with
 TypesRegistry information

---
 src/fdb5/api/helpers/ListIterator.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/fdb5/api/helpers/ListIterator.cc b/src/fdb5/api/helpers/ListIterator.cc
index 15ca1bb2f..bf4c5c4ff 100644
--- a/src/fdb5/api/helpers/ListIterator.cc
+++ b/src/fdb5/api/helpers/ListIterator.cc
@@ -38,6 +38,11 @@ Key ListElement::combinedKey() const {
             combined.set(kv.first, kv.second);
         }
     }
+<<<<<<< HEAD
+=======
+    combined.registry(keyParts_[2].registry());
+
+>>>>>>> 8a0c3a00 (FDB-303: Remove Rule from the Key, and replace with TypesRegistry information)
     return combined;
 }
 

From ed0a9747a730ac004d261c5fd1de9bc2398a4e52 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 22 Mar 2024 14:12:00 +0000
Subject: [PATCH 10/23] fix merge

---
 src/fdb5/api/helpers/ListIterator.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/fdb5/api/helpers/ListIterator.cc b/src/fdb5/api/helpers/ListIterator.cc
index bf4c5c4ff..15ca1bb2f 100644
--- a/src/fdb5/api/helpers/ListIterator.cc
+++ b/src/fdb5/api/helpers/ListIterator.cc
@@ -38,11 +38,6 @@ Key ListElement::combinedKey() const {
             combined.set(kv.first, kv.second);
         }
     }
-<<<<<<< HEAD
-=======
-    combined.registry(keyParts_[2].registry());
-
->>>>>>> 8a0c3a00 (FDB-303: Remove Rule from the Key, and replace with TypesRegistry information)
     return combined;
 }
 

From 873736964b39bab0ac30df8466275a8360a82715 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 22 Mar 2024 14:25:06 +0000
Subject: [PATCH 11/23] fix merge

---
 src/fdb5/api/FDB.cc | 34 +++++++++++++---------------------
 1 file changed, 13 insertions(+), 21 deletions(-)

diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc
index 54f1dd3c2..ec5059960 100644
--- a/src/fdb5/api/FDB.cc
+++ b/src/fdb5/api/FDB.cc
@@ -29,6 +29,7 @@
 #include "fdb5/database/Key.h"
 #include "fdb5/io/HandleGatherer.h"
 #include "fdb5/message/MessageDecoder.h"
+#include "fdb5/types/Type.h"
 
 namespace fdb5 {
 
@@ -100,33 +101,24 @@ void FDB::archive(const Key& key, const void* data, size_t length) {
     eckit::Timer timer;
     timer.start();
 
-<<<<<<< HEAD
-    auto stepunit = key.find("stepunits");
-    if (stepunit != key.end()) {
-        Key k;
-        for (auto it : key) {
-            if (it.first == "step" && stepunit->second.size()>0 && stepunit->second[0]!='h') {
-                // TODO - enable canonical representation of step (as soon as Metkit supports it)
-                std::string canonicalStep = it.second+stepunit->second; // k.registry().lookupType("step").toKey("step", it.second+stepunit->second);
-                k.set(it.first, canonicalStep);
-            } else {
-                if (it.first != "stepunits") {
-                    k.set(it.first, it.second);
-                }
-            }
-        }
-        internal_->archive(k, data, length);
-    } else {
-        internal_->archive(key, data, length);
-    }
-=======
     // This is the API entrypoint. Keys supplied by the user may not have type registry info attached (so
     // serialisation won't work properly...)
     Key keyInternal(key);
     keyInternal.registry(config().schema().registry());
 
+    // step in archival requests from the model is just an integer. We need to include the stepunit
+    auto stepunit = keyInternal.find("stepunits");
+    if (stepunit != keyInternal.end()) {
+        if (stepunit->second.size()>0 && stepunit->second[0]!='h') {
+            auto step = keyInternal.find("step");
+            if (step != keyInternal.end()) {
+                std::string canonicalStep = keyInternal.registry().lookupType("step").toKey("step", step->second+stepunit->second);
+            }
+        }
+        keyInternal.unset("stepunits");
+    }
+
     internal_->archive(keyInternal, data, length);
->>>>>>> c062d99a (FDB-303: Remove Rule from the Key, and replace with TypesRegistry information)
     dirty_ = true;
 
     timer.stop();

From 5de3b25e4769ab18ff3930eb5218e0ffa5554090 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 22 Mar 2024 16:52:34 +0000
Subject: [PATCH 12/23] added missing include

---
 src/fdb5/database/Key.h | 1 +
 src/fdb5/rules/Rule.h   | 1 +
 src/fdb5/rules/Schema.h | 1 +
 3 files changed, 3 insertions(+)

diff --git a/src/fdb5/database/Key.h b/src/fdb5/database/Key.h
index 54c94f88b..76672e3f4 100644
--- a/src/fdb5/database/Key.h
+++ b/src/fdb5/database/Key.h
@@ -20,6 +20,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "eckit/types/Types.h"
 
diff --git a/src/fdb5/rules/Rule.h b/src/fdb5/rules/Rule.h
index beef3e5ea..ba6867905 100644
--- a/src/fdb5/rules/Rule.h
+++ b/src/fdb5/rules/Rule.h
@@ -18,6 +18,7 @@
 
 #include 
 #include 
+#include 
 
 #include "eckit/memory/NonCopyable.h"
 #include "eckit/types/Types.h"
diff --git a/src/fdb5/rules/Schema.h b/src/fdb5/rules/Schema.h
index b0cc9d1e4..5b5740716 100644
--- a/src/fdb5/rules/Schema.h
+++ b/src/fdb5/rules/Schema.h
@@ -18,6 +18,7 @@
 
 #include 
 #include 
+#include 
 
 #include "eckit/exception/Exceptions.h"
 #include "eckit/filesystem/PathName.h"

From 10dac709fea8af37ce6501482af3d61d99cf7d79 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 22 Mar 2024 20:10:48 +0000
Subject: [PATCH 13/23] missing header

---
 src/fdb5/types/TypesRegistry.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/fdb5/types/TypesRegistry.h b/src/fdb5/types/TypesRegistry.h
index 6cd143858..949aa27c8 100644
--- a/src/fdb5/types/TypesRegistry.h
+++ b/src/fdb5/types/TypesRegistry.h
@@ -18,6 +18,7 @@
 
 #include 
 #include 
+#include 
 
 #include "eckit/memory/NonCopyable.h"
 

From ab19780a5e416bd399b8fec8f12161f5a74c69bd Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 3 Apr 2024 06:59:03 +0200
Subject: [PATCH 14/23] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 7a979be0b..dd0ad7ae6 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.11.30
+5.12.0

From 861fdac7c2c9d7d9039f2f116321f67ae370edd7 Mon Sep 17 00:00:00 2001
From: Chris Bradley 
Date: Thu, 4 Apr 2024 17:53:04 +0100
Subject: [PATCH 15/23] Add method for copying the axis map

---
 src/fdb5/database/IndexAxis.cc | 11 +++++++++++
 src/fdb5/database/IndexAxis.h  |  2 ++
 2 files changed, 13 insertions(+)

diff --git a/src/fdb5/database/IndexAxis.cc b/src/fdb5/database/IndexAxis.cc
index aec44d4e9..a5c436f30 100755
--- a/src/fdb5/database/IndexAxis.cc
+++ b/src/fdb5/database/IndexAxis.cc
@@ -321,6 +321,17 @@ const eckit::DenseSet &IndexAxis::values(const std::string &keyword
     return *(i->second);
 }
 
+std::map> IndexAxis::map() const {
+
+    // Make a copy of the axis map
+    std::map> result;
+
+    for (const auto& kv : axis_) {
+        result[kv.first] = eckit::DenseSet(*kv.second);
+    }
+    return result;
+}
+
 void IndexAxis::print(std::ostream &out) const {
     out << "IndexAxis["
         <<  "axis=";
diff --git a/src/fdb5/database/IndexAxis.h b/src/fdb5/database/IndexAxis.h
index e581dbe3b..e4fc4aaac 100644
--- a/src/fdb5/database/IndexAxis.h
+++ b/src/fdb5/database/IndexAxis.h
@@ -66,6 +66,8 @@ class IndexAxis : private eckit::NonCopyable {
     bool has(const std::string &keyword) const;
     const eckit::DenseSet &values(const std::string &keyword) const;
 
+    std::map> map() const;
+
     void dump(std::ostream &out, const char* indent) const;
 
     bool partialMatch(const metkit::mars::MarsRequest& request) const;

From 1ec03f5ab5b4b5e8cc55e72fcf2a5e0185cde46b Mon Sep 17 00:00:00 2001
From: Chris Bradley 
Date: Thu, 4 Apr 2024 19:15:05 +0100
Subject: [PATCH 16/23] Add unit test

---
 src/fdb5/database/IndexAxis.cc       |  2 +-
 tests/fdb/database/test_indexaxis.cc | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/src/fdb5/database/IndexAxis.cc b/src/fdb5/database/IndexAxis.cc
index a5c436f30..32dee0583 100755
--- a/src/fdb5/database/IndexAxis.cc
+++ b/src/fdb5/database/IndexAxis.cc
@@ -327,7 +327,7 @@ std::map> IndexAxis::map() const {
     std::map> result;
 
     for (const auto& kv : axis_) {
-        result[kv.first] = eckit::DenseSet(*kv.second);
+        result.emplace(kv.first, *kv.second);
     }
     return result;
 }
diff --git a/tests/fdb/database/test_indexaxis.cc b/tests/fdb/database/test_indexaxis.cc
index 190d50d2a..a8e27767d 100644
--- a/tests/fdb/database/test_indexaxis.cc
+++ b/tests/fdb/database/test_indexaxis.cc
@@ -145,6 +145,33 @@ CASE("Check that merging works correctly") {
     EXPECT(iatest != ia2);
 }
 
+CASE("Copy internal map") {
+
+    fdb5::IndexAxis ia;
+    ia.insert(EXAMPLE_K1);
+    ia.insert(EXAMPLE_K2);
+    ia.insert(EXAMPLE_K3);
+    ia.sort();
+
+    std::map> map = ia.map();
+
+    EXPECT(map.size() == 4);
+    for (const auto& [k, v] : map) {
+        EXPECT(v == ia.values(k));
+    }
+
+    // Make sure it's not a shallow copy
+    map["class"].insert("new1");
+    map["class"].sort();
+    EXPECT(map["class"].contains("new1"));
+    EXPECT(!ia.values("class").contains("new1"));
+
+    ia.insert(fdb5::Key{{{"class", "new2"}}});
+    ia.sort();
+    EXPECT(ia.values("class").contains("new2"));
+    EXPECT(!map["class"].contains("new2"));
+}
+
 //----------------------------------------------------------------------------------------------------------------------
 
 } // anonymous namespace

From ead8f24d75d221c6706407930241f33ae5c1c9a7 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 5 Apr 2024 08:19:47 +0200
Subject: [PATCH 17/23] fix merge (axis test)

---
 src/fdb5/database/Key.cc             | 13 +++++++++----
 tests/fdb/database/test_indexaxis.cc |  6 +++---
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/fdb5/database/Key.cc b/src/fdb5/database/Key.cc
index fcf2ed7d6..93828217a 100644
--- a/src/fdb5/database/Key.cc
+++ b/src/fdb5/database/Key.cc
@@ -32,7 +32,9 @@ Key::Key(const std::shared_ptr reg) :
 
 Key::Key(const std::string &s, const Rule *rule) :
     keys_(),
-    registry_(rule ? rule->registry() : nullptr), canonical_(false) {
+    registry_(rule ? rule->registry() : nullptr),
+    canonical_(false) {
+
     eckit::Tokenizer parse(":", true);
     eckit::StringList values;
     parse(s, values);
@@ -43,7 +45,8 @@ Key::Key(const std::string &s, const Rule *rule) :
 
 Key::Key(const eckit::StringDict &keys, const std::shared_ptr reg) :
     keys_(keys),
-    registry_(reg), canonical_(false) {
+    registry_(reg),
+    canonical_(false) {
 
     eckit::StringDict::const_iterator it = keys.begin();
     eckit::StringDict::const_iterator end = keys.end();
@@ -53,13 +56,15 @@ Key::Key(const eckit::StringDict &keys, const std::shared_ptr reg
 }
 
 Key::Key(eckit::Stream& s, const std::shared_ptr reg) :
-    registry_(reg), canonical_(reg==nullptr) {
+    registry_(reg),
+    canonical_(reg == nullptr) {
     decode(s);
 }
 
 Key::Key(std::initializer_list> l, const std::shared_ptr reg) :
     keys_(l),
-    registry_(reg), canonical_(false) {
+    registry_(reg),
+    canonical_(reg == nullptr) {
 
     for (const auto& kv : keys_) {
         names_.emplace_back(kv.first);
diff --git a/tests/fdb/database/test_indexaxis.cc b/tests/fdb/database/test_indexaxis.cc
index a8e27767d..de056df6d 100644
--- a/tests/fdb/database/test_indexaxis.cc
+++ b/tests/fdb/database/test_indexaxis.cc
@@ -18,8 +18,8 @@ fdb5::Key EXAMPLE_K1{{
 
 fdb5::Key EXAMPLE_K2{{
     {"class", "rd"},
-     {"expver", "0001"},
-     {"time", "1200"}
+    {"expver", "0001"},
+    {"time", "1200"}
 }};
 
 fdb5::Key EXAMPLE_K3{{
@@ -50,9 +50,9 @@ CASE("Insertion and comparison") {
     EXPECT(ia1 != ia2);
 
     ia2.insert(EXAMPLE_K1);
-
     EXPECT(!(ia1 == ia2));
     EXPECT(ia1 != ia2);
+    
     ia1.sort();
     ia2.sort();
     EXPECT(ia1 == ia2);

From 7dd352f0d19ac6eb11bf602bf4cbae44dba06357 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 5 Apr 2024 12:41:48 +0200
Subject: [PATCH 18/23] fix case sensitive filename

---
 src/fdb5/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 525cf8837..147439afd 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -32,7 +32,7 @@ list( APPEND fdb5_srcs
     api/SelectFDB.h
 
     api/helpers/APIIterator.h
-    api/helpers/Axesiterator.cc
+    api/helpers/AxesIterator.cc
     api/helpers/AxesIterator.h
     api/helpers/ControlIterator.cc
     api/helpers/ControlIterator.h

From 905c036e04032430769fe2992eeac3332214367b Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 5 Apr 2024 12:41:48 +0200
Subject: [PATCH 19/23] fix case sensitive filename

---
 src/fdb5/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
index 525cf8837..147439afd 100644
--- a/src/fdb5/CMakeLists.txt
+++ b/src/fdb5/CMakeLists.txt
@@ -32,7 +32,7 @@ list( APPEND fdb5_srcs
     api/SelectFDB.h
 
     api/helpers/APIIterator.h
-    api/helpers/Axesiterator.cc
+    api/helpers/AxesIterator.cc
     api/helpers/AxesIterator.h
     api/helpers/ControlIterator.cc
     api/helpers/ControlIterator.h

From 81da2f02006fd24b66188d4513ebc15a56c73afd Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 10 Apr 2024 14:20:15 +0200
Subject: [PATCH 20/23] fix test to adapt to metkit 1.11.10

---
 tests/fdb/type/test_toKey.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc
index 1cb91f70e..4bdc797ab 100644
--- a/tests/fdb/type/test_toKey.cc
+++ b/tests/fdb/type/test_toKey.cc
@@ -155,11 +155,11 @@ CASE( "Step & ClimateDaily - expansion" ) {
 
     key.set("step", "30m-60m");
     // std::cout << key.get("step") << " " << key.canonicalValue("step") << std::endl;
-    EXPECT(key.canonicalValue("step") == "30m-60m");
+    EXPECT(key.canonicalValue("step") == "30m-1");
 
     key.set("step", "30m-1");
     // std::cout << key.get("step") << " " << key.canonicalValue("step") << std::endl;
-    EXPECT(key.canonicalValue("step") == "30m-60m");
+    EXPECT(key.canonicalValue("step") == "30m-1");
 
     key.set("step", "60m-120m");
     // std::cout << key.get("step") << " " << key.canonicalValue("step") << std::endl;

From 8243342389eff4d8ad0116e5a6ecc90024bb99d3 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Wed, 10 Apr 2024 19:59:55 +0200
Subject: [PATCH 21/23] updated step-range canonical unit

---
 src/fdb5/api/fdb_c.cc        | 38 +++++++++++++++-
 src/fdb5/api/fdb_c.h         | 15 ++++++
 tests/fdb/api/test_fdb_c.cc  | 88 ++++++++++++++++++++++++++++++++++++
 tests/fdb/type/test_toKey.cc |  6 ---
 4 files changed, 140 insertions(+), 7 deletions(-)

diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc
index 9c2fa3363..79aa7fc84 100644
--- a/src/fdb5/api/fdb_c.cc
+++ b/src/fdb5/api/fdb_c.cc
@@ -14,6 +14,8 @@
 #include "eckit/runtime/Main.h"
 
 #include "metkit/mars/MarsRequest.h"
+#include "metkit/mars/MarsExpension.h"
+#include "eckit/utils/Tokenizer.h"
 
 #include "fdb5/fdb5_version.h"
 #include "fdb5/api/FDB.h"
@@ -45,11 +47,27 @@ struct fdb_request_t {
     fdb_request_t(std::string str) {
         request_ = metkit::mars::MarsRequest(str);
     }
+    size_t values(const char* name, char** values[]) {
+        std::string n(name);
+        std::vector vv = request_.values(name);
+
+        *values = new char*[vv.size()];
+        for (size_t i = 0; i < vv.size(); i++) {
+            (*values)[i] = new char[vv[i].size()+1];
+            strncpy((*values)[i], vv[i].c_str(), vv[i].size());
+            (*values)[i][vv[i].size()] = '\0';
+        }
+        return vv.size();
+    }
     void values(const char* name, const char* values[], int numValues) {
         std::string n(name);
         std::vector vv;
+        Tokenizer parse("/");
+
         for (int i=0; i result;
+        	parse(values[i], result);
+            vv.insert(std::end(vv), std::begin(result), std::end(result));
         }
         request_.values(n, vv);
     }
@@ -58,6 +76,12 @@ struct fdb_request_t {
         req->request_ = metkit::mars::MarsRequest::parse(str);
         return req;
     }
+    void expand() {
+        bool inherit = false;
+        bool strict = true;
+        metkit::mars::MarsExpension expand(inherit, strict);
+        request_ = expand.expand(request_);
+    }
     const metkit::mars::MarsRequest request() const { return request_; }
 private:
     metkit::mars::MarsRequest request_;
@@ -403,6 +427,18 @@ int fdb_request_add(fdb_request_t* req, const char* param, const char* values[],
         req->values(param, values, numValues);
     });
 }
+int fdb_request_get(fdb_request_t* req, const char* param, char** values[], size_t* numValues) {
+    return wrapApiFunction([req, param, values, numValues] {
+        ASSERT(req);
+        ASSERT(param);
+        *numValues = req->values(param, values);
+    });
+}
+int fdb_expand_request(fdb_request_t* req) {
+    return wrapApiFunction([req]{
+        req->expand();
+    });
+}
 int fdb_delete_request(fdb_request_t* req) {
     return wrapApiFunction([req]{
         ASSERT(req);
diff --git a/src/fdb5/api/fdb_c.h b/src/fdb5/api/fdb_c.h
index dc725a757..e3371c61a 100644
--- a/src/fdb5/api/fdb_c.h
+++ b/src/fdb5/api/fdb_c.h
@@ -139,6 +139,21 @@ int fdb_new_request(fdb_request_t** req);
  */
 int fdb_request_add(fdb_request_t* req, const char* param, const char* values[], int numValues);
 
+/** Get the Metadata values associated to a Request metadata
+ * \param req Request instance
+ * \param param Metadata name
+ * \param values Metadata values
+ * \param numValues number of metadata values
+ * \returns Return code (#FdbErrorValues)
+ */
+int fdb_request_get(fdb_request_t* req, const char* param, char** values[], size_t* numValues);
+
+/** Expand a Request
+ * \param req Request instance
+ * \returns Return code (#FdbErrorValues)
+ */
+int fdb_expand_request(fdb_request_t* req);
+
 /** Deallocates Request object and associated resources.
  * \param req Request instance
  * \returns Return code (#FdbErrorValues)
diff --git a/tests/fdb/api/test_fdb_c.cc b/tests/fdb/api/test_fdb_c.cc
index cda453e04..57c1f486c 100644
--- a/tests/fdb/api/test_fdb_c.cc
+++ b/tests/fdb/api/test_fdb_c.cc
@@ -449,6 +449,94 @@ CASE( "fdb_c - retrieve" ) {
 }
 
 
+CASE( "fdb_c - expand" ) {
+
+    fdb_handle_t* fdb;
+    fdb_new_handle(&fdb);
+    fdb_request_t* request;
+    fdb_new_request(&request);
+    fdb_request_add1(request, "domain", "g");
+    fdb_request_add1(request, "stream", "oper");
+    fdb_request_add1(request, "levtype", "pl");
+    fdb_request_add1(request, "levelist", "300");
+    const char* dates[] = {"20191110", "to", "20191111"};
+    fdb_request_add(request, "date", dates, 3);
+    fdb_request_add1(request, "time", "0000");
+    fdb_request_add1(request, "step", "0");
+    fdb_request_add1(request, "param", "138");
+    fdb_request_add1(request, "class", "rd");
+    fdb_request_add1(request, "type", "an");
+    fdb_request_add1(request, "expver", "xxxx");
+
+    char buf[1000];
+    char grib[4];
+    long read = 0;
+    long size;
+    fdb_datareader_t* dr;
+    fdb_new_datareader(&dr);
+    EXPECT_EQUAL(fdb_retrieve(fdb, request, dr), FDB_ERROR_GENERAL_EXCEPTION);
+
+    EXPECT_EQUAL(fdb_expand_request(request), FDB_SUCCESS);
+
+    size_t numValues;
+    char** values;
+    
+    fdb_request_get(request, "date", &values, &numValues);
+    EXPECT_EQUAL(numValues, 2);
+    EXPECT_EQUAL(0, strncmp(values[0], "20191110", 8));
+    EXPECT_EQUAL(0, strncmp(values[1], "20191111", 8));
+    delete values[0];
+    delete values[1];
+    delete values;
+
+    EXPECT(fdb_retrieve(fdb, request, dr) == FDB_SUCCESS);
+    fdb_datareader_open(dr, &size);
+    EXPECT_NOT_EQUAL(0, size);
+    fdb_datareader_read(dr, grib, 4, &read);
+    EXPECT_EQUAL(4, read);
+    EXPECT_EQUAL(0, strncmp(grib, "GRIB", 4));
+    fdb_datareader_tell(dr, &read);
+    EXPECT_EQUAL(4, read);
+    fdb_datareader_seek(dr, 3);
+    fdb_datareader_tell(dr, &read);
+    EXPECT_EQUAL(3, read);
+    fdb_datareader_skip(dr, 3);
+    fdb_datareader_tell(dr, &read);
+    EXPECT_EQUAL(6, read);
+    fdb_datareader_read(dr, buf, 1000, &read);
+    EXPECT_EQUAL(1000, read);
+    fdb_datareader_tell(dr, &read);
+    EXPECT_EQUAL(1006, read);
+    fdb_delete_datareader(dr);
+
+    fdb_request_add1(request, "date", "20191110/to/20191115/by/2");
+
+    fdb_request_get(request, "date", &values, &numValues);
+    EXPECT_EQUAL(numValues, 5);
+    EXPECT_EQUAL(0, strncmp(values[0], "20191110", 8));
+    EXPECT_EQUAL(0, strncmp(values[1], "to", 2));
+    EXPECT_EQUAL(0, strncmp(values[2], "20191115", 8));
+    EXPECT_EQUAL(0, strncmp(values[3], "by", 2));
+    EXPECT_EQUAL(0, strncmp(values[4], "2", 1));
+    for (size_t i = 0; iregistry());
 
-//    std::cout << key.valuesToString() << std::endl;
     EXPECT(key.canonicalValue("date") == t(now.yyyymmdd()));
     EXPECT(key.valuesToString() == "od:0001:oper:ofb:"+t(now.yyyymmdd())+":0000:MHS:3001");
 

From b32c391b988bf64a5897653c3bcf251b3df8edc2 Mon Sep 17 00:00:00 2001
From: Emanuele Danovaro 
Date: Fri, 12 Apr 2024 23:37:35 +0200
Subject: [PATCH 22/23] version bump

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index dd0ad7ae6..61ef74965 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-5.12.0
+5.12.1

From 94e229cfa82d15986ff2381f61b88c06d557be72 Mon Sep 17 00:00:00 2001
From: Christopher Bradley 
Date: Thu, 2 May 2024 13:25:41 +0100
Subject: [PATCH 23/23] Add GitHub Action to sync to Bitbucket

---
 .github/workflows/sync.yml | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 .github/workflows/sync.yml

diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
new file mode 100644
index 000000000..50b8478ee
--- /dev/null
+++ b/.github/workflows/sync.yml
@@ -0,0 +1,26 @@
+name: sync
+ 
+# Controls when the workflow will run
+on:
+ 
+  # Trigger the workflow on all pushes
+  push:
+    branches:
+    - '**'
+    tags:
+    - '**'
+ 
+  # Trigger the workflow when a branch or tag is deleted
+  delete: ~
+ 
+jobs:
+ 
+  # Calls a reusable CI workflow to sync the current with a remote repository.
+  #   It will correctly handle addition of any new and removal of existing Git objects.
+  sync:
+    name: sync
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/sync.yml@v2
+    secrets:
+      target_repository: mars/fdb5
+      target_username: ClonedDuck
+      target_token: ${{ secrets.BITBUCKET_PAT }}