Merge branch 'develop' of https://github.com/ecmwf/fdb into develop

.github/workflows/sync.yml

@@ -0,0 +1,26 @@
+name: sync
+
+# Controls when the workflow will run
+on:
+
+  # Trigger the workflow on all pushes
+  push:
+    branches:
+    - '**'
+    tags:
+    - '**'
+
+  # Trigger the workflow when a branch or tag is deleted
+  delete: ~
+
+jobs:
+
+  # Calls a reusable CI workflow to sync the current with a remote repository.
+  #   It will correctly handle addition of any new and removal of existing Git objects.
+  sync:
+    name: sync
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/sync.yml@v2
+    secrets:
+      target_repository: mars/fdb5
+      target_username: ClonedDuck
+      target_token: ${{ secrets.BITBUCKET_PAT }}

VERSION

@@ -1 +1 @@
-5.11.30
+5.12.1

src/fdb5/CMakeLists.txt

@@ -32,6 +32,8 @@ list( APPEND fdb5_srcs
     api/SelectFDB.h
 
     api/helpers/APIIterator.h
+    api/helpers/AxesIterator.cc
+    api/helpers/AxesIterator.h
     api/helpers/ControlIterator.cc
     api/helpers/ControlIterator.h
     api/helpers/FDBToolRequest.cc
@@ -50,6 +52,8 @@ list( APPEND fdb5_srcs
     api/local/QueryVisitor.h
     api/local/QueueStringLogTarget.h
     api/local/ListVisitor.h
+    api/local/AxesVisitor.cc
+    api/local/AxesVisitor.h
     api/local/ControlVisitor.cc
     api/local/ControlVisitor.h
     api/local/DumpVisitor.h
@@ -408,6 +412,7 @@ if(HAVE_FDB_BUILD_TOOLS)
 endif()
 
 list( APPEND fdb5_tools
+    fdb-axes
     fdb-write
     fdb-copy
     fdb-dump

src/fdb5/api/FDB.cc

@@ -295,6 +295,16 @@ void FDB::flush() {
     }
 }
 
+IndexAxis FDB::axes(const FDBToolRequest& request, int level) {
+    IndexAxis axes;
+    AxesElement elem;
+    auto it = internal_->axes(request, level);
+    while (it.next(elem)) {
+        axes.merge(elem.axes());
+    }
+    return axes;
+}
+
 bool FDB::dirty() const {
     return dirty_;
 }

src/fdb5/api/FDB.h

@@ -109,6 +109,9 @@ class FDB {
     ControlIterator control(const FDBToolRequest& request,
                             ControlAction action,
                             ControlIdentifiers identifiers);
+
+    IndexAxis axes(const FDBToolRequest& request, int level=3);
+
     bool enabled(const ControlIdentifier& controlIdentifier) const;
 
     bool dirty() const;

src/fdb5/api/FDBFactory.h

@@ -28,7 +28,7 @@
 #include "fdb5/database/DB.h"
 #include "fdb5/config/Config.h"
 #include "fdb5/api/FDBStats.h"
-#include "fdb5/api/helpers/ListIterator.h"
+#include "fdb5/api/helpers/AxesIterator.h"
 #include "fdb5/api/helpers/ListIterator.h"
 #include "fdb5/api/helpers/ControlIterator.h"
 #include "fdb5/api/helpers/DumpIterator.h"
@@ -88,6 +88,8 @@ class FDBBase : private eckit::NonCopyable {
 
     virtual MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) = 0;
 
+    virtual AxesIterator axes(const FDBToolRequest& request, int axes) { NOTIMP; }
+
     // -------------- API management ----------------------------
 
     /// ID used for hashing in the Rendezvous hash. Should be unique amongst those used

src/fdb5/api/LocalFDB.cc

@@ -29,6 +29,7 @@
 #include "fdb5/rules/Schema.h"
 #include "fdb5/LibFdb5.h"
 
+#include "fdb5/api/local/AxesVisitor.h"
 #include "fdb5/api/local/ControlVisitor.h"
 #include "fdb5/api/local/DumpVisitor.h"
 #include "fdb5/api/local/ListVisitor.h"
@@ -123,6 +124,11 @@ ControlIterator LocalFDB::control(const FDBToolRequest& request,
     return queryInternal<ControlVisitor>(request, action, identifiers);
 }
 
+AxesIterator LocalFDB::axes(const FDBToolRequest& request, int level) {
+    LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::axes() : " << request << std::endl;
+    return queryInternal<AxesVisitor>(request, config_, level);
+}
+
 void LocalFDB::flush() {
     if (archiver_) {
         archiver_->flush();

src/fdb5/api/LocalFDB.h

@@ -59,6 +59,8 @@ class LocalFDB : public FDBBase {
 
     MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override;
 
+    AxesIterator axes(const FDBToolRequest& request, int axes) override;
+
     void flush() override;
 
 private: // methods

src/fdb5/api/fdb_c.cc

@@ -14,6 +14,8 @@
 #include "eckit/runtime/Main.h"
 
 #include "metkit/mars/MarsRequest.h"
+#include "metkit/mars/MarsExpension.h"
+#include "eckit/utils/Tokenizer.h"
 
 #include "fdb5/fdb5_version.h"
 #include "fdb5/api/FDB.h"
@@ -45,11 +47,27 @@ struct fdb_request_t {
     fdb_request_t(std::string str) {
         request_ = metkit::mars::MarsRequest(str);
     }
+    size_t values(const char* name, char** values[]) {
+        std::string n(name);
+        std::vector<std::string> vv = request_.values(name);
+
+        *values = new char*[vv.size()];
+        for (size_t i = 0; i < vv.size(); i++) {
+            (*values)[i] = new char[vv[i].size()+1];
+            strncpy((*values)[i], vv[i].c_str(), vv[i].size());
+            (*values)[i][vv[i].size()] = '\0';
+        }
+        return vv.size();
+    }
     void values(const char* name, const char* values[], int numValues) {
         std::string n(name);
         std::vector<std::string> vv;
+        Tokenizer parse("/");
+
         for (int i=0; i<numValues; i++) {
-            vv.push_back(std::string(values[i]));
+        	std::vector<std::string> result;
+        	parse(values[i], result);
+            vv.insert(std::end(vv), std::begin(result), std::end(result));
         }
         request_.values(n, vv);
     }
@@ -58,6 +76,12 @@ struct fdb_request_t {
         req->request_ = metkit::mars::MarsRequest::parse(str);
         return req;
     }
+    void expand() {
+        bool inherit = false;
+        bool strict = true;
+        metkit::mars::MarsExpension expand(inherit, strict);
+        request_ = expand.expand(request_);
+    }
     const metkit::mars::MarsRequest request() const { return request_; }
 private:
     metkit::mars::MarsRequest request_;
@@ -403,6 +427,18 @@ int fdb_request_add(fdb_request_t* req, const char* param, const char* values[],
         req->values(param, values, numValues);
     });
 }
+int fdb_request_get(fdb_request_t* req, const char* param, char** values[], size_t* numValues) {
+    return wrapApiFunction([req, param, values, numValues] {
+        ASSERT(req);
+        ASSERT(param);
+        *numValues = req->values(param, values);
+    });
+}
+int fdb_expand_request(fdb_request_t* req) {
+    return wrapApiFunction([req]{
+        req->expand();
+    });
+}
 int fdb_delete_request(fdb_request_t* req) {
     return wrapApiFunction([req]{
         ASSERT(req);

src/fdb5/api/fdb_c.h

@@ -139,6 +139,21 @@ int fdb_new_request(fdb_request_t** req);
  */
 int fdb_request_add(fdb_request_t* req, const char* param, const char* values[], int numValues);
 
+/** Get the Metadata values associated to a Request metadata
+ * \param req Request instance
+ * \param param Metadata name
+ * \param values Metadata values
+ * \param numValues number of metadata values
+ * \returns Return code (#FdbErrorValues)
+ */
+int fdb_request_get(fdb_request_t* req, const char* param, char** values[], size_t* numValues);
+
+/** Expand a Request
+ * \param req Request instance
+ * \returns Return code (#FdbErrorValues)
+ */
+int fdb_expand_request(fdb_request_t* req);
+
 /** Deallocates Request object and associated resources.
  * \param req Request instance
  * \returns Return code (#FdbErrorValues)

src/fdb5/api/helpers/AxesIterator.cc

@@ -0,0 +1,41 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the EC H2020 funded project NextGenIO
+ * (Project ID: 671951) www.nextgenio.eu
+ */
+
+#include "fdb5/api/helpers/AxesIterator.h"
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+AxesElement::AxesElement(Key&& dbKey, IndexAxis&& axes) :
+        dbKey_(std::move(dbKey)), axes_(std::move(axes)) {}
+
+AxesElement::AxesElement(eckit::Stream& s) {
+    s >> dbKey_;
+    axes_ = IndexAxis(s, IndexAxis::currentVersion());
+}
+
+void AxesElement::print(std::ostream& out) const {
+    out << "Axes(db=" << dbKey_ << ", axes=" << axes_ << ")";
+}
+
+void AxesElement::encode(eckit::Stream& s) const {
+    s << dbKey_;
+    axes_.encode(s, IndexAxis::currentVersion());
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+} // namespace fdb5

src/fdb5/api/helpers/AxesIterator.h

@@ -0,0 +1,74 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/// @author Simon Smart
+/// @date   January 2024
+
+#pragma once
+
+#include "fdb5/database/Key.h"
+#include "fdb5/database/IndexAxis.h"
+#include "fdb5/api/helpers/APIIterator.h"
+
+namespace eckit {
+class Stream;
+class JSON;
+}
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class AxesElement {
+public: // methods
+
+    AxesElement() = default;
+    AxesElement(Key&& dbKey, IndexAxis&& axis);
+    explicit AxesElement(eckit::Stream& s);
+
+    [[ nodiscard ]]
+    const Key& key() const { return dbKey_; }
+
+    [[ nodiscard ]]
+    const IndexAxis& axes() const { return axes_; }
+
+    void print(std::ostream& out) const;
+
+private: // methods
+
+    void encode(eckit::Stream& s) const;
+
+    friend std::ostream& operator<<(std::ostream& os, const AxesElement& e) {
+        e.print(os);
+        return os;
+    }
+
+    friend eckit::Stream& operator<<(eckit::Stream& s, const AxesElement& r) {
+        r.encode(s);
+        return s;
+    }
+
+private: // members
+
+    Key dbKey_;
+    IndexAxis axes_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+using AxesAggregateIterator = APIAggregateIterator<AxesElement>;
+
+using AxesAsyncIterator = APIAsyncIterator<AxesElement>;
+
+using AxesIterator = APIIterator<AxesElement>;
+
+//----------------------------------------------------------------------------------------------------------------------
+
+} // namespace fdb5

src/fdb5/api/helpers/ListIterator.cc

@@ -38,7 +38,6 @@ Key ListElement::combinedKey() const {
             combined.set(kv.first, kv.second);
         }
     }
-
     return combined;
 }