Merge pull request #42 from ecmwf/feature/grib-jump-changes

Feature/grib jump changes

src/metkit/codes/GribAccessor.h

@@ -47,19 +47,20 @@ class GribAccessor : private GribAccessorBase {
 private: // members
 
     std::string name_;
+    bool quiet_;
 
 public: // methods
 
-    GribAccessor(const std::string& name): name_(name) {}
+    GribAccessor(const std::string& name, bool quiet = false): name_(name), quiet_(quiet) {}
 
     T value(const GribHandle& h) const
     {
         T value;
-        grib_get_value(h, name_, value);
+        grib_get_value(h, name_, value, quiet_);
         return value;
     }
 
-    T value(const GribHandle& h,T def) const
+    T value(const GribHandle& h, T def) const
     {
         T value = def;
         grib_get_value(h, name_, value, true);

src/metkit/codes/GribHandle.cc

@@ -80,6 +80,29 @@ GribHandle::GribHandle(eckit::DataHandle& handle):
     CODES_CALL(err);
     ASSERT(h);
     handle_ = h;
+
+    fclose(f);
+}
+
+GribHandle::GribHandle(eckit::DataHandle& handle, eckit::Offset offset):
+    handle_(nullptr),
+    owned_(true) {
+
+    codes_handle* h = nullptr;
+    int err = 0;
+
+    FILE* f = handle.openf();
+    ASSERT(f);
+
+    handle.seek(offset);
+
+    h = codes_handle_new_from_file(0, f, PRODUCT_GRIB, &err);
+
+    CODES_CALL(err);
+    ASSERT(h);
+    handle_ = h;
+
+    fclose(f);
 }
 
 GribHandle::~GribHandle() noexcept(false) {

src/metkit/codes/GribHandle.h

@@ -53,6 +53,9 @@ class GribHandle : private eckit::NonCopyable {
     /// constructor creating a grib_handle from a DataHandle
     explicit GribHandle(eckit::DataHandle&);
 
+    // Constructor creating a grib_handle from a DataHandle starting from a given offset
+    explicit GribHandle(eckit::DataHandle&, eckit::Offset);
+
     /// destructor will delete the grib_handle if we own it
     ~GribHandle() noexcept(false);
 

src/metkit/mars/DHSProtocol.cc

@@ -370,7 +370,7 @@ Length DHSProtocol::retrieve(const MarsRequest& request)
 {
     Endpoint callbackEndpoint = callback_->endpoint();
 
-    Log::debug() << "DHSProtocol: call back on " << callbackEndpoint << std::endl;
+    LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol: call back on " << callbackEndpoint << std::endl;
 
     task_.reset(new ClientTask(request, RequestEnvironment::instance().request(),
                                   callbackEndpoint.host(), callbackEndpoint.port()));
@@ -385,16 +385,16 @@ Length DHSProtocol::retrieve(const MarsRequest& request)
     while (wait(result)) {
     }
 
-    Log::debug() << "DHSProtocol::retrieve " << result << std::endl;
+    LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol::retrieve " << result << std::endl;
     return result;
 }
 
 void DHSProtocol::archive(const MarsRequest& request, const Length& size)
 {
     Endpoint callbackEndpoint = callback_->endpoint();
 
-    Log::debug() << "DHSProtocol::archive " << size << std::endl;
-    Log::debug() << "DHSProtocol: call back on " << callbackEndpoint << std::endl;
+    LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol::archive " << size << std::endl;
+    LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol: call back on " << callbackEndpoint << std::endl;
 
     task_.reset(new ClientTask(request, RequestEnvironment::instance().request(),
                                   callbackEndpoint.host(), callbackEndpoint.port()));
@@ -408,7 +408,7 @@ void DHSProtocol::archive(const MarsRequest& request, const Length& size)
     Length result = size;
     while (wait(result)) {
     }
-    Log::debug() << "DHSProtocol: archive completed." << std::endl;
+    LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol: archive completed." << std::endl;
 }
 
 void DHSProtocol::cleanup()
@@ -488,7 +488,7 @@ bool DHSProtocol::wait(Length& size)
 
         char code = task_->receive(s);
 
-        Log::debug() << "DHSProtocol: code [" << code << "]" << std::endl;
+        LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol: code [" << code << "]" << std::endl;
 
         std::string msg;
         long long bytes;
@@ -502,7 +502,7 @@ bool DHSProtocol::wait(Length& size)
         /* read source */
         case 'r':
             bytes = size;
-            Log::debug() << "DHSProtocol:r [" << bytes << "]" << std::endl;
+            LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol:r [" << bytes << "]" << std::endl;
             s << bytes;
             sending_ = true;
             return false;
@@ -514,7 +514,7 @@ bool DHSProtocol::wait(Length& size)
 
         case 'w':
             s >> bytes;
-            Log::debug() << "DHSProtocol:w " << bytes << std::endl;
+            LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol:w " << bytes << std::endl;
             size = bytes;
             return false;
 
@@ -555,7 +555,7 @@ bool DHSProtocol::wait(Length& size)
 
         case 'D': /* debug */
             s >> msg;
-            Log::debug() << msg << " [" << name_ << "]" << std::endl;
+            LOG_DEBUG_LIB(LibMetkit) << msg << " [" << name_ << "]" << std::endl;
             if (forward_) {
                 Log::userInfo() << msg << " [" << name_ << "]" << std::endl;
             }

src/metkit/mars/MarsRequest.cc

@@ -333,9 +333,64 @@ MarsRequest::operator eckit::Value() const {
     NOTIMP;
 }
 
+// recursively expand along keys in expvalues
+void expand_along_keys(
+    const MarsRequest& prototype,
+    const std::vector<std::pair<std::string, std::vector<std::string>>>& expvalues,
+    std::vector<MarsRequest>& requests,
+    size_t i) {
+
+    if(i == expvalues.size()) {
+        requests.push_back(prototype);
+        return;
+    }
+
+    const std::string& key = expvalues[i].first;
+    const std::vector<std::string>& values = expvalues[i].second;
+
+    MarsRequest req(prototype);
+    for (auto& value : values) {
+        req.setValue(key, value);
+        expand_along_keys(req, expvalues, requests, i+1);
+    }
+}
+
+std::vector<MarsRequest> MarsRequest::split(const std::vector<std::string>& keys) const {
+
+    size_t n = 1;
+
+    LOG_DEBUG_LIB(LibMetkit) << "Splitting request with keys" << keys << std::endl;
+
+    std::vector<std::pair<std::string, std::vector<std::string>>> expvalues;
+    for (auto& key : keys) {
+        std::vector<std::string> v = values(key, true); // ok to be empty
+        LOG_DEBUG_LIB(LibMetkit) << "splitting along key " << key << " n values " << v.size() <<  " values " << v << std::endl;
+        if (v.empty()) continue;
+        n *= v.size();
+        expvalues.emplace_back(std::make_pair(key, v));
+    }
+
+    std::vector<MarsRequest> requests;
+    requests.reserve(n);
+
+    if(n == 1) {
+        requests.push_back(*this);
+        return requests;
+    }
+
+    expand_along_keys(*this, expvalues, requests, 0);
+
+    return requests;
+}
+
+std::vector<MarsRequest> MarsRequest::split(const std::string& key) const {
+    std::vector<std::string> keys = { key };
+    return split( keys );
+}
+
 void MarsRequest::merge(const MarsRequest& other) {
     for (auto& param : params_) {
-        eckit::Log::debug<LibMetkit>() << "Merging parameter " << param << std::endl;
+        LOG_DEBUG_LIB(LibMetkit) << "Merging parameter " << param << std::endl;
         auto it = other.find(param.name());
         if (it != other.params_.end())
             param.merge(*it);
@@ -410,6 +465,11 @@ void MarsRequest::erase(const std::string& name) {
     }
 }
 
+std::string MarsRequest::asString() const {
+    std::ostringstream oss;
+    oss << *this;
+    return oss.str();
+}
 //----------------------------------------------------------------------------------------------------------------------
 
 std::vector<MarsRequest> MarsRequest::parse(std::istream& in, bool strict) {

src/metkit/mars/MarsRequest.h

@@ -38,9 +38,12 @@ class MarsRequest;
 
 class MarsRequest {
 public:  // methods
+
     MarsRequest();
+
     explicit MarsRequest(const std::string&);
     explicit MarsRequest(eckit::Stream&, bool lowercase = false);
+
     MarsRequest(const std::string&, const std::map<std::string, std::string>&);
     MarsRequest(const std::string&, const eckit::Value&);
 
@@ -60,12 +63,10 @@ class MarsRequest {
     size_t countValues(const std::string&) const;
     bool has(const std::string&) const;
 
-
     bool is(const std::string& param, const std::string& value) const;
 
     const std::vector<std::string>& values(const std::string&, bool emptyOk = false) const;
 
-
     template <class T>
     size_t getValues(const std::string& name, std::vector<T>& v, bool emptyOk = false) const;
 
@@ -83,6 +84,12 @@ class MarsRequest {
 
     void unsetValues(const std::string&);
 
+    /// Splits a MARS request into multiple requests along the provided key
+    std::vector<MarsRequest> split(const std::string& keys) const;
+
+    /// Splits a MARS request into multiple requests along the indicated keys
+    std::vector<MarsRequest> split(const std::vector<std::string>& keys) const;
+
     /// Merges one MarsRequest into another
     /// @todo Improve performance -- uses O(N^2) search / merge in std::list's
     void merge(const MarsRequest& other);
@@ -108,7 +115,10 @@ class MarsRequest {
 
     void erase(const std::string& param);
 
+    std::string asString() const;
+
 public:  // static methods
+
     static MarsRequest parse(const std::string& s, bool strict = false);
     static std::vector<MarsRequest> parse(std::istream&, bool strict = false);
 

src/metkit/mars/MarsRequestHandle.cc

@@ -12,6 +12,8 @@
 // Baudouin Raoult - (c) ECMWF Feb 12
 
 #include "metkit/mars/MarsRequestHandle.h"
+#include "metkit/config/LibMetkit.h"
+
 #include "eckit/utils/StringTools.h"
 #include "eckit/types/Types.h"
 
@@ -53,7 +55,7 @@ MarsRequestHandle::MarsRequestHandle(const MarsRequest& request,
     protocol_(protocol),
     opened_(false)
 {
-    eckit::Log::debug() << "MarsRequestHandle::MarsRequestHandle: request: " << request << " protocol: " << protocol << std::endl;
+    LOG_DEBUG_LIB(LibMetkit) << "MarsRequestHandle::MarsRequestHandle: request: " << request << " protocol: " << protocol << std::endl;
     ASSERT(protocol);
 }
 

src/metkit/mars/ParamID.h

@@ -115,7 +115,7 @@ void ParamID::normalise(const REQUEST_T& request,
                 newreq.push_back(p);
             }
 
-            eckit::Log::debug<LibMetkit>() << "useGRIBParamID p=" << p
+            LOG_DEBUG_LIB(LibMetkit) << "useGRIBParamID p=" << p
                                << ", alt=" << alt
                                << ", choice=" << newreq.back() << std::endl;
 
@@ -174,7 +174,7 @@ void ParamID::normalise(const REQUEST_T& request,
                 if (!wantVO) req.push_back(windVO);
                 if (!wantD)  req.push_back(windD);
 
-                eckit::Log::debug<LibMetkit>() << "U/V conversion requested U=" << windU << ", V=" << windV << ", VO=" << windVO << ", D=" << windD << std::endl;
+                LOG_DEBUG_LIB(LibMetkit)<< "U/V conversion requested U=" << windU << ", V=" << windV << ", VO=" << windVO << ", D=" << windD << std::endl;
                 windConversion = true;
             }
         }

src/metkit/mars/TypeParam.cc

@@ -140,12 +140,7 @@ Rule::Rule(const eckit::Value& matchers, const eckit::Value& values, const eckit
 
         if (aliases.isNil()) {
 
-            Log::debug<LibMetkit>()
-                    << "No aliases for "
-                    << id
-                    << " "
-                    << *this
-                    << std::endl;
+            LOG_DEBUG_LIB(LibMetkit) << "No aliases for " << id << " " << *this << std::endl;
             continue;
         }
 
@@ -158,8 +153,7 @@ Rule::Rule(const eckit::Value& matchers, const eckit::Value& values, const eckit
 
                 if (precedence[v] <= j) {
 
-                    Log::debug<LibMetkit>()
-                            << "Redefinition ignored: param "
+                    LOG_DEBUG_LIB(LibMetkit) << "Redefinition ignored: param "
                             << v
                             << "='"
                             << first
@@ -172,8 +166,7 @@ Rule::Rule(const eckit::Value& matchers, const eckit::Value& values, const eckit
                 }
                 else {
 
-                    Log::debug<LibMetkit>()
-                            << "Redefinition of param "
+                    LOG_DEBUG_LIB(LibMetkit) << "Redefinition of param "
                             << v
                             << "='"
                             << first

src/metkit/odb/IdMapper.cc

@@ -84,7 +84,7 @@ IdMap::IdMap(const std::string& configFile,
              size_t alphanumericIndex) {
 
     PathName configPath = codes_path() / configFile;
-    Log::debug<LibMetkit>() << "GribCodesBase::GribCodesBase: config file:" << configPath << std::endl;
+    LOG_DEBUG_LIB(LibMetkit) << "GribCodesBase::GribCodesBase: config file:" << configPath << std::endl;
 
 	numeric2alpha_.clear();
 
@@ -99,7 +99,7 @@ IdMap::IdMap(const std::string& configFile,
             long num = eckit::Translator<std::string, long>()(StringTools::trim(words[numericIndex]));
             std::string alpha (StringTools::trim(words[alphanumericIndex]));
 			numeric2alpha_[num] = StringTools::lower(alpha);
-            Log::debug<LibMetkit>() << "GribCodesBase::readConfig: num='" << num << "' alpha='" << alpha << "'" << std::endl;
+            LOG_DEBUG_LIB(LibMetkit) << "GribCodesBase::readConfig: num='" << num << "' alpha='" << alpha << "'" << std::endl;
 		}
 	}
 }