Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/grib jump changes #42

Merged
merged 5 commits into from
Mar 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions src/metkit/codes/GribAccessor.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,19 +47,20 @@ class GribAccessor : private GribAccessorBase {
private: // members

std::string name_;
bool quiet_;

public: // methods

GribAccessor(const std::string& name): name_(name) {}
GribAccessor(const std::string& name, bool quiet = false): name_(name), quiet_(quiet) {}

T value(const GribHandle& h) const
{
T value;
grib_get_value(h, name_, value);
grib_get_value(h, name_, value, quiet_);
return value;
}

T value(const GribHandle& h,T def) const
T value(const GribHandle& h, T def) const
{
T value = def;
grib_get_value(h, name_, value, true);
Expand Down
23 changes: 23 additions & 0 deletions src/metkit/codes/GribHandle.cc
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,29 @@ GribHandle::GribHandle(eckit::DataHandle& handle):
CODES_CALL(err);
ASSERT(h);
handle_ = h;

fclose(f);
}

GribHandle::GribHandle(eckit::DataHandle& handle, eckit::Offset offset):
handle_(nullptr),
owned_(true) {

codes_handle* h = nullptr;
int err = 0;

FILE* f = handle.openf();
ASSERT(f);

handle.seek(offset);

h = codes_handle_new_from_file(0, f, PRODUCT_GRIB, &err);

CODES_CALL(err);
ASSERT(h);
handle_ = h;

fclose(f);
}

GribHandle::~GribHandle() noexcept(false) {
Expand Down
3 changes: 3 additions & 0 deletions src/metkit/codes/GribHandle.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ class GribHandle : private eckit::NonCopyable {
/// constructor creating a grib_handle from a DataHandle
explicit GribHandle(eckit::DataHandle&);

// Constructor creating a grib_handle from a DataHandle starting from a given offset
explicit GribHandle(eckit::DataHandle&, eckit::Offset);

/// destructor will delete the grib_handle if we own it
~GribHandle() noexcept(false);

Expand Down
18 changes: 9 additions & 9 deletions src/metkit/mars/DHSProtocol.cc
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,7 @@ Length DHSProtocol::retrieve(const MarsRequest& request)
{
Endpoint callbackEndpoint = callback_->endpoint();

Log::debug() << "DHSProtocol: call back on " << callbackEndpoint << std::endl;
LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol: call back on " << callbackEndpoint << std::endl;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this slightly changes the log behaviour: it now depends on the env variable METKIT_DEBUG instead of DEBUG.
It might be even better...

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it is more efficient

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

and correct


task_.reset(new ClientTask(request, RequestEnvironment::instance().request(),
callbackEndpoint.host(), callbackEndpoint.port()));
Expand All @@ -385,16 +385,16 @@ Length DHSProtocol::retrieve(const MarsRequest& request)
while (wait(result)) {
}

Log::debug() << "DHSProtocol::retrieve " << result << std::endl;
LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol::retrieve " << result << std::endl;
return result;
}

void DHSProtocol::archive(const MarsRequest& request, const Length& size)
{
Endpoint callbackEndpoint = callback_->endpoint();

Log::debug() << "DHSProtocol::archive " << size << std::endl;
Log::debug() << "DHSProtocol: call back on " << callbackEndpoint << std::endl;
LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol::archive " << size << std::endl;
LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol: call back on " << callbackEndpoint << std::endl;

task_.reset(new ClientTask(request, RequestEnvironment::instance().request(),
callbackEndpoint.host(), callbackEndpoint.port()));
Expand All @@ -408,7 +408,7 @@ void DHSProtocol::archive(const MarsRequest& request, const Length& size)
Length result = size;
while (wait(result)) {
}
Log::debug() << "DHSProtocol: archive completed." << std::endl;
LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol: archive completed." << std::endl;
}

void DHSProtocol::cleanup()
Expand Down Expand Up @@ -488,7 +488,7 @@ bool DHSProtocol::wait(Length& size)

char code = task_->receive(s);

Log::debug() << "DHSProtocol: code [" << code << "]" << std::endl;
LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol: code [" << code << "]" << std::endl;

std::string msg;
long long bytes;
Expand All @@ -502,7 +502,7 @@ bool DHSProtocol::wait(Length& size)
/* read source */
case 'r':
bytes = size;
Log::debug() << "DHSProtocol:r [" << bytes << "]" << std::endl;
LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol:r [" << bytes << "]" << std::endl;
s << bytes;
sending_ = true;
return false;
Expand All @@ -514,7 +514,7 @@ bool DHSProtocol::wait(Length& size)

case 'w':
s >> bytes;
Log::debug() << "DHSProtocol:w " << bytes << std::endl;
LOG_DEBUG_LIB(LibMetkit) << "DHSProtocol:w " << bytes << std::endl;
size = bytes;
return false;

Expand Down Expand Up @@ -555,7 +555,7 @@ bool DHSProtocol::wait(Length& size)

case 'D': /* debug */
s >> msg;
Log::debug() << msg << " [" << name_ << "]" << std::endl;
LOG_DEBUG_LIB(LibMetkit) << msg << " [" << name_ << "]" << std::endl;
if (forward_) {
Log::userInfo() << msg << " [" << name_ << "]" << std::endl;
}
Expand Down
62 changes: 61 additions & 1 deletion src/metkit/mars/MarsRequest.cc
Original file line number Diff line number Diff line change
Expand Up @@ -333,9 +333,64 @@ MarsRequest::operator eckit::Value() const {
NOTIMP;
}

// recursively expand along keys in expvalues
void expand_along_keys(
const MarsRequest& prototype,
const std::vector<std::pair<std::string, std::vector<std::string>>>& expvalues,
std::vector<MarsRequest>& requests,
size_t i) {

if(i == expvalues.size()) {
requests.push_back(prototype);
return;
}

const std::string& key = expvalues[i].first;
const std::vector<std::string>& values = expvalues[i].second;

MarsRequest req(prototype);
for (auto& value : values) {
req.setValue(key, value);
expand_along_keys(req, expvalues, requests, i+1);
}
}

std::vector<MarsRequest> MarsRequest::split(const std::vector<std::string>& keys) const {

size_t n = 1;

LOG_DEBUG_LIB(LibMetkit) << "Splitting request with keys" << keys << std::endl;

std::vector<std::pair<std::string, std::vector<std::string>>> expvalues;
for (auto& key : keys) {
std::vector<std::string> v = values(key, true); // ok to be empty
LOG_DEBUG_LIB(LibMetkit) << "splitting along key " << key << " n values " << v.size() << " values " << v << std::endl;
if (v.empty()) continue;
n *= v.size();
expvalues.emplace_back(std::make_pair(key, v));
}

std::vector<MarsRequest> requests;
requests.reserve(n);

if(n == 1) {
requests.push_back(*this);
return requests;
}

expand_along_keys(*this, expvalues, requests, 0);

return requests;
}

std::vector<MarsRequest> MarsRequest::split(const std::string& key) const {
std::vector<std::string> keys = { key };
return split( keys );
}

void MarsRequest::merge(const MarsRequest& other) {
for (auto& param : params_) {
eckit::Log::debug<LibMetkit>() << "Merging parameter " << param << std::endl;
LOG_DEBUG_LIB(LibMetkit) << "Merging parameter " << param << std::endl;
auto it = other.find(param.name());
if (it != other.params_.end())
param.merge(*it);
Expand Down Expand Up @@ -410,6 +465,11 @@ void MarsRequest::erase(const std::string& name) {
}
}

std::string MarsRequest::asString() const {
std::ostringstream oss;
oss << *this;
return oss.str();
}
//----------------------------------------------------------------------------------------------------------------------

std::vector<MarsRequest> MarsRequest::parse(std::istream& in, bool strict) {
Expand Down
14 changes: 12 additions & 2 deletions src/metkit/mars/MarsRequest.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,12 @@ class MarsRequest;

class MarsRequest {
public: // methods

MarsRequest();

explicit MarsRequest(const std::string&);
explicit MarsRequest(eckit::Stream&, bool lowercase = false);

MarsRequest(const std::string&, const std::map<std::string, std::string>&);
MarsRequest(const std::string&, const eckit::Value&);

Expand All @@ -60,12 +63,10 @@ class MarsRequest {
size_t countValues(const std::string&) const;
bool has(const std::string&) const;


bool is(const std::string& param, const std::string& value) const;

const std::vector<std::string>& values(const std::string&, bool emptyOk = false) const;


template <class T>
size_t getValues(const std::string& name, std::vector<T>& v, bool emptyOk = false) const;

Expand All @@ -83,6 +84,12 @@ class MarsRequest {

void unsetValues(const std::string&);

/// Splits a MARS request into multiple requests along the provided key
std::vector<MarsRequest> split(const std::string& keys) const;

/// Splits a MARS request into multiple requests along the indicated keys
std::vector<MarsRequest> split(const std::vector<std::string>& keys) const;

/// Merges one MarsRequest into another
/// @todo Improve performance -- uses O(N^2) search / merge in std::list's
void merge(const MarsRequest& other);
Expand All @@ -108,7 +115,10 @@ class MarsRequest {

void erase(const std::string& param);

std::string asString() const;

public: // static methods

static MarsRequest parse(const std::string& s, bool strict = false);
static std::vector<MarsRequest> parse(std::istream&, bool strict = false);

Expand Down
4 changes: 3 additions & 1 deletion src/metkit/mars/MarsRequestHandle.cc
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
// Baudouin Raoult - (c) ECMWF Feb 12

#include "metkit/mars/MarsRequestHandle.h"
#include "metkit/config/LibMetkit.h"

#include "eckit/utils/StringTools.h"
#include "eckit/types/Types.h"

Expand Down Expand Up @@ -53,7 +55,7 @@ MarsRequestHandle::MarsRequestHandle(const MarsRequest& request,
protocol_(protocol),
opened_(false)
{
eckit::Log::debug() << "MarsRequestHandle::MarsRequestHandle: request: " << request << " protocol: " << protocol << std::endl;
LOG_DEBUG_LIB(LibMetkit) << "MarsRequestHandle::MarsRequestHandle: request: " << request << " protocol: " << protocol << std::endl;
ASSERT(protocol);
}

Expand Down
4 changes: 2 additions & 2 deletions src/metkit/mars/ParamID.h
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ void ParamID::normalise(const REQUEST_T& request,
newreq.push_back(p);
}

eckit::Log::debug<LibMetkit>() << "useGRIBParamID p=" << p
LOG_DEBUG_LIB(LibMetkit) << "useGRIBParamID p=" << p
<< ", alt=" << alt
<< ", choice=" << newreq.back() << std::endl;

Expand Down Expand Up @@ -174,7 +174,7 @@ void ParamID::normalise(const REQUEST_T& request,
if (!wantVO) req.push_back(windVO);
if (!wantD) req.push_back(windD);

eckit::Log::debug<LibMetkit>() << "U/V conversion requested U=" << windU << ", V=" << windV << ", VO=" << windVO << ", D=" << windD << std::endl;
LOG_DEBUG_LIB(LibMetkit)<< "U/V conversion requested U=" << windU << ", V=" << windV << ", VO=" << windVO << ", D=" << windD << std::endl;
windConversion = true;
}
}
Expand Down
13 changes: 3 additions & 10 deletions src/metkit/mars/TypeParam.cc
Original file line number Diff line number Diff line change
Expand Up @@ -140,12 +140,7 @@ Rule::Rule(const eckit::Value& matchers, const eckit::Value& values, const eckit

if (aliases.isNil()) {

Log::debug<LibMetkit>()
<< "No aliases for "
<< id
<< " "
<< *this
<< std::endl;
LOG_DEBUG_LIB(LibMetkit) << "No aliases for " << id << " " << *this << std::endl;
continue;
}

Expand All @@ -158,8 +153,7 @@ Rule::Rule(const eckit::Value& matchers, const eckit::Value& values, const eckit

if (precedence[v] <= j) {

Log::debug<LibMetkit>()
<< "Redefinition ignored: param "
LOG_DEBUG_LIB(LibMetkit) << "Redefinition ignored: param "
<< v
<< "='"
<< first
Expand All @@ -172,8 +166,7 @@ Rule::Rule(const eckit::Value& matchers, const eckit::Value& values, const eckit
}
else {

Log::debug<LibMetkit>()
<< "Redefinition of param "
LOG_DEBUG_LIB(LibMetkit) << "Redefinition of param "
<< v
<< "='"
<< first
Expand Down
4 changes: 2 additions & 2 deletions src/metkit/odb/IdMapper.cc
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ IdMap::IdMap(const std::string& configFile,
size_t alphanumericIndex) {

PathName configPath = codes_path() / configFile;
Log::debug<LibMetkit>() << "GribCodesBase::GribCodesBase: config file:" << configPath << std::endl;
LOG_DEBUG_LIB(LibMetkit) << "GribCodesBase::GribCodesBase: config file:" << configPath << std::endl;

numeric2alpha_.clear();

Expand All @@ -99,7 +99,7 @@ IdMap::IdMap(const std::string& configFile,
long num = eckit::Translator<std::string, long>()(StringTools::trim(words[numericIndex]));
std::string alpha (StringTools::trim(words[alphanumericIndex]));
numeric2alpha_[num] = StringTools::lower(alpha);
Log::debug<LibMetkit>() << "GribCodesBase::readConfig: num='" << num << "' alpha='" << alpha << "'" << std::endl;
LOG_DEBUG_LIB(LibMetkit) << "GribCodesBase::readConfig: num='" << num << "' alpha='" << alpha << "'" << std::endl;
}
}
}
Expand Down
Loading