From 4b8c28cc8b3a6e5c0d5fedceaa387cac2602661a Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Thu, 14 Dec 2023 10:11:25 +0000 Subject: [PATCH 1/4] METK-118 added resource to disable context check during param expansion --- src/metkit/mars/TypeParam.cc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/metkit/mars/TypeParam.cc b/src/metkit/mars/TypeParam.cc index af060751..d343329f 100644 --- a/src/metkit/mars/TypeParam.cc +++ b/src/metkit/mars/TypeParam.cc @@ -107,6 +107,15 @@ class Rule : public metkit::mars::MarsExpandContext { sep = ","; } out << "}"; + + // for (auto v:values_) { + // out << v; + // auto m = mapping_.find(v); + // if (m!=mapping_.end()) { + // out << "("<second<< ")"; + // } + // out << ","; + // } } void info(std::ostream& out) const { @@ -315,6 +324,15 @@ static void init() { const eckit::Value ids = eckit::YAMLParser::decodeFile(LibMetkit::paramIDYamlFile()); ASSERT(ids.isOrderedMap()); + bool metkitRawParam = eckit::Resource("metkitRawParam;$METKIT_RAW_PARAM", false); + if (metkitRawParam) { + (*rules).push_back(Rule(eckit::Value::makeMap(), ids.keys(), ids)); + // for (int i=0; isize(); i++) { + // std::cout << "rule "<at(i) << std::endl; + // } + return; + } + eckit::Value r = eckit::YAMLParser::decodeFile(LibMetkit::paramYamlFile()); ASSERT(r.isList()); // r.dump(std::cout) << std::endl; From 36406fd51f1d4c0f644fde9f68cda4d23dc956d6 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Thu, 4 Jan 2024 21:41:26 +0000 Subject: [PATCH 2/4] DHS protocol - multiple hosts --- src/metkit/mars/DHSProtocol.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/metkit/mars/DHSProtocol.cc b/src/metkit/mars/DHSProtocol.cc index 5706f749..7d74fc69 100644 --- a/src/metkit/mars/DHSProtocol.cc +++ b/src/metkit/mars/DHSProtocol.cc @@ -321,13 +321,19 @@ DHSProtocol::DHSProtocol(const Configuration& params): BaseProtocol(params), callback_(BaseCallbackConnection::build(params)), name_(params.getString("name")), - host_(params.getString("host")), port_(params.getInt("port", 9000)), done_(false), error_(false), sending_(false), forward_(false) { + if (params.has("hosts")) { + std::vector hosts = params.getStringVector("hosts"); + host_ = hosts.at(std::rand() % hosts.size()); + } else { + ASSERT(params.has("host")); + host_ = params.getString("host"); + } } DHSProtocol::DHSProtocol(Stream& s): From 2cfe5878b063172e725cdf65bbd83928171c7975 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Fri, 5 Jan 2024 09:38:41 +0000 Subject: [PATCH 3/4] DHS protocol - select same host for passive proxy --- src/metkit/mars/DHSProtocol.cc | 11 +++++++---- src/metkit/mars/DHSProtocol.h | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/metkit/mars/DHSProtocol.cc b/src/metkit/mars/DHSProtocol.cc index 7d74fc69..5451485f 100644 --- a/src/metkit/mars/DHSProtocol.cc +++ b/src/metkit/mars/DHSProtocol.cc @@ -72,7 +72,7 @@ namespace { return selectProxyHost(config.getStringVector("proxyHosts")); } - throw UserError("Neither proxyHost nor proxyPort specified in configuration"); + throw UserError("Neither proxyHosts nor proxyHost specified in configuration"); } } @@ -278,9 +278,12 @@ const ClassSpec& BaseCallbackConnection::classSpec() { return spec; } -BaseCallbackConnection* BaseCallbackConnection::build(const Configuration& config) { - if (config.has("proxyHost") || config.has("proxyHosts")) { +BaseCallbackConnection* BaseCallbackConnection::build(const Configuration& config, const std::string& host) { + if (config.has("proxyHost") || config.has("proxyHosts") || (config.getBool("passiveProxy", true) && config.getBool("useHostAsProxy", false))) { if (config.getBool("passiveProxy", true)) { + if (config.getBool("useHostAsProxy", false)) { + return new PassiveProxyCallback{unpackHostPort(host)}; + } return new PassiveProxyCallback{config}; } return new ProxyCallback{config}; @@ -319,7 +322,6 @@ DHSProtocol::DHSProtocol(const std::string& name, DHSProtocol::DHSProtocol(const Configuration& params): BaseProtocol(params), - callback_(BaseCallbackConnection::build(params)), name_(params.getString("name")), port_(params.getInt("port", 9000)), done_(false), @@ -334,6 +336,7 @@ DHSProtocol::DHSProtocol(const Configuration& params): ASSERT(params.has("host")); host_ = params.getString("host"); } + callback_.reset(BaseCallbackConnection::build(params, host_)); } DHSProtocol::DHSProtocol(Stream& s): diff --git a/src/metkit/mars/DHSProtocol.h b/src/metkit/mars/DHSProtocol.h index 924b08e9..5bd98eea 100644 --- a/src/metkit/mars/DHSProtocol.h +++ b/src/metkit/mars/DHSProtocol.h @@ -36,7 +36,7 @@ class BaseCallbackConnection : public eckit::Streamable { BaseCallbackConnection() {} virtual ~BaseCallbackConnection() {} - static BaseCallbackConnection* build(const eckit::Configuration& config); + static BaseCallbackConnection* build(const eckit::Configuration& config, const std::string& host = ""); virtual const eckit::net::Endpoint& endpoint() const = 0; From 01056957c119b35f72b7299a2a22ea7f5a4283c6 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Tue, 9 Jan 2024 13:39:14 +0000 Subject: [PATCH 4/4] cleanup --- src/metkit/mars/TypeParam.cc | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/metkit/mars/TypeParam.cc b/src/metkit/mars/TypeParam.cc index d343329f..480bd970 100644 --- a/src/metkit/mars/TypeParam.cc +++ b/src/metkit/mars/TypeParam.cc @@ -68,9 +68,6 @@ bool Matcher::match(const metkit::mars::MarsRequest& request, bool partial) cons return partial; } - // std::cout << vals << std::endl; - - for (size_t i = 0; i < values_.size(); i++) { std::string v = values_[i]; @@ -107,15 +104,6 @@ class Rule : public metkit::mars::MarsExpandContext { sep = ","; } out << "}"; - - // for (auto v:values_) { - // out << v; - // auto m = mapping_.find(v); - // if (m!=mapping_.end()) { - // out << "("<second<< ")"; - // } - // out << ","; - // } } void info(std::ostream& out) const { @@ -274,16 +262,12 @@ std::string Rule::lookup(const MarsExpandContext& ctx, const std::string & s, bo } } - // std::cout << "OK " << ok << " " << param << std::endl; - if (ok && param > 0) { std::ostringstream oss; if (table == 128) { table = 0; } - // std::cerr << "Param " << param << " " << table << std::endl; - oss << table * 1000 + param; // return metkit::mars::MarsLanguage::bestMatch(oss.str(), values_, fail, false, mapping_, this); @@ -327,15 +311,11 @@ static void init() { bool metkitRawParam = eckit::Resource("metkitRawParam;$METKIT_RAW_PARAM", false); if (metkitRawParam) { (*rules).push_back(Rule(eckit::Value::makeMap(), ids.keys(), ids)); - // for (int i=0; isize(); i++) { - // std::cout << "rule "<at(i) << std::endl; - // } return; } eckit::Value r = eckit::YAMLParser::decodeFile(LibMetkit::paramYamlFile()); ASSERT(r.isList()); - // r.dump(std::cout) << std::endl; const eckit::Value rs = eckit::YAMLParser::decodeFile(LibMetkit::paramStaticYamlFile()); ASSERT(rs.isList()); @@ -484,7 +464,6 @@ bool TypeParam::expand(const MarsExpandContext& ctx, const MarsRequest& request, void TypeParam::pass2(const MarsExpandContext& ctx, MarsRequest& request) { - // std::cout << request << std::endl; std::vector values = request.values(name_, true); expand(ctx, request, values, true); request.setValuesTyped(this, values);