bes  Updated for version 3.20.8
DmrppRequestHandler.cc
1 // DmrppRequestHandler.cc
2 
3 // Copyright (c) 2016 OPeNDAP, Inc. Author: James Gallagher
4 // <jgallagher@opendap.org>, Patrick West <pwest@opendap.org>
5 // Nathan Potter <npotter@opendap.org>
6 //
7 // modify it under the terms of the GNU Lesser General Public License
8 // as published by the Free Software Foundation; either version 2.1 of
9 // the License, or (at your option) any later version.
10 //
11 // This library is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 // Lesser General Public License for more details.
15 //
16 // License along with this library; if not, write to the Free Software
17 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
18 // 02110-1301 U\ SA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI.
21 // 02874-0112.
22 
23 #include "config.h"
24 
25 #include <string>
26 #include <memory>
27 #include <sstream>
28 
29 #include <curl/curl.h>
30 #include <stdlib.h>
31 
32 #include <Ancillary.h>
33 #include <ObjMemCache.h>
34 #include <DMR.h>
35 #include <D4Group.h>
36 #include <DAS.h>
37 
38 #include <InternalErr.h>
39 #include <mime_util.h> // for name_path
40 
41 #include <BESResponseHandler.h>
42 #include <BESResponseNames.h>
43 #include <BESDapNames.h>
44 #include <BESDataNames.h>
45 #include <BESDASResponse.h>
46 #include <BESDDSResponse.h>
47 #include <BESDataDDSResponse.h>
48 #include <BESVersionInfo.h>
49 #include <BESTextInfo.h>
50 #include <BESContainer.h>
51 
52 #include <BESDMRResponse.h>
53 
54 #include <BESConstraintFuncs.h>
55 #include <BESServiceRegistry.h>
56 #include <BESUtil.h>
57 #include <BESLog.h>
58 #include <TheBESKeys.h>
59 
60 #include <BESDapError.h>
61 #include <BESInternalFatalError.h>
62 #include <BESDebug.h>
63 #include <BESStopWatch.h>
64 
65 #include "DMRpp.h"
66 #include "DmrppTypeFactory.h"
67 #include "DmrppParserSax2.h"
68 #include "DmrppRequestHandler.h"
69 #include "CurlHandlePool.h"
70 #include "DmrppMetadataStore.h"
71 #include "CredentialsManager.h"
72 
73 using namespace bes;
74 using namespace libdap;
75 using namespace std;
76 
77 #define prolog std::string("DmrppRequestHandler::").append(__func__).append("() - ")
78 #define MODULE "dmrpp"
79 
80 namespace dmrpp {
81 
82 
83 ObjMemCache *DmrppRequestHandler::das_cache = 0;
84 ObjMemCache *DmrppRequestHandler::dds_cache = 0;
85 ObjMemCache *DmrppRequestHandler::dmr_cache = 0;
86 
87 // This is used to maintain a pool of reusable curl handles that enable connection
88 // reuse. jhrg
89 CurlHandlePool *DmrppRequestHandler::curl_handle_pool = 0;
90 
91 bool DmrppRequestHandler::d_use_parallel_transfers = true;
92 unsigned int DmrppRequestHandler::d_max_parallel_transfers = 8;
93 
94 // Default minimum value is 2MB: 2 * (1024*1024)
95 unsigned int DmrppRequestHandler::d_min_size = 2097152;
96 
97 static void read_key_value(const std::string &key_name, bool &key_value)
98 {
99  bool key_found = false;
100  string value;
101  TheBESKeys::TheKeys()->get_value(key_name, value, key_found);
102  if (key_found) {
103  value = BESUtil::lowercase(value);
104  key_value = (value == "true" || value == "yes");
105  }
106 }
107 
108 static void read_key_value(const std::string &key_name, unsigned int &key_value)
109 {
110  bool key_found = false;
111  string value;
112  TheBESKeys::TheKeys()->get_value(key_name, value, key_found);
113  if (key_found) {
114  istringstream iss(value);
115  iss >> key_value;
116  }
117 }
118 
123 DmrppRequestHandler::DmrppRequestHandler(const string &name) :
124  BESRequestHandler(name)
125 {
126  add_method(DMR_RESPONSE, dap_build_dmr);
127  add_method(DAP4DATA_RESPONSE, dap_build_dap4data);
128  add_method(DAS_RESPONSE, dap_build_das);
129  add_method(DDS_RESPONSE, dap_build_dds);
130  add_method(DATA_RESPONSE, dap_build_dap2data);
131 
132  add_method(VERS_RESPONSE, dap_build_vers);
133  add_method(HELP_RESPONSE, dap_build_help);
134 
135  read_key_value("DMRPP.UseParallelTransfers", d_use_parallel_transfers);
136  read_key_value("DMRPP.MaxParallelTransfers", d_max_parallel_transfers);
137 
138 #if !HAVE_CURL_MULTI_API
139  if (DmrppRequestHandler::d_use_parallel_transfers)
140  ERROR_LOG("The DMR++ handler is configured to use parallel transfers, but the libcurl Multi API is not present, defaulting to serial transfers");
141 #endif
142 
143  CredentialsManager::theCM()->load_credentials();
144 
145  if (!curl_handle_pool)
146  curl_handle_pool = new CurlHandlePool(d_max_parallel_transfers);
147 
148  // This and the matching cleanup function can be called many times as long as
149  // they are called in balanced pairs. jhrg 9/3/20
150  curl_global_init(CURL_GLOBAL_DEFAULT);
151 }
152 
153 DmrppRequestHandler::~DmrppRequestHandler()
154 {
155  delete curl_handle_pool;
156  curl_global_cleanup();
157 }
158 
159 void DmrppRequestHandler::build_dmr_from_file(BESContainer *container, DMR* dmr)
160 {
161  string data_pathname = container->access();
162 
163  dmr->set_filename(data_pathname);
164  dmr->set_name(name_path(data_pathname));
165 
166  DmrppTypeFactory BaseFactory; // Use the factory for this handler's types
167  dmr->set_factory(&BaseFactory);
168 
169  DmrppParserSax2 parser;
170  ifstream in(data_pathname.c_str(), ios::in);
171 
172  parser.intern(in, dmr);
173 
174  dmr->set_factory(0);
175 }
176 
190 {
191  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
192 
193  BESResponseObject *response = dhi.response_handler->get_response_object();
194  BESDMRResponse *bdmr = dynamic_cast<BESDMRResponse *>(response);
195  if (!bdmr) throw BESInternalError("Cast error, expected a BESDDSResponse object.", __FILE__, __LINE__);
196 
197  try {
198  build_dmr_from_file(dhi.container, bdmr->get_dmr());
199 
200  bdmr->set_dap4_constraint(dhi);
201  bdmr->set_dap4_function(dhi);
202  }
203  catch (BESError &e) {
204  throw e;
205  }
206  catch (InternalErr & e) {
207  throw BESDapError(e.get_error_message(), true, e.get_error_code(), __FILE__, __LINE__);
208  }
209  catch (Error & e) {
210  throw BESDapError(e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
211  }
212  catch (...) {
213  throw BESInternalFatalError("Unknown exception caught building a DMR", __FILE__, __LINE__);
214  }
215 
216  BESDEBUG(MODULE, prolog << "END" << endl);
217 
218  return true;
219 }
220 
221 bool DmrppRequestHandler::dap_build_dap4data(BESDataHandlerInterface &dhi)
222 {
223  BESStopWatch sw;
224  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
225 
226  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
227 
228  BESResponseObject *response = dhi.response_handler->get_response_object();
229  BESDMRResponse *bdmr = dynamic_cast<BESDMRResponse *>(response);
230  if (!bdmr) throw BESInternalError("Cast error, expected a BESDMRResponse object.", __FILE__, __LINE__);
231 
232  try {
233  // Check the Container to see if the handler should get the response from the MDS.
234  if (dhi.container->get_attributes().find(MDS_HAS_DMRPP) != string::npos) {
235  DmrppMetadataStore *mds = DmrppMetadataStore::get_instance();
236  if (!mds)
237  throw BESInternalError("MDS configuration error: The DMR++ module could not find the MDS", __FILE__, __LINE__);
238 
240  if (!dmrpp)
241  throw BESInternalError("DMR++ module error: Null DMR++ object read from the MDS", __FILE__, __LINE__);
242 
243  delete bdmr->get_dmr();
244  bdmr->set_dmr(dmrpp);
245  }
246  else {
247  build_dmr_from_file(dhi.container, bdmr->get_dmr());
248  }
249 
250  bdmr->set_dap4_constraint(dhi);
251  bdmr->set_dap4_function(dhi);
252  }
253  catch (BESError &e) {
254  throw e;
255  }
256  catch (InternalErr & e) {
257  throw BESDapError(e.get_error_message(), true, e.get_error_code(), __FILE__, __LINE__);
258  }
259  catch (Error & e) {
260  throw BESDapError(e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
261  }
262  catch (...) {
263  throw BESInternalFatalError("Unknown exception caught building DAP4 Data response", __FILE__, __LINE__);
264  }
265 
266  BESDEBUG(MODULE, prolog << "END" << endl);
267 
268  return false;
269 }
270 
275 {
276  BESStopWatch sw;
277  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
278 
279  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
280 
281  BESResponseObject *response = dhi.response_handler->get_response_object();
282  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(response);
283  if (!bdds) throw BESInternalError("Cast error, expected a BESDataDDSResponse object.", __FILE__, __LINE__);
284 
285  try {
286  string container_name_str = bdds->get_explicit_containers() ? dhi.container->get_symbolic_name() : "";
287 
288  DDS *dds = bdds->get_dds();
289  if (!container_name_str.empty()) dds->container_name(container_name_str);
290  string accessed = dhi.container->access();
291 
292  // Look in memory cache, if it's initialized
293  DDS *cached_dds_ptr = 0;
294  if (dds_cache && (cached_dds_ptr = static_cast<DDS*>(dds_cache->get(accessed)))) {
295  // copy the cached DAS into the BES response object
296  BESDEBUG(MODULE, prolog << "DDS Cached hit for : " << accessed << endl);
297  *dds = *cached_dds_ptr;
298  bdds->set_constraint(dhi);
299  }
300  else {
301  // Not in the local binary cache, make one...
302  DMR *dmr = NULL;
303 
304  // Check the Container to see if the handler should get the response from the MDS.
305  if (dhi.container->get_attributes().find(MDS_HAS_DMRPP) != string::npos) {
306  DmrppMetadataStore *mds = DmrppMetadataStore::get_instance();
307  if (!mds)
308  throw BESInternalError("MDS configuration error: The DMR++ module could not find the MDS", __FILE__, __LINE__);
309 
310  dmr = mds->get_dmrpp_object(dhi.container->get_relative_name());
311  if (!dmr)
312  throw BESInternalError("DMR++ module error: Null DMR++ object read from the MDS", __FILE__, __LINE__);
313  }
314  else {
315  dmr = new DMR();
316  build_dmr_from_file(dhi.container, dmr);
317  }
318 
319  // delete the current one;
320  delete dds;
321  // assign the new one.
322  dds = dmr->getDDS();
323 
324  // Stuff it into the response.
325  bdds->set_dds(dds);
326  bdds->set_constraint(dhi);
327 
328  delete dmr;
329 
330  // Cache it, if the cache is active.
331  if (dds_cache) {
332  dds_cache->add(new DDS(*dds), accessed);
333  }
334  }
335 
336  bdds->clear_container();
337  }
338  catch (BESError &e) {
339  throw;
340  }
341  catch (InternalErr & e) {
342  BESDapError ex(e.get_error_message(), true, e.get_error_code(), __FILE__, __LINE__);
343  throw ex;
344  }
345  catch (Error & e) {
346  BESDapError ex(e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
347  throw ex;
348  }
349  catch (std::exception &e) {
350  string s = string("C++ Exception: ") + e.what();
351  BESInternalFatalError ex(s, __FILE__, __LINE__);
352  throw ex;
353  }
354  catch (...) {
355  string s = "unknown exception caught building DDS";
356  BESInternalFatalError ex(s, __FILE__, __LINE__);
357  throw ex;
358  }
359 
360  BESDEBUG(MODULE, prolog << "END" << endl);
361  return true;
362 }
363 
368 {
369  BESStopWatch sw;
370  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
371 
372  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
373 
374  BESResponseObject *response = dhi.response_handler->get_response_object();
375  BESDDSResponse *bdds = dynamic_cast<BESDDSResponse *>(response);
376  if (!bdds) throw BESInternalError("Cast error, expected a BESDDSResponse object.", __FILE__, __LINE__);
377 
378  try {
379  string container_name_str = bdds->get_explicit_containers() ? dhi.container->get_symbolic_name() : "";
380 
381  DDS *dds = bdds->get_dds();
382  if (!container_name_str.empty()) dds->container_name(container_name_str);
383  string accessed = dhi.container->access();
384 
385  // Look in memory cache, if it's initialized
386  DDS *cached_dds_ptr = 0;
387  if (dds_cache && (cached_dds_ptr = static_cast<DDS*>(dds_cache->get(accessed)))) {
388  // copy the cached DAS into the BES response object
389  BESDEBUG(MODULE, prolog << "DDS Cached hit for : " << accessed << endl);
390  *dds = *cached_dds_ptr;
391  }
392  else {
393  // Not in cache, make one...
394  DMR *dmr = new DMR(); // FIXME is this leaked? jhrg 6/1/18
395  build_dmr_from_file(dhi.container, dmr);
396 
397  // delete the current one;
398  delete dds;
399  dds = 0;
400 
401  // assign the new one.
402  dds = dmr->getDDS();
403 
404  // Stuff it into the response.
405  bdds->set_dds(dds);
406  bdds->set_constraint(dhi);
407 
408  // Cache it, if the cache is active.
409  if (dds_cache) {
410  dds_cache->add(new DDS(*dds), accessed);
411  }
412  }
413 
414  bdds->clear_container();
415  }
416  catch (BESError &e) {
417  throw;
418  }
419  catch (InternalErr & e) {
420  BESDapError ex(e.get_error_message(), true, e.get_error_code(), __FILE__, __LINE__);
421  throw ex;
422  }
423  catch (Error & e) {
424  BESDapError ex(e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
425  throw ex;
426  }
427  catch (std::exception &e) {
428  string s = string("C++ Exception: ") + e.what();
429  BESInternalFatalError ex(s, __FILE__, __LINE__);
430  throw ex;
431  }
432  catch (...) {
433  string s = "unknown exception caught building DDS";
434  BESInternalFatalError ex(s, __FILE__, __LINE__);
435  throw ex;
436  }
437 
438  BESDEBUG(MODULE, prolog << "END" << endl);
439  return true;
440 }
441 
447 {
448  BESStopWatch sw;
449  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start(prolog + "timer" , dhi.data[REQUEST_ID]);
450 
451  BESResponseObject *response = dhi.response_handler->get_response_object();
452  BESDASResponse *bdas = dynamic_cast<BESDASResponse *>(response);
453  if (!bdas) throw BESInternalError("Cast error, expected a BESDASResponse object.", __FILE__, __LINE__);
454 
455  try {
456  string container_name_str = bdas->get_explicit_containers() ? dhi.container->get_symbolic_name() : "";
457 
458  DAS *das = bdas->get_das();
459  if (!container_name_str.empty()) das->container_name(container_name_str);
460  string accessed = dhi.container->access();
461 
462  // Look in memory cache (if it's initialized)
463  DAS *cached_das_ptr = 0;
464  if (das_cache && (cached_das_ptr = static_cast<DAS*>(das_cache->get(accessed)))) {
465  // copy the cached DAS into the BES response object
466  *das = *cached_das_ptr;
467  }
468  else {
469  // Not in cache, better make one!
470  // 1) Build a DMR
471  DMR *dmr = new DMR();
472  build_dmr_from_file(dhi.container, dmr);
473 
474  // Get a DDS from the DMR
475  DDS *dds = dmr->getDDS();
476 
477  // Load the BESDASResponse DAS from the DDS
478  dds->get_das(das);
479 
480  delete dds;
481  delete dmr;
482 
483  Ancillary::read_ancillary_das(*das, accessed);
484  // Add to cache if cache is active
485  if (das_cache) {
486  das_cache->add(new DAS(*das), accessed);
487  }
488  }
489 
490  bdas->clear_container();
491  }
492  catch (BESError &e) {
493  throw;
494  }
495  catch (InternalErr & e) {
496  BESDapError ex(e.get_error_message(), true, e.get_error_code(), __FILE__, __LINE__);
497  throw ex;
498  }
499  catch (Error & e) {
500  BESDapError ex(e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
501  throw ex;
502  }
503  catch (std::exception &e) {
504  string s = string("C++ Exception: ") + e.what();
505  BESInternalFatalError ex(s, __FILE__, __LINE__);
506  throw ex;
507  }
508  catch (...) {
509  string s = "unknown exception caught building DAS";
510  BESInternalFatalError ex(s, __FILE__, __LINE__);
511  throw ex;
512  }
513 
514  BESDEBUG(MODULE, prolog << "END" << endl);
515  return true;
516 }
517 
518 
519 bool DmrppRequestHandler::dap_build_vers(BESDataHandlerInterface &dhi)
520 {
521  BESVersionInfo *info = dynamic_cast<BESVersionInfo *>(dhi.response_handler->get_response_object());
522  if (!info) throw BESInternalFatalError("Expected a BESVersionInfo instance.", __FILE__, __LINE__);
523 
524  info->add_module(MODULE_NAME, MODULE_VERSION);
525  return true;
526 }
527 
528 bool DmrppRequestHandler::dap_build_help(BESDataHandlerInterface &dhi)
529 {
530  BESInfo *info = dynamic_cast<BESInfo *>(dhi.response_handler->get_response_object());
531  if (!info) throw BESInternalFatalError("Expected a BESVersionInfo instance.", __FILE__, __LINE__);
532 
533  // This is an example. If you had a help file you could load it like
534  // this and if your handler handled the following responses.
535  map<string, string> attrs;
536  attrs["name"] = MODULE_NAME /* PACKAGE_NAME */;
537  attrs["version"] = MODULE_VERSION /* PACKAGE_VERSION */;
538  list<string> services;
539  BESServiceRegistry::TheRegistry()->services_handled(MODULE, services);
540  if (services.size() > 0) {
541  string handles = BESUtil::implode(services, ',');
542  attrs["handles"] = handles;
543  }
544  info->begin_tag("module", &attrs);
545  info->end_tag("module");
546 
547  return true;
548 }
549 
550 void DmrppRequestHandler::dump(ostream &strm) const
551 {
552  strm << BESIndent::LMarg << "DmrppRequestHandler::dump - (" << (void *) this << ")" << endl;
553  BESIndent::Indent();
555  BESIndent::UnIndent();
556 }
557 
558 } // namespace dmrpp
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:65
std::string get_attributes() const
retrieve the attributes desired from this container
Definition: BESContainer.h:242
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
std::string get_relative_name() const
Get the relative name of the object in this container.
Definition: BESContainer.h:186
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
Holds a DDS object within the BES.
void set_dds(libdap::DDS *ddsIn)
virtual void clear_container()
clear the container in the DAP response object
libdap::DDS * get_dds()
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_dds(libdap::DDS *ddsIn)
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:160
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
informational response object
Definition: BESInfo.h:63
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual bool add_method(const std::string &name, p_request_handler_method method)
add a handler method to the request handler that knows how to fill in a specific response object
virtual void dump(std::ostream &strm) const
dumps information about this object
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:200
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:638
An in-memory cache for DapObj (DAS, DDS, ...) objects.
Definition: ObjMemCache.h:84
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
Definition: ObjMemCache.cc:63
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
Definition: ObjMemCache.cc:105
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:339
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
Store the DAP DMR++ metadata responses.
virtual dmrpp::DMRpp * get_dmrpp_object(const std::string &name)
Build a DMR++ object from the cached Response.
Provide a way to print the DMR++ response.
Definition: DMRpp.h:42
static bool dap_build_dds(BESDataHandlerInterface &dhi)
static bool dap_build_dap2data(BESDataHandlerInterface &dhi)
static bool dap_build_dmr(BESDataHandlerInterface &dhi)
static bool dap_build_das(BESDataHandlerInterface &dhi)
virtual void dump(std::ostream &strm) const
dumps information about this object