bes  Updated for version 3.20.5
DDSLoader.cc
1 // This file is part of the "NcML Module" project, a BES module designed
3 // to allow NcML files to be used to be used as a wrapper to add
4 // AIS to existing datasets of any format.
5 //
6 // Copyright (c) 2009 OPeNDAP, Inc.
7 // Author: Michael Johnson <m.johnson@opendap.org>
8 //
9 // For more information, please also see the main website: http://opendap.org/
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26 //
27 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29 #include "config.h"
30 
31 #include <sstream>
32 #include <algorithm>
33 
34 #include <DataDDS.h>
35 
36 #include <BESConstraintFuncs.h>
37 #include <BESContainerStorage.h>
38 #include <BESContainerStorageList.h>
39 #include <BESDapNames.h>
40 #include <BESDapResponse.h>
41 #include <BESDataDDSResponse.h>
42 #include <BESDataHandlerInterface.h>
43 #include <BESDDSResponse.h>
44 #include <BESStopWatch.h>
45 #include <BESInternalError.h>
46 #include <BESResponseHandler.h>
47 #include <BESResponseNames.h>
48 #include <BESRequestHandlerList.h>
49 #include <BESServiceRegistry.h>
50 #include <BESTextInfo.h>
51 #include <BESUtil.h>
52 #include <BESVersionInfo.h>
53 
54 #include <BESDebug.h>
55 #include <BESLog.h>
56 
57 #include "DDSLoader.h"
58 #include "NCMLDebug.h"
59 #include "NCMLUtil.h"
60 
61 using namespace std;
62 using namespace agg_util;
63 using namespace libdap;
64 
65 // Rep Init
66 
67 /* static */
68 long DDSLoader::_gensymID = 0L;
69 
70 // Impl
71 
72 DDSLoader::DDSLoader(BESDataHandlerInterface& dhi) :
73  _dhi(dhi), /*d_saved_dhi(0),*/_hijacked(false), _filename(""), _store(0), _containerSymbol(""), _origAction(""), _origActionName(
74  ""), _origContainer(0), _origResponse(0)
75 {
76 }
77 
78 // WE ONLY COPY THE DHI! I got forced to impl this.
79 DDSLoader::DDSLoader(const DDSLoader& proto) :
80  _dhi(proto._dhi), /*d_saved_dhi(0),*/_hijacked(false), _filename(""), _store(0), _containerSymbol(""), _origAction(
81  ""), _origActionName(""), _origContainer(0), _origResponse(0)
82 {
83 }
84 
85 DDSLoader&
86 DDSLoader::operator=(const DDSLoader& rhs)
87 {
88  BESDEBUG("ncml", "DDSLoader::operator=: " << endl);
89 
90  if (&rhs == this) {
91  return *this;
92  }
93 
94  // First cleanup any state
95 #if 1
96  // Old comment, written in the midst of fixing bug #2176...
97  // I removed this call because ensureClean() will call restoreDHI()
98  // and then we will call the BESDataHandlerInterface::clone() method
99  // which will take the DHI from the DDSLoader passed in and clone it.
100  // ...no sense setting it twice.
101  //
102  // After the fix...
103  // Added it back in because I think it might be used after all. In many
104  // (all?) cases, the rhs._dhi is the same object as this->_dhi, so the
105  // clone() method will never get called. However, the 'saved state' of
106  // the DHI instance might be needed. It's not needed for the current
107  // operations (no tests fail when it is removed), but future versions
108  // might make use of the saved state.
109  // jhrg 4/18/14
110  ensureClean();
111 #else
112  removeContainerFromStorage();
113 #endif
114 
115  // Now copy the dhi only, since
116  // we assume we'll be using this fresh.
117  // Normally we don't want to copy these
118  // but I got forced into it.
119  //
120  // Update. Fix for bug #2176. With clang-503.0.40 calling
121  // BESDataHandlerInterface::make_copy() was inexplicably nulling the 'data'
122  // map member of the DHI. This was happening because the two maps were the same
123  // because the two DHI instances were the same - that is the 'rhs._dhi' field
124  // and this' _dhi field were/are one and the same. I'm going to leave this
125  // test here even though the code in BESDataHandlerInterface has been fixed to
126  // test for this case - and new copy ctor and operator=() methods added.
127  // jhrg 4/18/14
128  if (&_dhi != &rhs._dhi) _dhi.make_copy(rhs._dhi);
129 
130  return *this;
131 }
132 
134 {
135  ensureClean();
136 }
137 
138 #if 0
139 // Never used. 10/16/15 jhrg
140 auto_ptr<BESDapResponse> DDSLoader::load(const string& location, ResponseType type)
141 {
142  // We need to make the proper response object as well, since in this call the dhi is coming in with the
143  // response object for the original ncml request.
144  std::auto_ptr<BESDapResponse> response = makeResponseForType(type);
145  loadInto(location, type, response.get());
146  return response; // relinquish
147 }
148 #endif
149 
150 void DDSLoader::loadInto(const std::string& location, ResponseType type, BESDapResponse* pResponse)
151 {
152  VALID_PTR(pResponse);
153  VALID_PTR(_dhi.response_handler);
154 
155  // Just be sure we're cleaned up before doing anything, in case the caller calls load again after exception
156  // and before dtor.
157  ensureClean();
158 
159  _filename = location;
160 
161  // Remember current state of dhi before we touch it -- _hijacked is now true!!
162  snapshotDHI();
163 
164  // Add a new symbol to the storage list and return container for it.
165  // We will remove this new container on the way out.
166  BESContainer* container = addNewContainerToStorage();
167 
168  // Take over the dhi
169  // Container is allocated using ptr_duplicate. Must free existing container. See RestoreDHI. jhrg 6/19/19
170  _dhi.container = container;
171  _dhi.response_handler->set_response_object(pResponse);
172 
173  // Choose the proper request type...
174  _dhi.action = getActionForType(type);
175  _dhi.action_name = getActionNameForType(type);
176 
177  // Figure out which underlying type of response it is to get the DDS (or DataDDS via DDS super).
179  if (!pDDS) {
180  THROW_NCML_INTERNAL_ERROR("DDSLoader::load expected BESDDSResponse or BESDataDDSResponse but got neither!");
181  }
182  pDDS->set_request_xml_base(pResponse->get_request_xml_base());
183 
184  // DO IT!
185  try {
186  BESDEBUG("ncml", "Before BESRequestHandlerList::TheList()->execute_current" << endl);
187  BESDEBUG("ncml", "Handler name: " << BESRequestHandlerList::TheList()->get_handler_names() << endl);
188 
189  BESRequestHandlerList::TheList()->execute_current(_dhi);
190 
191  BESDEBUG("ncml", "After BESRequestHandlerList::TheList()->execute_current" << endl);
192  }
193  catch (BESError &e) {
194  *(BESLog::TheLog()) << "WARNING - " << string(__PRETTY_FUNCTION__) << ": " << e.get_file() << ":" << e.get_line() << ": "
195  << e.get_message() << " (the exception was re-thrown)."<< endl;
196  throw e;
197  }
198 
199  // Put back the dhi state we hijacked
200  restoreDHI();
201 
202  // Get rid of the container we added.
203  removeContainerFromStorage();
204 
205  _filename = "";
206 
207  // We should be clean here too.
208  ensureClean();
209 }
210 
212 {
213  ensureClean();
214 }
215 
216 bool is_url(std::string location){
217  std::string http("http://");
218  std::string https("https://");
219 
220  // case insensitive check
221  std::string tip = location.substr(0,http.size());
222  std::transform(tip.begin(), tip.end(), tip.begin(), ::tolower);
223  bool result = http.compare(tip)==0;
224 
225  // case insensitive check
226  tip = location.substr(0,https.size());
227  std::transform(tip.begin(), tip.end(), tip.begin(), ::tolower);
228 
229  result = result || http.compare(tip)==0;
230 
231  return result;
232 
233 
234 }
235 
237 DDSLoader::addNewContainerToStorage()
238 {
239  // Make sure we can find the storage
240  BESContainerStorageList *store_list = BESContainerStorageList::TheList();
241  VALID_PTR(store_list);
242 
243  // Check for URL in the _filename, 'cause that means gateway!
244  BESContainerStorage* store;
245  if(is_url(_filename)){
246  BESDEBUG("ncml", __func__ << "() - GATEWAY CONTAINER!" << endl);
247  store = store_list->find_persistence("gateway");
248  }
249  else {
250  store = store_list->find_persistence("catalog");
251  }
252  if (!store) {
253  throw BESInternalError("couldn't find the catalog storage", __FILE__, __LINE__);
254  }
255 
256  // Make a new symbol from the ncml filename
257  // NCML_ASSERT_MSG(_dhi.container, "DDSLoader::addNewContainerToStorage(): null container!");
258  // string newSymbol = _dhi.container->get_symbolic_name() + "_location_" + _filename;
259  string newSymbol = getNextContainerName() + "__" + _filename;
260 
261  // this will throw an exception if the location isn't found in the
262  // catalog. Might want to catch this. Wish the add returned the
263  // container object created. Might want to change it.
264  store->add_container(newSymbol, _filename, "");
265 
266  // If we were successful, note the store location and symbol we added for removal later.
267  _store = store;
268  _containerSymbol = newSymbol;
269 
270  // Now look up the symbol we added
271  BESContainer *container = store->look_for(_containerSymbol);
272  if (!container) {
273  throw BESInternalError("couldn't find the container we just added:" + newSymbol, __FILE__, __LINE__);
274  }
275 
276  return container;
277 }
278 
279 void DDSLoader::removeContainerFromStorage()
280 {
281  // If we have non-null _store, we added the container symbol,
282  // so get rid of it
283  if (_store) {
284  try {
285  // This should go through, but if there's an exception, we could unwind through the dtor,
286  // so make sure we don't.
287  _store->del_container(_containerSymbol);
288  }
289  catch (BESError& besErr) {
290  *(BESLog::TheLog()) << "WARNING: tried to remove symbol " << _containerSymbol
291  << " from singleton but unexpectedly it was not there." << endl;
292  }
293  _containerSymbol = "";
294  _store = 0;
295  }
296 }
297 
298 void DDSLoader::snapshotDHI()
299 {
300  VALID_PTR(_dhi.response_handler);
301 
302  BESDEBUG( "ncml", "DDSLoader::snapshotDHI() - Taking snapshot of DataHAndlerInterface for (action: " << _dhi.action << " action_name: " << _dhi.action_name << ")" << endl );
303  BESDEBUG( "ncml_verbose", "original dhi = " << _dhi << endl );
304 
305  // Store off the container for the original ncml file call and replace with the new one
306  _origContainer = _dhi.container;
307  _origAction = _dhi.action;
308  _origActionName = _dhi.action_name;
309 
310  _origResponse = _dhi.response_handler->get_response_object();
311 
312  BESDEBUG( "ncml", "DDSLoader::snapshotDHI() - Replaced with DataHAndlerInterface for (action: " << _dhi.action << " action_name: " << _dhi.action_name << ")" << endl );
313 
314  _hijacked = true;
315 }
316 
317 void DDSLoader::restoreDHI()
318 {
319  VALID_PTR(_dhi.response_handler);
320 
321  // Make sure we have state before we go mucking
322  if (!_hijacked) {
323  return;
324  }
325 
326  // Before we overwrite the 'high jacked' DHI's container, call
327  // release(). If this is a simple file, it's no big deal to
328  // skip this (because the file is closed elsewhere). But, if the
329  // DHI is working with a _compressed_ file, it is a big deal
330  // because this is the call that closes the cached uncompressed
331  // file and frees the lock. This was the bug associated with
332  // ticket HR-64. jhrg 10/16/15
333  _dhi.container->release();
334  // Leak. Allocated locally by addNewContainerToStorage() in loadInto(). jhrg 6/19/19
335  delete _dhi.container;
336 
337  // Restore saved state
338  _dhi.container = _origContainer;
339  _dhi.action = _origAction;
340  _dhi.action_name = _origActionName;
341 
342  _dhi.response_handler->set_response_object(_origResponse);
343 
344  BESDEBUG( "ncml", "DDSLoader::restoreDHI() - Restored of DataHAndlerInterface for (action: " << _dhi.action << " action_name: " << _dhi.action_name << ")" << endl );
345 
346  BESDEBUG( "ncml_verbose", "restored dhi = " << _dhi << endl );
347 
348  // clear our copy of saved state
349  _origAction = "";
350  _origActionName = "";
351  _origResponse = 0;
352  _origContainer = 0;
353  _filename = "";
354 
355  _hijacked = false;
356 }
357 
358 void DDSLoader::ensureClean()
359 {
360  // If we're still hijacked here, there was an exception in load, so clean
361  // up if needed.
362  if (_hijacked) {
363  restoreDHI();
364  }
365 
366  // Make sure we've removed the new symbol from the container list as well.
367  removeContainerFromStorage();
368 }
369 
370 /* static */
371 std::string DDSLoader::getNextContainerName()
372 {
373  static const string _sPrefix = "__DDSLoader_Container_ID_";
374  _gensymID++;
375  std::ostringstream oss;
376  oss << _sPrefix << (_gensymID);
377  return oss.str();
378 }
379 
380 std::auto_ptr<BESDapResponse> DDSLoader::makeResponseForType(ResponseType type)
381 {
382  if (type == eRT_RequestDDX) {
383  // The BaseTypeFactory is leaked. jhrg 6/19/19
384  return auto_ptr<BESDapResponse>(new BESDDSResponse(new DDS(0 /*new BaseTypeFactory()*/, "virtual")));
385  }
386  else if (type == eRT_RequestDataDDS) {
387  // Leak fix jhrg 6/19/19
388  return auto_ptr<BESDapResponse>(new BESDataDDSResponse(new DDS(0 /*new BaseTypeFactory()*/, "virtual")));
389  }
390  else {
391  THROW_NCML_INTERNAL_ERROR("DDSLoader::makeResponseForType() got unknown type!");
392  }
393 }
394 
396 {
397  if (type == eRT_RequestDDX) {
398  return DDS_RESPONSE;
399  }
400  else if (type == eRT_RequestDataDDS) {
401  return DATA_RESPONSE;
402  }
403 
404  THROW_NCML_INTERNAL_ERROR("DDSLoader::getActionForType(): unknown type!");
405 }
406 
408 {
409  if (type == eRT_RequestDDX) {
410  return DDX_RESPONSE_STR;
411  }
412  else if (type == eRT_RequestDataDDS) {
413  return DATA_RESPONSE_STR;
414  }
415 
416  THROW_NCML_INTERNAL_ERROR("DDSLoader::getActionNameForType(): unknown type!");
417 }
418 
420 {
421  if (type == eRT_RequestDDX) {
422  return dynamic_cast<BESDDSResponse*>(pResponse);
423  }
424  else if (type == eRT_RequestDataDDS) {
425  return dynamic_cast<BESDataDDSResponse*>(pResponse);
426  }
427  else {
428  return false;
429  }
430 }
431 
static std::auto_ptr< BESDapResponse > makeResponseForType(ResponseType type)
Definition: DDSLoader.cc:380
provides persistent storage for data storage information represented by a container.
exception thrown if inernal error encountered
static std::string getActionNameForType(ResponseType type)
Definition: DDSLoader.cc:407
Holds a DDS object within the BES.
static bool checkResponseIsValidType(ResponseType type, BESDapResponse *pResponse)
Definition: DDSLoader.cc:419
virtual std::string get_message()
get the error message for this exception
Definition: BESError.h:99
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
static std::string getActionForType(ResponseType type)
Definition: DDSLoader.cc:395
static libdap::DDS * getDDSFromEitherResponse(BESDapResponse *response)
Definition: NCMLUtil.cc:354
virtual BESContainerStorage * find_persistence(const std::string &persist_name)
find the persistence store with the given name
virtual BESResponseObject * set_response_object(BESResponseObject *o)
replaces the current response object with the specified one, returning the current response object
virtual BESContainer * look_for(const std::string &sym_name)=0
looks for a container in this persistent store
void make_copy(const BESDataHandlerInterface &copy_from)
deprecated
virtual BESResponseObject * get_response_object()
return the current response object
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
virtual bool del_container(const std::string &s_name)=0
removes a container with the given symbolic name
void loadInto(const std::string &location, ResponseType type, BESDapResponse *pResponse)
Load a DDX or DataDDS response into the given pResponse object, which must be non-null.
Definition: DDSLoader.cc:150
Provides a mechanism for accessing container information from different container stores registered w...
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
virtual ~DDSLoader()
Dtor restores the state of dhi Restores the state of the dhi to what it was when object if it is stil...
Definition: DDSLoader.cc:133
Represents an OPeNDAP DAP response object within the BES.
Structure storing information used by the BES to handle the request.
virtual std::string get_file()
get the file name where the exception was thrown
Definition: BESError.h:107
string get_request_xml_base() const
Return the xml:base URL for this request.
virtual void execute_current(BESDataHandlerInterface &dhi)
Execute a single method for the current container that will fill in the response object rather than i...
virtual void add_container(const std::string &sym_name, const std::string &real_name, const std::string &type)=0
adds a container with the provided information
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:68
DDSLoader(BESDataHandlerInterface &dhi)
Create a loader that will hijack dhi on a load call, then restore it's state.
Definition: DDSLoader.cc:72
string action
the response object requested, e.g. das, dds
BESContainer * container
pointer to current container in this interface
virtual int get_line()
get the line number where the exception was thrown
Definition: BESError.h:115
void cleanup()
restore dhi to clean state
Definition: DDSLoader.cc:211