39 #ifdef HAVE_TR1_FUNCTIONAL 40 #include <tr1/functional> 44 #include <ConstraintEvaluator.h> 45 #include <DDXParserSAX2.h> 47 #include <XDRStreamMarshaller.h> 48 #include <XDRStreamUnMarshaller.h> 49 #include <XDRFileUnMarshaller.h> 51 #include <D4StreamMarshaller.h> 52 #include <D4StreamUnMarshaller.h> 57 #include <mime_util.h> 60 #include "CacheTypeFactory.h" 61 #include "CacheMarshaller.h" 62 #include "CacheUnMarshaller.h" 64 #include "BESDapFunctionResponseCache.h" 65 #include "BESDapResponseBuilder.h" 66 #include "BESInternalError.h" 69 #include "TheBESKeys.h" 73 #define DEBUG_KEY "response_cache" 75 #ifdef HAVE_TR1_FUNCTIONAL 76 #define HASH_OBJ std::tr1::hash 78 #define HASH_OBJ std::hash 84 const string DATA_MARK =
"--DATA:";
87 const unsigned int max_cacheable_ce_len = 4096;
88 const unsigned int max_collisions = 50;
90 const unsigned int default_cache_size = 20;
91 const string default_cache_prefix =
"rc";
92 const string default_cache_dir =
"";
94 const string BESDapFunctionResponseCache::PATH_KEY =
"DAP.FunctionResponseCache.path";
95 const string BESDapFunctionResponseCache::PREFIX_KEY =
"DAP.FunctionResponseCache.prefix";
96 const string BESDapFunctionResponseCache::SIZE_KEY =
"DAP.FunctionResponseCache.size";
100 unsigned long BESDapFunctionResponseCache::get_cache_size_from_config()
104 unsigned long size_in_megabytes = default_cache_size;
108 "BESDapFunctionResponseCache::getCacheSizeFromConfig(): Located BES key " << SIZE_KEY<<
"=" << size << endl);
109 istringstream iss(size);
110 iss >> size_in_megabytes;
113 return size_in_megabytes;
116 string BESDapFunctionResponseCache::get_cache_prefix_from_config()
119 string prefix = default_cache_prefix;
123 "BESDapFunctionResponseCache::getCachePrefixFromConfig(): Located BES key " << PREFIX_KEY<<
"=" << prefix << endl);
131 string BESDapFunctionResponseCache::get_cache_dir_from_config()
135 string cacheDir = default_cache_dir;
139 "BESDapFunctionResponseCache::getCacheDirFromConfig(): Located BES key " << PATH_KEY<<
"=" << cacheDir << endl);
163 BESDapFunctionResponseCache::get_instance(
const string &cache_dir,
const string &prefix,
unsigned long long size)
165 if (d_instance == 0) {
166 if (!cache_dir.empty() && dir_exists(cache_dir)) {
169 atexit(delete_instance);
175 "BESDapFunctionResponseCache::get_instance(dir,prefix,size) - d_instance: " << d_instance << endl);
181 BESDapFunctionResponseCache::get_instance()
183 if (d_instance == 0) {
184 string cache_dir = get_cache_dir_from_config();
185 if (!cache_dir.empty() && dir_exists(cache_dir)) {
187 get_cache_size_from_config());
189 atexit(delete_instance);
194 BESDEBUG(DEBUG_KEY,
"BESDapFunctionResponseCache::get_instance() - d_instance: " << d_instance << endl);
209 bool BESDapFunctionResponseCache::is_valid(
const string &cache_file_name,
const string &dataset)
215 off_t entry_size = 0;
216 time_t entry_time = 0;
218 if (stat(cache_file_name.c_str(), &buf) == 0) {
219 entry_size = buf.st_size;
220 entry_time = buf.st_mtime;
226 if (entry_size == 0)
return false;
228 time_t dataset_time = entry_time;
229 if (stat(dataset.c_str(), &buf) == 0) {
230 dataset_time = buf.st_mtime;
237 if (dataset_time > entry_time)
return false;
242 string BESDapFunctionResponseCache::get_resource_id(DDS *dds,
const string &constraint)
244 return dds->filename() +
"#" + constraint;
247 bool BESDapFunctionResponseCache::can_be_cached(DDS *dds,
const string &constraint)
249 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" constraint + dds->filename() length: " 250 << constraint.length() + dds->filename().size() << endl);
252 return (constraint.length() + dds->filename().size() <= max_cacheable_ce_len);
262 string BESDapFunctionResponseCache::get_hash_basename(
const string &resource_id)
265 HASH_OBJ<string> str_hash;
266 size_t hashValue = str_hash(resource_id);
267 stringstream hashed_id;
268 hashed_id << hashValue;
269 string cache_file_name = getCacheDirectory();
270 cache_file_name.append(
"/").append(getCacheFilePrefix()).append(hashed_id.str());
272 return cache_file_name;
301 string resourceId = dds->filename() +
"#" + constraint;
303 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" resourceId: '" << resourceId <<
"'" << endl);
306 HASH_OBJ<string> str_hash;
309 size_t hashValue = str_hash(resourceId);
310 stringstream hashed_id;
311 hashed_id << hashValue;
313 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" hashed_id: '" << hashed_id.str() <<
"'" << endl);
319 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" cache_file_name: '" << cache_file_name <<
"'" << endl);
325 if ((ret_dds = load_from_cache(resourceId, cache_file_name))) {
326 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" Data loaded from cache file: " << cache_file_name << endl);
327 ret_dds->filename(dds->filename());
329 else if ((ret_dds = write_dataset_to_cache(dds, resourceId, constraint, cache_file_name))) {
330 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" Data written to cache file: " << cache_file_name << endl);
334 else if ((ret_dds = load_from_cache(resourceId, cache_file_name))) {
335 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" Data loaded from cache file (2nd try): " << cache_file_name << endl);
336 ret_dds->filename(dds->filename());
339 BESDEBUG(DEBUG_KEY,__FUNCTION__ <<
" Used cache_file_name: " << cache_file_name <<
" for resource ID: " << resourceId << endl);
361 BESDapFunctionResponseCache::load_from_cache(
const string &resource_id,
string &cache_file_name)
363 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" resource_id: " << resource_id << endl);
367 unsigned long suffix_counter = 0;
368 bool keep_looking =
true;
370 if (suffix_counter > max_collisions) {
372 ss <<
"Cache error! There are " << suffix_counter <<
" hash collisions for the resource '" << resource_id
373 <<
"' And that is a bad bad thing.";
379 cfname << cache_file_name <<
"_" << suffix_counter++;
381 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" candidate cache_file_name: " << cfname.str() << endl);
384 if (!get_read_lock(cfname.str(), fd)) {
385 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" !get_read_lock(cfname.str(), fd): " << fd << endl);
388 keep_looking =
false;
391 cache_file_name = cfname.str();
398 ifstream cache_file_istream(cfname.str().c_str());
399 char line[max_cacheable_ce_len];
400 cache_file_istream.getline(line, max_cacheable_ce_len);
401 string cached_resource_id;
402 cached_resource_id.assign(line);
404 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" cached_resource_id: " << cached_resource_id << endl);
406 if (cached_resource_id.compare(resource_id) == 0) {
408 BESDEBUG(DEBUG_KEY,
"BESDapFunctionResponseCache::load_from_cache() - Cache Hit!" << endl);
411 cached_dds = read_cached_data(cache_file_istream);
414 unlock_and_close(cfname.str());
416 }
while (!cached_dds && keep_looking);
418 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" Cache " << (cached_dds!=0?
"HIT":
"MISS") <<
" for: " << cache_file_name << endl);
428 BESDapFunctionResponseCache::read_cached_data(istream &cached_data)
432 DDS *fdds =
new DDS(&factory);
434 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" - BEGIN" << endl);
437 DDXParser ddx_parser(fdds->get_factory());
443 ddx_parser.intern_stream(cached_data, fdds, data_cid, DATA_MARK);
451 for (DDS::Vars_iter i = fdds->var_begin(), e = fdds->var_end(); i != e; ++i) {
452 (*i)->deserialize(um, fdds);
457 for (DDS::Vars_iter i = fdds->var_begin(), e = fdds->var_end(); i != e; ++i) {
458 (*i)->set_read_p(
true);
459 (*i)->set_send_p(
true);
465 if ((*i)->type() == dods_sequence_c) {
466 static_cast<Sequence*
>(*i)->reset_row_number(
true);
470 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" - END." << endl);
472 fdds->set_factory(0);
493 BESDapFunctionResponseCache::write_dataset_to_cache(DDS *dds,
const string &resource_id,
const string &func_ce,
494 const string &cache_file_name)
496 BESDEBUG(DEBUG_KEY, __FUNCTION__ <<
" BEGIN " << resource_id <<
": " 497 << func_ce <<
": " << cache_file_name << endl);
502 if (create_and_lock(cache_file_name, fd)) {
505 BESDEBUG(DEBUG_KEY,__FUNCTION__ <<
" Caching " << resource_id <<
", func_ce: " << func_ce << endl);
508 ofstream cache_file_ostream(cache_file_name.c_str(), ios::out|ios::app|ios::binary);
509 if (!cache_file_ostream.is_open())
510 throw BESInternalError(
"Could not open '" + cache_file_name +
"' to write cached response.", __FILE__, __LINE__);
514 cache_file_ostream << resource_id << endl;
517 ConstraintEvaluator func_eval;
518 func_eval.parse_constraint(func_ce, *dds);
519 fdds = func_eval.eval_function_clauses(*dds);
521 fdds->print_xml_writer(cache_file_ostream,
true,
"");
523 cache_file_ostream << DATA_MARK << endl;
529 ConstraintEvaluator new_ce;
532 for (DDS::Vars_iter i = fdds->var_begin(); i != fdds->var_end(); i++) {
533 if ((*i)->send_p()) {
534 (*i)->serialize(new_ce, *fdds, m,
false);
542 exclusive_to_shared_lock(fd);
547 unsigned long long size = update_cache_info(cache_file_name);
548 if (cache_too_big(size)) update_and_purge(cache_file_name);
550 unlock_and_close(cache_file_name);
554 cache_file_ostream.close();
555 this->purge_file(cache_file_name);
556 unlock_and_close(cache_file_name);
Marshaller that knows how serialize dap data objects to a disk cache This class can be used with libd...
exception thrown if inernal error encountered
static string lowercase(const string &s)
virtual libdap::DDS * get_or_cache_dataset(libdap::DDS *dds, const std::string &constraint)
Return a DDS loaded with data that can be serialized back to a client.
virtual string get_cache_file_name(const string &src, bool mangle=true)
void get_value(const string &s, string &val, bool &found)
Retrieve the value of a given key, if set.
static BESKeys * TheKeys()
Cache the results from server functions.
UnMarshaller that knows how to deserialize dap objects.