Field3D
SparseFile.h
Go to the documentation of this file.
1 //----------------------------------------------------------------------------//
2 
3 /*
4  * Copyright (c) 2009 Sony Pictures Imageworks Inc
5  *
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in the
16  * documentation and/or other materials provided with the
17  * distribution. Neither the name of Sony Pictures Imageworks nor the
18  * names of its contributors may be used to endorse or promote
19  * products derived from this software without specific prior written
20  * permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
33  * OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 //----------------------------------------------------------------------------//
37 
42 //----------------------------------------------------------------------------//
43 
44 #ifndef _INCLUDED_Field3D_SparseFile_H_
45 #define _INCLUDED_Field3D_SparseFile_H_
46 
47 //----------------------------------------------------------------------------//
48 
49 #include <vector>
50 #include <list>
51 
52 #include <hdf5.h>
53 
54 #include "Exception.h"
55 #include "Hdf5Util.h"
56 #include "SparseDataReader.h"
57 #include "Traits.h"
58 
59 //----------------------------------------------------------------------------//
60 
61 #include "ns.h"
62 
64 
65 //----------------------------------------------------------------------------//
66 // Forward declarations
67 //----------------------------------------------------------------------------//
68 
69 namespace Sparse {
70 
71  template <typename Data_T>
72  struct SparseBlock;
73 
74 }
75 
76 template <typename Data_T>
77 class SparseField;
78 
79 //----------------------------------------------------------------------------//
80 
81 namespace SparseFile {
82 
83 //----------------------------------------------------------------------------//
84 // Reference
85 //----------------------------------------------------------------------------//
86 
92 //----------------------------------------------------------------------------//
93 
94 template <class Data_T>
95 class Reference
96 {
97 public:
98 
99  // Typedefs ------------------------------------------------------------------
100 
101  typedef std::vector<Sparse::SparseBlock<Data_T>*> BlockPtrs;
102 
103  // Public data members -------------------------------------------------------
104 
105  std::string filename;
106  std::string layerPath;
109 
111  std::vector<int> fileBlockIndices;
114  std::vector<int> blockLoaded;
121  std::vector<bool> blockUsed;
124  std::vector<int> loadCounts;
128  std::vector<int> refCounts;
132  boost::mutex *blockMutex;
133 
134  // Ctors, dtor ---------------------------------------------------------------
135 
137  Reference(const std::string filename, const std::string layerPath);
138  ~Reference();
139 
141  Reference(const Reference &o);
142 
144  Reference & operator=(const Reference &o);
145 
146  // Main methods --------------------------------------------------------------
147 
149  bool fileIsOpen();
151  void setNumBlocks(int numBlocks);
155  void openFile();
159  void loadBlock(int blockIdx);
161  void unloadBlock(int blockIdx);
164  void incBlockRef(int blockIdx);
166  void decBlockRef(int blockIdx);
168  int blockSize(int blockIdx) const;
171  int totalLoads() const;
174  int numLoadedBlocks() const;
177  int totalLoadedBlocks() const;
180  float averageLoads() const;
182  void resetCacheStatistics();
183 
184 private:
185 
188 
192 
196 
198  boost::mutex m_mutex;
199 
200 };
201 
202 //----------------------------------------------------------------------------//
203 // References
204 //----------------------------------------------------------------------------//
205 
207 {
208 public:
209 
210  // Main methods --------------------------------------------------------------
211 
214  template <class Data_T>
215  Reference<Data_T>& ref(int idx);
216 
219  template <class Data_T>
220  int append(const Reference<Data_T>& ref);
221 
223  template <class Data_T>
224  int numRefs() const;
225 
226 private:
227 
228  // Data members --------------------------------------------------------------
229 
230  std::vector<Reference<half> > m_hRefs;
231  std::vector<Reference<V3h> > m_vhRefs;
232  std::vector<Reference<float> > m_fRefs;
233  std::vector<Reference<V3f> > m_vfRefs;
234  std::vector<Reference<double> > m_dRefs;
235  std::vector<Reference<V3d> > m_vdRefs;
236 
237 };
238 
239 //----------------------------------------------------------------------------//
240 
241 class CacheBlock {
242 public:
244  int refIdx;
245  int blockIdx;
246  CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn) :
247  blockType(blockTypeIn), refIdx(refIdxIn), blockIdx(blockIdxIn)
248  { }
249 };
250 
251 //----------------------------------------------------------------------------//
252 
253 } // namespace SparseFile
254 
255 //----------------------------------------------------------------------------//
256 // SparseFileManager
257 //----------------------------------------------------------------------------//
258 
314 //----------------------------------------------------------------------------//
315 
317 {
318 
319 public:
320 
321  template <class Data_T>
322  friend class SparseField;
323 
324  // typedefs ------------------------------------------------------------------
325 
326  typedef std::list<SparseFile::CacheBlock> CacheList;
327 
328  // Main methods --------------------------------------------------------------
329 
331  static SparseFileManager &singleton();
332 
335  void setLimitMemUse(bool enabled);
336 
339  bool doLimitMemUse() const;
340 
342  void setMaxMemUse(float maxMemUse);
343 
346  void flushCache();
347 
349  long long totalLoads();
350 
352  long long numLoadedBlocks();
353 
355  long long totalLoadedBlocks();
356 
359  float cacheFractionLoaded();
360 
362  float cacheLoadsPerBlock();
363 
366  float cacheEfficiency();
367 
369  void resetCacheStatistics();
370 
371  //--------------------------------------------------------------------------//
372  // Utility functions
373 
378  template <class Data_T>
379  void incBlockRef(int fileId, int blockIdx);
380 
385  template <class Data_T>
386  void decBlockRef(int fileId, int blockIdx);
387 
391  template <class Data_T>
392  void activateBlock(int fileId, int blockIdx);
393 
394 protected:
395 
397  template <class Data_T>
399 
402  template <class Data_T>
403  int getNextId(const std::string filename, const std::string layerPath);
404 
405  template <class Data_T>
406  void removeFieldFromCache(int refIdx);
407 
408 private:
409 
412 
415 
417  void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx);
418 
421  void deallocateBlocks(int bytesNeeded);
422 
425  template <class Data_T>
427 
429  template <class Data_T>
430  void deallocateBlock(CacheList::iterator &it);
431 
433  float m_maxMemUse;
434 
437 
439  int m_memUse;
440 
444 
448 
455 
458  CacheList::iterator m_nextBlock;
459 
462  boost::mutex m_mutex;
463 
464 };
465 
466 //----------------------------------------------------------------------------//
467 // Reference implementations
468 //----------------------------------------------------------------------------//
469 
470 namespace SparseFile {
471 
472 //----------------------------------------------------------------------------//
473 
474 template <class Data_T>
475 Reference<Data_T>::Reference(const std::string a_filename,
476  const std::string a_layerPath)
477  : filename(a_filename), layerPath(a_layerPath),
478  valuesPerBlock(-1), occupiedBlocks(-1),
479  blockMutex(NULL), m_fileHandle(-1), m_reader(NULL) {
480  /* Empty */
481 }
482 
483 //----------------------------------------------------------------------------//
484 
485 template <class Data_T>
487 {
488  if (m_reader)
489  delete m_reader;
490 
491  if (blockMutex)
492  delete [] blockMutex;
493 }
494 
495 //----------------------------------------------------------------------------//
496 
497 template <class Data_T>
499 {
500  m_reader = NULL;
501  blockMutex = NULL;
502  *this = o;
503 }
504 
505 //----------------------------------------------------------------------------//
506 
507 template <class Data_T>
510 {
511  // Copy public member variables (where appropriate)
512  filename = o.filename;
513  layerPath = o.layerPath;
514  valuesPerBlock = o.valuesPerBlock;
515  occupiedBlocks = o.occupiedBlocks;
516  fileBlockIndices = o.fileBlockIndices;
517  blockLoaded = o.blockLoaded;
518  blocks = o.blocks;
519  blockUsed = o.blockUsed;
520  loadCounts = o.loadCounts;
521  refCounts = o.refCounts;
522  if (blockMutex)
523  delete[] blockMutex;
524  blockMutex = new boost::mutex[blocks.size()];
525 
526  // Copy private member variables (where appropriate)
527  m_fileHandle = o.m_fileHandle;
528  // Don't copy id, let hdf5 generate a new one.
529  if (m_fileHandle >= 0) {
530  m_layerGroup.open(m_fileHandle, layerPath.c_str());
531  }
532 
533  if (m_reader)
534  delete m_reader;
535  m_reader = NULL;
536 
537  return *this;
538 }
539 
540 //----------------------------------------------------------------------------//
541 
542 template <class Data_T>
544 {
545  return m_fileHandle >= 0;
546 }
547 
548 //----------------------------------------------------------------------------//
549 
550 template <class Data_T>
552 {
553  boost::mutex::scoped_lock lock(m_mutex);
554 
555  fileBlockIndices.resize(numBlocks);
556  blockLoaded.resize(numBlocks, 0);
557  blocks.resize(numBlocks, 0);
558  blockUsed.resize(numBlocks, false);
559  loadCounts.resize(numBlocks, 0);
560  refCounts.resize(numBlocks, 0);
561  if (blockMutex)
562  delete[] blockMutex;
563  blockMutex = new boost::mutex[numBlocks];
564 }
565 
566 //----------------------------------------------------------------------------//
567 
568 template <class Data_T>
570 {
571  using namespace Exc;
572  using namespace Hdf5Util;
573 
574  boost::mutex::scoped_lock lock_A(m_mutex);
575 
576  // check that the file wasn't already opened before obtaining the lock
577  if (fileIsOpen()) {
578  return;
579  }
580 
581  m_fileHandle = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
582  if (m_fileHandle < 0)
583  throw NoSuchFileException(filename);
584 
585  m_layerGroup.open(m_fileHandle, layerPath.c_str());
586  if (m_layerGroup.id() < 0) {
587  Msg::print(Msg::SevWarning, "In SparseFile::Reference::openFile: "
588  "Couldn't find layer group " + layerPath +
589  " in .f3d file ");
590  throw FileIntegrityException(filename);
591  }
592 
593  m_reader = new SparseDataReader<Data_T>(m_layerGroup.id(), valuesPerBlock,
594  occupiedBlocks);
595 }
596 
597 //----------------------------------------------------------------------------//
598 
599 template <class Data_T>
601 {
602  boost::mutex::scoped_lock lock(m_mutex);
603 
604  // Allocate the block
605  blocks[blockIdx]->resize(valuesPerBlock);
606  assert(blocks[blockIdx]->data.size() > 0);
607  // Read the data
608  assert(m_reader);
609  m_reader->readBlock(fileBlockIndices[blockIdx], blocks[blockIdx]->dataRef());
610  // Mark block as loaded
611  blockLoaded[blockIdx] = 1;
612 }
613 
614 //----------------------------------------------------------------------------//
615 
616 template <class Data_T>
618 {
619  // Deallocate the block
620  blocks[blockIdx]->clear();
621 
622  // Mark block as unloaded
623  blockLoaded[blockIdx] = 0;
624 }
625 
626 //----------------------------------------------------------------------------//
627 
628 template <class Data_T>
630 {
631  boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
632  ++refCounts[blockIdx];
633 }
634 
635 //----------------------------------------------------------------------------//
636 
637 template <class Data_T>
639 {
640  boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
641  --refCounts[blockIdx];
642 }
643 
644 //----------------------------------------------------------------------------//
645 
646 template <class Data_T>
647 int Reference<Data_T>::blockSize(int /* blockIdx */) const
648 {
649  return valuesPerBlock * sizeof(Data_T);
650 }
651 
652 //----------------------------------------------------------------------------//
653 
654 template <class Data_T>
656 {
657  std::vector<int>::const_iterator i = loadCounts.begin();
658  std::vector<int>::const_iterator end = loadCounts.end();
659  int numLoads = 0;
660  for (; i != end; ++i)
661  numLoads += *i;
662 
663  return numLoads;
664 }
665 
666 //----------------------------------------------------------------------------//
667 
668 template <class Data_T>
670 {
671  std::vector<int>::const_iterator i = blockLoaded.begin();
672  std::vector<int>::const_iterator end = blockLoaded.end();
673  int numBlocks = 0;
674  for (; i != end; ++i)
675  if (*i)
676  numBlocks++;
677 
678  return numBlocks;
679 }
680 
681 //----------------------------------------------------------------------------//
682 
683 template <class Data_T>
685 {
686  std::vector<int>::const_iterator i = loadCounts.begin();
687  std::vector<int>::const_iterator li = blockLoaded.begin();
688  std::vector<int>::const_iterator end = loadCounts.end();
689  int numBlocks = 0;
690 
691  if (blockLoaded.size() == 0) {
692  for (; i != end; ++i)
693  if (*i)
694  numBlocks++;
695  } else {
696  assert(loadCounts.size() == blockLoaded.size());
697 
698  for (; i != end; ++i, ++li)
699  if (*i || *li)
700  numBlocks++;
701  }
702 
703  return numBlocks;
704 }
705 
706 //----------------------------------------------------------------------------//
707 
708 template <class Data_T>
710 {
711  std::vector<int>::const_iterator i = loadCounts.begin();
712  std::vector<int>::const_iterator end = loadCounts.end();
713  int numLoads = 0, numBlocks = 0;
714  for (; i != end; ++i) {
715  if (*i) {
716  numLoads += *i;
717  numBlocks++;
718  }
719  }
720 
721  return (float)numLoads / std::max(1, numBlocks);
722 }
723 
724 //----------------------------------------------------------------------------//
725 
726 template <class Data_T>
728 {
729  std::vector<int>::iterator li = loadCounts.begin();
730  std::vector<int>::iterator lend = loadCounts.end();
731  for (; li != lend; ++li)
732  *li = 0;
733 }
734 
735 //----------------------------------------------------------------------------//
736 
737 } // namespace SparseFile
738 
739 //----------------------------------------------------------------------------//
740 // Specializations for FileReferences
741 //----------------------------------------------------------------------------//
742 
743 namespace SparseFile {
744 
745 //----------------------------------------------------------------------------//
746 
747 template <>
748 inline Reference<half>&
750 {
751  return m_hRefs[idx];
752 }
753 
754 //----------------------------------------------------------------------------//
755 
756 template <>
757 inline Reference<V3h>&
759 {
760  return m_vhRefs[idx];
761 }
762 
763 //----------------------------------------------------------------------------//
764 
765 template <>
766 inline Reference<float>&
768 {
769  return m_fRefs[idx];
770 }
771 
772 //----------------------------------------------------------------------------//
773 
774 template <>
775 inline Reference<V3f>&
777 {
778  return m_vfRefs[idx];
779 }
780 
781 //----------------------------------------------------------------------------//
782 
783 template <>
784 inline Reference<double>&
786 {
787  return m_dRefs[idx];
788 }
789 
790 //----------------------------------------------------------------------------//
791 
792 template <>
793 inline Reference<V3d>&
795 {
796  return m_vdRefs[idx];
797 }
798 
799 //----------------------------------------------------------------------------//
800 
801 template <>
803 {
804  m_hRefs.push_back(ref);
805  return m_hRefs.size() - 1;
806 }
807 
808 //----------------------------------------------------------------------------//
809 
810 template <>
812 {
813  m_vhRefs.push_back(ref);
814  return m_vhRefs.size() - 1;
815 }
816 
817 //----------------------------------------------------------------------------//
818 
819 template <>
821 {
822  m_fRefs.push_back(ref);
823  return m_fRefs.size() - 1;
824 }
825 
826 //----------------------------------------------------------------------------//
827 
828 template <>
830 {
831  m_vfRefs.push_back(ref);
832  return m_vfRefs.size() - 1;
833 }
834 
835 //----------------------------------------------------------------------------//
836 
837 template <>
839 {
840  m_dRefs.push_back(ref);
841  return m_dRefs.size() - 1;
842 }
843 
844 //----------------------------------------------------------------------------//
845 
846 template <>
848 {
849  m_vdRefs.push_back(ref);
850  return m_vdRefs.size() - 1;
851 }
852 
853 //----------------------------------------------------------------------------//
854 
855 template <>
856 inline int FileReferences::numRefs<half>() const
857 {
858  return m_hRefs.size();
859 }
860 
861 //----------------------------------------------------------------------------//
862 
863 template <>
864 inline int FileReferences::numRefs<V3h>() const
865 {
866  return m_vhRefs.size();
867 }
868 
869 //----------------------------------------------------------------------------//
870 
871 template <>
872 inline int FileReferences::numRefs<float>() const
873 {
874  return m_fRefs.size();
875 }
876 
877 //----------------------------------------------------------------------------//
878 
879 template <>
880 inline int FileReferences::numRefs<V3f>() const
881 {
882  return m_vfRefs.size();
883 }
884 
885 //----------------------------------------------------------------------------//
886 
887 template <>
888 inline int FileReferences::numRefs<double>() const
889 {
890  return m_dRefs.size();
891 }
892 
893 //----------------------------------------------------------------------------//
894 
895 template <>
896 inline int FileReferences::numRefs<V3d>() const
897 {
898  return m_vdRefs.size();
899 }
900 
901 //----------------------------------------------------------------------------//
902 // Implementations for FileReferences
903 //----------------------------------------------------------------------------//
904 
905 template <class Data_T>
907 {
908  assert(false && "Do not use memory limiting on sparse fields that aren't "
909  "simple scalars or vectors!");
911  "FileReferences::ref(): Do not use memory limiting on sparse "
912  "fields that aren't simple scalars or vectors!");
913  static Reference<Data_T> dummy("", "");
914  return dummy;
915 }
916 
917 //----------------------------------------------------------------------------//
918 
919 template <class Data_T>
921 {
922  assert(false && "Do not use memory limiting on sparse fields that aren't "
923  "simple scalars or vectors!");
925  "FileReferences::append(): Do not use memory limiting on sparse "
926  "fields that aren't simple scalars or vectors!");
927  return -1;
928 }
929 
930 //----------------------------------------------------------------------------//
931 
932 template <class Data_T>
934 {
935  assert(false && "Do not use memory limiting on sparse fields that aren't "
936  "simple scalars or vectors!");
938  "FileReferences::numRefs(): "
939  "Do not use memory limiting on sparse "
940  "fields that aren't "
941  "simple scalars or vectors!");
942  return -1;
943 }
944 
945 //----------------------------------------------------------------------------//
946 
947 } // namespace SparseFile
948 
949 //----------------------------------------------------------------------------//
950 // SparseFileManager implementations
951 //----------------------------------------------------------------------------//
952 
953 template <class Data_T>
954 int
955 SparseFileManager::getNextId(const std::string filename,
956  const std::string layerPath)
957 {
958  using namespace SparseFile;
959 
960  int id = m_fileData.append(Reference<Data_T>(filename, layerPath));
961  return id;
962 }
963 
964 //----------------------------------------------------------------------------//
965 
966 template <class Data_T>
967 void
969 {
970  boost::mutex::scoped_lock lock(m_mutex);
971 
973  SparseFile::Reference<Data_T> &reference = m_fileData.ref<Data_T>(refIdx);
974 
975  CacheList::iterator it = m_blockCacheList.begin();
976  CacheList::iterator end = m_blockCacheList.end();
977  CacheList::iterator next;
978 
979  int bytesFreed = 0;
980 
981  while (it != end) {
982  if (it->blockType == blockType && it->refIdx == refIdx) {
983  if (it == m_nextBlock) {
984  ++m_nextBlock;
985  }
986  next = it;
987  ++next;
988  bytesFreed += reference.blockSize(it->blockIdx);
989  m_blockCacheList.erase(it);
990  it = next;
991  } else {
992  ++it;
993  }
994  }
995  m_memUse -= bytesFreed;
996 
997  // reset the block indices to -1, to ensure that the cache manager
998  // won't try to activate a block
999  reference.fileBlockIndices.clear();
1000  reference.fileBlockIndices.resize(reference.blocks.size(), -1);
1001  // clear the reference's pointers into the field, and relevant
1002  reference.blocks.clear();
1003  reference.blockLoaded.clear();
1004  reference.blockUsed.clear();
1005 }
1006 
1007 //----------------------------------------------------------------------------//
1008 
1009 template <class Data_T>
1012 {
1013  return m_fileData.ref<Data_T>(index);
1014 }
1015 
1016 //----------------------------------------------------------------------------//
1017 
1018 template <class Data_T>
1019 void
1020 SparseFileManager::activateBlock(int fileId, int blockIdx)
1021 {
1022  SparseFile::Reference<Data_T> &reference = m_fileData.ref<Data_T>(fileId);
1023 
1024  if (reference.fileBlockIndices[blockIdx] >= 0) {
1025  if (!reference.blockLoaded[blockIdx]) {
1026  int blockSize = reference.blockSize(blockIdx);
1027  if (m_limitMemUse) {
1028  // if we already have enough free memory, deallocateBlocks()
1029  // will just return
1030  deallocateBlocks(blockSize);
1031  }
1032 
1033  if (!reference.fileIsOpen()) {
1034  reference.openFile();
1035  }
1036 
1037  boost::mutex::scoped_lock lock_A(m_mutex);
1038  boost::mutex::scoped_lock lock_B(reference.blockMutex[blockIdx]);
1039  // check to see if it was loaded between when the function
1040  // started and we got the lock on the block
1041  if (!reference.blockLoaded[blockIdx]) {
1042  reference.loadBlock(blockIdx);
1043  reference.loadCounts[blockIdx]++;
1044  addBlockToCache(DataTypeTraits<Data_T>::typeEnum(), fileId, blockIdx);
1045  m_memUse += blockSize;
1046  }
1047  }
1048  }
1049  reference.blockUsed[blockIdx] = true;
1050 }
1051 
1052 //----------------------------------------------------------------------------//
1053 
1054 template <class Data_T>
1055 void
1056 SparseFileManager::incBlockRef(int fileId, int blockIdx)
1057 {
1058  SparseFile::Reference<Data_T> &reference = m_fileData.ref<Data_T>(fileId);
1059 
1060  if (reference.fileBlockIndices[blockIdx] >= 0) {
1061  reference.incBlockRef(blockIdx);
1062  }
1063 }
1064 
1065 //----------------------------------------------------------------------------//
1066 
1067 template <class Data_T>
1068 void
1069 SparseFileManager::decBlockRef(int fileId, int blockIdx)
1070 {
1071  SparseFile::Reference<Data_T> &reference = m_fileData.ref<Data_T>(fileId);
1072 
1073  if (reference.fileBlockIndices[blockIdx] >= 0) {
1074  reference.decBlockRef(blockIdx);
1075  }
1076 }
1077 
1078 //----------------------------------------------------------------------------//
1079 
1081 
1082 //----------------------------------------------------------------------------//
1083 
1084 #endif