44 #ifndef _INCLUDED_Field3D_SparseFile_H_ 45 #define _INCLUDED_Field3D_SparseFile_H_ 71 template <
typename Data_T>
76 template <
typename Data_T>
94 template <
class Data_T>
101 typedef std::vector<Sparse::SparseBlock<Data_T>*>
BlockPtrs;
137 Reference(
const std::string filename,
const std::string layerPath);
151 void setNumBlocks(
int numBlocks);
159 void loadBlock(
int blockIdx);
161 void unloadBlock(
int blockIdx);
164 void incBlockRef(
int blockIdx);
166 void decBlockRef(
int blockIdx);
168 int blockSize(
int blockIdx)
const;
171 int totalLoads()
const;
174 int numLoadedBlocks()
const;
177 int totalLoadedBlocks()
const;
180 float averageLoads()
const;
182 void resetCacheStatistics();
214 template <
class Data_T>
219 template <
class Data_T>
223 template <
class Data_T>
247 blockType(blockTypeIn), refIdx(refIdxIn), blockIdx(blockIdxIn)
321 template <
class Data_T>
335 void setLimitMemUse(
bool enabled);
339 bool doLimitMemUse()
const;
342 void setMaxMemUse(
float maxMemUse);
349 long long totalLoads();
352 long long numLoadedBlocks();
355 long long totalLoadedBlocks();
359 float cacheFractionLoaded();
362 float cacheLoadsPerBlock();
366 float cacheEfficiency();
369 void resetCacheStatistics();
378 template <
class Data_T>
379 void incBlockRef(
int fileId,
int blockIdx);
385 template <
class Data_T>
386 void decBlockRef(
int fileId,
int blockIdx);
391 template <
class Data_T>
392 void activateBlock(
int fileId,
int blockIdx);
397 template <
class Data_T>
402 template <
class Data_T>
403 int getNextId(
const std::string filename,
const std::string layerPath);
405 template <
class Data_T>
406 void removeFieldFromCache(
int refIdx);
417 void addBlockToCache(
DataTypeEnum blockType,
int fileId,
int blockIdx);
421 void deallocateBlocks(
int bytesNeeded);
425 template <
class Data_T>
429 template <
class Data_T>
430 void deallocateBlock(CacheList::iterator &it);
474 template <
class Data_T>
476 const std::string a_layerPath)
477 : filename(a_filename), layerPath(a_layerPath),
478 valuesPerBlock(-1), occupiedBlocks(-1),
479 blockMutex(NULL), m_fileHandle(-1), m_reader(NULL) {
485 template <
class Data_T>
497 template <
class Data_T>
507 template <
class Data_T>
542 template <
class Data_T>
550 template <
class Data_T>
553 boost::mutex::scoped_lock lock(
m_mutex);
557 blocks.resize(numBlocks, 0);
568 template <
class Data_T>
574 boost::mutex::scoped_lock lock_A(
m_mutex);
583 throw NoSuchFileException(
filename);
588 "Couldn't find layer group " +
layerPath +
590 throw FileIntegrityException(
filename);
599 template <
class Data_T>
602 boost::mutex::scoped_lock lock(
m_mutex);
606 assert(
blocks[blockIdx]->data.size() > 0);
616 template <
class Data_T>
620 blocks[blockIdx]->clear();
628 template <
class Data_T>
631 boost::mutex::scoped_lock lock(
blockMutex[blockIdx]);
637 template <
class Data_T>
640 boost::mutex::scoped_lock lock(
blockMutex[blockIdx]);
646 template <
class Data_T>
654 template <
class Data_T>
657 std::vector<int>::const_iterator i =
loadCounts.begin();
658 std::vector<int>::const_iterator end =
loadCounts.end();
660 for (; i != end; ++i)
668 template <
class Data_T>
671 std::vector<int>::const_iterator i =
blockLoaded.begin();
672 std::vector<int>::const_iterator end =
blockLoaded.end();
674 for (; i != end; ++i)
683 template <
class Data_T>
686 std::vector<int>::const_iterator i =
loadCounts.begin();
687 std::vector<int>::const_iterator li =
blockLoaded.begin();
688 std::vector<int>::const_iterator end =
loadCounts.end();
692 for (; i != end; ++i)
698 for (; i != end; ++i, ++li)
708 template <
class Data_T>
711 std::vector<int>::const_iterator i =
loadCounts.begin();
712 std::vector<int>::const_iterator end =
loadCounts.end();
713 int numLoads = 0, numBlocks = 0;
714 for (; i != end; ++i) {
721 return (
float)numLoads / std::max(1, numBlocks);
726 template <
class Data_T>
729 std::vector<int>::iterator li =
loadCounts.begin();
730 std::vector<int>::iterator lend =
loadCounts.end();
731 for (; li != lend; ++li)
760 return m_vhRefs[idx];
778 return m_vfRefs[idx];
796 return m_vdRefs[idx];
804 m_hRefs.push_back(ref);
805 return m_hRefs.size() - 1;
813 m_vhRefs.push_back(ref);
814 return m_vhRefs.size() - 1;
822 m_fRefs.push_back(ref);
823 return m_fRefs.size() - 1;
831 m_vfRefs.push_back(ref);
832 return m_vfRefs.size() - 1;
840 m_dRefs.push_back(ref);
841 return m_dRefs.size() - 1;
849 m_vdRefs.push_back(ref);
850 return m_vdRefs.size() - 1;
856 inline int FileReferences::numRefs<half>()
const 858 return m_hRefs.size();
864 inline int FileReferences::numRefs<V3h>()
const 866 return m_vhRefs.size();
872 inline int FileReferences::numRefs<float>()
const 874 return m_fRefs.size();
880 inline int FileReferences::numRefs<V3f>()
const 882 return m_vfRefs.size();
888 inline int FileReferences::numRefs<double>()
const 890 return m_dRefs.size();
896 inline int FileReferences::numRefs<V3d>()
const 898 return m_vdRefs.size();
905 template <
class Data_T>
908 assert(
false &&
"Do not use memory limiting on sparse fields that aren't " 909 "simple scalars or vectors!");
911 "FileReferences::ref(): Do not use memory limiting on sparse " 912 "fields that aren't simple scalars or vectors!");
919 template <
class Data_T>
922 assert(
false &&
"Do not use memory limiting on sparse fields that aren't " 923 "simple scalars or vectors!");
925 "FileReferences::append(): Do not use memory limiting on sparse " 926 "fields that aren't simple scalars or vectors!");
932 template <
class Data_T>
935 assert(
false &&
"Do not use memory limiting on sparse fields that aren't " 936 "simple scalars or vectors!");
938 "FileReferences::numRefs(): " 939 "Do not use memory limiting on sparse " 940 "fields that aren't " 941 "simple scalars or vectors!");
953 template <
class Data_T>
966 template <
class Data_T>
970 boost::mutex::scoped_lock lock(
m_mutex);
975 CacheList::iterator it = m_blockCacheList.begin();
976 CacheList::iterator end = m_blockCacheList.end();
977 CacheList::iterator next;
982 if (it->blockType == blockType && it->refIdx == refIdx) {
983 if (it == m_nextBlock) {
988 bytesFreed += reference.
blockSize(it->blockIdx);
989 m_blockCacheList.erase(it);
995 m_memUse -= bytesFreed;
1002 reference.
blocks.clear();
1009 template <
class Data_T>
1013 return m_fileData.ref<Data_T>(index);
1018 template <
class Data_T>
1027 if (m_limitMemUse) {
1030 deallocateBlocks(blockSize);
1037 boost::mutex::scoped_lock lock_A(
m_mutex);
1038 boost::mutex::scoped_lock lock_B(reference.
blockMutex[blockIdx]);
1054 template <
class Data_T>
1067 template <
class Data_T>
int numLoadedBlocks() const
Returns the total number of blocks that are currently loaded, for statistics.
void removeFieldFromCache(int refIdx)
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Contains utility functions and classes for Hdf5 files.
hid_t m_fileHandle
Holds the Hdf5 handle to the file.
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full...
Namespace for Exception objects.
void activateBlock(int fileId, int blockIdx)
Called by SparseField when it's about to read from a block. This should not be called by the user...
std::vector< int > blockLoaded
Whether each block is loaded. We don't use bools since vector<bool> is weird.
void incBlockRef(int fileId, int blockIdx)
Increments the usage reference count on the specified block, to prevent it from getting unloaded whil...
void incBlockRef(int blockIdx)
Increment reference count on a block, indicates the block is currently in use, so prevents it from be...
int m_memUse
Current amount of memory in use in bytes.
std::vector< bool > blockUsed
Flags of whether the blocks have been accessed since they were last considered for deallocation by th...
CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn)
std::vector< Sparse::SparseBlock< Data_T > * > BlockPtrs
Reference< Data_T > & ref(int idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed...
int numRefs() const
Returns the number of file references of the corresponding collection.
void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity. ...
std::vector< int > fileBlockIndices
Index in file for each block.
int m_maxMemUseInBytes
Max amount om memory to use in bytes.
void setNumBlocks(int numBlocks)
Sets the number of blocks used by the SparseField we're supporting.
static DataTypeEnum typeEnum()
void decBlockRef(int blockIdx)
Decrement reference count on a block.
std::vector< int > refCounts
Per-block counts of the number of current references to the blocks. If a block's ref count is non-zer...
Contains various utility functions for Hdf5.
std::vector< Reference< double > > m_dRefs
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
float m_maxMemUse
Max amount om memory to use in megabytes.
SparseDataReader< Data_T > * m_reader
Pointer to the reader object. NULL at construction time. Created in openFile().
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
int blockSize(int blockIdx) const
Returns the number of bytes used by the data in the block.
std::vector< Reference< V3f > > m_vfRefs
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
std::vector< Reference< V3h > > m_vhRefs
static SparseFileManager * ms_singleton
Pointer to singleton.
This class gets used by SparseFieldIO and SparseFileManager to read the block data. On creation it will open the data set and not close it until the object is destroyed.
std::vector< Reference< half > > m_hRefs
BlockPtrs blocks
Pointers to each block. This is so we can go in and manipulate them as we please. ...
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
void unloadBlock(int blockIdx)
Unloads the block with the given index from memory.
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
Namespace for sparse field specifics.
std::list< SparseFile::CacheBlock > CacheList
Contains functions controlling the loading of sparse fields.
boost::mutex m_mutex
Mutex to prevent two threads from modifying conflicting data.
void resetCacheStatistics()
Resets counts of total block loads.
Hdf5Util::H5ScopedGopen m_layerGroup
Hold the group containing the data open for the duration of the Reference's existence.
SparseFile::Reference< Data_T > & reference(int index)
Returns a reference to the Reference object with the given index.
std::vector< Reference< float > > m_fRefs
int getNextId(const std::string filename, const std::string layerPath)
Returns the id of the next cache item. This is stored in the SparseField in order to reference its fi...
void decBlockRef(int fileId, int blockIdx)
Decrements the usage reference count on the specified block, after its value is no longer being used ...
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
Reference & operator=(const Reference &o)
Assignment operator. Clears ref counts and rebuilds mutex array.
float averageLoads() const
Returns the average number of loads per accessed block in this file, for cache statistics.
int totalLoadedBlocks() const
Returns the total number of blocks that were ever loaded (max 1 per block, not the number of blocks)...
Reference(const std::string filename, const std::string layerPath)
Constructor. Requires the filename and layer path of the field to be known.
bool fileIsOpen()
Checks if the file used by this reference is open already.
std::vector< int > loadCounts
Per-block counts of the number of times each block has been loaded, for cache statistics.
hid_t id() const
Query the hid_t value.
This Field subclass stores voxel data in block-allocated arrays.
Scoped object - opens a group on creation and closes it on destruction.
Contains Exception base class.
std::vector< Reference< V3d > > m_vdRefs
boost::mutex * blockMutex
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-s...
void open(hid_t parentLocation, const std::string &name)
int totalLoads() const
Returns the total number of loads of the blocks of this file, for cache statistics.
int append(const Reference< Data_T > &ref)
Appends a reference to the collection. This is specialized so that the correct data member is accesse...