casacore
TSMCube.h
Go to the documentation of this file.
1 //# TSMCube.h: Tiled hypercube in a table
2 //# Copyright (C) 1995,1996,1997,1999,2000,2001,2002
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id: TSMCube.h 21521 2014-12-10 08:06:42Z gervandiepen $
27 
28 #ifndef TABLES_TSMCUBE_H
29 #define TABLES_TSMCUBE_H
30 
31 
32 //# Includes
33 #include <casacore/casa/aips.h>
34 #include <casacore/tables/DataMan/TSMShape.h>
35 #include <casacore/casa/Containers/Record.h>
36 #include <casacore/casa/Arrays/IPosition.h>
37 #include <casacore/casa/OS/Conversion.h>
38 #include <casacore/casa/iosfwd.h>
39 
40 namespace casacore { //# NAMESPACE CASACORE - BEGIN
41 
42 //# Forward declarations
43 class TiledStMan;
44 class TSMFile;
45 class TSMColumn;
46 class BucketCache;
47 template<class T> class Block;
48 
49 // <summary>
50 // Tiled hypercube in a table
51 // </summary>
52 
53 // <use visibility=local>
54 
55 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
56 // </reviewed>
57 
58 // <prerequisite>
59 //# Classes you should understand before using this one.
60 // <li> <linkto class=TiledStMan>TiledStMan</linkto>
61 // <li> <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
62 // for a discussion of the maximum cache size
63 // <li> <linkto class=TSMFile>TSMFile</linkto>
64 // <li> <linkto class=BucketCache>BucketCache</linkto>
65 // </prerequisite>
66 
67 // <etymology>
68 // TSMCube represents a hypercube in the Tiled Storage Manager.
69 // </etymology>
70 
71 // <synopsis>
72 // TSMCube defines a tiled hypercube. The data is stored in a TSMFile
73 // object and accessed using a BucketCache object. The hypercube can
74 // be extensible in its last dimension to support tables with a size
75 // which is not known in advance.
76 // <br>
77 // Normally hypercubes share the same TSMFile object, but extensible
78 // hypercubes have their own TSMFile object (to be extensible).
79 // If the hypercolumn has multiple data columns, their cells share the same
80 // tiles. Per tile data column A appears first, thereafter B, etc..
81 // <br>
82 // The data in the cache is held in external format and is converted
83 // when accessed. The alternative would be to hold it in the cache in
84 // local format and convert it when read/written from the file. It was
85 // felt that the latter approach would generate more needless conversions.
86 // <p>
87 // The possible id and coordinate values are stored in a Record
88 // object. They are written in the main hypercube AipsIO file.
89 // <p>
90 // TSMCube uses the maximum cache size set for a Tiled Storage manager.
91 // The description of class
92 // <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
93 // contains a discussion about the effect of setting the maximum cache size.
94 // </synopsis>
95 
96 // <motivation>
97 // TSMCube encapsulates all operations on a hypercube.
98 // </motivation>
99 
100 //# <todo asof="$DATE:$">
101 //# A List of bugs, limitations, extensions or planned refinements.
102 //# </todo>
103 
104 
105 class TSMCube
106 {
107 public:
108  // Define the possible access types for TSMDataColumn.
109  enum AccessType {
115  };
116 
117  // Construct the hypercube using the given file with the given shape.
118  // The record contains the id and possible coordinate values.
119  // <br>If the cubeshape is empty, the hypercube is still undefined and
120  // can be added later with setShape. That is only used by TiledCellStMan.
121  // <br> The fileOffset argument is meant for class TiledFileAccess.
122  TSMCube (TiledStMan* stman, TSMFile* file,
123  const IPosition& cubeShape,
124  const IPosition& tileShape,
125  const Record& values,
126  Int64 fileOffset,
127  Bool useDerived = False);
128 
129  // Reconstruct the hypercube by reading its data from the AipsIO stream.
130  // It will link itself to the correct TSMFile. The TSMFile objects
131  // must have been reconstructed in advance.
132  TSMCube (TiledStMan* stman, AipsIO& ios,
133  Bool useDerived = False);
134 
135  virtual ~TSMCube();
136 
137  // Flush the data in the cache.
138  virtual void flushCache();
139 
140  // Clear the cache, so data will be reread.
141  // If wanted, the data is flushed before the cache is cleared.
142  void clearCache (Bool doFlush = True);
143 
144  // Empty the cache.
145  // It will flush the cache as needed and remove all buckets from it
146  // resulting in a possibly large drop in memory used.
147  // It'll also clear the <src>userSetCache_p</src> flag.
148  void emptyCache();
149 
150  // Show the cache statistics.
151  virtual void showCacheStatistics (ostream& os) const;
152 
153  // Put the data of the object into the AipsIO stream.
154  void putObject (AipsIO& ios);
155 
156  // Get the data of the object from the AipsIO stream.
157  // It returns the data manager sequence number, which is -1 if
158  // no file is attached to the cube (for cells without a value).
159  Int getObject (AipsIO& ios);
160 
161  // Resync the object with the data file.
162  // It reads the object, and adjusts the cache.
163  virtual void resync (AipsIO& ios);
164 
165  // Is the hypercube extensible?
166  Bool isExtensible() const;
167 
168  // Get the bucket size (bytes).
169  // It is the length of a tile in external format.
170  uInt bucketSize() const;
171 
172  // Get the length of a tile (in bytes) in local format.
173  uInt localTileLength() const;
174 
175  // Set the hypercube shape.
176  // This is only possible if the shape was not defined yet.
177  virtual void setShape (const IPosition& cubeShape,
178  const IPosition& tileShape);
179 
180  // Get the shape of the hypercube.
181  const IPosition& cubeShape() const;
182 
183  // Get the shape of the tiles.
184  const IPosition& tileShape() const;
185 
186  // Get the shape of the data cells in the cube.
187  IPosition cellShape() const;
188 
189  // Get the size of a coordinate (i.e. the number of values in it).
190  // If not defined, it returns zero.
191  uInt coordinateSize (const String& coordinateName) const;
192 
193  // Get the record containing the id and coordinate values.
194  // It is used by TSMIdColumn and TSMCoordColumn.
195  // <group>
196  const Record& valueRecord() const;
198  // </group>
199 
200  // Test if the id values match.
201  Bool matches (const PtrBlock<TSMColumn*>& idColSet,
202  const Record& idValues);
203 
204  // Extend the last dimension of the cube with the given number.
205  // The record can contain the coordinates of the elements added.
206  virtual void extend (uInt nr, const Record& coordValues,
207  const TSMColumn* lastCoordColumn);
208 
209  // Extend the coordinates vector for the given coordinate
210  // to the given length with the given coordValues.
211  // It will be initialized to zero if no coordValues are given.
212  // If the coordinate vector does not exist yet, it will be created.
213  void extendCoordinates (const Record& coordValues,
214  const String& coordName, uInt length);
215 
216  // Read or write a section in the cube.
217  // It is assumed that the section buffer is long enough.
218  virtual void accessSection (const IPosition& start, const IPosition& end,
219  char* section, uInt colnr,
220  uInt localPixelSize, uInt externalPixelSize,
221  Bool writeFlag);
222 
223  // Read or write a section in a strided way.
224  // It is assumed that the section buffer is long enough.
225  virtual void accessStrided (const IPosition& start, const IPosition& end,
226  const IPosition& stride,
227  char* section, uInt colnr,
228  uInt localPixelSize, uInt externalPixelSize,
229  Bool writeFlag);
230 
231  // Get the current cache size (in buckets).
232  uInt cacheSize() const;
233 
234  // Calculate the cache size (in buckets) for the given slice
235  // and access path.
236  // <group>
237  uInt calcCacheSize (const IPosition& sliceShape,
238  const IPosition& windowStart,
239  const IPosition& windowLength,
240  const IPosition& axisPath) const;
241  static uInt calcCacheSize (const IPosition& cubeShape,
242  const IPosition& tileShape,
243  Bool extensible,
244  const IPosition& sliceShape,
245  const IPosition& windowStart,
246  const IPosition& windowLength,
247  const IPosition& axisPath,
248  uInt maxCacheSizeMiB, uInt bucketSize);
249  // </group>
250 
251  // Set the cache size for the given slice and access path.
252  virtual void setCacheSize (const IPosition& sliceShape,
253  const IPosition& windowStart,
254  const IPosition& windowLength,
255  const IPosition& axisPath,
256  Bool forceSmaller, Bool userSet);
257 
258  // Resize the cache object.
259  // If forceSmaller is False, the cache will only be resized when it grows.
260  // If the given size exceeds the maximum size with more
261  // than 10%, the maximum size will be used.
262  // The cacheSize has to be given in buckets.
263  // <br>The flag <src>userSet</src> inidicates if the cache size is set by
264  // the user (by an Accessor object) or automatically (by TSMDataColumn).
265  virtual void setCacheSize (uInt cacheSize, Bool forceSmaller, Bool userSet);
266 
267  // Validate the cache size (in buckets).
268  // This means it will return the given cache size (in buckets) if
269  // smaller than the maximum cache size (given in MiB).
270  // Otherwise the maximum is returned.
271  // <group>
273  static uInt validateCacheSize (uInt cacheSize, uInt maxSizeMiB,
274  uInt bucketSize);
275  // </group>
276 
277  // Determine if the user set the cache size (using setCacheSize).
278  Bool userSetCache() const;
279 
280  // Functions for TSMDataColumn to keep track of the last type of
281  // access to a hypercube. It uses it to determine if the cache
282  // has to be reset.
283  // <group>
285  const IPosition& getLastColSlice() const;
286  void setLastColAccess (AccessType type);
287  void setLastColSlice (const IPosition& slice);
288  // </group>
289 
290 protected:
291  // Initialize the various variables.
292  // <group>
293  void setup();
294  void setupNrTiles();
295  // </group>
296 
297  // Adjust the tile shape to the hypercube shape.
298  // A size of 0 gets set to 1.
299  // A tile size > cube size gets set to the cube size.
301  const IPosition& tileShape) const;
302 
303  // Resize the IPosition member variables used in accessSection()
304  // if nrdim_p changes value.
305  void resizeTileSections();
306 
307 private:
308  // Forbid copy constructor.
309  TSMCube (const TSMCube&);
310 
311  // Forbid assignment.
312  TSMCube& operator= (const TSMCube&);
313 
314  // Get the cache object.
315  // This will construct the cache object if not present yet.
317 
318  // Construct the cache object (if not constructed yet).
319  virtual void makeCache();
320 
321  // Resync the cache object.
322  virtual void resyncCache();
323 
324  // Delete the cache object.
325  virtual void deleteCache();
326 
327  // Access a line in a more optimized way.
328  void accessLine (char* section, uInt pixelOffset,
329  uInt localPixelSize,
330  Bool writeFlag, BucketCache* cachePtr,
331  const IPosition& startTile, uInt endTile,
332  const IPosition& startPixelInFirstTile,
333  uInt endPixelInLastTile,
334  uInt lineIndex);
335 
336  // Define the callback functions for the BucketCache.
337  // <group>
338  static char* readCallBack (void* owner, const char* external);
339  static void writeCallBack (void* owner, char* external,
340  const char* local);
341  static char* initCallBack (void* owner);
342  static void deleteCallBack (void* owner, char* buffer);
343  // </group>
344 
345  // Define the functions doing the actual read and write of the
346  // data in the tile and converting it to/from local format.
347  // <group>
348  char* readTile (const char* external);
349  void writeTile (char* external, const char* local);
350  // </group>
351 
352 protected:
353  //# Declare member variables.
354 
355  char * cachedTile_p; // optimization to hold one tile chunk
356 
357  // Pointer to the parent storage manager.
359  // Is the class used directly or only by a derived class only?
361  // The values of the possible id and coordinate columns.
363  // Is the hypercube extensible?
365  // Dimensionality of the hypercube.
367  // Number of tiles in the hypercube.
369  // The shape of the hypercube.
371  // The shape of the tiles in the hypercube.
373  // The number of tiles in each hypercube dimension.
375  // Precomputed tileShape information.
377  // Precomputed tilesPerDim information.
379  // Number of tiles in all but last dimension (used when extending).
381  // The tilesize in bytes.
383  // Pointer to the TSMFile object holding the data.
385  // Offset in the TSMFile object where the data of this hypercube starts.
387  // Offset for each data column in a tile (in external format).
389  // Offset for each data column in a tile (in local format).
391  // The bucket size in bytes (is equal to tile size in bytes).
393  // The tile size in bytes in local format.
395  // The bucket cache.
397  // Did the user set the cache size?
399  // Was the last column access to a cell, slice, or column?
401  // The slice shape of the last column access to a slice.
403 
404  // IPosition variables used in accessSection(); declared here
405  // as member variables to avoid significant construction and
406  // desctruction overhead if they are local to accessSection()
407  // #tiles needed for the section
409  // First tile needed
411  // Last tile needed
413  // First pixel in first tile
415  // Last pixel in first tile
417  // Last pixel in last tile
419 };
420 
421 
422 
424 {
425  if (cache_p == 0) {
426  makeCache();
427  }
428  return cache_p;
429 }
430 inline uInt TSMCube::bucketSize() const
431 {
432  return bucketSize_p;
433 }
435 {
436  return localTileLength_p;
437 }
438 inline const IPosition& TSMCube::cubeShape() const
439 {
440  return cubeShape_p;
441 }
442 inline const IPosition& TSMCube::tileShape() const
443 {
444  return tileShape_p;
445 }
446 inline const Record& TSMCube::valueRecord() const
447 {
448  return values_p;
449 }
451 {
452  return values_p;
453 }
455 {
456  return userSetCache_p;
457 }
459 {
460  return lastColAccess_p;
461 }
462 inline const IPosition& TSMCube::getLastColSlice() const
463 {
464  return lastColSlice_p;
465 }
467 {
468  lastColAccess_p = type;
469 }
470 inline void TSMCube::setLastColSlice (const IPosition& slice)
471 {
472  lastColSlice_p.resize (slice.nelements());
473  lastColSlice_p = slice;
474 }
475 
476 
477 
478 
479 } //# NAMESPACE CASACORE - END
480 
481 #endif
static void deleteCallBack(void *owner, char *buffer)
A Vector of integers, for indexing into Array<T> objects.
Definition: IPosition.h:119
Bool extensible_p
Is the hypercube extensible?
Definition: TSMCube.h:364
void setLastColAccess(AccessType type)
Definition: TSMCube.h:466
IPosition lastColSlice_p
The slice shape of the last column access to a slice.
Definition: TSMCube.h:402
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition: aipsxtype.h:38
int Int
Definition: aipstype.h:50
virtual void showCacheStatistics(ostream &os) const
Show the cache statistics.
Cache for buckets in a part of a file.
Definition: BucketCache.h:217
void accessLine(char *section, uInt pixelOffset, uInt localPixelSize, Bool writeFlag, BucketCache *cachePtr, const IPosition &startTile, uInt endTile, const IPosition &startPixelInFirstTile, uInt endPixelInLastTile, uInt lineIndex)
Access a line in a more optimized way.
const Record & valueRecord() const
Get the record containing the id and coordinate values.
Definition: TSMCube.h:446
Bool isExtensible() const
Is the hypercube extensible?
Bool useDerived_p
Is the class used directly or only by a derived class only?
Definition: TSMCube.h:360
IPosition cellShape() const
Get the shape of the data cells in the cube.
Tiled hypercube in a table.
Definition: TSMCube.h:105
IPosition adjustTileShape(const IPosition &cubeShape, const IPosition &tileShape) const
Adjust the tile shape to the hypercube shape.
TSMCube(TiledStMan *stman, TSMFile *file, const IPosition &cubeShape, const IPosition &tileShape, const Record &values, Int64 fileOffset, Bool useDerived=False)
Construct the hypercube using the given file with the given shape.
AipsIO is the object persistency mechanism of Casacore.
Definition: AipsIO.h:168
TiledStMan * stmanPtr_p
Pointer to the parent storage manager.
Definition: TSMCube.h:358
IPosition endPixelInLastTile_p
Last pixel in last tile.
Definition: TSMCube.h:418
AccessType
Define the possible access types for TSMDataColumn.
Definition: TSMCube.h:109
Base class for Tiled Storage Manager classes.
Definition: TiledStMan.h:107
IPosition cubeShape_p
The shape of the hypercube.
Definition: TSMCube.h:370
uInt localTileLength_p
The tile size in bytes in local format.
Definition: TSMCube.h:394
uInt bucketSize() const
Get the bucket size (bytes).
Definition: TSMCube.h:430
static char * initCallBack(void *owner)
uInt localTileLength() const
Get the length of a tile (in bytes) in local format.
Definition: TSMCube.h:434
IPosition tileShape_p
The shape of the tiles in the hypercube.
Definition: TSMCube.h:372
AccessType getLastColAccess() const
Functions for TSMDataColumn to keep track of the last type of access to a hypercube.
Definition: TSMCube.h:458
TSMShape expandedTilesPerDim_p
Precomputed tilesPerDim information.
Definition: TSMCube.h:378
Block< uInt > localOffset_p
Offset for each data column in a tile (in local format).
Definition: TSMCube.h:390
void extendCoordinates(const Record &coordValues, const String &coordName, uInt length)
Extend the coordinates vector for the given coordinate to the given length with the given coordValues...
Bool userSetCache() const
Determine if the user set the cache size (using setCacheSize).
Definition: TSMCube.h:454
IPosition nrTileSection_p
IPosition variables used in accessSection(); declared here as member variables to avoid significant c...
Definition: TSMCube.h:408
void setup()
Initialize the various variables.
IPosition tilesPerDim_p
The number of tiles in each hypercube dimension.
Definition: TSMCube.h:374
IPosition startPixelInFirstTile_p
First pixel in first tile.
Definition: TSMCube.h:414
virtual void accessSection(const IPosition &start, const IPosition &end, char *section, uInt colnr, uInt localPixelSize, uInt externalPixelSize, Bool writeFlag)
Read or write a section in the cube.
uInt nelements() const
The number of elements in this IPosition.
Definition: IPosition.h:544
uInt nrdim_p
Dimensionality of the hypercube.
Definition: TSMCube.h:366
uInt nrTilesSubCube_p
Number of tiles in all but last dimension (used when extending).
Definition: TSMCube.h:380
uInt cacheSize() const
Get the current cache size (in buckets).
Int64 fileOffset_p
Offset in the TSMFile object where the data of this hypercube starts.
Definition: TSMCube.h:386
Int getObject(AipsIO &ios)
Get the data of the object from the AipsIO stream.
virtual void resync(AipsIO &ios)
Resync the object with the data file.
virtual void extend(uInt nr, const Record &coordValues, const TSMColumn *lastCoordColumn)
Extend the last dimension of the cube with the given number.
virtual void flushCache()
Flush the data in the cache.
AccessType lastColAccess_p
Was the last column access to a cell, slice, or column?
Definition: TSMCube.h:400
BucketCache * getCache()
Get the cache object.
Definition: TSMCube.h:423
Bool userSetCache_p
Did the user set the cache size?
Definition: TSMCube.h:398
LatticeExprNode length(const LatticeExprNode &expr, const LatticeExprNode &axis)
2-argument function to get the length of an axis.
void resizeTileSections()
Resize the IPosition member variables used in accessSection() if nrdim_p changes value.
void clearCache(Bool doFlush=True)
Clear the cache, so data will be reread.
uInt calcCacheSize(const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath) const
Calculate the cache size (in buckets) for the given slice and access path.
A hierarchical collection of named fields of various types.
Definition: Record.h:180
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
const IPosition & tileShape() const
Get the shape of the tiles.
Definition: TSMCube.h:442
IPosition startTile_p
First tile needed.
Definition: TSMCube.h:410
virtual ~TSMCube()
A column in the Tiled Storage Manager.
Definition: TSMColumn.h:97
TSMShape expandedTileShape_p
Precomputed tileShape information.
Definition: TSMCube.h:376
File object for Tiled Storage Manager.
Definition: TSMFile.h:81
void putObject(AipsIO &ios)
Put the data of the object into the AipsIO stream.
uInt validateCacheSize(uInt cacheSize) const
Validate the cache size (in buckets).
const Bool False
Definition: aipstype.h:44
uInt coordinateSize(const String &coordinateName) const
Get the size of a coordinate (i.e.
A drop-in replacement for Block<T*>.
Definition: Block.h:814
virtual void resyncCache()
Resync the cache object.
const IPosition & getLastColSlice() const
Definition: TSMCube.h:462
char * cachedTile_p
Definition: TSMCube.h:355
TSMCube & operator=(const TSMCube &)
Forbid assignment.
uInt nrTiles_p
Number of tiles in the hypercube.
Definition: TSMCube.h:368
IPosition endPixelInFirstTile_p
Last pixel in first tile.
Definition: TSMCube.h:416
uInt tileSize_p
The tilesize in bytes.
Definition: TSMCube.h:382
Expanded IPosition for shapes.
Definition: TSMShape.h:77
BucketCache * cache_p
The bucket cache.
Definition: TSMCube.h:396
virtual void makeCache()
Construct the cache object (if not constructed yet).
virtual void accessStrided(const IPosition &start, const IPosition &end, const IPosition &stride, char *section, uInt colnr, uInt localPixelSize, uInt externalPixelSize, Bool writeFlag)
Read or write a section in a strided way.
TSMFile * filePtr_p
Pointer to the TSMFile object holding the data.
Definition: TSMCube.h:384
const IPosition & cubeShape() const
Get the shape of the hypercube.
Definition: TSMCube.h:438
virtual void setCacheSize(const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath, Bool forceSmaller, Bool userSet)
Set the cache size for the given slice and access path.
void setLastColSlice(const IPosition &slice)
Definition: TSMCube.h:470
virtual void deleteCache()
Delete the cache object.
String: the storage and methods of handling collections of characters.
Definition: String.h:223
char * readTile(const char *external)
Define the functions doing the actual read and write of the data in the tile and converting it to/fro...
IPosition endTile_p
Last tile needed.
Definition: TSMCube.h:412
void resize(uInt newSize, Bool copy=True)
Old values are copied on resize if copy==True.
virtual void setShape(const IPosition &cubeShape, const IPosition &tileShape)
Set the hypercube shape.
static char * readCallBack(void *owner, const char *external)
Define the callback functions for the BucketCache.
static void writeCallBack(void *owner, char *external, const char *local)
Bool matches(const PtrBlock< TSMColumn *> &idColSet, const Record &idValues)
Test if the id values match.
Record values_p
The values of the possible id and coordinate columns.
Definition: TSMCube.h:362
void emptyCache()
Empty the cache.
Record & rwValueRecord()
Definition: TSMCube.h:450
uInt bucketSize_p
The bucket size in bytes (is equal to tile size in bytes).
Definition: TSMCube.h:392
const Bool True
Definition: aipstype.h:43
this file contains all the compiler specific defines
Definition: mainpage.dox:28
Block< uInt > externalOffset_p
Offset for each data column in a tile (in external format).
Definition: TSMCube.h:388
void writeTile(char *external, const char *local)
unsigned int uInt
Definition: aipstype.h:51