casacore
TableParse.h
Go to the documentation of this file.
1 //# TableParse.h: Classes to hold results from table grammar parser
2 //# Copyright (C) 1994,1995,1997,1998,1999,2000,2001,2003
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id$
27 
28 #ifndef TABLES_TABLEPARSE_H
29 #define TABLES_TABLEPARSE_H
30 
31 //# Includes
32 #include <casacore/casa/aips.h>
33 #include <casacore/tables/Tables/Table.h>
34 #include <casacore/tables/Tables/TableDesc.h>
35 #include <casacore/tables/TaQL/ExprNode.h>
36 #include <casacore/tables/TaQL/TaQLResult.h>
37 #include <casacore/tables/TaQL/ExprGroup.h>
38 #include <casacore/casa/BasicSL/String.h>
39 #include <casacore/casa/Utilities/Sort.h>
40 #include <casacore/casa/Containers/Record.h>
41 #include <casacore/casa/Containers/Block.h>
42 #include <map>
43 #include <vector>
44 #include <limits>
45 
46 namespace casacore { //# NAMESPACE CASACORE - BEGIN
47 
48 //# Forward Declarations
49 class TableExprNodeSet;
50 class TableExprNodeSetElem;
51 class TableExprNodeIndex;
52 class TableColumn;
53 class AipsIO;
54 template<class T> class Vector;
55 template<class T> class ArrayColumn;
56 
57 
58 // <summary>
59 // Class to hold values from table grammar parser
60 // </summary>
61 
62 // <use visibility=local>
63 
64 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="tTableGram">
65 // </reviewed>
66 
67 // <prerequisite>
68 //# Classes you should understand before using this one.
69 // </prerequisite>
70 
71 // <etymology>
72 // TableParse is the class used to parse a table command.
73 // </etymology>
74 
75 // <synopsis>
76 // TableParse is used by the parser of table select statements.
77 // The parser is written in Bison and Flex in files TableGram.y and .l.
78 // The statements in there use the routines in this file to act
79 // upon a reduced rule.
80 // Since multiple tables can be given (with a shorthand), the table
81 // names are stored in a container. The variable names can be qualified
82 // by the table name and will be looked up in the appropriate table.
83 //
84 // A select command is similar to SQL and can look like:
85 // SELECT columns FROM tab1 sh1, tab2 sh2, tab3 WHERE
86 // sh1.field == 3*sh1.field2 ... ORDERBY columns GIVING table
87 // This is described in more detail in TableGram.l.
88 //
89 // The class TableParse only contains information about a table
90 // used in the table command.
91 //
92 // Global functions are used to operate on the information.
93 // The main function is the global function tableCommand.
94 // It executes the given TaQL command and returns the resulting table.
95 // This is, in fact, the only function to be used by a user.
96 // </synopsis>
97 
98 // <motivation>
99 // It is necessary to be able to give a table select command in ASCII.
100 // This can be used in a CLI or in the table browser to get a subset
101 // of a table or to sort a table.
102 // </motivation>
103 
104 //# <todo asof="$DATE:$">
105 //# A List of bugs, limitations, extensions or planned refinements.
106 //# </todo>
107 
108 
110 {
111 
112 public:
113  // Default constructor for container class.
114  TableParse();
115 
116  // Copy constructor (copy semantics).
117  TableParse (const TableParse&);
118 
119  // Assignment (copy semantics).
121 
122  // Associate the table and the shorthand.
123  TableParse (const Table& table, const String& shorthand);
124 
125  // Test if shorthand matches.
126  Bool test (const String& shortHand) const;
127 
128  // Get the shorthand.
129  const String& shorthand() const;
130 
131  // Get table object.
132  const Table& table() const;
133 
134 private:
137 };
138 
139 
140 
141 // <synopsis>
142 // Parse and execute the given command.
143 // It will open (and close) all tables needed.
144 // It returns the resulting table.
145 // The command type (select or update) and the selected or updated
146 // column names can be returned.
147 // Zero or more temporary tables can be used in the command
148 // using the $nnn syntax.
149 // </synopsis>
150 // <group name=tableCommand>
151 TaQLResult tableCommand (const String& command);
152 
153 TaQLResult tableCommand (const String& command,
154  const Table& tempTable);
155 TaQLResult tableCommand (const String& command,
156  const std::vector<const Table*>& tempTables);
157 TaQLResult tableCommand (const String& command,
158  Vector<String>& columnNames);
159 TaQLResult tableCommand (const String& command,
160  Vector<String>& columnNames,
161  String& commandType);
162 TaQLResult tableCommand (const String& command,
163  const std::vector<const Table*>& tempTables,
164  Vector<String>& columnNames);
165 TaQLResult tableCommand (const String& command,
166  const std::vector<const Table*>& tempTables,
167  Vector<String>& columnNames,
168  String& commandType);
169 // </group>
170 
171 
172 
173 
174 // <summary>
175 // Helper class for sort keys in TableParse
176 // </summary>
177 
178 // <use visibility=local>
179 
180 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
181 // </reviewed>
182 
183 // <prerequisite>
184 //# Classes you should understand before using this one.
185 // <li> TableParse
186 // </prerequisite>
187 
188 // <etymology>
189 // TableParseSort holds a sort expression and order.
190 // </etymology>
191 
192 // <synopsis>
193 // A table command is parsed.
194 // An object of this class is used to hold the sort expression
195 // and sort order.
196 // </synopsis>
197 
198 
200 {
201 public:
202  // Construct from a given expression.
203  // The order is not given.
204  TableParseSort();
205 
206  // Construct from a given expression.
207  // The order is not given.
208  explicit TableParseSort (const TableExprNode&);
209 
210  // Construct from a given expression and for the given order.
212 
213  ~TableParseSort();
214 
215  // Get the expression node.
216  const TableExprNode& node() const;
217 
218  // Get the sort order.
219  Sort::Order order() const;
220 
221  // Is the order given?
222  Bool orderGiven() const;
223 
224 private:
225  // Check if the node results in a scalar and does not contain
226  // aggregate functions.
227  void checkNode() const;
228 
232 };
233 
234 
235 
236 
237 // <summary>
238 // Helper class for updates in TableParse
239 // </summary>
240 
241 // <use visibility=local>
242 
243 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
244 // </reviewed>
245 
246 // <prerequisite>
247 //# Classes you should understand before using this one.
248 // <li> TableParse
249 // </prerequisite>
250 
251 // <etymology>
252 // TableParseUpdate holds a column name, optional indices, optional mask,
253 // and an update expression.
254 // </etymology>
255 
256 // <synopsis>
257 // A table command is parsed.
258 // An object of this class is used to hold the column name, optional indices,
259 // and value expression for the UPDATE command.
260 // </synopsis>
261 
262 
264 {
265 public:
267  : indexPtr_p(0) {}
268 
269  // Construct from a column name and expression.
270  // By default it checks if no aggregate functions are used.
271  TableParseUpdate (const String& columnName,
272  const String& columnNameMask,
273  const TableExprNode&,
274  Bool checkAggr=True);
275 
276  // Construct from a column name, subscripts or mask, and expression.
277  // It checks if no aggregate functions are used.
278  TableParseUpdate (const String& columnName,
279  const String& columnNameMask,
280  const TableExprNodeSet& indices,
281  const TableExprNode&,
282  const TaQLStyle&);
283 
284  // Construct from a column name, subscripts and mask, and expression.
285  // It checks if no aggregate functions are used.
286  // It checks if one of the indices represents subscripts, the other a mask.
287  TableParseUpdate (const String& columnName,
288  const String& columnNameMask,
289  const TableExprNodeSet& indices1,
290  const TableExprNodeSet& indices2,
291  const TableExprNode&,
292  const TaQLStyle&);
293  // Handle the subscripts or mask.
294  // It checks if subscripts or mask was not already used.
295  void handleIndices (const TableExprNodeSet& indices,
296  const TaQLStyle& style);
297  ~TableParseUpdate();
298 
299  // Set the column name.
300  void setColumnName (const String& name);
301 
302  // Set the column name forthe mask.
303  void setColumnNameMask (const String& name);
304 
305  // Get the column name.
306  const String& columnName() const;
307 
308  // Get the possible column name for the mask.
309  const String& columnNameMask() const;
310 
311  // Tell if the mask is given first (i.e., before slice).
312  Bool maskFirst() const
313  { return maskFirst_p; }
314 
315  // Get the pointer to the indices.
316  TableExprNodeIndex* indexPtr() const;
317 
318  // Get the index expression node.
319  const TableExprNode& indexNode() const;
320 
321  // Get the expression node.
322  // <group>
323  const TableExprNode& node() const;
324  TableExprNode& node();
325  // </group>
326 
327  // Get the mask.
328  const TableExprNode& mask() const
329  { return mask_p; }
330 
331  // Adapt the possible unit of the expression to the possible unit
332  // of the column.
333  void adaptUnit (const Unit& columnUnit);
334 
335 private:
338  Bool maskFirst_p; //# True = mask is given before slice
339  TableExprNodeIndex* indexPtr_p; //# copy of pointer in indexNode_p
343 };
344 
345 
346 
347 
348 // <summary>
349 // Select-class for flex/bison scanner/parser for TableParse
350 // </summary>
351 
352 // <use visibility=local>
353 
354 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
355 // </reviewed>
356 
357 // <prerequisite>
358 //# Classes you should understand before using this one.
359 // <li> TableParse
360 // <li> TableGram.l and .y (flex and bison grammar)
361 // </prerequisite>
362 
363 // <synopsis>
364 // This class is needed for the the actions in the flex scanner
365 // and bison parser.
366 // This stores the information by constructing TableParse objects
367 // as needed and storing them in a vector.
368 // </synopsis>
369 
370 // <motivation>
371 // It is necessary to be able to give a table select command in ASCII.
372 // This can be used in a CLI or in the table browser to get a subset
373 // of a table or to sort a table.
374 // </motivation>
375 
376 //# <todo asof="$DATE:$">
377 //# A List of bugs, limitations, extensions or planned refinements.
378 //# </todo>
379 
380 
382 {
383 public:
384  enum CommandType {
393  PSHOW
394  };
395 
397  GROUPBY=1,
398  AGGR_FUNCS=2,
399  ONLY_COUNTALL=4
400  };
401 
402  // Construct.
404 
405  // Destructor.
406  ~TableParseSelect();
407 
408  // Return the command type.
410  { return commandType_p; }
411 
412  // Return the expression node.
414  { return node_p; }
415 
416  // Create a temporary table in no tables are given in FROM.
417  void makeTableNoFrom (const vector<TableParseSelect*>& stack);
418 
419  // Execute the select command (select/sort/projection/groupby/having/giving).
420  // The setInGiving flag tells if a set in the GIVING part is allowed.
421  // The mustSelect flag tells if a SELECT command must do something.
422  // Usually that is required, but not for a SELECT in an INSERT command.
423  // Optionally the maximum nr of rows to be selected can be given.
424  // It will be used as the default value for the LIMIT clause.
425  // 0 = no maximum.
426  void execute (Bool showTimings, Bool setInGiving,
427  Bool mustSelect, uInt maxRow, Bool doTracing=False);
428 
429  // Execute a query in a from clause resulting in a Table.
430  Table doFromQuery (Bool showTimings);
431 
432  // Execute a subquery and create an appropriate node for the result.
433  TableExprNode doSubQuery (Bool showTimings);
434 
435  // Test if a subquery has sufficient elements.
436  // It uses default LIMIT=1, but that can be overidden in the subquery.
437  // The flag tells if NOT EXISTS or EXISTS was given.
438  TableExprNode doExists (Bool noexists, Bool showTimings);
439 
440  // Show the expression tree.
441  void show (ostream& os) const;
442 
443  // Keep the selection expression.
444  void handleWhere (const TableExprNode&);
445 
446  // Keep the groupby expressions.
447  // It checks if they are all scalar expressions.
448  void handleGroupby (const vector<TableExprNode>&, Bool rollup);
449 
450  // Keep the having expression.
451  void handleHaving (const TableExprNode&);
452 
453  // Keep the expression of a calculate command.
454  void handleCalcComm (const TableExprNode&);
455 
456  // Keep the create table command.
457  void handleCreTab (const Record& dmInfo);
458 
459  // Keep the column specification in a create table command.
460  void handleColSpec (const String& columnName, const String& dataType,
461  const Record& spec, Bool isCOrder=False);
462 
463  // Reopen the table (for update) used in the ALTER TABLE command.
464  void handleAltTab();
465 
466  // Add columns to the table of ALTER TABLE.
467  // The column descriptions have already been added to tableDesc_p.
468  void handleAddCol (const Record& dmInfo);
469 
470  // Add a keyword or replace a keyword with the value of another keyword.
471  // The keywords can be table or column keywords (col::key).
472  ValueHolder getRecFld (const String& name);
473 
474  // Define a field with the given data type in the Record.
475  static void setRecFld (RecordInterface& rec,
476  const String& name,
477  const String& dtype,
478  const ValueHolder& vh);
479 
480  // Get the type string. If empty, it is made from the given
481  // data type.
482  static String getTypeString (const String& typeStr, DataType type);
483 
484  // Add a keyword or replace a keyword with a value.
485  // The keyword can be a table or column keyword (col::key).
486  // The data type string can be empty leaving the data type unchanged.
487  void handleSetKey (const String& name, const String& dtype,
488  const ValueHolder& value);
489 
490  // Rename a table or column keyword.
491  void handleRenameKey (const String& oldName, const String& newName);
492 
493  // Remove a table or column keyword.
494  void handleRemoveKey (const String& name);
495 
496  // Split the given name into optional shorthand, column and fields.
497  // Find the keywordset for it and fill in the final keyword name.
498  // It is a helper function for handleSetKey, etc.
499  TableRecord& findKeyword (const String& name, String& keyName);
500 
501  // Add an update object.
502  void addUpdate (const CountedPtr<TableParseUpdate>& upd);
503 
504  // Set the insert expressions for all rows.
505  void setInsertExprs (const std::vector<TableExprNode> exprs)
506  { insertExprs_p = exprs; }
507 
508  // Keep the update expressions.
509  void handleUpdate();
510 
511  // Make ready for the insert expression.
512  // The first one uses values (added via addUpdate),
513  // the second one a subquery.
514  // <group>
515  void handleInsert();
516  void handleInsert (TableParseSelect* sel);
517  // </group>
518 
519  // Make ready for a COUNT command.
520  // It checks if all column expressions are scalar.
521  void handleCount();
522 
523  // Keep the sort expressions.
524  void handleSort (const std::vector<TableParseSort>& sortList,
525  Bool noDuplicates, Sort::Order defaultSortOrder);
526 
527  // Evaluate and keep limit/offset/stride given as start:end:incr
528  void handleLimit (const TableExprNodeSetElem& expr);
529 
530  // Evaluate and keep the limit value.
531  void handleLimit (const TableExprNode& expr);
532 
533  // Evaluate and keep the offset value.
534  void handleOffset (const TableExprNode& expr);
535 
536  // Evaluate and add the rows.
537  void handleAddRow (const TableExprNode& expr);
538 
539  // Add a table nr, name, or object to the container.
540  void addTable (Int tabnr, const String& name,
541  const Table& table,
542  const String& shorthand,
543  Bool addToFromList,
544  const vector<const Table*> tempTables,
545  const vector<TableParseSelect*>& stack);
546 
547  // Make a Table object for given name, seqnr or so.
548  // If <src>alwaysOpen=False</src> the table will only be looked up,
549  // but not opened if not found. This is meant for concatenated tables
550  // in TaQLNodeHandler.
551  Table makeTable (Int tabnr, const String& name,
552  const Table& ftab,
553  const String& shorthand,
554  const vector<const Table*> tempTables,
555  const vector<TableParseSelect*>& stack,
556  Bool alwaysOpen=True);
557 
558  // Replace the first table (used by CALC command).
559  void replaceTable (const Table& table);
560 
561  // Find the keyword or column name and create a TableExprNode from it.
562  // If <src>tryProj=True</src> it is first tried if the column is a coluymn
563  // in the projected table (i.e., result from the SELECT part).
564  TableExprNode handleKeyCol (const String& name, Bool tryProj);
565 
566  // Handle a slice operator.
567  static TableExprNode handleSlice (const TableExprNode& array,
568  const TableExprNodeSet& indices,
569  const TaQLStyle&);
570 
571  // Handle a function.
572  TableExprNode handleFunc (const String& name,
573  const TableExprNodeSet& arguments,
574  const TaQLStyle&);
575 
576  // Make a function object node for the given function name and arguments.
577  // The ignoreFuncs vector contains invalid function codes.
578  static TableExprNode makeFuncNode (TableParseSelect*,
579  const String& name,
580  const TableExprNodeSet& arguments,
581  const Vector<int>& ignoreFuncs,
582  const Table& table,
583  const TaQLStyle&);
584 
585  // Add a column to the list of column names.
586  void handleColumn (Int type, const String& name, const TableExprNode& expr,
587  const String& newName, const String& nameMask,
588  const String& newDtype);
589 
590  // Finish the addition of columns to the list of column names.
591  void handleColumnFinish (Bool distinct);
592 
593  // Set the DataManager info for a new table.
594  void setDMInfo (const Record& dminfo)
595  { dminfo_p = dminfo;}
596 
597  // Handle the name and type given in a GIVING clause.
598  void handleGiving (const String& name, const Record& type);
599 
600  // Handle the set given in a GIVING clause.
601  void handleGiving (const TableExprNodeSet&);
602 
603  // Get the projected column names.
604  const Block<String>& getColumnNames() const;
605 
606  // Get the resulting table.
607  const Table& getTable() const;
608 
609  // An exception is thrown if the node uses an aggregate function.
610  static void checkAggrFuncs (const TableExprNode& node);
611 
612  // Show the structure of fromTables_p[0] using the options given in parts[2:].
613  String getTableInfo (const Vector<String>& parts, const TaQLStyle& style);
614 
615  // Split a name into its parts (shorthand, column and field names).
616  // True is returned if the name contained a keyword part.
617  // In that case fieldNames contains the keyword name and the possible
618  // subfields. The possible shorthand and the column name are
619  // filled in if it is a column keyword.
620  // If the name represents a column, fieldNames contains the subfields
621  // of the column (for the case where the column contains records).
622  // If the name is invalid, an exception is thrown if checkError=True.
623  // Otherwise the name is treated as a normal name without keyword.
624  // If allowEmtpy is True, :: is allowed, otherwise an error is thrown.
625  static Bool splitName (String& shorthand, String& columnName,
626  Vector<String>& fieldNames, const String& name,
627  Bool checkError, Bool isKeyword, Bool allowNoKey);
628 
629 private:
630  // Test if groupby or aggregate functions are given.
631  // <br> bit 0: on = groupby is given
632  // <br> bit 1: on = aggregate functions are given
633  // <br> bit 2: on = only select count(*) aggregate function is given
634  Int testGroupAggr (std::vector<TableExprNodeRep*>& aggr) const;
635 
636  // Get the aggregate functions used in SELECT and HAVING.
637  vector<TableExprNodeRep*> getAggrNodes() const;
638 
639  // Try to make a UDF function node for the given function name and arguments.
640  static TableExprNode makeUDFNode (TableParseSelect*,
641  const String& name,
642  const TableExprNodeSet& arguments,
643  const Table& table,
644  const TaQLStyle&);
645 
646  // Find the function code belonging to a function name.
647  // Functions to be ignored can be given (as function type values).
648  // If the function name is unknown, NRFUNC is returned.
649  static TableExprFuncNode::FunctionType findFunc (const String& name,
650  uInt narguments,
651  const Vector<Int>& ignoreFuncs);
652 
653  // Do the update step.
654  // Rows 0,1,2,.. in UpdTable are updated from the expression result
655  // for the rows in the given rownrs vector.
656  void doUpdate (Bool showTimings, const Table& origTable,
657  Table& updTable, const Vector<uInt>& rownrs,
658  const CountedPtr<TableExprGroupResult>& groups =
660 
661  // Do the insert step and return a selection containing the new rows.
662  Table doInsert (Bool showTimings, Table& table);
663 
664  // Do the delete step.
665  void doDelete (Bool showTimings, Table& table);
666 
667  // Do the count step returning a memory table containing the unique
668  // column values and the counts of the column values.
669  Table doCount (Bool showTimings, const Table&);
670 
671  // Do the projection step returning a table containing the projection.
672  Table doProject (Bool showTimings, const Table&,
673  const CountedPtr<TableExprGroupResult>& groups =
675 
676  // Do the projection containing column expressions.
677  // Use the selected or unselected columns depending on <src>useSel</src>.
678  Table doProjectExpr (Bool useSel,
679  const CountedPtr<TableExprGroupResult>& groups);
680 
681  // Create a table using the given parameters.
682  // The variables set by handleGiven are used for name and type.
683  Table createTable (const TableDesc& td,
684  Int64 nrow, const Record& dmInfo);
685 
686  // Make the (empty) table for the epxression in the SELECT clause.
687  void makeProjectExprTable();
688 
689  // Fill projectExprSelColumn_p telling the columns to be projected
690  // at the first stage.
691  void makeProjectExprSel();
692 
693  // Add a column node to applySelNodes_p.
694  void addApplySelNode (const TableExprNode& node)
695  { applySelNodes_p.push_back (node); }
696 
697  // Set the selected rows for the column objects in applySelNodes_p.
698  // These nodes refer the original table. They requires different row
699  // numbers than the selected groups and projected columns.
700  // rownrs_p is changed to use row 0..n.
701  // It returns the Table containing the subset of rows in the input Table.
702  Table adjustApplySelNodes (const Table&);
703 
704  // Do the groupby/aggregate step and return its result.
706  (bool showTimings, const std::vector<TableExprNodeRep*> aggrNodes,
707  Int groupAggrUsed);
708 
709  // Do the HAVING step.
710  void doHaving (Bool showTimings,
711  const CountedPtr<TableExprGroupResult>& groups);
712 
713  // Do a groupby/aggregate step that only does a 'select count(*)'.
714  CountedPtr<TableExprGroupResult> doOnlyCountAll (TableExprNodeRep* aggrNode);
715 
716  // Do a full groupby/aggregate step.
718  (const std::vector<TableExprNodeRep*>& aggrNodes);
719 
720  // Do the sort step.
721  void doSort (Bool showTimings);
722 
723  // Do the limit/offset step.
724  void doLimOff (Bool showTimings);
725  Table doLimOff (Bool showTimings, const Table& table);
726 
727  // Do the 'select distinct' step.
728  Table doDistinct (Bool showTimings, const Table& table);
729 
730  // Finish the table (rename, copy, and/or flush).
731  Table doFinish (Bool showTimings, Table& table);
732 
733  // Update the values in the columns (helpers of doUpdate).
734  // <group>
735  template<typename TCOL, typename TNODE>
736  void updateValue (uInt row, const TableExprId& rowid,
737  Bool isScalarCol, const TableExprNode& node,
738  const Array<Bool>& mask, Bool maskFirst,
739  TableColumn& col, const Slicer* slicerPtr,
740  ArrayColumn<Bool>& maskCol);
741  template<typename TCOL, typename TNODE>
742  void updateScalar (uInt row, const TableExprId& rowid,
743  const TableExprNode& node,
744  TableColumn& col);
745  template<typename TCOL, typename TNODE>
746  void updateArray (uInt row, const TableExprId& rowid,
747  const TableExprNode& node,
748  const Array<TNODE>& res,
749  ArrayColumn<TCOL>& col);
750  template<typename TCOL, typename TNODE>
751  void updateSlice (uInt row, const TableExprId& rowid,
752  const TableExprNode& node,
753  const Array<TNODE>& res,
754  const Slicer& slice,
755  ArrayColumn<TCOL>& col);
756  template<typename TCOL, typename TNODE>
757  void copyMaskedValue (uInt row, ArrayColumn<TCOL>& acol,
758  const Slicer* slicerPtr,
759  const TNODE* val,
760  uInt incr, const Array<Bool>& mask);
761  Array<Bool> makeMaskSlice (const Array<Bool>& mask,
762  Bool maskFirst,
763  const IPosition& shapeCol,
764  const Slicer* slicerPtr);
765  void checkMaskColumn (Bool hasMask,
766  const ArrayColumn<Bool>& maskCol,
767  const TableColumn& col);
768  // </group>
769 
770  // Make a data type from the string.
771  // It checks if it is compatible with the given (expression) data type.
772  DataType makeDataType (DataType dtype, const String& dtstr,
773  const String& colName);
774 
775  // Get the order for this key. Use the default order_p if not
776  // explicitly given with the key.
777  Sort::Order getOrder (const TableParseSort& key) const;
778 
779  // Make an array from the contents of a column in a subquery.
780  TableExprNode getColSet();
781 
782  // Make a set from the results of the subquery.
783  TableExprNode makeSubSet() const;
784 
785  // Evaluate an int scalar expression.
786  Int64 evalIntScaExpr (const TableExprNode& expr) const;
787 
788  // Find a table for the given shorthand.
789  // Optionally the WITH tables are searched as well.
790  // If no shorthand is given, the first table is returned (if there).
791  // If not found, a null Table object is returned.
792  Table findTable (const String& shorthand, Bool doWith) const;
793 
794  // Handle the selection of a wildcarded column name.
795  void handleWildColumn (Int stringType, const String& name);
796 
797  // Add the description of a column to the table description.
798  // ndim < 0 means a scalar column.
799  void addColumnDesc (TableDesc& td, DataType dtype,
800  const String& colName, Int options,
801  Int ndim, const IPosition& shape,
802  const String& dmType, const String& dmGroup,
803  const String& comment,
804  const TableRecord& keywordSet,
805  const Vector<String>& unitName,
806  const Record& attributes);
807 
808  // Find the names of all stored columns in a table.
809  Block<String> getStoredColumns (const Table& tab) const;
810 
811  // Try to find the keyword representing a table in one of the tables
812  // in any select block (from inner to outer).
813  // If not found, an exception is thrown.
814  static Table tableKey (const String& fullName,
815  const String& shorthand, const String& columnName,
816  const Vector<String>& fieldNames,
817  const vector<TableParseSelect*>& stack);
818 
819  // Try to find the keyword representing a table in the given table.
820  // If the columnName is empty, the keyword is a table keyword.
821  // If not found, a null Table object is returned.
822  static Table findTableKey (const Table& table, const String& columnName,
823  const Vector<String>& keyNames);
824 
825  // Check if the tables used in selection columns have the same
826  // size as the first table given in FROM.
827  void checkTableProjSizes() const;
828 
829  // Create the set of aggregate functions and groupby keys in case
830  // a single groupby key is given.
831  // This offers much faster map access then doGroupByAggrMultiple.
832  template<typename T>
833  std::vector<CountedPtr<TableExprGroupFuncSet> > doGroupByAggrSingleKey
834  (const vector<TableExprNodeRep*>& aggrNodes)
835  {
836  // We have to group the data according to the (possibly empty) groupby.
837  // We step through the table in the normal order which may not be the
838  // groupby order.
839  // A map<key,int> is used to keep track of the results where the int
840  // is the index in a vector of a set of aggregate function objects.
841  vector<CountedPtr<TableExprGroupFuncSet> > funcSets;
842  std::map<T, int> keyFuncMap;
843  T lastKey = std::numeric_limits<T>::max();
844  int groupnr = -1;
845  // Loop through all rows.
846  // For each row generate the key to get the right entry.
847  TableExprId rowid(0);
848  T key;
849  for (uInt i=0; i<rownrs_p.size(); ++i) {
850  rowid.setRownr (rownrs_p[i]);
851  groupbyNodes_p[0].get (rowid, key);
852  if (key != lastKey) {
853  typename std::map<T, int>::iterator iter = keyFuncMap.find (key);
854  if (iter == keyFuncMap.end()) {
855  groupnr = funcSets.size();
856  keyFuncMap[key] = groupnr;
857  funcSets.push_back (new TableExprGroupFuncSet (aggrNodes));
858  } else {
859  groupnr = iter->second;
860  }
861  }
862  rowid.setRownr (rownrs_p[i]);
863  funcSets[groupnr]->apply (rowid);
864  }
865  return funcSets;
866  }
867 
868  // Create the set of aggregate functions and groupby keys in case
869  // multiple keys are given.
870  std::vector<CountedPtr<TableExprGroupFuncSet> > doGroupByAggrMultipleKeys
871  (const vector<TableExprNodeRep*>& aggrNodes);
872 
873  //# Command type.
875  //# Table description for a series of column descriptions.
877  //# Vector of TableParse objects (from WITH and FROM clause).
878  //# This is needed for the functions above, otherwise they have no
879  //# way to communicate.
880  vector<TableParse> withTables_p;
881  vector<TableParse> fromTables_p;
882  //# Block of selected column names (new name in case of select).
884  //# Block of selected mask column names (for masked arrays).
886  //# Block of selected column expressions.
888  //# The old name for a selected column.
890  //# The new data type for a column.
892  //# The keywords used in a column.
894  //# Number of real expressions used in selected columns.
896  //# Distinct values in output?
898  //# Name and type of the resulting table (from GIVING part).
900  uInt resultType_p; //# 0-unknown 1=memory 2=scratch 3=plain
901  Bool resultCreated_p; //# Has the result table been created?
906  //# Resulting set (from GIVING part).
908  //# The WHERE expression tree.
910  //# The GROUPBY expressions.
911  vector<TableExprNode> groupbyNodes_p;
912  Bool groupbyRollup_p; //# use ROLLUP in GROUPBY?
913  //# The HAVING expression.
915  //# The possible limit (= max nr of selected rows) (0 means no limit).
917  //# The possible last row (0 means no end; can be <0).
918  //# limit_p and endrow_p cannot be both !=0.
920  //# The possible offset (= nr of selected rows to skip).
922  //# The possible stride in offset:endrow:stride.
924  //# The update and insert list.
925  std::vector<CountedPtr<TableParseUpdate>> update_p;
926  //# The insert expressions (possibly for multiple rows).
927  std::vector<TableExprNode> insertExprs_p;
928  //# The table selection to be inserted.
930  //# The sort list.
931  std::vector<TableParseSort> sort_p;
932  //# The noDuplicates sort switch.
934  //# The default sort order.
936  //# All nodes that need to be adjusted for a selection of rownrs.
937  //# It can consist of column nodes and the rowid function node.
938  //# Some nodes (in aggregate functions) can later be disabled for adjustment.
939  vector<TableExprNode> applySelNodes_p;
940  //# The resulting table.
942  //# The first table used when creating a column object.
943  //# All other tables used for them should have the same size.
946  //# The table resulting from a projection with expressions.
948  //# The projected columns used in the HAVING and ORDERBY clauses.
951  //# The resulting row numbers.
953 };
954 
955 
956 
957 //# Implement the inline functions.
958 inline Bool TableParse::test (const String& str) const
959  { return (shorthand_p == str ? True : False); }
960 
961 inline const String& TableParse::shorthand() const
962  { return shorthand_p; }
963 
964 inline const Table& TableParse::table() const
965  { return table_p; }
966 
967 
968 inline void TableParseUpdate::setColumnName (const String& name)
969  { columnName_p = name; }
971  { columnNameMask_p = name; }
973  { return columnName_p; }
975  { return columnNameMask_p; }
977  { return indexPtr_p; }
979  { return indexNode_p; }
981  { return node_p; }
983  { return node_p; }
984 inline void TableParseUpdate::adaptUnit (const Unit& columnUnit)
985  { node_p.adaptUnit (columnUnit); }
986 
987 inline const TableExprNode& TableParseSort::node() const
988  { return node_p; }
990  { return given_p; }
992  { return order_p; }
993 
994 
996  { return columnNames_p; }
997 
998 inline const Table& TableParseSelect::getTable() const
999  { return table_p; }
1000 
1002  { update_p.push_back (upd); }
1003 
1005  { return (key.orderGiven() ? key.order() : order_p); }
1006 
1007 
1008 } //# NAMESPACE CASACORE - END
1009 
1010 #endif
A Vector of integers, for indexing into Array<T> objects.
Definition: IPosition.h:119
Block< Bool > projectExprSelColumn_p
Definition: TableParse.h:950
A 1-D Specialization of the Array class.
Definition: ArrayIO.h:45
vector< TableExprNode > groupbyNodes_p
Definition: TableParse.h:911
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition: aipsxtype.h:38
void setColumnName(const String &name)
Set the column name.
Definition: TableParse.h:968
int Int
Definition: aipstype.h:50
TableExprNodeSet * resultSet_p
Definition: TableParse.h:907
Bool orderGiven() const
Is the order given?
Definition: TableParse.h:989
EndianFormat
Define the possible endian formats in which table data can be stored.
Definition: Table.h:192
Main interface class to a read/write table.
Definition: Table.h:153
TableExprNodeIndex * indexPtr_p
Definition: TableParse.h:339
LatticeExprNode mask(const LatticeExprNode &expr)
This function returns the mask of the given expression.
std::vector< TableExprNode > insertExprs_p
Definition: TableParse.h:927
vector< TableParse > withTables_p
Definition: TableParse.h:880
TableExprNode indexNode_p
Definition: TableParse.h:340
TableExprNode array(const TableExprNode &values, const TableExprNodeSet &shape)
Create an array of the given shape and fill it with the values.
Definition: ExprNode.h:1886
Class to hold multiple table expression nodes.
Definition: ExprNodeSet.h:311
LatticeExprNode max(const LatticeExprNode &left, const LatticeExprNode &right)
Handle class for a table column expression tree.
Definition: ExprNode.h:155
TableExprNode node_p
Definition: TableParse.h:229
Order
Enumerate the sort order:
Definition: Sort.h:260
void setRownr(uInt rownr)
Set the row number.
Definition: TableExprId.h:185
void setColumnNameMask(const String &name)
Set the column name forthe mask.
Definition: TableParse.h:970
TableExprNodeIndex * indexPtr() const
Get the pointer to the indices.
Definition: TableParse.h:976
Sort::Order getOrder(const TableParseSort &key) const
Get the order for this key.
Definition: TableParse.h:1004
void adaptUnit(const Unit &columnUnit)
Adapt the possible unit of the expression to the possible unit of the column.
Definition: TableParse.h:984
Abstract base class for a node in a table column expression tree.
Definition: ExprNodeRep.h:157
TableParse & operator=(const TableParse &)
Assignment (copy semantics).
Block< String > columnNameMasks_p
Definition: TableParse.h:885
const Table & table() const
Get table object.
Definition: TableParse.h:964
Options defining how table files are organized.
Definition: StorageOption.h:76
defines physical units
Definition: Unit.h:189
vector< TableExprNode > applySelNodes_p
Definition: TableParse.h:939
vector< TableParse > fromTables_p
Definition: TableParse.h:881
Select-class for flex/bison scanner/parser for TableParse.
Definition: TableParse.h:381
Sort::Order order() const
Get the sort order.
Definition: TableParse.h:991
Referenced counted pointer for constant data.
Definition: CountedPtr.h:80
Bool maskFirst() const
Tell if the mask is given first (i.e., before slice).
Definition: TableParse.h:312
const TableExprNode & node() const
Get the expression node.
Definition: TableParse.h:980
Table::EndianFormat endianFormat_p
Definition: TableParse.h:903
Class to hold the table expression nodes for an element in a set.
Definition: ExprNodeSet.h:94
Class to hold the result of a TaQL command.
Definition: TaQLResult.h:67
Vector< uInt > rownrs_p
Definition: TableParse.h:952
A holder for a value of any basic Casacore data type.
Definition: ValueHolder.h:67
Class with static members defining the TaQL style.
Definition: TaQLStyle.h:64
LatticeExprNode ndim(const LatticeExprNode &expr)
1-argument function to get the dimensionality of a lattice.
CommandType commandType() const
Return the command type.
Definition: TableParse.h:409
Block< uInt > projectExprSubset_p
Definition: TableParse.h:949
Block< String > columnDtypes_p
Definition: TableParse.h:891
const TableExprNode & mask() const
Get the mask.
Definition: TableParse.h:328
const Block< String > & getColumnNames() const
Get the projected column names.
Definition: TableParse.h:995
const String & columnNameMask() const
Get the possible column name for the mask.
Definition: TableParse.h:974
Block< TableRecord > columnKeywords_p
Definition: TableParse.h:893
A hierarchical collection of named fields of various types.
Definition: Record.h:180
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
Bool test(const String &shortHand) const
Test if shorthand matches.
Definition: TableParse.h:958
Read/write access to a table column.
Definition: TableColumn.h:98
Class containing the results of aggregated values in a group.
Definition: ExprGroup.h:801
Block< String > columnNames_p
Definition: TableParse.h:883
const Bool False
Definition: aipstype.h:44
TableExprNode shape(const TableExprNode &array)
Function operating on any scalar or array resulting in a Double array containing the shape...
Definition: ExprNode.h:1944
A hierarchical collection of named fields of various types.
Definition: TableRecord.h:182
Block< TableExprNode > columnExpr_p
Definition: TableParse.h:887
TableExprNode getNode() const
Return the expression node.
Definition: TableParse.h:413
TableParse()
Default constructor for container class.
Helper class for sort keys in TableParse.
Definition: TableParse.h:199
Specify which elements to extract from an n-dimensional array.
Definition: Slicer.h:289
simple 1-D array
Definition: ArrayIO.h:47
const TableExprNode & node() const
Get the expression node.
Definition: TableParse.h:987
void addUpdate(const CountedPtr< TableParseUpdate > &upd)
Add an update object.
Definition: TableParse.h:1001
Class to hold values from table grammar parser.
Definition: TableParse.h:109
TableExprNode havingNode_p
Definition: TableParse.h:914
TableParseSelect * insSel_p
Definition: TableParse.h:929
The identification of a TaQL selection subject.
Definition: TableExprId.h:97
std::vector< CountedPtr< TableParseUpdate > > update_p
Definition: TableParse.h:925
void setInsertExprs(const std::vector< TableExprNode > exprs)
Set the insert expressions for all rows.
Definition: TableParse.h:505
Block< String > columnOldNames_p
Definition: TableParse.h:889
const TableExprNode & indexNode() const
Get the index expression node.
Definition: TableParse.h:978
String: the storage and methods of handling collections of characters.
Definition: String.h:223
Define the structure of a Casacore table.
Definition: TableDesc.h:187
const Table & getTable() const
Get the resulting table.
Definition: TableParse.h:998
TaQLResult tableCommand(const String &command)
Abstract base class for Record classes.
StorageOption storageOption_p
Definition: TableParse.h:902
const String & shorthand() const
Get the shorthand.
Definition: TableParse.h:961
The index of an array element in a table select expression.
void addApplySelNode(const TableExprNode &node)
Add a column node to applySelNodes_p.
Definition: TableParse.h:694
const Bool True
Definition: aipstype.h:43
const String & columnName() const
Get the column name.
Definition: TableParse.h:972
this file contains all the compiler specific defines
Definition: mainpage.dox:28
LatticeExprNode value(const LatticeExprNode &expr)
This function returns the value of the expression without a mask.
unsigned int uInt
Definition: aipstype.h:51
Helper class for updates in TableParse.
Definition: TableParse.h:263
std::vector< TableParseSort > sort_p
Definition: TableParse.h:931
void setDMInfo(const Record &dminfo)
Set the DataManager info for a new table.
Definition: TableParse.h:594